Skip to content

Instantly share code, notes, and snippets.

@lukebakken
Last active August 29, 2015 14:11
Show Gist options
  • Save lukebakken/65cff6499315dc66d7f7 to your computer and use it in GitHub Desktop.
Save lukebakken/65cff6499315dc66d7f7 to your computer and use it in GitHub Desktop.
Load Riak Memory backend
#!/usr/bin/env bash
set -o errexit
set -o nounset
declare -r curl_cmd='curl --ipv4'
declare -r bucket_name='memtest'
declare -r random_data_file=/tmp/random_ascii_data.txt
declare -r load_done_file=/tmp/load_data_done.$$
declare -r myfifo=/tmp/test-memory-backend.fifo.$$
declare -r script_path="$(realpath $0)"
declare -r script_dir="$(dirname $script_path)"
declare -r riak_dev_dir="$HOME/Projects/basho/riak-memtest/dev"
function on_exit
{
cleanup
exec 4>&-
}
trap on_exit EXIT
function cleanup
{
rm -vf $random_data_file
rm -vf $load_done_file
rm -vf $myfifo
}
function setup
{
cleanup
set +o errexit
for dev in dev{1..4}
do
mkdir "$script_dir/$dev"
done
set -o errexit
}
function cluster_info
{
set +o errexit
while [[ ! -f $load_done_file ]]
do
sleep 600
now=$(date '+%s')
for dev in $riak_dev_dir/dev*
do
devdir=${dev##*/}
$dev/bin/riak-admin cluster-info $script_dir/$devdir/cluster-info-$now.html local
done
done
set -o errexit
}
function collect_stats
{
while [[ ! -f $load_done_file ]]
do
now=$(date '+%s')
for dev in $riak_dev_dir/dev*
do
devdir=${dev##*/}
$dev/bin/riak-admin status > $devdir/$now.out 2>&1 &
done
sleep 60
done
}
function generate_random_ascii_data
{
local -i data_count=$(((RANDOM % 4) + 8))
dd if=/dev/urandom bs=1024 count=$data_count 2>/dev/null | openssl enc -base64 > $random_data_file
}
function load_data
{
local -rai ports=(10018 10028 10038 10048)
local -ri ports_len=${#ports[@]}
local -ri count=8388608
local -ri max_curl_procs=256
local -i running_curl_procs=0
local -i cpid=0
mkfifo -m 0700 "$myfifo"
exec 4<> "$myfifo" # hook pipe up to descriptor 4 for read/write
generate_random_ascii_data
local -i i=0
for ((i = 0; i < count; ++i))
do
if (( (i % 65536) == 0 ))
then
generate_random_ascii_data
fi
while (( running_curl_procs >= max_curl_procs ))
do
cpid=0
if read -u 4 cpid
then
wait $cpid
(( --running_curl_procs ))
fi
done
(
local -i ports_idx=$((RANDOM % ports_len))
local -i port=${ports[$ports_idx]}
$curl_cmd -XPUT -H'Content-type: text/plain' "localhost:$port/buckets/$bucket_name/keys/$i" --data-ascii @$random_data_file
echo $BASHPID 1>&4
) &
(( ++running_curl_procs ))
done
touch $load_done_file
}
# storage_backend = memory
# memory_backend.ttl = 90s
# memory_backend.max_memory_per_vnode = 1MB
#
# $curl_cmd -vvv -XPUT -H'Content-type: application/json' localhost:10018/buckets/$bucket_name/props --data '{"props":{"backend":"mem_backend"}}'
# $curl_cmd -vvv -XGET localhost:10018/buckets/$bucket_name/props
setup
load_data &
sleep 5
collect_stats &
cluster_info &
# TODO [erlang:garbage_collect(Pid) || Pid <- processes()].
echo ''
echo -n 'Loading data and collecting stats...'
wait
echo DONE
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment