gistfile1.py

#!/usr/bin/python

### Example usage: $0 {symbolic-host1} {s-h2} ... {s-host12}

from be_setup import *
from wrap_systest_cluster_membership import *

import sys, time
symbolic_hosts = sys.argv[1:]

print 'Symbolic hosts are %s' % symbolic_hosts
if len(symbolic_hosts) < 12:
    print 'Sorry, argv list contains less than 12 hosts.'
    exit(1)
elif len(symbolic_hosts) > 12:
    print ''
    print 'NOTE: Host list is length %d' % len(symbolic_hosts)
    print '      Only the first 12 hosts in that list will be used.'
    print ''
    time.sleep(3)

verbose = 1,
timeout = 180

## It would be cool/"fast" to be able to do this in parallel, but I would
## rather not do threading stuff in Python....
duts = []
duts_host_info = []
duts_os = []
for symb in symbolic_hosts:
    (dut, dut_host_info, dut_os) = login_to_symbolic_host(symb)
    print 'Logged in to %s' % symb
    dut.verbose(verbose)
    if timeout == -1:
        timeout = dut.timeout
    dut.timeout = timeout
    duts.append(dut)
    duts_host_info.append(dut_host_info)
    duts_os.append(dut_os)

from basho_expect import *
from repos_map import *
from systest_python_unit_tests import *  ## for app.config mangling helpers
from install_riak_package import *
from uninstall_riak_package import *

def repeat_read_of_all_keys_all_nodes(some_duts, num_test_keys,
                                      bucket, body_prefix, rep_limit = 14):
    for dut in some_duts:
        go_again = True
        i = 0
        while (go_again and i < rep_limit):
            i = i + 1
            n = get_some_keys(dut, num_test_keys, bucket = bucket,
                              body_prefix = body_prefix, must_succeed = False)
            dut.status_msg('get_some_keys', 'iter %d: got %d of %d keys' % (i, n, num_test_keys))
            go_again = (n != num_test_keys)
            if go_again:
                dut.cmd('/tmp/riak-repl status')
                dut.ok_cmd('sleep 10')
        if i >= rep_limit and (n != num_test_keys):
            dut.admin_transfers(must_succeed = False)
            raise IOError ('repeat read limit reached')

do_install_from_scratch = True,
do_assemble_from_scratch = True,
current_version = True

for i in range(0, len(duts)):
    duts[i].step('Uninstall riak package harshly on %s' % duts[i].symbolic_host)
    try:
        uninstall_riak_package(duts[i], duts_host_info[i], duts_os[i],
                               delete_riak_dirs = True)
    except:
        True
    duts[i].step('Install riak package on %s' % duts[i].symbolic_host)
    install_riak_package(duts[i], duts_host_info[i], duts_os[i])

n_tries = 8
settle_timeout = 601
dut0 = duts[0]
(duts_pkg_details, duts_hostname, duts_nodename) = \
    get_duts_pkg_details(duts, duts_host_info, duts_os)

dut0.step('Edit app.config, then start Riak on range(0, 6)')

for i in range(0, 6):
    systest_python_copy_conf_port_bindings_all(
        duts[i], duts_host_info[i], duts_os[i],
        app_config_edits = [("riak_core", "ring_creation_size", "512")])
    duts[i].control_start()

dut0.step('Join riak nodes range(1, 6) to 0')

for i in range(0, 6):
    duts[i].admin_join('%s@%s' % (duts_pkg_details[0]['node_lhs'],
                                  duts_host_info[0]['host']))
cookie = get_erlang_vm_cookie(dut0)
## Hrm, this one seems flakey?  Perhaps I should remove this func?
## wait_for_ring_to_settle(cookie, duts_nodename[0:6], timeout = settle_timeout)
wait_until_ringready(duts[0])
        
dut0.step('Write some known data to the cluster')

num_test_keys = 600
bucket = default_systest_bucket()
magic_number = time.time()
body_prefix = 'body: magic-number=%s, i=' % magic_number

put_some_keys(duts[0], num_test_keys, bucket = bucket, body_prefix = body_prefix)

dut0.step('Read the known data from all members of the cluster')

repeat_read_of_all_keys_all_nodes(duts[0:6], num_test_keys, bucket, body_prefix)

dut0.step('Leave the nodes in range(3, 6)')

#for i in range(3, 6):
#    duts[i].admin_leave()
#    wait_until_ringready(duts[0])
#    wait_until_transfers_done(duts[0])
#
#dut0.step('Join nodes range(6, 9)')
#
#for i in range(6, 9):
#    systest_python_copy_conf_port_bindings_all(
#        duts[i], duts_host_info[i], duts_os[i],
#        app_config_edits = [("riak_core", "ring_creation_size", "512")])
#    duts[i].control_start()
#    duts[i].ok_cmd('sleep 3')
#    duts[i].admin_join('%s@%s' % (duts_pkg_details[0]['node_lhs'],
#                                  duts_host_info[0]['host']))
#
#dut0.step('Wait for the ringready, then read all keys on range(6, 9)')
#
#wait_until_ringready(duts[0])
#
# NOTE: what I actually did was more like:
for i in range(3, 6):
    duts[i].admin_leave()
    wait_until_ringready(duts[0])

for i in range(3, 6):
    wait_until_transfers_done(duts[0])

dut0.step('Join nodes range(6, 9)')

for i in range(6, 9):
    systest_python_copy_conf_port_bindings_all(
        duts[i], duts_host_info[i], duts_os[i],
        app_config_edits = [("riak_core", "ring_creation_size", "512")])
    duts[i].control_start()
    duts[i].ok_cmd('sleep 3')
    duts[i].admin_join('%s@%s' % (duts_pkg_details[0]['node_lhs'],
                                  duts_host_info[0]['host']))
    wait_until_ringready(duts[0])

dut0.step('Wait for the ringready, then read all keys on range(6, 9)')

# End change

repeat_read_of_all_keys_all_nodes(duts[6:9], num_test_keys, bucket, body_prefix)

dut0.step('Leave the nodes in range(0, 3)')

#for i in range(0, 3):
#    duts[i].admin_leave()
#    wait_until_ringready(duts[6])
#    wait_until_transfers_done(duts[6])
#
# NOTE: again, my actually steps were more like:
for i in range(0, 3):
    duts[i].admin_leave()
    wait_until_ringready(duts[6])

for i in range(0, 3):
    wait_until_transfers_done(duts[6])

dut0.step('Wait for ringready and transfers to finish')

wait_until_ringready(duts[6])
wait_until_transfers_done(duts[6])

dut0.step('One last check that all keys & values match')

repeat_read_of_all_keys_all_nodes(duts[6:9], num_test_keys, bucket, body_prefix)

dut0.step('Test status: PASS')
exit(0)