test_reproducer.py

import logging
import time
from iptables import IPTable, IPTableRule

from dtest_class import Tester, get_ip_from_node

logger = logging.getLogger(__name__)


def retry_till_success(fun, *args, **kwargs):
    timeout = kwargs.pop('timeout', 60)
    bypassed_exception = kwargs.pop('bypassed_exception', Exception)

    deadline = time.time() + timeout
    while True:
        try:
            return fun(*args, **kwargs)
        except bypassed_exception:
            if time.time() > deadline:
                raise
            else:
                # brief pause before next attempt
                time.sleep(0.015)


class TestBootstrap(Tester):

    def test_reproducer_for_982(self, request):

        """ reproducer scylladb/scylla-operator#982 """
        logger.info("populating cluster with three nodes")
        cluster = self.cluster
        cluster.populate(3)
        logger.info("starting cluster")
        cluster.start(wait_for_binary_proto=True, wait_other_notice=True)

        logger.info("stopping node3")
        node1, node2, node3 = cluster.nodelist()
        node3.stop(gently=False)

        iptables_obj = IPTable(chain_name=__name__)
        iptables_obj.create_new_chain()

        node1_ip_address = get_ip_from_node(node=node1)
        node2_ip_address = get_ip_from_node(node=node2)

        rule1 = IPTableRule(protocol='tcp', source=f'{node1_ip_address}/32', destination_port=7000, target='DROP')
        rule2 = IPTableRule(protocol='tcp', source=f'{node2_ip_address}/32', destination_port=7000, target='DROP')

        iptables_obj.add_rule(rule1)
        iptables_obj.add_rule(rule2)

        logger.info("decommission node3")
        node3.start(wait_for_binary_proto=True, wait_other_notice=False)
        for n in cluster.nodelist():
            stdout, stderr = n.nodetool("status")
            logger.info(stdout)
            logger.info(stderr)

        retry_till_success(node3.decommission, timeout=120)

        iptables_obj.delete_chain()

        logger.info("add node4")
        node4 = cluster.new_node(4)
        node4.start(wait_for_binary_proto=True)
        logger.info("done")