cassandra_cluster.py

# fab tasks for creating a multi-node Cassandra cluster on a single machine.
#
# This script is very opinionated. It makes the following assumptions:
#
#   - Your supervisor is systemd (this was tested on fedora-17)
#   - Cassandra is installed via the Datastax-supplied RPMs, which means there are
#     some naive assumptions about variables to be changed, such as paths. See below.
#
# Each instance consists of:
#
#   - An 'instance ID' starting at 1
#   - A directory tree for config, data, and logs: /cass/instance-<instance ID>/
#   - Thrift will listen on 127.0.0.x  where x = instance ID
#   - JMX_PORT will be JMX_PORT_START + instance ID, since JMX can't be easily
#     coerced to listen on a specific address.
#   - Runs as the 'cassandra' user.
#
# Examples:
# ---------
#
# Create a 3 node cluster:
#
#   fab -f cassandra-cluster-setup.py create_cluster:nodes=3
#
# Start up all nodes:
#
#   fab -f cassandra-cluster-setup.py start_cluster:nodes=3
#
# Shutdown and delete cluster:
#
#   fab -f cassandra-cluster-setup.py remove_cluster:nodes=3
#
# Author
# ------
# Joe Miller - https://github.com/joemiller / https://twitter.com/miller_joe
#

from fabric.api import *
from fabric.contrib.files import *

from clint.textui import colored, indent, puts


#######
# config/globals
CASSANDRA_BASE_PATH = '/cass'
JMX_PORT_START = 12000
IP_TEMPLATE = "127.0.0.%s"


###########
# Helpers
def sed(filename, before, after, options='', backup_ext=''):
    local("perl -p -i%s -e 's@%s@%s@%s' %s" % (backup_ext, before, after, options, filename))


def create_systemd_unit_template(path='/etc/systemd/system/cassandra_cluster@.service'):
    systemd_service_unit = """
[Unit]
Description=Cassandra

[Service]
User=cassandra
Group=cassandra
Environment=CASSANDRA_HOME=/usr/share/cassandra/
Environment=CASSANDRA_INCLUDE={cassandra_base_path}/instance-%i/conf/cassandra.in.sh
Environment=CASSANDRA_CONF={cassandra_base_path}/instance-%i/conf
Environment=MAX_HEAP_SIZE=512M
Environment=HEAP_NEWSIZE=100M
ExecStart=/usr/sbin/cassandra -f
KillMode=process
Restart=always
RestartSec=5min
LimitNOFILE=32768

[Install]
WantedBy=multi-user.target
        """.format(cassandra_base_path=CASSANDRA_BASE_PATH)
    with file(path, 'w') as f:
        f.write(systemd_service_unit)


#############
# Tasks
@task
def create_cluster(nodes=2, cluster_name='test_cluster'):
    local('mkdir -p %s' % CASSANDRA_BASE_PATH)
    create_systemd_unit_template()

    # instances will be named/numbered starting at 1: 1, 2, 3, ... <nodes>
    for instance_id in range(1, int(nodes) + 1):
        jmx_port = JMX_PORT_START + instance_id
        instance_path = '%s/instance-%s' % (CASSANDRA_BASE_PATH, instance_id)  # eg: /cass/instance-1
        instance_ip = IP_TEMPLATE % instance_id

        create_instance(instance_id, cluster_name, instance_path, instance_ip, jmx_port)


@task
def create_instance(instance_id, cluster_name, instance_path, instance_ip, jmx_port):
    """Create a new instance of cassandra"""
    puts(colored.green("Creating new Cassandra instance:"))
    puts(colored.green("  - instance_id               : %s" % instance_id))
    puts(colored.green("  - cluster_name              : %s" % cluster_name))
    puts(colored.green("  - instance_ip               : %s" % instance_ip))
    puts(colored.green("  - JMX port (for nodetool -p): %s" % jmx_port))
    puts(colored.green("  - instance_path             : %s" % instance_path))

    cassandra_yaml_file = '%s/conf/cassandra.yaml' % instance_path
    cassandra_env_file = '%s/conf/cassandra-env.sh' % instance_path
    cassandra_inc_file = '%s/conf/cassandra.in.sh' % instance_path
    log4j_conf_file = '%s/conf/log4j-server.properties' % instance_path

    local('mkdir -p %s' % instance_path)
    local('mkdir -p %s/log' % instance_path)
    local('mkdir -p %s/conf' % instance_path)
    local('mkdir -p %s/data' % instance_path)
    local('mkdir -p %s/commitlog' % instance_path)
    local('mkdir -p %s/saved_caches' % instance_path)
    local('cp /etc/cassandra/conf/* %s/conf/' % instance_path)

    sed(cassandra_inc_file, '^CASSANDRA_CONF=.+$', 'CASSANDRA_CONF=%s/conf' % instance_path)
    # set JMX port.
    sed(cassandra_env_file, '^JMX_PORT=.+$', 'JMX_PORT=%s' % jmx_port)

    sed(cassandra_yaml_file, '^cluster_name: .+$', 'cluster_name: %s' % cluster_name)
    sed(cassandra_yaml_file, '^listen_address: .+$', 'listen_address: %s' % instance_ip)
    sed(cassandra_yaml_file, '^rpc_address: .+$', 'rpc_address: %s' % instance_ip)
    # the 'seed' node is set to the first instance, eg: 127.0.0.1
    seed_node = IP_TEMPLATE % 1
    sed(cassandra_yaml_file, '^(\s*- seeds:)\s*.+$', '$1 "%s"' % seed_node)

    # note: this is a somewhat naive way to change the various paths in the config file.
    # it will work with the rpm-based cassandra.yaml because all of the paths we are
    # interested in will begin with '/var/lib/cassandra', but it may not work with
    # other cassandra.yaml files
    sed(cassandra_yaml_file, '/var/lib/cassandra', instance_path)
    sed(log4j_conf_file, '/var/log/cassandra', '%s/log' % instance_path)

    local('chown -R cassandra:cassandra %s' % instance_path)
    local('ln -s /etc/systemd/system/cassandra_cluster@.service /etc/systemd/system/cassandra_cluster@%s.service' % instance_id)
    local('systemctl daemon-reload')


@task
def start_cluster(nodes=2):
    for i in range(1, int(nodes) + 1):
        start_instance(i)

@task
def start_instance(instance_id):
    local('systemctl start cassandra_cluster@%s.service' % instance_id)

@task
def stop_cluster(nodes=2):
    for i in range(1, int(nodes) + 1):
        stop_instance(i)

@task
def stop_instance(instance_id):
    local('systemctl stop cassandra_cluster@%s.service' % instance_id)

@task
def remove_cluster(nodes=2):
    for instance_id in range(1, int(nodes) + 1):
        instance_path = '%s/instance-%s' % (CASSANDRA_BASE_PATH, instance_id)
        remove_instance(instance_id, instance_path)
    local('rm -f /etc/systemd/system/cassandra_cluster@.service')
    local('systemctl daemon-reload')

@task
def remove_instance(instance_id, instance_path):
    stop_instance(instance_id)
    local('rm -rf -- "%s"' % instance_path)
    local('rm -f -- "/etc/systemd/system/cassandra_cluster@%s.service"' % instance_id)