rpm builder for vitess

build.sh

#!/bin/sh -ex

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

vitess_version='2.1.0-alpha'

source /opt/rh/devtoolset-4/enable
source /opt/rh/python27/enable

export GOROOT="/usr/local/go"
export PATH="${GOROOT}/bin:${PATH}"
export MYSQL_FLAVOR=MySQL56

mkdir tmp
export TMPDIR="$PWD/tmp"

# This is to provide the dependency for bootstrap.sh below, it is not packaged
# If you already have mysql installed somewhere, you can remove all of the below
# mysql lines and set VT_MYSQL_ROOT env var to your mysql install location
mysql_repo_url="https://repo.percona.com/centos/6/os/x86_64/"
mysql_server_rpm="Percona-Server-server-57-5.7.16-10.1.el6.x86_64.rpm"
mysql_client_rpm="Percona-Server-client-57-5.7.16-10.1.el6.x86_64.rpm"

wget --progress='dot:mega' -O "${mysql_server_rpm}" "${mysql_repo_url}/${mysql_server_rpm}"
wget --progress='dot:mega' -O "${mysql_client_rpm}" "${mysql_repo_url}/${mysql_client_rpm}"

mkdir extracted-mysql-rpm
pushd extracted-mysql-rpm
rpm2cpio "../${mysql_server_rpm}" | cpio -idm
rpm2cpio "../${mysql_client_rpm}" | cpio -idm
popd

export VT_MYSQL_ROOT="$PWD/extracted-mysql-rpm/usr"

mkdir vitess-build
pushd vitess-build

GOPATH="$PWD" go get -v -u github.com/kardianos/govendor

clone_dir="$PWD/src/github.com/youtube/vitess"
git clone "https://github.com/youtube/vitess.git" "$clone_dir"
pushd "$clone_dir"
git checkout "tags/v${vitess_version}"

./bootstrap.sh
source ./dev.env
make build
popd

mkdir -p ./rpmroot/opt/vitess
cp -r ./bin ./rpmroot/opt/vitess/bin
cp -r "${clone_dir}/web" ./rpmroot/opt/vitess/web

mkdir -p ./rpmroot/etc/sysconfig
cp $DIR/vitess-vttablet.sysconfig ./rpmroot/etc/sysconfig/vitess-vttablet

# RPM versions can't have hyphens in them
vitess_version=$(echo "$vitess_version" | sed -e 's/-/./g')

fpm -s dir -t rpm \
    --name vitess \
    --vendor YouTube \
    --version "${vitess_version}" \
    --iteration "${BUILD_NUMBER}" \
    --category "Applications/Databases" \
    --url "https://github.com/youtube/vitess" \
    --description "Vitess is a database clustering system for horizontal scaling of MySQL." \
    --rpm-autoreq \
    --provides 'config(%{name}) = %{version}-%{release}' \
    --rpm-os linux \
    --architecture x86_64 \
    --rpm-init "$DIR/vitess-vttablet.init" \
    --config-files etc/sysconfig/vitess-vttablet \
    -C ./rpmroot

rpm_name="vitess-${vitess_version}-${BUILD_NUMBER}.x86_64.rpm"

echo "RPM Info:"
rpm -qpi "$rpm_name"
echo "RPM depends on:"
rpm -qp --requires "$rpm_name"
echo "Files included in RPM:"
rpm -qpl "$rpm_name"

vitess-vttablet.init

#!/bin/sh
### BEGIN INIT INFO
# Provides:
# Required-Start:    $remote_fs $syslog
# Required-Stop:     $remote_fs $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Starts a Vitess vttablet instance
# Description:       The vitess vttablet is used for communication between a mysql instance and the vitess system
### END INIT INFO

. /etc/sysconfig/vitess-vttablet

cmd="/opt/vitess/bin/vttablet \
    -log_dir $LOG_DIR \
    -tablet-path $ALIAS \
    -tablet_hostname "$HOSTNAME" \
    -init_db_name_override $KEYSPACE \
    -init_keyspace $KEYSPACE \
    -init_shard $SHARD \
    -init_tablet_type replica \
    -health_check_interval 5s \
    -enable_replication_reporter \
    -backup_storage_implementation file \
    -file_backup_storage_root $BACKUP_DIR \
    -port $WEB_PORT \
    -grpc_port $GRPC_PORT \
    -service_map grpc-queryservice,grpc-tabletmanager,grpc-updatestream \
    -pid_file $PID_FILE \
    -db-config-app-uname $APP_UNAME \
    -db-config-app-dbname $KEYSPACE \
    -db-config-app-charset utf8 \
    -db-config-allprivs-uname $ALLPRIVS_UNAME \
    -db-config-allprivs-dbname $KEYSPACE \
    -db-config-allprivs-charset utf8 \
    -db-config-dba-uname $DBA_UNAME \
    -db-config-dba-charset utf8 \
    -db-config-repl-uname $REPL_UNAME \
    -db-config-repl-dbname $KEYSPACE \
    -db-config-repl-charset utf8 \
    -db-config-filtered-uname $FILTERED_UNAME \
    -db-config-filtered-dbname $KEYSPACE \
    -db-config-filtered-charset utf8 \
    -db-credentials-server file \
    -db-credentials-file $CREDENTIALS_FILE \
    -db-config-app-unixsocket $MYSQL_SOCKET \
    -db-config-filtered-unixsocket $MYSQL_SOCKET \
    -zk.local-cell $CELL \
    -mycnf-file $MY_CNF_FILE"

dir="/opt/vitess/bin/"
name=`basename $0`
pid_file="/var/run/$name.pid"
stdout_log="$LOG_DIR/$name.out"
stderr_log="$LOG_DIR/$name.err"

get_pid() {
    cat "$pid_file"
}

is_running() {
    [ -f "$pid_file" ] && ps `get_pid` > /dev/null 2>&1
}

case "$1" in
    start)
    if is_running; then
        echo "Already started"
    else
        echo "Starting $name"
        if [ -z "$VITESS_USER" ]; then
            sudo $cmd >> "$stdout_log" 2>> "$stderr_log" &
        else
            sudo -E -u "$VITESS_USER" $cmd >> "$stdout_log" 2>> "$stderr_log" &
        fi
        echo $! > "$pid_file"
        if ! is_running; then
            echo "Unable to start, see $stdout_log and $stderr_log"
            exit 1
        fi
    fi
    ;;
    stop)
    if is_running; then
        echo -n "Stopping $name.."
        kill `get_pid`
        for i in {1..10}
        do
            if ! is_running; then
                break
            fi

            echo -n "."
            sleep 1
        done
        echo

        if is_running; then
            echo "Not stopped; may still be shutting down or shutdown may have failed"
            exit 1
        else
            echo "Stopped"
            if [ -f "$pid_file" ]; then
                rm "$pid_file"
            fi
        fi
    else
        echo "Not running"
    fi
    ;;
    restart)
    $0 stop
    if is_running; then
        echo "Unable to stop, will not attempt to start"
        exit 1
    fi
    $0 start
    ;;
    status)
    if is_running; then
        echo "Running"
    else
        echo "Stopped"
        exit 1
    fi
    ;;
    *)
    echo "Usage: $0 {start|stop|restart|status}"
    exit 1
    ;;
esac

exit 0

vitess-vttablet.sysconfig

# 
# The following environment variables are necessary for filling out 
# the vttablet binary command in /etc/init.d/vitess-vttablet
#
# This file should be modified, substituting in your own values for all variables below
#

# If set, run the vitess process as this user. Vitess will panic if run as root
VITESS_USER=vitess

#
# Vitess network settings
#

# Your cell is your datacenter
CELL=
# Override if you'd like to not use dns, for instance
HOSTNAME=$(hostname -f)
# The port the Web UI and debug variables are available on
WEB_PORT=8083
# The port used for internal vitess communication
GRPC_PORT=8084

#
# Configs for linking vitess to the installed mysql
#

# A keyspace in Vitess loosely maps to a database, there should be one vttablet per database, so point this at the database you'd like
KEYSPACE=
# If your DB is sharded, make this the appropriate index for the shard you're using. 0 if not sharded
SHARD=
# Server id is a mysql identifier, you can get it from your my.cnf or hardcode here
SERVER_ID=$(grep server-id /etc/my.cnf | grep -o -E "[0-9]+")
# Your alias is used to identify this shard within the vitess UIs and commands. It seems to need to map to cell + server_id as below
ALIAS="$CELL-$SERVER_ID"
# Used for local connections to the mysql process
MYSQL_SOCKET=
# Your my.cnf file for the paired mysql process
MY_CNF_FILE=/etc/my.cnf

#
# Vitess directories and config files
#

# Root directory for vitess
VTDATAROOT=/mnt/vitess
# where to store logs
LOG_DIR=$VTDATAROOT/log
# Where to store backups, for file-based backup
BACKUP_DIR=$VTDATAROOT/backup
# Where to store file-based configs
CONFIG_DIR=$VTDATAROOT/config
# If your vitess mysql users have passwords, this file should be a json map from "user":["password"]
CREDENTIALS_FILE=$CONFIG_DIR/.my_Vitess_Creds
# Pid file for vttablet, tracked separately from the init.d pid file
PID_FILE=$VTDATAROOT/$KEYSPACE-vttablet.pid

#
# Vitess mysql users
#

# Used for general queries and such
APP_UNAME=vt_app
# For certain administrative functions
ALLPRIVS_UNAME=vt_allprivs
# For running things like schema changes, and stopping/starting a mysql
DBA_UNAME=vt_dba
# For replication
REPL_UNAME=vt_repl
# For vitess's filtered replication, which enabled things like vertical and horizontal sharding, and schema swaps
FILTERED_UNAME=vt_filtered

#
# Exports necessary for within the vttablet process
#

# For finding the mysql binaries, used by vitess for executing certain migrations
export VT_MYSQL_ROOT=/usr
# For finding your zk quorums, which vitess uses to point at the local and global zk clusters
export ZK_CLIENT_CONFIG=$CONFIG_DIR/zk.json