Skip to content

Instantly share code, notes, and snippets.

@crazytaxii
Last active December 22, 2021 05:26
Show Gist options
  • Save crazytaxii/f3292e86846b6cafdba3778392241403 to your computer and use it in GitHub Desktop.
Save crazytaxii/f3292e86846b6cafdba3778392241403 to your computer and use it in GitHub Desktop.
Deploy 3 nodes Ceph cluster for testing.
#!/bin/bash
set -e
CEPH_VER=${CEPH_VER:-"15.2.1"}
OSD_DEV=${OSD_DEV:-/dev/sdb}
ssh_test() {
if [[ $(ssh root@$1 exit) -ne 0 ]]
then
echo "failed to login $1 as root" && exit 1
fi
}
append_hosts() {
sed -i -e /$2/d /etc/hosts
echo "$1 $2" >> /etc/hosts
}
docker() {
yum install -y yum-utils
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install docker-ce docker-ce-cli containerd.io -y
systemctl start docker && systemctl enable docker
}
deps() {
ceph_ver=$1
yum install -y wget python3 epel-release
rm -f /etc/yum.repos.d/ceph.repo
cat <<EOF > /etc/yum.repos.d/ceph.repo
[Ceph]
name=Ceph basearch
baseurl=https://download.ceph.com/rpm-$ceph_ver/el7/basearch
enabled=1
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.gpg
[Ceph-noarch]
name=Ceph noarch
baseurl=https://download.ceph.com/rpm-$ceph_ver/el7/noarch
enabled=1
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.gpg
[Ceph-source]
name=Ceph SRPMS
baseurl=https://download.ceph.com/rpm-$ceph_ver/el7/SRPMS
enabled=1
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.gpg
EOF
sed -i -e 's/basearch/\$basearch/g' /etc/yum.repos.d/ceph.repo
rpm --import 'https://download.ceph.com/keys/release.asc'
yum makecache
yum install ceph-common -y
rm -rf /usr/local/bin/cephadm
wget -P /usr/local/bin https://raw.githubusercontent.com/ceph/ceph/v$ceph_ver/src/cephadm/cephadm && chmod +x /usr/local/bin/cephadm
}
chrony() {
cat <<EOF > /etc/chrony.conf
server $1 iburst
# Ignor source level
stratumweight 0
# Record the rate at which the system clock gains/losses time.
driftfile /var/lib/chrony/drift
# Allow the system clock to be stepped in the first three updates
# if its offset is larger than 1 second.
makestep 1.0 3
# Enable kernel synchronization of the real-time clock (RTC).
rtcsync
# Enable hardware timestamping on all interfaces that support it.
#hwtimestamp *
# Increase the minimum number of selectable sources required to adjust
# the system clock.
#minsources 2
# Allow NTP client access from local network.
allow 0.0.0.0/0
bindcmdaddress 127.0.0.1
bindcmdaddress ::1
# Serve time even if not synchronized to a time source.
EOF
if [[ $1 == "localhost" ]]
then
echo "local stratum 10" >> /etc/chrony.conf
fi
cat <<EOF >> /etc/chrony.conf
# Specify file containing keys for NTP authentication.
keyfile /etc/chrony.keys
# Specify directory for log files.
logdir /var/log/chrony
# Select which information is logged.
#log measurements statistics tracking
noclientlog
EOF
systemctl restart chronyd && systemctl enable chronyd
}
main() {
# check
if [[ -z $1 ]]
then
echo "missing IP/hostname of node2" && exit 1
fi
if [[ -z $2 ]]
then
echo "missing IP/hostname of node3" && exit
fi
# IP address of host
ip_addr=$(hostname -I | cut -d ' ' -f 1)
# test ssh connection
for host in localhost $1 $2
do
ssh_test $host
done
# stop and disable firewalld
for host in localhost $1 $2
do
ssh root@$host "systemctl stop firewalld && systemctl disable firewalld"
done
# set up chrony docker and install dependencies
chrony localhost
docker
deps $CEPH_VER
for host in $1 $2
do
ssh root@$host << EOF
$(typeset -f chrony)
chrony $ip_addr
$(typeset -f docker)
docker
$(typeset -f deps)
deps $CEPH_VER
EOF
done
host1=$(hostname)
host2=$(ssh root@$1 "hostname")
host3=$(ssh root@$2 "hostname")
for host in $ip_addr $1 $2
do
ssh root@$host << EOF
$(typeset -f append_hosts)
append_hosts $ip_addr $host1
append_hosts $1 $host2
append_hosts $2 $host3
EOF
done
# bootstrap
export CEPHADM_IMAGE=docker.io/ceph/ceph:v$CEPH_VER
cephadm bootstrap --mon-ip $ip_addr
# wait for 5 mins
echo "wait for 5 mins"
sleep 5m
for host in $host2 $host3
do
ssh-copy-id -f -i /etc/ceph/ceph.pub root@$host
ceph orch host add $host
done
# wait for 5 mins
echo "wait for 5 mins again"
sleep 5m
for host in $host1 $host2 $host3
do
ceph orch daemon add osd $host:$OSD_DEV
done
# wait for 5 mins
sleep 5m
echo "mission complete!"
}
main "$@"
@crazytaxii
Copy link
Author

crazytaxii commented Dec 22, 2021

Only for CentOS!
Make sure node1 ssh to node2 and node3 with key pair.

Run this on first node(e.g. 10.211.55.123):

$ curl -OsL https://gist.githubusercontent.com/crazytaxii/f3292e86846b6cafdba3778392241403/raw/111cd1fa522b9b5363fe854cd83e20eec70123f6/ceph3nodes.sh && chmod +x ./ceph3nodes.sh
$ ./ceph3nodes.sh 10.211.55.124 10.211.55.125

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment