Last active
December 22, 2021 05:26
-
-
Save crazytaxii/f3292e86846b6cafdba3778392241403 to your computer and use it in GitHub Desktop.
Deploy 3 nodes Ceph cluster for testing.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
set -e | |
CEPH_VER=${CEPH_VER:-"15.2.1"} | |
OSD_DEV=${OSD_DEV:-/dev/sdb} | |
ssh_test() { | |
if [[ $(ssh root@$1 exit) -ne 0 ]] | |
then | |
echo "failed to login $1 as root" && exit 1 | |
fi | |
} | |
append_hosts() { | |
sed -i -e /$2/d /etc/hosts | |
echo "$1 $2" >> /etc/hosts | |
} | |
docker() { | |
yum install -y yum-utils | |
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo | |
yum install docker-ce docker-ce-cli containerd.io -y | |
systemctl start docker && systemctl enable docker | |
} | |
deps() { | |
ceph_ver=$1 | |
yum install -y wget python3 epel-release | |
rm -f /etc/yum.repos.d/ceph.repo | |
cat <<EOF > /etc/yum.repos.d/ceph.repo | |
[Ceph] | |
name=Ceph basearch | |
baseurl=https://download.ceph.com/rpm-$ceph_ver/el7/basearch | |
enabled=1 | |
gpgcheck=1 | |
gpgkey=https://download.ceph.com/keys/release.gpg | |
[Ceph-noarch] | |
name=Ceph noarch | |
baseurl=https://download.ceph.com/rpm-$ceph_ver/el7/noarch | |
enabled=1 | |
gpgcheck=1 | |
gpgkey=https://download.ceph.com/keys/release.gpg | |
[Ceph-source] | |
name=Ceph SRPMS | |
baseurl=https://download.ceph.com/rpm-$ceph_ver/el7/SRPMS | |
enabled=1 | |
gpgcheck=1 | |
gpgkey=https://download.ceph.com/keys/release.gpg | |
EOF | |
sed -i -e 's/basearch/\$basearch/g' /etc/yum.repos.d/ceph.repo | |
rpm --import 'https://download.ceph.com/keys/release.asc' | |
yum makecache | |
yum install ceph-common -y | |
rm -rf /usr/local/bin/cephadm | |
wget -P /usr/local/bin https://raw.githubusercontent.com/ceph/ceph/v$ceph_ver/src/cephadm/cephadm && chmod +x /usr/local/bin/cephadm | |
} | |
chrony() { | |
cat <<EOF > /etc/chrony.conf | |
server $1 iburst | |
# Ignor source level | |
stratumweight 0 | |
# Record the rate at which the system clock gains/losses time. | |
driftfile /var/lib/chrony/drift | |
# Allow the system clock to be stepped in the first three updates | |
# if its offset is larger than 1 second. | |
makestep 1.0 3 | |
# Enable kernel synchronization of the real-time clock (RTC). | |
rtcsync | |
# Enable hardware timestamping on all interfaces that support it. | |
#hwtimestamp * | |
# Increase the minimum number of selectable sources required to adjust | |
# the system clock. | |
#minsources 2 | |
# Allow NTP client access from local network. | |
allow 0.0.0.0/0 | |
bindcmdaddress 127.0.0.1 | |
bindcmdaddress ::1 | |
# Serve time even if not synchronized to a time source. | |
EOF | |
if [[ $1 == "localhost" ]] | |
then | |
echo "local stratum 10" >> /etc/chrony.conf | |
fi | |
cat <<EOF >> /etc/chrony.conf | |
# Specify file containing keys for NTP authentication. | |
keyfile /etc/chrony.keys | |
# Specify directory for log files. | |
logdir /var/log/chrony | |
# Select which information is logged. | |
#log measurements statistics tracking | |
noclientlog | |
EOF | |
systemctl restart chronyd && systemctl enable chronyd | |
} | |
main() { | |
# check | |
if [[ -z $1 ]] | |
then | |
echo "missing IP/hostname of node2" && exit 1 | |
fi | |
if [[ -z $2 ]] | |
then | |
echo "missing IP/hostname of node3" && exit | |
fi | |
# IP address of host | |
ip_addr=$(hostname -I | cut -d ' ' -f 1) | |
# test ssh connection | |
for host in localhost $1 $2 | |
do | |
ssh_test $host | |
done | |
# stop and disable firewalld | |
for host in localhost $1 $2 | |
do | |
ssh root@$host "systemctl stop firewalld && systemctl disable firewalld" | |
done | |
# set up chrony docker and install dependencies | |
chrony localhost | |
docker | |
deps $CEPH_VER | |
for host in $1 $2 | |
do | |
ssh root@$host << EOF | |
$(typeset -f chrony) | |
chrony $ip_addr | |
$(typeset -f docker) | |
docker | |
$(typeset -f deps) | |
deps $CEPH_VER | |
EOF | |
done | |
host1=$(hostname) | |
host2=$(ssh root@$1 "hostname") | |
host3=$(ssh root@$2 "hostname") | |
for host in $ip_addr $1 $2 | |
do | |
ssh root@$host << EOF | |
$(typeset -f append_hosts) | |
append_hosts $ip_addr $host1 | |
append_hosts $1 $host2 | |
append_hosts $2 $host3 | |
EOF | |
done | |
# bootstrap | |
export CEPHADM_IMAGE=docker.io/ceph/ceph:v$CEPH_VER | |
cephadm bootstrap --mon-ip $ip_addr | |
# wait for 5 mins | |
echo "wait for 5 mins" | |
sleep 5m | |
for host in $host2 $host3 | |
do | |
ssh-copy-id -f -i /etc/ceph/ceph.pub root@$host | |
ceph orch host add $host | |
done | |
# wait for 5 mins | |
echo "wait for 5 mins again" | |
sleep 5m | |
for host in $host1 $host2 $host3 | |
do | |
ceph orch daemon add osd $host:$OSD_DEV | |
done | |
# wait for 5 mins | |
sleep 5m | |
echo "mission complete!" | |
} | |
main "$@" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Only for CentOS!
Make sure node1 ssh to node2 and node3 with key pair.
Run this on first node(e.g. 10.211.55.123):
$ curl -OsL https://gist.githubusercontent.com/crazytaxii/f3292e86846b6cafdba3778392241403/raw/111cd1fa522b9b5363fe854cd83e20eec70123f6/ceph3nodes.sh && chmod +x ./ceph3nodes.sh $ ./ceph3nodes.sh 10.211.55.124 10.211.55.125