ceph-src-dir: /home/github/ceph
ceph-build-dir: /home/github/ceph/build-master
# cd <ceph-build-dir>
# export PATH=<ceph-build-dir>/bin:$PATH
create the first cluster (do not (yet) change the cluster name from "cluster1/2")
# MON=1 OSD=1 MDS=1 MGR=1 ../src/mstart.sh cluster1 -n
create the second cluster
# MON=1 OSD=1 MDS=1 MGR=1 ../src/mstart.sh cluster2 -n
apply the following diff to the file under ceph-src-dir
NOTE: This is because my build is not a full build, but missing some parts and hence mstart/vstart has some errors
diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index 36d712f85b..8654905381 100755
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -213,7 +213,7 @@ setup_cluster()
{
local cluster=$1
- CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${cluster} -n ${RBD_MIRROR_VARGS}
+ #CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${cluster} -n ${RBD_MIRROR_VARGS}
cd ${CEPH_ROOT}
rm -f ${TEMPDIR}/${cluster}.conf
create the following script under <ceph-src-dir>/qa/workunits/rbd/rbd_mirror_setup.sh
#!/bin/sh -ex
MIRROR_POOL_MODE=image
MIRROR_IMAGE_MODE=snapshot
RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror
RBD_MIRROR_NOCLEANUP=1
RBD_MIRROR_INSTANCES=1
. $(dirname $0)/rbd_mirror_helpers.sh
setup
start_mirrors ${CLUSTER1}
start_mirrors ${CLUSTER2}
create the tmp directory
# mkdir /tmp/tmp.rbd_mirror
create the mirror pools and start the mirror daemons
# ../qa/workunits/rbd/rbd_mirror_setup.sh
execute further ceph commands as follows
# ./bin/ceph -c ./run/cluster1/ceph.conf osd lspools
OR
# ./bin/ceph -c ./run/cluster2/ceph.conf osd lspools
-
RBD pools that are mirrored are: mirror and mirror_parent
-
Create an image and set mirror property, and in general play around
-
To simulate IO to the RBD image, use options like function write_image() in
<ceph-src-dir>/qa/workunits/rbd/rbd_mirror_helpers.sh(if using a script that will call the function, your PWD should be /tmp/tmp.rbd_mirror (so that the --cluster option works)) -
To simulate RBD mirror daemon failure/restart/etc, kill the respective rbd-mirror daemons
-
To restart a killed daemon,
# cd /tmp/tmp.rbd_mirror# rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true
# cd -
-
To cleanup the setup,
# pkill rbd-mirror# ../src/mstop.sh cluster1# ../src/mstop.sh cluster2# rm -rf ./run/*# rm -rf /tmp/tmp.rbd-mirror/*