Skip to content

Instantly share code, notes, and snippets.

@cnp96
Created November 3, 2020 18:55
Show Gist options
  • Save cnp96/7be1756f7eb76ea78c9b832966e84dbf to your computer and use it in GitHub Desktop.
Save cnp96/7be1756f7eb76ea78c9b832966e84dbf to your computer and use it in GitHub Desktop.
Sync MongoDB instances
#!/bin/bash
# DB info
old_db="source_database_name"
# Connection info
uri="source_db_connection_string"
to_uri="target_db_connection_string"
# Storage info
now=$(date +"%Y-%m-%dT%TZ")
root_dir="$(pwd)/cron"
lock_file="${root_dir}/sync.lock"
sync_log="${root_dir}/sync.log"
out_dir="${root_dir}/dump"
sync_dir="${out_dir}/${now}"
# Create files and directories
mkdir -p $sync_dir
touch $sync_log
# Log info -- pwd refers to the user home directory
from=$1
last_sync_time=$(tail -n 1 $sync_log)
if [[ $from == "" ]]; then
if [[ $last_sync_time == "" ]]; then
echo "FATAL: From time is not specified. Updating the log registry with current timestamp and exiting!"
echo $(date +"%Y-%m-%dT%TZ") >>$sync_log
exit 1
else
from=$last_sync_time
fi
fi
to=$2
if [[ $to == "" ]]; then to=$(date +"%Y-%m-%dT%TZ"); fi
# Setup
if [[ -f $lock_file ]]; then
echo "FATAL: A sync is already in progress for timestamp ${last_sync_time}. Exiting!"
exit 1
fi
# Create Mutex
touch $lock_file
echo $to >>$sync_log
echo "INFO: Updated log registry to use new timestamp on next run."
# Ops
mkdir -p $sync_dir
echo "INFO: Created sync directory: ${sync_dir}"
echo "Fetching oplog in range [${from} - ${to}]"
query="{\"wall\": {\"\$gte\": {\"\$date\": \"$from\"}, \"\$lte\": {\"\$date\": \"$to\"} }, \"ns\": {\"\$regex\": \"$old_db\"}}"
echo $query >"${root_dir}/query.json"
if $(mongodump --uri=$uri --collection \"oplog.rs\" --queryFile "${root_dir}/query.json" --gzip -v --out=$sync_dir); then
echo "INFO: Dump success!"
echo "INFO: Replaying oplogs..."
if $(mongorestore --uri=$to_uri --oplogReplay --noIndexRestore --gzip -vv $sync_dir); then
echo "INFO: Restore success!"
else
rm -rf $sync_dir
sed -i '$ d' $sync_log
fi
else
rm -rf $sync_dir
sed -i '$ d' $sync_log
echo "ERROR: Dump failed!"
fi
# Clear Mutex
rm $lock_file
# Run this script as
# ./delta-sync.sh from_epoch_in_ms
@fzhougithub
Copy link

Thanks a lot!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment