Remote directory syncronization using rsync and lsynd

exclude_list

# Install this file as ~/mirrors/exclude_list

RCS
SCCS
CVS
CVS.adm
RCSLOG
cvslog.*
tags
TAGS
.make.state
.nse_depinfo
*~
#*
.#*
,*
_$*
*$
*.old
*.bak
*.BAK
*.orig
*.rej
.del-*
*.a
*.olb
*.o
*.so
*.exe
*.Z
*.elc
*.ln
.svn/
.git/
.hg/
.bzr/

Mirror.md

Description

This is a simple script that will rsync down a directory tree from a remote system, via ssh, and then keep the remote copy in sync with the local copy using lsyncd, as you edit it. In each case, the newest version of the file is always kept. It is intended for cases when editing remotely is too uncomfortable, and git pull/push is too manual.

lsyncd uses inotify to watch folders efficiently, for changes, new files, etc. It then mirrors those changes on the server using ssh commands, and/or rsync to upload file deltas.

Once the initial sync is complete, the synchronization is not bidirectional. If a change is made to the remote copy, then mirror <name> will pull down those changes.

An external exclude_list file is used to control the download sync, and lsyncd's exclude_cvs option is used the control the upload sync.

All the mirrors live in $HOME/mirrors/<name>.

Usage:

mirror resume

• First, kill any existing lsyncd processes
• Then, rsync all newer files from the server.
• Finally, restart all client-to-server lsyncd daemons.

mirror <name>

• Like "resume", but for a specific mirrored folder

mirror <name> <host:directory> [init]

• Rsync the remote directory, minus VCS data, to ~/mirrors/.
• And start lsyncd to keep the remote files up-to-date with local changes.
• Append "init" to have it overwrite local changes

mirror.sh

#!/bin/bash

# This is a simple script that will rsync down a directory tree from a remote
# system, via ssh, and then keep the remote copy in sync with the local copy
# using lsyncd, as you edit it. In each case, the newest version of the file
# is always kept. It is intended for cases when editing remotely is too
# uncomfortable, and git pull/push is too manual.
#
# lsyncd uses inotify to watch folders efficiently, for changes, new files,
# etc. It then mirrors those changes on the server using ssh commands, and/or
# rsync to upload file deltas.
#
# Once the initial sync is complete, the synchronization is not bidirectional.
# If a change is made to the remote copy, then "mirror <name>" will pull down
# those changes.
#
# An external 'exclude_list' file is used to control the download sync,
# and lsyncd's exclude_cvs option is used the control the upload sync.
#
# All the mirrors live in $HOME/mirrors/<name>.

# Usage:
#
# mirror resume
#      First, kill any existing lsyncd processes
#      Then, rsync all *newer* files from the server.
#      Finally, restart all client-to-server lsyncd daemons.
#
# mirror <name>
#      Like "resume", but for a specific mirrored folder
#
# mirror <name> <host:directory> [init]
#      Rsync the remote directory, minus VCS data, to ~/mirrors/<name>.
#      And start lsyncd to keep the remote files up-to-date with local changes.
#      Append "init" to have it overwrite local changes

function kill_lsyncd () {
  local local_name="$1"

  # Kill existing lsyncd, if any
  ps -o pid,args -u `whoami` | grep "l[s]yncd.*$local_name\.mirror" | sed 's/^ *//' | cut -d' ' -f1 | xargs --no-run-if-empty kill
}

function start_mirror () {
  local local_name="$1"
  local remote_dir="${2#*:}"
  local remote_host="${2%%:*}"
  local init="$3"
  local update

  set -e

  kill_lsyncd "$local_name"

  # Sync down from the server.
  case "$init" in
  init) update="--checksum --delete" ;;
  *)    update=--update ;;
  esac
  echo "Pull $local_name ..."
  rsync -a $update --info=progress2 --exclude-from=exclude_list "$remote_host:$remote_dir/" "$local_name/"

  # Start upward synchronization
  lsyncd "$local_name.mirror"
}

function resume_mirror () {
  local local_name="$1"
  local remote_dir
  local remote_host

  # reload settings from the Lua file.
  remote_host=$(grep -Po '(?<=host=").*(?=",)' "$local_name")
  remote_dir=$(grep -Po '(?<=targetdir=").*(?=",)' "$local_name")
  start_mirror "${local_name%.mirror}" "$remote_host:$remote_dir"
}

function create_mirror () {
  local_name="$1"
  remote_dir="${2#*:}"
  remote_host="${2%%:*}"
  init="$3"

  # Create a Lua config file
  cat > "$local_name.mirror" <<EOF
sync {
     default.rsyncssh,
     delay = 0,
     source="/home/$(whoami)/mirrors/$local_name",
     host="$remote_host",
     targetdir="$remote_dir",
     rsync = {
       archive = true,
       cvs_exclude = true
     }
}
EOF

  start_mirror "$local_name" "$remote_host:$remote_dir" "$init"
}

cd ~/mirrors

if [ "$1" == "resume" ]; then
  # resume all existing mirrors
  for mirror in *.mirror; do
    resume_mirror "$mirror"
  done
elif [ "$1" == "stop" ] && [ -f "$2.mirror" ]; then
  kill_lsyncd "$2"
elif [ "$2" == "" ] && [ -f "$1.mirror" ]; then
  # Resume the specific mirror
  resume_mirror "$1.mirror"
else
  # Start a new mirror, or restart an existing one
  create_mirror "$1" "$2" "$3"
fi