Rsync push backup client (python)

gistfile1.py

#!/usr/bin/python

import fcntl
import time
import sys
import os
import yaml
import subprocess

basedir=os.path.dirname(os.path.realpath(__file__))

# lockfile - ensure that we do not run more than one instance
f = open (basedir+'/backup_rsync_push.lock', 'w')
try: fcntl.lockf (f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except:
	sys.stderr.write ('[%s] Script already running.\n' % time.strftime ('%c') )
	sys.exit (-1)

# read config
stream = open(basedir+'/backup_rsync_push.conf', 'r')
conf=yaml.load(stream)
stream.close()

# backup server to use
server=conf['server']

# loop over all volumes
for volume in conf['volumes']:
	# volume requires properties 'name' and 'path'
	volname=volume['name']
	sys.stderr.write ('[%s] (%s) starting backup\n' % (time.strftime ('%c'),volname) )

	volpath=volume['path']
	if not volpath.endswith("/"):
		volpath+="/"

	# if volume.readyfile_check!=false check the presence of the file .ready_for_backup in the backup source directory.
	# this is enabled by default to check if the expected content is there (rather than overwrite existing backups with an empty folder or wrong data)
	if not ('readyfile_check' in volume) or volume['readyfile_check']:
		if not os.path.exists(volpath+".ready_for_backup"):
			sys.stderr.write ('[%s] (%s) testfile ".ready_for_backup" does not exist in "%s" - skipping this backup.\n' % (time.strftime ('%c'),volname,volpath) )
			continue
		# if volume.readyfile_max_age is set to a certain number of minutes, backup will be skipped if the file .ready_for_backup is older
		# this is useful if something like database dumps must be completed before we run the backups
		if 'readyfile_max_age' in volume and volume['readyfile_max_age']>0:
			maxage=volume['readyfile_max_age']
			age=(time.time() - os.path.getmtime(volpath+".ready_for_backup")) / 60
			if age > maxage:
				sys.stderr.write ('[%s] (%s) testfile ".ready_for_backup" is %d minutes old wich is more than the allowed %d minutes - skipping this backup.\n' % (time.strftime ('%c'),volname,age,maxage) )
				continue

	# build rsync command
	cmd=["/usr/bin/nice","-n","19","/usr/bin/ionice","-c","3"]

	maxDuration=0
	# restrict backup to max duration minutes
	if 'max_duration' in volume and volume['max_duration']>0:
		maxDuration=volume['max_duration']
		cmd.append("/usr/bin/timeout")
		cmd.append("%dm" % maxDuration)

	cmd.append("/usr/bin/rsync")

	# various options
	cmd.append("-avr")
	cmd.append("--numeric-ids")
	cmd.append("--delete-during")
	cmd.append("--acls")
	cmd.append("--xattrs")
	cmd.append("--sparse")

	# one exclude option for each excluded folder/file
	if 'exclude' in volume:
		for exl in volume['exclude']:
			cmd.append("--exclude")
			cmd.append(exl)

	# source
	cmd.append(volpath)
	# destination
	cmd.append("backup-rsync-push@%s:%s" % (server,volname))

	# ensure that we use our own key for backup, not the one passed via ssh agent by the current user
	myenv=os.environ.copy();
	myenv['SSH_AUTH_SOCK']=""

	rsyncExitValue=-1

	# execute the rsync command
	try:
		p=subprocess.Popen(cmd,env=myenv)
		p.wait()
		rsyncExitValue=p.returncode
	except KeyboardInterrupt:
		rsyncExitValue=20
		pass
	finally:
		try:
			p.terminate()
			time.sleep(1)
			p.kill()
		except ProcessLookupError:
			pass # Process might already be terminated

	if maxDuration>0 and rsyncExitValue==124:
		sys.stderr.write ('[%s] (%s) rsync does not finish within %d minutes - this backup was canceled.\n' % (time.strftime ('%c'),volname,maxDuration) )
		continue

	# rsync exit code 0 - everything was ok
	# rsync exit code 24 - everything was ok but some files changed during sync
	if rsyncExitValue!=0 and rsyncExitValue!=24:
		sys.stderr.write ('[%s] (%s) rsync exited with code %d - this backup is failed.\n' % (time.strftime ('%c'),volname,rsyncExitValue) )
		continue

	# everything is done. now tell the remote server that the backup is finished
	sys.stderr.write ('[%s] (%s) backup ok - tell the server that we are done.\n' % (time.strftime ('%c'),volname) )
	cmd=["/usr/bin/ssh"]
	# destination
	cmd.append("backup-rsync-push@%s" % (server))
	cmd.append("FINISH_BACKUP")
	cmd.append(volname)
	subprocess.call(cmd,env=myenv)