rclone python config wrapper

jobs.yml

tasks:
  - name: backup_something
    local: "/path/to/sync"
    remote: "memset_memstore:path/to/remote"
    operation: sync
  - name: backup_something_else
    local: "/another/path/to/sync"
    remote: "memset_memstore:another/path/to/remote"
    operation: sync

rclone.conf

[memset_memstore]
type = swift
user = backup
key = **SECRET KEY**
auth = https://auth.storage.memset.com/v2.0
domain =
tenant = **memstorename (eg. mstestyaa1)**
tenant_domain =
region =
storage_url =
auth_version =

rclonesync.py

#!/usr/bin/env python
import yaml
import logging
import subprocess
import socket
import sys
import time
import select
import re
from multiprocessing.pool import ThreadPool

class RemoveSummaryFilter(logging.Filter):
    summarywords = ['Errors', 'Checks', 'Transferred', 'Elapsed time', 'Encrypted Swift container', 'Waiting for deletions to finish']
    def filter(self, record):
        return not any(record.getMessage().startswith(x) for x in self.summarywords)

class RemoveRcloneDatetimeFilter(logging.Filter):
    regex = re.compile(r".*\s:\s")
    def filter(self, record):
        msg = record.msg
        out = self.regex.sub("", msg)
        record.msg = out
        return True

class RemoveEmptyLineFilter(logging.Filter):
    def filter(self, record):
        return bool(record.msg.isspace() or bool(record.msg.strip()))

generic_formatter = logging.Formatter('%(asctime)s %(levelname)-6s: %(message)s', datefmt='%Y/%m/%d %H:%M:%S')
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.addFilter(RemoveRcloneDatetimeFilter())
logger.addFilter(RemoveSummaryFilter())
logger.addFilter(RemoveEmptyLineFilter())

fileHandler = logging.FileHandler("/var/log/backup.log")
fileHandler.setFormatter(generic_formatter)
logger.addHandler(fileHandler)

consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(generic_formatter)

# TODO: MAKE THIS DETECT THAT IT IS RUNNING ON A TTY AND SHUT UP IF NOT
logger.addHandler(consoleHandler) # Comment out this line if your cron is noisy and you're running this at /etc/cron.hourly/rclonesync.py

class LOCK(object):
    locksock = None
    def get_lock(self, process_name):
        # Without holding a reference to our socket somewhere it gets garbage
        # collected when the function exits
        self.locksock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
        try:
            self.locksock.bind('\0' + process_name)
            return True
        except socket.error:
            return False
        return False

    def destroy(self):
        self.locksock.shutdown(socket.SHUT_RDWR)
        self.locksock.close()


def call(popenargs, logger, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs):
    """
    Variant of subprocess.call that accepts a logger instead of stdout/stderr,
    and logs stdout messages via logger.debug and stderr messages via
    logger.error.
    """
    child = subprocess.Popen(popenargs, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, **kwargs)
    poll = select.poll()
    poll.register(child.stdout, select.POLLIN | select.POLLHUP)
    poll.register(child.stderr, select.POLLIN | select.POLLHUP)
    pollc = 2
    events = poll.poll()
    while pollc > 0 and len(events) > 0:
        for rfd, event in events:
            if event & select.POLLIN:
                if rfd == child.stdout.fileno():
                    line = child.stdout.readline()
                    if len(line) > 0:
                        logger.log(stdout_log_level, line[:-1])
                if rfd == child.stderr.fileno():
                    line = child.stderr.readline()
                    if len(line) > 0:
                        logger.log(stderr_log_level, line[:-1])
            if event & select.POLLHUP:
                poll.unregister(rfd)
                pollc -= 1
            if pollc > 0:
                events = poll.poll()
    return child.wait()

def loadConfig():
    try:
        with open("/etc/rclone/jobs.yml", 'r') as ymlfile:
            return yaml.load(ymlfile)
    except IOError:
        logger.fatal('Config was not found. Ensure /etc/rclone/jobs.yml exists and is valid')
        exit(1)

def runTask(task):
    base = ['/usr/bin/rclone', '-v', '--config', '/etc/rclone/rclone.conf']
    exclude = []
    if 'exclude' in task:
        for filename in task['exclude']:
            exclude += ['--exclude', filename]
    things = [task['operation'], '--delete-after', task['local'], task['remote']]
    command = base + exclude + things
    logger.debug('Command: %s' % command)
    try:
        call(command, logger, stdout_log_level=logging.INFO, stderr_log_level=logging.INFO, close_fds=True)
        logger.info('Finished: %s' % task['name'])
    except subprocess.CalledProcessError:
        logger.fatal('FAILED: %s' % task['name'])
    except OSError:
        logger.fatal('FAILED: %s - ensure rclone is present and installed at /usr/bin/rclone' % task['name'])
    task['locksock'].destroy()

def main():
    config = loadConfig()
    tasks = config['tasks']
    todo = []
    for task in tasks:
        name = task['name']
        lock_name = 'rclonesync-%s' % name
        task['locksock'] = LOCK()
        if task['locksock'].get_lock(lock_name):
            logger.debug('Got the lock (%s)' % lock_name)
            logger.info('Started: %s' % name)
            todo.append(task)
        else:
            logger.error('Task %s was already running. Skipping.' % name)
    p = ThreadPool(5)
    p.map(runTask, todo)

if __name__ == '__main__':
    main()