Restart Datadog agent automatically inside docker when python script detects integration changes

dogsitter.py

#!/opt/datadog-agent/embedded/bin/python

import os
import sys
import time
import re
import subprocess
import logging

# To keep this lightweight we assume the following
# sys.argv[1] is the directory to watch

# global vars
DD_CONF_DIR = '/etc/dd-agent/conf.d/'
dogObj = {}  # create dogObj dictionary
dogObj['directory'] = sys.argv[1]
dogObj['filetype'] = 'yaml'
dogObj['heartbeat'] = int(os.environ.get('DD_HEARTBEAT', 120))  # in seconds
dogObj['watched_files'] = []
logpath = '/log/dogsitter'
loglevel = str(os.environ.get('DD_LOGLEVEL', 'WARNING'))  # setting log level default
logformat = '%(asctime)s %(levelname)s: %(message)s'

# Set logging module log level
if loglevel == 'DEBUG':
    llevel = logging.DEBUG
elif loglevel == 'INFO':
    llevel = logging.INFO
elif loglevel == 'WARNING':
    llevel = logging.WARNING
else:
    llevel = logging.ERROR

try:
    os.makedirs(logpath, 0755)
except OSError:
    if not os.path.isdir(logpath):
        raise

logging.basicConfig(filename=logpath + '/dogsitter.log', format=logformat, level=llevel)


def restart():
    logging.warning('Restarting now')
    subprocess.call(['/opt/datadog-agent/bin/supervisorctl', '-s',
                     'unix:///opt/datadog-agent/run/datadog-supervisor.sock',
                     'restart', 'datadog-agent:'])


def find_files_to_watch(directory, filetype):
    match = []
    for root, dirs, files in os.walk(directory):
        for f in files:
            pattern = re.compile(".*" + filetype + "$")
            if pattern.match(f):
                found = {}
                found['name'] = f
                found['path'] = root + '/' + f
                found['st_mtime'] = get_modified_time(found['path'])
                match.append(found)
    logging.debug('Found files to watch %r' % match)
    return match


def compare_watched_files(args):
    changed = 0
    oldscan = args['watched_files']
    rescan = find_files_to_watch(args['directory'], args['filetype'])
    if len(args['watched_files']) != len(rescan):
        logging.warning("There are now %s files.", str(len(rescan)))
        changed = 1
    else:
        logging.info("Same number of files.")
        count = 0
        while (count < len(rescan)):
            if rescan[count]['st_mtime'] != oldscan[count]['st_mtime']:
                logging.warning("Time changed from %s to %s.", oldscan[count]['st_mtime'], rescan[count]['st_mtime'])
                changed = 2
                break
            else:
                logging.info("No time change detected on %s", rescan[count]['path'])
                count = count + 1
    return changed


def update_configuration():
    for link in os.listdir(DD_CONF_DIR):
        remove_symlink(DD_CONF_DIR + link)
    # directory is cleared, begin update
    dogObj['watched_files'] = find_files_to_watch(dogObj['directory'], dogObj['filetype'])
    for wf in dogObj['watched_files']:
        # This creates a symbolic link
        logging.warning("Symlinking %s to %s", wf['path'], DD_CONF_DIR + wf['name'])
        os.symlink(wf['path'], DD_CONF_DIR + wf['name'])


def remove_symlink(path):
    if os.path.islink(path):
        logging.warning("Removing %s symlink", path)
        os.unlink(path)


def get_modified_time(args):
    statinfo = os.stat(args)
    return statinfo.st_mtime


def dogsitter():
    dogObj['watched_files'] = []
    update_configuration()
    while len(dogObj['watched_files']) >= 0:
        condition = compare_watched_files(dogObj)
        if condition == 0:
            logging.info("No change in watched files.")
        elif condition == 1:
            logging.warning("Number of watched files have changed.")
            update_configuration()
            restart()
        elif condition == 2:
            logging.warning("A watched file was modified.")
            dogObj['watched_files'] = find_files_to_watch(dogObj['directory'], dogObj['filetype'])
            restart()
        else:
            logging.error("! How did you get here.")

        time.sleep(dogObj['heartbeat'])
        logging.info("slept for %s seconds", str(dogObj['heartbeat']))

if __name__ == '__main__':
    dogsitter()

supervisor.conf

[supervisorctl]
serverurl = unix:///opt/datadog-agent/run/datadog-supervisor.sock

[unix_http_server]
file=/opt/datadog-agent/run/datadog-supervisor.sock

[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisord]
http_port = /opt/datadog-agent/run/datadog-supervisor.sock
minfds = 1024
minprocs = 200
loglevel = info
logfile = /var/log/datadog/supervisord.log
logfile_maxbytes = 50MB
nodaemon = false
pidfile = /opt/datadog-agent/run/datadog-supervisord.pid
logfile_backups = 10
environment=PYTHONPATH=/opt/datadog-agent/agent:/opt/datadog-agent/agent/checks,LANG=POSIX

[program:collector]
command=/opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/agent.py foreground --use-local-forwarder
stdout_logfile=NONE
stderr_logfile=NONE
priority=999
startsecs=5
startretries=3
environment=PYTHONPATH='/opt/datadog-agent/agent:/opt/datadog-agent/agent/checks/libs:$PYTHONPATH'

[program:forwarder]
command=/opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/ddagent.py
stdout_logfile=NONE
stderr_logfile=NONE
startsecs=5
startretries=3
priority=998

[program:dogstatsd]
command=/opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/dogstatsd.py --use-local-forwarder
stdout_logfile=NONE
stderr_logfile=NONE
startsecs=5
startretries=3
priority=998

[program:jmxfetch]
command=/opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/jmxfetch.py
stdout_logfile=NONE
stderr_logfile=NONE
redirect_stderr=true
priority=999
startsecs=3

[program:dogsitter]
command=/opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/dogsitter.py /devops-vol/monitoring/integrations
stdout_logfile=NONE
stderr_logfile=NONE
redirect_stderr=true
priority=1
startsecs=3

[group:datadog-agent]
programs=forwarder,collector,dogstatsd,jmxfetch