Polls a file hosted at a URL and downloads it initially and if it changes.

PollingFileDownloader.py

"""Polls a file hosted at a URL and downloads it initially and if it changes."""

# Should be fairly robust to web server issues (in fact, it would only
# be a handful of lines were it not for error handling)

import requests
import time
import sys

FILE_URL = "http://<mywebserver>/<myfile>"
WRITE_FILE_PATH = "/tmp/<myfile>"
POLLING_INTERVAL_SEC = 60
CONNECT_AND_READ_TIMEOUT = 3

# Check last_modified using a head request, then pull down the file when it
# changes.

class MethodEnum:
    HEAD = 1
    GET  = 2

def writeFile(content):
    with open(WRITE_FILE_PATH,'wb') as destFile:
        destFile.write(content)

def doRequest(url, method):
    response = None
    try:
        if MethodEnum.GET == method:
            response = requests.get(url=url, timeout=CONNECT_AND_READ_TIMEOUT)
        elif MethodEnum.HEAD == method:
            response = requests.head(url=url, timeout=CONNECT_AND_READ_TIMEOUT)
        else:
            raise("Developer error... unhandled method:  " + method)
    except requests.exceptions.ConnectionError as e:
        print("Problem connecting:  " + str(e))
    except requests.exceptions.ConnectTimeout as e:
        print("Slow connection:  " + str(e))
    return response

def pollForever():
    # Initial pass... pull down the file.
    try:
        lastModified, priorLastModified = '',''
        resp = doRequest(FILE_URL, MethodEnum.GET)
        if resp:
            writeFile(resp.content)
            print("Initial download completed to {0}".format(WRITE_FILE_PATH))
            priorLastModified = resp.headers['last-modified']
        time.sleep(POLLING_INTERVAL_SEC)
    except Exception as initDownloadError:
        print("Problem with initial download: " + str(initDownloadError))

    while(True):
        try:
            resp = doRequest(FILE_URL, MethodEnum.HEAD)
            if resp:
                lastModified = resp.headers['last-modified']
            if priorLastModified and lastModified:
                if lastModified != priorLastModified:
                    print("Later version detected [{0}]... downloading it to  {1}".format(
                        lastModified, WRITE_FILE_PATH))
                    resp = doRequest(FILE_URL, MethodEnum.GET)
                    if resp:
                        writeFile(resp.content)
                        priorLastModified = resp.headers['last-modified']
            if lastModified:
                priorLastModified = lastModified
        except Exception as pollingLoopError:
            print("Problem with polling loop: " + str(pollingLoopError))

        time.sleep(POLLING_INTERVAL_SEC)

if __name__ == '__main__':
    print("""\
Starting polling file download with:
Download URL:           {0}
Save file path:         {1}
Connect/Read timeout:   {2}
Polling Interval (sec): {3}""".format(
    FILE_URL, WRITE_FILE_PATH, CONNECT_AND_READ_TIMEOUT, POLLING_INTERVAL_SEC)
)
    sys.exit(pollForever())