Download & updating "www_logs" from Hetzner's web hosting service (for cron jobs)

download_www_logs_from_hetzner.py

#!/usr/bin/env python
# Python 3

import datetime
import gzip
import os
import stat

import pysftp # pip install pysftp==0.2.8

USER = 'username'
PWD = 'password'
HOST = 'www.domain.xy'
TARGET = '/path/to/data'

def _parse_remote(file_list):
    return {
        attr.filename: {
            'fn': attr.filename,
            'mtime': attr.st_mtime,
            'size': attr.st_size,
        } for attr in file_list
    }

def _parse_local(file_list, local_path):
    return {
        fn: {
            'fn': fn,
            'mtime': attr.st_mtime,
            'size': attr.st_size,
        } for fn, attr in zip(file_list, (os.stat(os.path.join(local_path, item)) for item in file_list))
    }

def _update(sftp_handle, local_path):
    remote_dict = _parse_remote(sftp_handle.listdir_attr())
    local_dict = _parse_local(os.listdir(local_path), local_path)
    missing = remote_dict.keys() - local_dict.keys()
    present = remote_dict.keys() & local_dict.keys()
    updated = set()
    for fn in present:
        if local_dict[fn]['mtime'] != remote_dict[fn]['mtime'] or local_dict[fn]['size'] != remote_dict[fn]['size']:
            updated.add(fn)
    print(missing)
    for fn in missing:
        sftp_handle.get(
            fn,
            localpath = os.path.join(local_path, fn),
            preserve_mtime = True
            )
    print(updated)
    for fn in updated:
        os.unlink(os.path.join(local_path, fn))
        sftp_handle.get(
            fn,
            localpath = os.path.join(local_path, fn),
            preserve_mtime = True
            )

def main():
    with pysftp.Connection(HOST, username = USER, password = PWD) as sftp:
        with sftp.cd('www_logs'):
            for domain_handle in sftp.listdir_attr():
                domain_path = os.path.join(TARGET, domain_handle.filename)
                os.makedirs(domain_path, exist_ok = True)
                print(domain_handle.filename)
                with sftp.cd(domain_handle.filename):
                    _update(sftp, domain_path)

if __name__ == '__main__':
    main()