covid19.py

#!/usr/bin/env python3

import requests
import argparse
import pickle
import os

url = 'https://services.arcgis.com/CCZiGSEQbAxxFVh3/arcgis/rest/services/COVID19Portugal_view/FeatureServer/0/query'

import os, time, stat
import hashlib

CACHE_DIR = '.cache'

def file_age_in_seconds(pathname):
    '''
    https://stackoverflow.com/a/6879539
    '''
    return time.time() - os.stat(pathname)[stat.ST_MTIME]

class request_cache:
    def __init__(self, max_age):
        self.max_age = max_age

    def __call__(self, func):
        def wrapper(*args):
            if not os.path.exists(CACHE_DIR):
                os.makedirs(CACHE_DIR)

            h = hashlib.md5(str(args).encode('utf-8')).hexdigest()
            file_name = '{}/{}'.format(CACHE_DIR, h)

            # 12 hours
            if os.path.isfile(file_name) and file_age_in_seconds(file_name) < self.max_age:
                with open(file_name, 'rb') as f:
                    data = pickle.load(f)
                    return data

            data = func(*args)
            with open(file_name, 'wb') as f:
                pickle.dump(data, f)
            return data
        return wrapper

@request_cache(max_age=43200)
def request_data_json(where = 'objectid IS NOT NULL'):
    params = {
        'f' : 'json',
        'where' : where,
        'returnGeometry' : 'false',
        'spatialRel' : 'esriSpatialRelIntersects',
        'outFields' : '*',
        'orderByFields' : 'datarelatorio asc',
        'resultOffset' : '0',
        'resultRecordCount' : '1000',
        'cacheHint' : 'true',
    }

    resp = requests.get(url=url, params=params)
    return resp.json()

def get_attributes():
    data = request_data_json()

    for f in data['fields']:
        yield (f['alias'], f['name'])

def get_alias(attributes, name):
    for a, n in attributes:
        if n == name:
            return a

def get_attribute_data(attribute):
    data = request_data_json('{} IS NOT NULL'.format(attribute))

    attributes = get_attributes()
    datarelatorio_alias = get_alias(attributes, 'datarelatorio')
    attribute_alias = get_alias(attributes, attribute)

    yield (datarelatorio_alias, attribute_alias)

    for f in data['features']:
        timestamp = f['attributes']['datarelatorio']
        value = f['attributes'][attribute]
        # print('{},{}'.format(timestamp, value))

        yield (timestamp, value)

if __name__== "__main__":
    parser = argparse.ArgumentParser(description='Get Portugal covid-19 stats')
    parser.add_argument('command', metavar='command', type=str, help='Commands (list|fetch)')
    parser.add_argument('--attribute', help='--attribute casosnovos')

    args = parser.parse_args()

    if args.command == 'list':
        for alias, name in get_attributes():
            print('{}: {}'.format(alias, name))

    elif args.command == 'csv':
        for timestamp, value in get_attribute_data(args.attribute):
            print('{},{}'.format(timestamp, value))

    elif args.command == 'plot':
        import pandas

        data = get_attribute_data(args.attribute)

        # for timestamp, value in get_attribute_data(args.attribute):
        #    print('{},{}'.format(timestamp, value))