dawa.py

"""
This tool is meant to process GTFS files from transport.data.gouv.fr,
convert them to the NeTEx format,
and upload them as community resources to transport.data.gouv.fr
"""

import urllib.request
import tempfile
import subprocess
import requests

CONVERTER = "/home/tristram/beta.gouv.fr/tartare-tools/target/release/gtfs2netexfr"
DATAGOUV_API = "https://demo.data.gouv.fr/api/1"
DATAGOUV_API_KEY = "KEY"
TRANSPORT_ORGANIZATION_ID = "5abca8d588ee386ee6ece479"

def download_gtfs(url):
    """
    Downloads the requested GTFS and saves it as local file.
    Returns the path to that file
    """
    local_filename, headers = urllib.request.urlretrieve(url)

    return local_filename

def convert(gtfs_src, publisher):
    """
    Converts a given gtfs file and returns the path to the generated netex zip file.
    The publisher is the name of the organization that published that dataset.
    """
    with tempfile.TemporaryDirectory() as netex_dir:
        ret = subprocess.run([CONVERTER, "--input", gtfs_src, "--output", netex_dir, "--participant", publisher])
        if ret.returncode == 0:
            netex_zip = f"{netex_dir}.zip"
            ret = subprocess.run([f"zip {netex_zip} -r {netex_dir}/*"], shell=True)
            if ret.returncode == 0:
                return netex_zip
            else:
                print("Unable to zip file")
        else:
            print("Unable to convert file")

def community_resources_exists(dataset_id, gtfs_name):
    """
    Checks if the a community resource already exists
    """
    headers = {'X-API-KEY': DATAGOUV_API_KEY}
    url = f"{DATAGOUV_API}/datasets/community_resources/"
    params = {
        'dataset': dataset_id,
        'organization': TRANSPORT_ORGANIZATION_ID
    }
    r = requests.get(url, params=params)
    print(r.json())

    return r.json()['data'][0]

def create_community_resource(dataset_id, gtfs_name):
    """
    Creates a community resource to that dataset.
    """
    headers = {'X-API-KEY': DATAGOUV_API_KEY}
    payload = { # Todo : on dirait que c’est pas pris en compte ça…
        'filename': 'netex.zip',
    }
    files = {'file': open('out.json','rb')}

    url = f"{DATAGOUV_API}/datasets/{dataset_id}/upload/community/"
    #files = {'file': open('surf.zip', 'rb')}
    r = requests.post(url, headers=headers, data=payload, files=files)
    if r.ok:
        print(r.json()['id'])
        print(r.json())
        return r.json()
    else:
        print(r.text)

def link_resource_to_dataset(dataset_id, resource):
    """
    Once the resource is uploaded,
    """
    url = f"{DATAGOUV_API}/datasets/community_resources/{resource['id']}/"
    resource['dataset'] = dataset_id
    resource['organization'] = TRANSPORT_ORGANIZATION_ID
    resource['description'] = "Converstion automatique du fichier code-code-code"
    resource['title'] = 'netex.zip'
    resource['format'] = 'NeTEx'

    headers = {'X-API-KEY': DATAGOUV_API_KEY}
    r = requests.put(url, headers=headers, json=resource)
    if r.ok:
        print(r.json())
    else:
        print(r.text)

def update_resource(resource_id, filename):
    """
    Replaces the file of an existing resource.

    After the call, and update to that resource is needed
    """
    url = f"{DATAGOUV_API}/datasets/community_resources/{resource_id}/upload/"
    headers = {'X-API-KEY': DATAGOUV_API_KEY}
    r = requests.post(url, headers=headers, files={'file': open(filename,'rb')} )
    if r.ok:
        print(r.json())
    else:
        print(r.text)


def main():
    """Main entry point"""
    # gtfs = download_gtfs("https://static.data.gouv.fr/resources/donnees-gtfs-sur-le-reseau-de-bus-surf-service-urbain-de-la-region-fougeraise/20200107-153800/07-01-2020-3.zip")
    # print("gtfs is : {}".format(gtfs))
    # netex = convert(gtfs, "fougèèère")
    # print("Hello, netex should be : {}".format(netex))
    print("fetching resource")
    r = community_resources_exists("588a238d88ee3846659b81a4", "msg.txt")
    print(f"found: {r}")
    #result = create_community_resource("588a238d88ee3846659b81a4", "surf.zip")
    link_resource_to_dataset("588a238d88ee3846659b81a4", r)
    #update_resource('a85f83fd-d5eb-4a70-8a31-810ce736cb38', 'msg.txt')


if __name__ == "__main__":
    main()