Biggest datasets and resources on data.gouv.fr (works on any udata instance)

biggest-datasets.csv

biggest-resources.csv

biggest.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import unicodecsv

from flask import url_for

from udata.app import create_app, standalone
from udata.models import Dataset

# Limit queryset length by keeping datasets
# with a resource having at least MIN_SIZE size
MIN_SIZE = 10000

MAX_DATASETS = 100
DATASETS_FILENAME = 'biggest-datasets.csv'

MAX_RESOURCES = 250
RESOURCES_FILENAME = 'biggest-resources.csv'

CONFIG = {
    'encoding': 'utf-8',
    'delimiter': b',',  # gist compatibility
    'quotechar': b'"',
}

app = standalone(create_app())


def get_writer(out, headers):
    '''Get a preconfigured CSV writer for a given output file'''
    return unicodecsv.DictWriter(out,
                                 quoting=unicodecsv.QUOTE_NONNUMERIC,
                                 fieldnames=headers, **CONFIG)


def dataset_url(id):
    return url_for('datasets.show', dataset=id, _external=True, _scheme='https')


# MongoDB aggregation pipeline listing biggest resources
datasets_pipeline = [
    {'$match': {'resources.filesize': {'$gt': MIN_SIZE}}},  # Faster
    {'$unwind': '$resources'},  # One document by resource
    {'$match': {'resources.filesize': {'$gt': MIN_SIZE}}},  # Faster
    {'$sort': {'resource.filesize': -1}},  # Store resources in order
    {'$group': {
        '_id': '$_id',
        'title': {'$first': '$title'},
        'maxsize': {'$max': '$resources.filesize'},  # Track max resource size for sorting
        'resources': {'$push': '$resources'},
    }},
    {'$sort': {'maxsize': -1}},
    {'$limit': MAX_DATASETS},
    {'$project': {
        'id': '$_id',
        'title': True,
        'resource': {'$arrayElemAt': ['$resources', 0]},  # only the first resource
    }},
]

# Export biggest datasets with their biggest resource as CSV
with open(DATASETS_FILENAME, 'wb') as out, app.app_context():
    fields = ['id', 'title', 'url', 'resource.id', 'resource.title', 'resource.url', 'resource.size']
    writer = get_writer(out, fields)
    writer.writeheader()
    for row in Dataset.objects.aggregate(*datasets_pipeline):
        writer.writerow({
            'id': row['id'],
            'title': row['title'],
            'url': dataset_url(row['id']),
            'resource.id': row['resource']['_id'],
            'resource.title': row['resource']['title'],
            'resource.url': row['resource']['url'],
            'resource.size': row['resource']['filesize'],
        })


# MongoDB aggregation pipeline listing biggest resources
resources_pipeline = [
    {'$match': {'resources.filesize': {'$gt': MIN_SIZE}}},
    {'$unwind': '$resources'},
    {'$sort': {'resources.filesize': -1}},
    {'$limit': MAX_RESOURCES},
]

# Export biggest resources with their dataset as CSV
with open(RESOURCES_FILENAME, 'wb') as out, app.app_context():
    fields = ['id', 'title', 'url', 'size', 'dataset.id', 'dataset.title', 'dataset.url']
    writer = get_writer(out, fields)
    writer.writeheader()
    for row in Dataset.objects.aggregate(*resources_pipeline):
        writer.writerow({
            'id': row['resources']['_id'],
            'title': row['resources']['title'],
            'url': row['resources']['url'],
            'size': row['resources']['filesize'],
            'dataset.id': row['_id'],
            'dataset.title': row['title'],
            'dataset.url': dataset_url(row['_id']),
        })

HTML table version