reader config file prototype for https://github.com/lemon24/reader/issues/177

config.py

from reader import make_reader
from reader._plugins import import_string, Loader, LoaderError
import json
import click
from dataclasses import dataclass


IMPORT_KWARGS = ('storage_cls', 'search_cls')
MERGE_KWARGS = ('plugins', )

def make_reader_from_config(*, plugins=None, **kwargs):
    """Like make_reader, but loads plugins and imports *_cls stuff."""
    plugins = plugins or {}

    for name in IMPORT_KWARGS:
        thing = kwargs.get(name)
        if thing and isinstance(thing, str):
            kwargs[name] = import_string(thing)

    reader = make_reader(**kwargs)

    try:
        loader = Loader(plugins)
        loader.load_plugins(reader)
    except LoaderError as e:
        reader.close()
        raise

    return reader

def merge_config(*configs):
    rv = {}
    to_merge = {}
    for config in configs:
        config = config.copy()
        for name in MERGE_KWARGS:
            if name in config:
                to_merge.setdefault(name, []).append(config.pop(name))
        rv.update(config)
    for name, dicts in to_merge.items():
        rv[name] = merge_config(*dicts)
    return rv


def read_config(*args):
    return {
        'reader': {
            'url': 'config-reader-url',
            'plugins': {'reader-config-plugin': None},
        },
        'cli': {
            #'url': 'config-cli-url',
            'plugins': {'cli-config-plugin': None},
            'defaults': {
                #'db': 'cli-default-db',
                'plugin': ('cli-reader-default-plugin',),
                'update': {'new_only': True},
                'serve': {'plugin': ('cli-serve-default-plugin',)}
            },
        },
        'app': {
            #'url': 'config-app-url',
            'plugins': {'app-config-plugin': None},
        }
    }


import functools

class Default: pass

class Default_bool(int, Default):
    # workaround for it not being possible to subclass bool.
    # however, this breaks Click, see wrap_param_default() for details.
    def __str__(self):
        return str(bool(self))

@functools.lru_cache()
def make_default_wrapper(cls):
    if cls is bool:
        return Default_bool
    return type('Default_' + cls.__name__, (cls, Default), {})

def wrap_default(thing):
    # assumes type(thing)(thing) will return another thing;
    # to support other constructor types, we could
    #   thing.__class__ = make_default_wrapper(type(thing))
    # but that works only for heap types.
    assert type(thing) in (int, bool, float, str, bytes, list, tuple, dict)
    return make_default_wrapper(type(thing))(thing)

def is_default(thing):
    return isinstance(thing, Default)

def split_defaults(dict):
    defaults = {}
    options = {}
    for k, v in dict.items():
        if not v:
            continue
        (defaults if is_default(v) else options)[k] = v
    return defaults, options

def wrap_param_default(param, value):
    # can't use our Default_bool for click defaults, because
    # BoolParamType calls isinstance(value, bool); if false,
    # it assumes it's a string that looks like a boolean ("true", "yes" etc.),
    # so we make our value look like that (while still marking it as default).
    if isinstance(value, bool) and isinstance(param.type, click.types.BoolParamType):
        return wrap_default(str(bool(value)).lower())
    return wrap_default(value)

def mark_command_defaults(command):
    # decorator for commands with options that end up in the config
    # passed to make_reader_from_config;
    # if not used, the config order won't work properly
    for param in command.params:
        if param.default is not None:
            param.default = wrap_param_default(param, param.default)
    return command

def mark_default_map_defaults(command, defaults):
    for param in command.params:
        if param.name in defaults:
            defaults[param.name] = wrap_param_default(param, defaults[param.name])
    for command_name, command in getattr(command, 'commands', {}).items():
        mark_default_map_defaults(command, defaults.get(command_name, {}))


def load_config_callback(ctx, _, config):
    ctx.default_map = config['cli'].pop('defaults', {})
    mark_default_map_defaults(ctx.command, ctx.default_map)
    ctx.obj = Config(config)
    return config


class Config(dict):

    def merge(self, *keys, options=None):
        default_options, user_options = split_defaults(
            merge_config(self.get('options', {}), options or {})
        )
        return merge_config(
            * [default_options] + [self[k] for k in keys] + [user_options]
        )


# NOTE: the defaults from default_map won't show up in --help;
# "show_default and default_map don't play nice together"
# https://github.com/pallets/click/issues/1548


@mark_command_defaults
@click.group()
@click.option('--db', default='~/.config/reader/db.sqlite', show_default=True)
@click.option(
    '--config',
    type=read_config, default='~/.config/reader/config.file',
    callback=load_config_callback, is_eager=True, expose_value=False,
)
@click.option('--plugin', multiple=True)
@click.pass_obj
def cli(config, db, plugin):
    # cli default < config[cli][defaults] < config[reader] < config[cli,app] < cli envvar,option

    # save options so other commands can use them
    config['options'] = {
        'url': db,
        'plugins': {p: None for p in plugin},
    }

    # wrap reader section with options; 
    # will be used by app to spawn non-app readers
    config['reader_final'] = config.merge('reader')

    # merge reader and cli, wrapping with options
    config['cli_final'] = config.merge('reader', 'cli')


@cli.command()
@click.option('--new-only/--no-new-only')
@click.pass_obj
def update(config, new_only):
    # the 1.5 reader._cli.pass_reader would do the merge for all of the commands
    print('new_only:', new_only)
    print('make_reader_from_config(cli_config) would get called with:')
    print(json.dumps(config['cli_final'], indent=4))


@mark_command_defaults
@cli.command()
@click.option('--plugin', multiple=True, default='aaa')
@click.pass_obj
def serve(config, plugin):
    # serve uses the app config section, not the cli one
    # (to mirror what WSGI would do)

    # merge reader and app, wrapping in our options and the cli-wide options
    config['app_final'] = config.merge('reader', 'app', options={
        'plugins': {p: None for p in plugin},
    })

    print('make_app_from_config(reader_config, app_config) would get called with:')
    print(json.dumps(config['reader_final'], indent=4))
    print(json.dumps(config['app_final'], indent=4))


def wsgi():
    # reader._app.wsgi
    import os
    import reader._app, readed._config
    config = reader._config.read_config(os.environ['READER_CONFIG'])
    app = reader._app.create_app(app)


if __name__ == '__main__':
    cli()

config.yaml

# using YAML because it's easy to write,
# may end up using a different format

# this is the "default" section
reader:
    # make_reader args,
    # per https://github.com/lemon24/reader/issues/168#issuecomment-642002049

    feed_root: /path/to/feeds

    url: db.sqlite
    # alternatively (a wrapper needs to import it from a string),
    storage_cls: reader._storage:Storage
    storage_arg: ...

    # optional
    search_url: 'sqlite-parasitic:'
    # alternatively,
    search_cls: package.module:Class
    search_arg: ...

    # an extension handled by the extension
    plugins:
        package.module:
        module2:
            key: value

cli:
    # more make_reader args; they replace the reader.* stuff
    feed_root: ''

    # ... with the exception of plugins; these get merged
    plugins:
        cli.module:

    # another special case, CLI defaults (become the Click default_map);
    # the CLI needs to pop this before merging with 'reader'
    defaults:
        verbose: 0  # --verbose, global verbosity

        # this provides the default to --plugin; using --plugin *replaces*
        # the default (merge happens only for config[$section][plugins]).
        # the command group extends config[cli][plugins] with the --plugin values
        plugin: [default_cli_plugin]

        add:  # add --update -v
            update: yes
            verbose: 1

        update:  # update --new-only --workers 10 -vv
            new_only: yes
            workers: 10
            verbose: 2

        serve:  # serve --port 8888 --plugin whatever
            port: 8888
            # similar to the global --plugin, except
            # serve extends config[app][plugins] with the --plugin values
            plugin: [default_app_plugi]

app:
    # more make_reader args, same as for cli

    # app plugins, merged into reader.plugins
    plugins:
        webapp.module:
        'webapp.module2:init_app':