Recursively merging dictionaries with boltons.iterutils.remap. Useful for @hynek's configs. https://twitter.com/hynek/status/696720593002041345

remerge.py

"""
This comes from Mahmoud Hashemi @mhashemi found at:
https://gist.github.com/mahmoud/db02d16ac89fa401b968

This is an extension of the technique first detailed here:

http://sedimental.org/remap.html#add_common_keys

In short, it calls remap on each container, back to front, using the accumulating
previous values as the default for the current iteration.

[NOTE: There is currently a bug/limitation where we cannot handle keys with
       arrays as values (c.f. 'zones' below).]
"""

from boltons.iterutils import remap, get_path, default_enter, default_visit


defaults = {'host': '127.0.0.1',
            'port': 8000,
            'endpoints': {'persistence': {'host': '127.0.0.1',
                                          'port': 8888},
                          'cache': {'host': '127.0.0.1',
                                    'port': 8889}}}


overlay = {'host': '127.0.0.1',
           'port': 8080,
           'zones': [{'a': 1}],
           'endpoints': {'persistence': {'host': '10.2.2.2',
                                         'port': 5433}},
           'overlay_version': '5.0'}


cache_host_override = {'endpoints': {'cache': {'host': '127.0.0.2'}}}


def remerge(target_list, sourced=False):
    """Takes a list of containers (e.g., dicts) and merges them using
    boltons.iterutils.remap. Containers later in the list take
    precedence (last-wins).

    By default, returns a new, merged top-level container. With the
    *sourced* option, `remerge` expects a list of (*name*, container*)
    pairs, and will return a source map: a dictionary mapping between
    path and the name of the container it came from.

    """

    if not sourced:
        target_list = [(id(t), t) for t in target_list]

    ret = None
    source_map = {}

    def remerge_enter(path, key, value):
        print("path={}, key={}, value={}".format(path, key, value))
        new_parent, new_items = default_enter(path, key, value)
        print("new_parent={}, new_items={}".format(new_parent, new_items))
        if ret and not path:
            print("updating new_parent to ret={}".format(ret))
            new_parent = ret
        try:
            cur_val = get_path(ret, path + (key,))
            print("cur_val={}".format(cur_val))
        except KeyError:
            print("KeyError")
            pass
        else:
            # TODO: type check?
            print("setting new_parent to {}".format(cur_val))
            new_parent = cur_val

        print("returning new_parent={}, new_items={}".format(new_parent, new_items))
        return new_parent, new_items

    for t_name, target in target_list:
        if sourced:
            def remerge_visit(path, key, value):
                source_map[path + (key,)] = t_name
                return True
        else:
            remerge_visit = default_visit

        ret = remap(target, enter=remerge_enter, visit=remerge_visit)

    if not sourced:
        return ret
    return ret, source_map


def main():
    from pprint import pprint
    import json

    merged, source_map = remerge([('defaults', defaults),
                                  ('overlay', overlay),
                                  ('cache_host_override', cache_host_override),
                                  ],
                                 sourced=True)

    assert merged['host'] == '127.0.0.1'
    assert merged['port'] == 8080
    assert merged['endpoints']['persistence']['host'] == '10.2.2.2'
    assert merged['endpoints']['persistence']['port'] == 5433
    assert merged['endpoints']['cache']['host'] == '127.0.0.2'
    assert merged['endpoints']['cache']['port'] == 8889
    assert merged['overlay_version'] == '5.0'

    pprint(merged)
    print()
    pprint(source_map)
    print(len(source_map), 'paths')

if __name__ == '__main__':
    main()

the_output.py

path=(), key=None, value={'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}
new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}})
KeyError
returning new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}})
path=(), key=host, value=127.0.0.1
new_parent=127.0.0.1, new_items=False
KeyError
returning new_parent=127.0.0.1, new_items=False
path=(), key=port, value=8000
new_parent=8000, new_items=False
KeyError
returning new_parent=8000, new_items=False
path=(), key=endpoints, value={'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}
new_parent={}, new_items=ItemsView({'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}})
KeyError
returning new_parent={}, new_items=ItemsView({'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}})
path=('endpoints',), key=persistence, value={'host': '127.0.0.1', 'port': 8888}
new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8888})
KeyError
returning new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8888})
path=('endpoints', 'persistence'), key=host, value=127.0.0.1
new_parent=127.0.0.1, new_items=False
KeyError
returning new_parent=127.0.0.1, new_items=False
path=('endpoints', 'persistence'), key=port, value=8888
new_parent=8888, new_items=False
KeyError
returning new_parent=8888, new_items=False
path=('endpoints',), key=cache, value={'host': '127.0.0.1', 'port': 8889}
new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8889})
KeyError
returning new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8889})
path=('endpoints', 'cache'), key=host, value=127.0.0.1
new_parent=127.0.0.1, new_items=False
KeyError
returning new_parent=127.0.0.1, new_items=False
path=('endpoints', 'cache'), key=port, value=8889
new_parent=8889, new_items=False
KeyError
returning new_parent=8889, new_items=False
path=(), key=None, value={'host': '127.0.0.1', 'port': 8080, 'overlay_version': '5.0', 'zones': [{'a': 1}], 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}}
new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8080, 'overlay_version': '5.0', 'zones': [{'a': 1}], 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}})
updating new_parent to ret={'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}
KeyError
returning new_parent={'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8080, 'overlay_version': '5.0', 'zones': [{'a': 1}], 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}})
path=(), key=host, value=127.0.0.1
new_parent=127.0.0.1, new_items=False
updating new_parent to ret={'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}
cur_val=127.0.0.1
setting new_parent to 127.0.0.1
returning new_parent=127.0.0.1, new_items=False
path=(), key=port, value=8080
new_parent=8080, new_items=False
updating new_parent to ret={'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}
cur_val=8000
setting new_parent to 8000
returning new_parent=8000, new_items=False
path=(), key=overlay_version, value=5.0
new_parent=5.0, new_items=False
updating new_parent to ret={'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}
KeyError
returning new_parent={'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}, new_items=False
path=(), key=zones, value=[{'a': 1}]
new_parent=[], new_items=<enumerate object at 0x7f5d2a0b4090>
updating new_parent to ret={'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}
KeyError
returning new_parent={'host': '127.0.0.1', 'port': 8000, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}, new_items=<enumerate object at 0x7f5d2a0b4090>
path=('zones',), key=0, value={'a': 1}
new_parent={}, new_items=ItemsView({'a': 1})
KeyError
returning new_parent={}, new_items=ItemsView({'a': 1})
path=('zones', 0), key=a, value=1
new_parent=1, new_items=False
KeyError
returning new_parent=1, new_items=False
path=(), key=endpoints, value={'persistence': {'host': '10.2.2.2', 'port': 5433}}
new_parent={}, new_items=ItemsView({'persistence': {'host': '10.2.2.2', 'port': 5433}})
updating new_parent to ret={'host': '127.0.0.1', 'port': 8000, 0: {'a': 1}, 'endpoints': {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}
cur_val={'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}
setting new_parent to {'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}
returning new_parent={'persistence': {'host': '127.0.0.1', 'port': 8888}, 'cache': {'host': '127.0.0.1', 'port': 8889}}, new_items=ItemsView({'persistence': {'host': '10.2.2.2', 'port': 5433}})
path=('endpoints',), key=persistence, value={'host': '10.2.2.2', 'port': 5433}
new_parent={}, new_items=ItemsView({'host': '10.2.2.2', 'port': 5433})
cur_val={'host': '127.0.0.1', 'port': 8888}
setting new_parent to {'host': '127.0.0.1', 'port': 8888}
returning new_parent={'host': '127.0.0.1', 'port': 8888}, new_items=ItemsView({'host': '10.2.2.2', 'port': 5433})
path=('endpoints', 'persistence'), key=host, value=10.2.2.2
new_parent=10.2.2.2, new_items=False
cur_val=127.0.0.1
setting new_parent to 127.0.0.1
returning new_parent=127.0.0.1, new_items=False
path=('endpoints', 'persistence'), key=port, value=5433
new_parent=5433, new_items=False
cur_val=8888
setting new_parent to 8888
returning new_parent=8888, new_items=False
path=(), key=None, value={'endpoints': {'cache': {'host': '127.0.0.2'}}}
new_parent={}, new_items=ItemsView({'endpoints': {'cache': {'host': '127.0.0.2'}}})
updating new_parent to ret={'overlay_version': '5.0', 'port': 8080, 'zones': {...}, 0: {'a': 1}, 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}
KeyError
returning new_parent={'overlay_version': '5.0', 'port': 8080, 'zones': {...}, 0: {'a': 1}, 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}, new_items=ItemsView({'endpoints': {'cache': {'host': '127.0.0.2'}}})
path=(), key=endpoints, value={'cache': {'host': '127.0.0.2'}}
new_parent={}, new_items=ItemsView({'cache': {'host': '127.0.0.2'}})
updating new_parent to ret={'overlay_version': '5.0', 'port': 8080, 'zones': {...}, 0: {'a': 1}, 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}, 'cache': {'host': '127.0.0.1', 'port': 8889}}}
cur_val={'persistence': {'host': '10.2.2.2', 'port': 5433}, 'cache': {'host': '127.0.0.1', 'port': 8889}}
setting new_parent to {'persistence': {'host': '10.2.2.2', 'port': 5433}, 'cache': {'host': '127.0.0.1', 'port': 8889}}
returning new_parent={'persistence': {'host': '10.2.2.2', 'port': 5433}, 'cache': {'host': '127.0.0.1', 'port': 8889}}, new_items=ItemsView({'cache': {'host': '127.0.0.2'}})
path=('endpoints',), key=cache, value={'host': '127.0.0.2'}
new_parent={}, new_items=ItemsView({'host': '127.0.0.2'})
cur_val={'host': '127.0.0.1', 'port': 8889}
setting new_parent to {'host': '127.0.0.1', 'port': 8889}
returning new_parent={'host': '127.0.0.1', 'port': 8889}, new_items=ItemsView({'host': '127.0.0.2'})
path=('endpoints', 'cache'), key=host, value=127.0.0.2
new_parent=127.0.0.2, new_items=False
cur_val=127.0.0.1
setting new_parent to 127.0.0.1
returning new_parent=127.0.0.1, new_items=False
{0: {'a': 1},
 'endpoints': {'cache': {'host': '127.0.0.2', 'port': 8889},
               'persistence': {'host': '10.2.2.2', 'port': 5433}},
 'host': '127.0.0.1',
 'overlay_version': '5.0',
 'port': 8080,
 'zones': <Recursion on dict with id=140038114041032>}

{('endpoints',): 'cache_host_override',
 ('endpoints', 'cache'): 'cache_host_override',
 ('endpoints', 'cache', 'host'): 'cache_host_override',
 ('endpoints', 'cache', 'port'): 'defaults',
 ('endpoints', 'persistence'): 'overlay',
 ('endpoints', 'persistence', 'host'): 'overlay',
 ('endpoints', 'persistence', 'port'): 'overlay',
 ('host',): 'overlay',
 ('overlay_version',): 'overlay',
 ('port',): 'overlay',
 ('zones',): 'overlay',
 ('zones', 0): 'overlay',
 ('zones', 0, 'a'): 'overlay'}
13 paths