Skip to content

Instantly share code, notes, and snippets.

@mahmoud

mahmoud/remerge.py

Forked from pleasantone/remerge.py
Last active June 15, 2016 09:57
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save mahmoud/2e4f200c0b6de752bfb18a0788ab360d to your computer and use it in GitHub Desktop.
Save mahmoud/2e4f200c0b6de752bfb18a0788ab360d to your computer and use it in GitHub Desktop.
Download ZIP
Recursively merging dictionaries with boltons.iterutils.remap. Useful for @hynek's configs. https://twitter.com/hynek/status/696720593002041345
Raw
remerge.py
"""
This comes from Mahmoud Hashemi @mhashemi found at:
https://gist.github.com/mahmoud/db02d16ac89fa401b968
This is an extension of the technique first detailed here:
http://sedimental.org/remap.html#add_common_keys
In short, it calls remap on each container, back to front, using the accumulating
previous values as the default for the current iteration.
[NOTE: There is currently a bug/limitation where we cannot handle keys with
arrays as values (c.f. 'zones' below).]
"""
from __future__ import print_function
from boltons.iterutils import remap, get_path, default_enter, default_visit, default_exit
defaults = {'host': '127.0.0.1',
'port': 8000,
'endpoints': {'persistence': {'host': '127.0.0.1',
'port': 8888},
'cache': {'host': '127.0.0.1',
'port': 8889}}}
overlay = {'host': '127.0.0.1',
'port': 8080,
'zones': [{'a': 1}],
'endpoints': {'persistence': {'host': '10.2.2.2',
'port': 5433}},
'overlay_version': '5.0'}
cache_host_override = {'endpoints': {'cache': {'host': '127.0.0.2'}}}
INDENT = []
real_print = print
def print(*a, **kw):
return real_print(' '.join(INDENT + ['']), *a, **kw)
def remerge(target_list, sourced=False):
"""Takes a list of containers (e.g., dicts) and merges them using
boltons.iterutils.remap. Containers later in the list take
precedence (last-wins).
By default, returns a new, merged top-level container. With the
*sourced* option, `remerge` expects a list of (*name*, container*)
pairs, and will return a source map: a dictionary mapping between
path and the name of the container it came from.
"""
if not sourced:
target_list = [(id(t), t) for t in target_list]
ret = None
source_map = {}
def remerge_enter(path, key, value):
print("- entering path={}, key={!r}, value={!r}".format(path, key, value))
new_parent, new_items = default_enter(path, key, value)
INDENT.append('')
print("- got new_parent={}, new_items={}".format(new_parent, new_items))
if ret and not path and key is None:
print("- updating new_parent to ret={}".format(ret))
new_parent = ret
try:
cur_val = get_path(ret, path + (key,))
print("- cur_val={}".format(cur_val))
except KeyError as ke:
print(" X {!r} -- {!s}".format(ret, ke))
pass
else:
# TODO: type check?
print("- setting new_parent to {}".format(cur_val))
new_parent = cur_val
print("- returning new_parent={}, new_items={}".format(new_parent, new_items))
return new_parent, new_items
def remerge_exit(path, key, old_parent, new_parent, new_items):
print("+ exiting path={}, key={}, old_parent={}, new_parent={}, new_items={}".format(path, key, old_parent, new_parent, new_items))
INDENT.pop()
return default_exit(path, key, old_parent, new_parent, new_items)
for t_name, target in target_list:
if sourced:
def remerge_visit(path, key, value):
source_map[path + (key,)] = t_name
return True
else:
remerge_visit = default_visit
ret = remap(target, enter=remerge_enter, visit=remerge_visit,
exit=remerge_exit)
if not sourced:
return ret
return ret, source_map
def main():
from pprint import pprint
import json
merged, source_map = remerge([('defaults', defaults),
('overlay', overlay),
('cache_host_override', cache_host_override),
],
sourced=True)
assert merged['host'] == '127.0.0.1'
assert merged['port'] == 8080
assert merged['endpoints']['persistence']['host'] == '10.2.2.2'
assert merged['endpoints']['persistence']['port'] == 5433
assert merged['endpoints']['cache']['host'] == '127.0.0.2'
assert merged['endpoints']['cache']['port'] == 8889
assert merged['overlay_version'] == '5.0'
print(' ')
print(' ')
pprint(merged)
print(' ')
# pprint(source_map)
# print(len(source_map), 'paths')
if __name__ == '__main__':
main()
Raw
the_output.py
- entering path=(), key=None, value={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000}
- got new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000})
X None -- could not access None from path (None,), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000})
- entering path=(), key='host', value='127.0.0.1'
- got new_parent=127.0.0.1, new_items=False
X None -- could not access 'host' from path ('host',), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent=127.0.0.1, new_items=False
- entering path=(), key='endpoints', value={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}
- got new_parent={}, new_items=ItemsView({'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}})
X None -- could not access 'endpoints' from path ('endpoints',), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent={}, new_items=ItemsView({'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}})
- entering path=('endpoints',), key='cache', value={'host': '127.0.0.1', 'port': 8889}
- got new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8889})
X None -- could not access 'endpoints' from path ('endpoints', 'cache'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8889})
- entering path=('endpoints', 'cache'), key='host', value='127.0.0.1'
- got new_parent=127.0.0.1, new_items=False
X None -- could not access 'endpoints' from path ('endpoints', 'cache', 'host'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent=127.0.0.1, new_items=False
- entering path=('endpoints', 'cache'), key='port', value=8889
- got new_parent=8889, new_items=False
X None -- could not access 'endpoints' from path ('endpoints', 'cache', 'port'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent=8889, new_items=False
+ exiting path=('endpoints',), key=cache, old_parent={'host': '127.0.0.1', 'port': 8889}, new_parent={}, new_items=[('host', '127.0.0.1'), ('port', 8889)]
- entering path=('endpoints',), key='persistence', value={'host': '127.0.0.1', 'port': 8888}
- got new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8888})
X None -- could not access 'endpoints' from path ('endpoints', 'persistence'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8888})
- entering path=('endpoints', 'persistence'), key='host', value='127.0.0.1'
- got new_parent=127.0.0.1, new_items=False
X None -- could not access 'endpoints' from path ('endpoints', 'persistence', 'host'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent=127.0.0.1, new_items=False
- entering path=('endpoints', 'persistence'), key='port', value=8888
- got new_parent=8888, new_items=False
X None -- could not access 'endpoints' from path ('endpoints', 'persistence', 'port'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent=8888, new_items=False
+ exiting path=('endpoints',), key=persistence, old_parent={'host': '127.0.0.1', 'port': 8888}, new_parent={}, new_items=[('host', '127.0.0.1'), ('port', 8888)]
+ exiting path=(), key=endpoints, old_parent={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, new_parent={}, new_items=[('cache', {'host': '127.0.0.1', 'port': 8889}), ('persistence', {'host': '127.0.0.1', 'port': 8888})]
- entering path=(), key='port', value=8000
- got new_parent=8000, new_items=False
X None -- could not access 'port' from path ('port',), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
- returning new_parent=8000, new_items=False
+ exiting path=(), key=None, old_parent={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000}, new_parent={}, new_items=[('host', '127.0.0.1'), ('endpoints', {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}), ('port', 8000)]
- entering path=(), key=None, value={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}
- got new_parent={}, new_items=ItemsView({'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080})
- updating new_parent to ret={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000}
X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000} -- could not access None from path (None,), got error: KeyError(None,)
- returning new_parent={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000}, new_items=ItemsView({'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080})
- entering path=(), key='zones', value=[{'a': 1}]
- got new_parent=[], new_items=<enumerate object at 0x7f67f7a2f6e0>
X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000} -- could not access 'zones' from path ('zones',), got error: KeyError('zones',)
- returning new_parent=[], new_items=<enumerate object at 0x7f67f7a2f6e0>
- entering path=('zones',), key=0, value={'a': 1}
- got new_parent={}, new_items=ItemsView({'a': 1})
X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000} -- could not access 'zones' from path ('zones', 0), got error: KeyError('zones',)
- returning new_parent={}, new_items=ItemsView({'a': 1})
- entering path=('zones', 0), key='a', value=1
- got new_parent=1, new_items=False
X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000} -- could not access 'zones' from path ('zones', 0, 'a'), got error: KeyError('zones',)
- returning new_parent=1, new_items=False
+ exiting path=('zones',), key=0, old_parent={'a': 1}, new_parent={}, new_items=[('a', 1)]
+ exiting path=(), key=zones, old_parent=[{'a': 1}], new_parent=[], new_items=[(0, {'a': 1})]
- entering path=(), key='host', value='127.0.0.1'
- got new_parent=127.0.0.1, new_items=False
- cur_val=127.0.0.1
- setting new_parent to 127.0.0.1
- returning new_parent=127.0.0.1, new_items=False
- entering path=(), key='endpoints', value={'persistence': {'host': '10.2.2.2', 'port': 5433}}
- got new_parent={}, new_items=ItemsView({'persistence': {'host': '10.2.2.2', 'port': 5433}})
- cur_val={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}
- setting new_parent to {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}
- returning new_parent={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, new_items=ItemsView({'persistence': {'host': '10.2.2.2', 'port': 5433}})
- entering path=('endpoints',), key='persistence', value={'host': '10.2.2.2', 'port': 5433}
- got new_parent={}, new_items=ItemsView({'host': '10.2.2.2', 'port': 5433})
- cur_val={'host': '127.0.0.1', 'port': 8888}
- setting new_parent to {'host': '127.0.0.1', 'port': 8888}
- returning new_parent={'host': '127.0.0.1', 'port': 8888}, new_items=ItemsView({'host': '10.2.2.2', 'port': 5433})
- entering path=('endpoints', 'persistence'), key='host', value='10.2.2.2'
- got new_parent=10.2.2.2, new_items=False
- cur_val=127.0.0.1
- setting new_parent to 127.0.0.1
- returning new_parent=127.0.0.1, new_items=False
- entering path=('endpoints', 'persistence'), key='port', value=5433
- got new_parent=5433, new_items=False
- cur_val=8888
- setting new_parent to 8888
- returning new_parent=8888, new_items=False
+ exiting path=('endpoints',), key=persistence, old_parent={'host': '10.2.2.2', 'port': 5433}, new_parent={'host': '127.0.0.1', 'port': 8888}, new_items=[('host', '10.2.2.2'), ('port', 5433)]
+ exiting path=(), key=endpoints, old_parent={'persistence': {'host': '10.2.2.2', 'port': 5433}}, new_parent={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, new_items=[('persistence', {'host': '10.2.2.2', 'port': 5433})]
- entering path=(), key='overlay_version', value='5.0'
- got new_parent=5.0, new_items=False
X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'port': 8000} -- could not access 'overlay_version' from path ('overlay_version',), got error: KeyError('overlay_version',)
- returning new_parent=5.0, new_items=False
- entering path=(), key='port', value=8080
- got new_parent=8080, new_items=False
- cur_val=8000
- setting new_parent to 8000
- returning new_parent=8000, new_items=False
+ exiting path=(), key=None, old_parent={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}, new_parent={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'port': 8000}, new_items=[('zones', [{'a': 1}]), ('host', '127.0.0.1'), ('endpoints', {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}), ('overlay_version', '5.0'), ('port', 8080)]
- entering path=(), key=None, value={'endpoints': {'cache': {'host': '127.0.0.2'}}}
- got new_parent={}, new_items=ItemsView({'endpoints': {'cache': {'host': '127.0.0.2'}}})
- updating new_parent to ret={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}
X {'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080} -- could not access None from path (None,), got error: KeyError(None,)
- returning new_parent={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}, new_items=ItemsView({'endpoints': {'cache': {'host': '127.0.0.2'}}})
- entering path=(), key='endpoints', value={'cache': {'host': '127.0.0.2'}}
- got new_parent={}, new_items=ItemsView({'cache': {'host': '127.0.0.2'}})
- cur_val={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}
- setting new_parent to {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}
- returning new_parent={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, new_items=ItemsView({'cache': {'host': '127.0.0.2'}})
- entering path=('endpoints',), key='cache', value={'host': '127.0.0.2'}
- got new_parent={}, new_items=ItemsView({'host': '127.0.0.2'})
- cur_val={'host': '127.0.0.1', 'port': 8889}
- setting new_parent to {'host': '127.0.0.1', 'port': 8889}
- returning new_parent={'host': '127.0.0.1', 'port': 8889}, new_items=ItemsView({'host': '127.0.0.2'})
- entering path=('endpoints', 'cache'), key='host', value='127.0.0.2'
- got new_parent=127.0.0.2, new_items=False
- cur_val=127.0.0.1
- setting new_parent to 127.0.0.1
- returning new_parent=127.0.0.1, new_items=False
+ exiting path=('endpoints',), key=cache, old_parent={'host': '127.0.0.2'}, new_parent={'host': '127.0.0.1', 'port': 8889}, new_items=[('host', '127.0.0.2')]
+ exiting path=(), key=endpoints, old_parent={'cache': {'host': '127.0.0.2'}}, new_parent={'cache': {'host': '127.0.0.2', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, new_items=[('cache', {'host': '127.0.0.2', 'port': 8889})]
+ exiting path=(), key=None, old_parent={'endpoints': {'cache': {'host': '127.0.0.2'}}}, new_parent={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.2', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}, new_items=[('endpoints', {'cache': {'host': '127.0.0.2', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}})]
{'endpoints': {'cache': {'host': '127.0.0.2', 'port': 8889},
'persistence': {'host': '10.2.2.2', 'port': 5433}},
'host': '127.0.0.1',
'overlay_version': '5.0',
'port': 8080,
'zones': [{'a': 1}]}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment