Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Recursive dictionary merge in Python
# Recursive dictionary merge
# Copyright (C) 2016 Paul Durivage <pauldurivage+github@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import collections
def dict_merge(dct, merge_dct):
""" Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
updating only top-level keys, dict_merge recurses down into dicts nested
to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
``dct``.
:param dct: dict onto which the merge is executed
:param merge_dct: dct merged into dct
:return: None
"""
for k, v in merge_dct.iteritems():
if (k in dct and isinstance(dct[k], dict)
and isinstance(merge_dct[k], collections.Mapping)):
dict_merge(dct[k], merge_dct[k])
else:
dct[k] = merge_dct[k]
@eligundry

This comment has been minimized.

Copy link

eligundry commented Feb 1, 2017

You da real MVP

@cetanu

This comment has been minimized.

Copy link

cetanu commented Nov 27, 2017

I love you, I appreciate you

@BrianAndersen78

This comment has been minimized.

Copy link

BrianAndersen78 commented Dec 25, 2017

I love you to!

@softwarevamp

This comment has been minimized.

Copy link

softwarevamp commented Jan 2, 2018

Is it possible to use this when calls yaml.load?

@hawksight

This comment has been minimized.

Copy link

hawksight commented Jan 19, 2018

Just what I needed, thank you.
Note for python3 users, turn .iteritems() to .items().

@softwarevamp - not sure in what area you want to load yaml, but here's an example, where the 'merge_dict' would be loaded from yaml prior to function call.

master_dict = {}
with open(</path/file.yaml>, 'r') as f: dict_from_yaml = yaml.safe_load(f)
dict_merge(master_dict, dict_from_yaml)
print(yaml.safe_dump(master_dict))

Although that assumes only one yaml document is in the file etc..
Hope that helps.

@wskinner

This comment has been minimized.

Copy link

wskinner commented Apr 4, 2018

@angstwad would you be willing to add a license to this nice code snippet? I would like to use it at my company, but without a license the lawyers will be very unhappy with me :)

@newmen

This comment has been minimized.

Copy link

newmen commented May 29, 2018

from toolz.dicttoolz import merge_with


def deep_merge(*ds):
    def combine(vals):
        if len(vals) == 1 or not all(isinstance(v, dict) for v in vals):
            return vals[-1]
        else:
            return deep_merge(*vals)
    return merge_with(combine, *ds)
@DomWeldon

This comment has been minimized.

Copy link

DomWeldon commented Jun 17, 2018

Here's a Python 3 version with a test case that: a) returns a new dictionary rather than updating the old ones, and b) controls whether to add in keys from merge_dct which are not in dct.

from unittest import TestCase
import collections


def dict_merge(dct, merge_dct, add_keys=True):
    """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
    updating only top-level keys, dict_merge recurses down into dicts nested
    to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
    ``dct``.

    This version will return a copy of the dictionary and leave the original
    arguments untouched.

    The optional argument ``add_keys``, determines whether keys which are
    present in ``merge_dict`` but not ``dct`` should be included in the
    new dict.

    Args:
        dct (dict) onto which the merge is executed
        merge_dct (dict): dct merged into dct
        add_keys (bool): whether to add new keys

    Returns:
        dict: updated dict
    """
    dct = dct.copy()
    if not add_keys:
        merge_dct = {
            k: merge_dct[k]
            for k in set(dct).intersection(set(merge_dct))
        }

    for k, v in merge_dct.items():
        if (k in dct and isinstance(dct[k], dict)
                and isinstance(merge_dct[k], collections.Mapping)):
            dct[k] = dict_merge(dct[k], merge_dct[k], add_keys=add_keys)
        else:
            dct[k] = merge_dct[k]

    return dct


class DictMergeTestCase(TestCase):
    def test_merges_dicts(self):
        a = {
            'a': 1,
            'b': {
                'b1': 2,
                'b2': 3,
            },
        }
        b = {
            'a': 1,
            'b': {
                'b1': 4,
            },
        }

        assert dict_merge(a, b)['a'] == 1
        assert dict_merge(a, b)['b']['b2'] == 3
        assert dict_merge(a, b)['b']['b1'] == 4

    def test_inserts_new_keys(self):
        """Will it insert new keys by default?"""
        a = {
            'a': 1,
            'b': {
                'b1': 2,
                'b2': 3,
            },
        }
        b = {
            'a': 1,
            'b': {
                'b1': 4,
                'b3': 5
            },
            'c': 6,
        }

        assert dict_merge(a, b)['a'] == 1
        assert dict_merge(a, b)['b']['b2'] == 3
        assert dict_merge(a, b)['b']['b1'] == 4
        assert dict_merge(a, b)['b']['b3'] == 5
        assert dict_merge(a, b)['c'] == 6

    def test_does_not_insert_new_keys(self):
        """Will it avoid inserting new keys when required?"""
        a = {
            'a': 1,
            'b': {
                'b1': 2,
                'b2': 3,
            },
        }
        b = {
            'a': 1,
            'b': {
                'b1': 4,
                'b3': 5,
            },
            'c': 6,
        }

        assert dict_merge(a, b, add_keys=False)['a'] == 1
        assert dict_merge(a, b, add_keys=False)['b']['b2'] == 3
        assert dict_merge(a, b, add_keys=False)['b']['b1'] == 4
        try:
            assert dict_merge(a, b, add_keys=False)['b']['b3'] == 5
        except KeyError:
            pass
        else:
            raise Exception('New keys added when they should not be')

        try:
            assert dict_merge(a, b, add_keys=False)['b']['b3'] == 6
        except KeyError:
            pass
        else:
            raise Exception('New keys added when they should not be')
@SylannBin

This comment has been minimized.

Copy link

SylannBin commented Jul 19, 2018

@angstwad Your method will fail to update lists.
By the way, why do you test if dct[k] is dict instead of collections.Mapping?

@DomWeldon you are replacing a reference of dct with a shallow copy of itself? This is essentially doing nothing.
Declare another variable and define it with a deepcopy instead:

from copy import deepcopy
dct2 = deepcopy(dct)
@jpopelka

This comment has been minimized.

Copy link

jpopelka commented Jul 25, 2018

  • You iterate over key, value in merge_dct but then throw the value away and get the value by index
    solution: use v instead of merge_dct[k]
  • k in dct and isinstance(dct[k], dict) can be simplified to isinstance(dct.get(k), dict)
    for k, v in merge_dct.items():
        if isinstance(dct.get(k), dict) and isinstance(v, collections.Mapping):
            dct[k] = dict_merge(dct[k], v, add_keys=add_keys)
        else:
            dct[k] = v

all @DomWeldon's tests pass with this

@mcw0

This comment has been minimized.

Copy link

mcw0 commented Jan 18, 2019

brilliant stuff, needed and will use that in next PoC

@Danielyan86

This comment has been minimized.

Copy link

Danielyan86 commented Oct 11, 2019

why does this code use collections.Mapping instead of dict type?

@CMeza99

This comment has been minimized.

Copy link

CMeza99 commented Nov 7, 2019

@Danielyan86, good question. It should be dict. dict is a type of Mapping, but so is Counter(https://docs.python.org/3/glossary.html#term-mapping). I think allowing a Counter to be valid would not result in the expected behavior.

@CMeza99

This comment has been minimized.

Copy link

CMeza99 commented Nov 7, 2019

Here is my take on the function:

def dict_merge(base_dct, merge_dct):
    base_dct.update({
        key: dict_merge(rtn_dct[key], merge_dct[key])
        if isinstance(base_dct.get(key), dict) and isinstance(merge_dct[key], dict)
        else merge_dct[key]
        for key in merge_dct.keys()
    })

And my take on @DomWeldon's implantation:

def dict_fmerge(base_dct, merge_dct, add_keys=True):
    rtn_dct = base_dct.copy()
    if add_keys is False:
        merge_dct = {key: merge_dct[key] for key in set(rtn_dct).intersection(set(merge_dct))}

    rtn_dct.update({
        key: dict_fmerge(rtn_dct[key], merge_dct[key], add_keys=add_keys)
        if isinstance(rtn_dct.get(key), dict) and isinstance(merge_dct[key], dict)
        else merge_dct[key]
        for key in merge_dct.keys()
    })
    return rtn_dct

https://gist.github.com/CMeza99/5eae3af0776bef32f945f34428669437

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.