mapper.py

from functools import reduce


TEST_PAYLOAD = {
    "NAME": "nylon stripe running shoes",
    "IDENTIFIER": "432jk4h23k4h23kjh4",
    "SUB CATEGORY": "sport",
    "IMAGE": "url_to_image",
    "MAIN CATEGORY": "shoes",
    "MASTER_CATALOG": "fall 2018",
    "SKU": "KHHKDHSJHKJH"
}

IMAGES_PAYLOAD = {
	'red': 155,
	'green': 147,
	'listing_image_id': 1086513044,
	'creation_tsz': 1479653274,
	'url_570xN': 'https://img0.etsystatic.com/143/2/12213581/il_570xN.1086513044_nnw9.jpg',
	'full_width': 1500,
	'url_fullxfull': 'https://img0.etsystatic.com/143/2/12213581/il_fullxfull.1086513044_nnw9.jpg',
	'is_black_and_white': False,
	'listing_id': 470854218,
	'url_170x135': 'https://img0.etsystatic.com/143/2/12213581/il_170x135.1086513044_nnw9.jpg',
	'hue': 51,
	'blue': 101,
	'hex_code': '9B9365',
	'saturation': 34,
	'full_height': 1001,
	'rank': 1,
	'brightness': 60,
	'url_75x75': 'https://img0.etsystatic.com/143/2/12213581/il_75x75.1086513044_nnw9.jpg'
}


"""
Use of special indicators in the data mapper.

- "&" tells the importer to move the value to another key:value. Already working.
- ":" tells the importer to just nest keys.  Already working.
- "[]" tells the importer to create an list.  Already working.
- "{}" tells the importer to create an object. Not working.
- "{}.somekey" tells the importer to create an object with somekey key. Not working.
- "[{}]" tells the importer to create a list of objects.  Not working.
"""
DATA_MAPPER_LIST_OF_OBJECTS = {
	'green': 'images[{}]',
	'full_width': 'images[{}]',
	'hex_code': 'images[{}]',
	'saturation': 'images[{}]',
	'red': 'images[{}]',
	'listing_image_id': 'images[{}]',
	'url_570xN': 'images[{}]',
	'full_height': 'images[{}]',
	'is_black_and_white': 'images[{}]',
	'rank': 'images[{}]',
	'blue': 'images[{}]',
	'creation_tsz': 'images[{}]',
	'url_75x75': 'images[{}]',
	'url_fullxfull': 'images[{}]',
	'brightness': 'images[{}]',
	'listing_id': 'listing_id',
	'url_170x135': 'images[{}]',
	'hue': 'images[{}]'
}

TEST_DATA_MAPPER = {
    "NAME": "name",
    "IDENTIFIER": "identifier",
    "SUB CATEGORY": "cat_sub&categories[]",
    "IMAGE": "img&assets:main_img",
    "MAIN CATEGORY": "cat_main&categories[]",
    "MASTER_CATALOG": "catalog&aws_classifiers:cat:master",
    "SKU": "sku"
}


def get_nested(dictionary, keys):
    """Get value of nested dict levels."""
    return reduce(lambda d, key: d.get(key, "") if isinstance(d, dict) else "",
                  keys, dictionary)


def form_doc(doc, map):
    """Form doc structure according to map."""
    data = {}
    ignored_parts = None

    for k, v in map.items():
        if k == 'ignored_parts':
            ignored_parts = v

            continue

        if not isinstance(v, str):
            return ''

        parts = v.split('&')
        payload_path = k.split(':')
        payload = get_nested(doc, payload_path)

        if isinstance(payload, str):
            payload = payload.strip()

        for part in parts:
            path = part.split(':')
            last = path.pop()
            is_list = False
            is_list_of_objects = False

            if last.endswith('[]'):
                last = last[:-2]
                is_list = True

            if last.endswith('[{}]'):
                last = last[:-4]
                is_list_of_objects = True

            if path:
                new_doc = set_nested(data, path)
            else:
                new_doc = data

            if is_list_of_objects:
                new_obj = {}
                new_obj[payload_path[0]] = payload

                if new_doc.get(last):
                    new_obj[payload_path[0]] = payload
                    new_doc[last].append(new_doc)
                else:
                    if isinstance(payload, list):
                        new_doc[last] = payload
                    else:
                        new_doc[last] = [payload]

            if is_list:
                if new_doc.get(last):
                    if isinstance(payload, list):
                        for item in payload:
                            new_doc[last].append(item)
                    else:
                        new_doc[last].append(payload)
                else:
                    if isinstance(payload, list):
                        new_doc[last] = payload
                    else:
                        new_doc[last] = [payload]

            else:
                new_doc[last] = payload

    if ignored_parts:
        data.update(ignored_parts)

    return data


def main():
    """
    This should generate the following structure.

    {
        "listing_id": 470854218,
        "images": [{
            "red": 155,
            "green": 147,
            "listing_image_id": 1086513044,
            "creation_tsz": 1479653274,
            "url_570xN": "https://img0.etsystatic.com/143/2/12213581/il_570xN.1086513044_nnw9.jpg",
            "full_width": 1500,
            "url_fullxfull": "https://img0.etsystatic.com/143/2/12213581/il_fullxfull.1086513044_nnw9.jpg",
            "is_black_and_white": false,
            "url_170x135": "https://img0.etsystatic.com/143/2/12213581/il_170x135.1086513044_nnw9.jpg",
            "hue": 51,
            "blue": 101,
            "hex_code": "9B9365",
            "saturation": 34,
            "full_height": 1001,
            "rank": 1,
            "brightness": 60,
            "url_75x75": "https://img0.etsystatic.com/143/2/12213581/il_75x75.1086513044_nnw9.jpg"
        }]
    }
    """
    mapped_document_images_list_of_objects = form_doc(IMAGES_PAYLOAD, DATA_MAPPER_LIST_OF_OBJECTS)

    """
    This should generate the following structure.  It already works.
    {
        "identifier": "432jk4h23k4h23kjh4",
        "img": "url_to_image",
        "assets": {
            "main_img": "url_to_image"
        },
        "sku": "KHHKDHSJHKJH",
        "cat_main": "shoes",
        "cat_sub": "sport",
        "name": "nylon stripe running shoes",
        "categories": ["shoes", "sport"],
        "catalog": "fall 2018",
        "aws_classifiers": {
            "cat": {
                "master": "fall 2018"
            }
        }
    }
    """
    test_mapped_data = form_doc(TEST_PAYLOAD, TEST_DATA_MAPPER)


if __name__== "__main__":
    main()