A Google App Engine optimization technique: Resolve references properties for a bunch of models in parallel, using a minimum number of datastore trips.

gistfile1.py

def find_unresolved_properties(entity):
    """ Returns a list of attribute names that are reference properties on an entity.
        It does it in a way that does not trigger a fetch.
    """
    reference_properties = {}
    try:
        for name, value in entity._entity.items():
            if isinstance(value, db.Key) and getattr(entity, "_RESOLVED_%s" % name, None):
                reference_properties[name] = value
    except AttributeError:
        pass
    return reference_properties

def resolve_references(entities):
    """ Resolves references for a bunch of entities in parallel. After calling this, access to a reference property 
        in any of the passed entities will not result in a datastore fetch.
        
        The SDK implements ReferenceProperty(s) such that access to any of them results
        in a fetch from the datastore.
        This method resolves reference properties for a bunch of entities in just one
        trip to the datastore!
    """
    unresolved_entities = {}
    keys = []
    # Find all unresolved reference properties
    for entity in entities:
        if not entity:
            continue
        unresolved = find_unresolved_properties(entity)
        unresolved_entities[entity.key()] = (entity, unresolved)
        keys.extend(unresolved.values())
    
    # Remove dupes
    keys = list(set(keys))
    # Fetch in parallel
    fetched = []
    for slice in miscutils.split_list(keys, 1000):
        fetched.extend(db.get(slice))
    # A dictionary that maps a reference entity key to fetched entity
    fetched_dict = dict(zip([e.key() for e in fetched if e], fetched))
    for entity, unresolved in unresolved_entities.values():
        for property_name, property_key in unresolved.items():
            setattr(entity, property_name, fetched_dict.get(property_key, None))