null_filter.py

"""
This is a program demonstrating that the elasticutils method .filter() will
break if you try and use a None value. It creates an index, indexes some data,
and does some searches, the last of which errors out.
"""

from elasticutils import get_es, S
 
 
HOST = 'localhost:9200'
INDEX = 'fooindex'
DOCTYPE = 'testdoc'
 
 
es = get_es(hosts=HOST, default_indexes=[INDEX])
 
es.delete_index_if_exists(INDEX)
 
mapping = {
    DOCTYPE: {
        'properties': {
            'id': {'type': 'integer'},
            'title': {'type': 'string'},
            'locale': {'type': 'string'},
            }
        }
    }
 
es.create_index(INDEX, settings={'mappings': mapping})

for mem in [
    {'id': 1,
     'title': 'Deleting cookies',
     'locale': 'xx'},
    {'id': 2,
     'title': 'What is a cookie?',
     'locale': 'xx'},
    {'id': 3,
     'title': 'Websites say cookies are blocked - Unblock them',
     'locale': 'xx'},
    {'id': 4,
     'title': 'Awesome Bar',
     'locale': 'xx'},
    {'id': 5,
     'title': 'Flash',
     'locale': None}]:

    es.index(mem, INDEX, DOCTYPE, id=mem['id'])

# ElasticSearch will refresh the indexes and make those documents
# available for querying in a second or so (it's configurable in
# ElasticSearch), but we want them available right now, so we refresh
# the index.
es.refresh(INDEX)

# Let's build a basic S that looks at the right instance of
# ElasticSearch, index, and doctype.
def get_s():
    return (S().es(hosts=[HOST])
               .indexes(INDEX)
               .doctypes(DOCTYPE)
               .values_dict())
 
print get_s().values_dict()
# Pretty printed output
# [
#    {u'locale': u'xx', u'id': 1, u'title': u'Deleting cookies'},
#    {u'locale': u'xx', u'id': 2, u'title': u'What is a cookie?'},
#    {u'locale': u'xx', u'id': 3, u'title': u'Websites say cookies are blocked - Unblock them'}
#    {u'locale': u'xx', u'id': 4, u'title': u'Awesome Bar'},
#    {u'locale': None, u'id': 5, u'title': u'Flash'},
# ]
# Note that locale is `None` for the last item.

# Basic filter, to demonstrate nothing is messed up.
print get_s().filter(locale='xx').values_dict()
# Pretty printed output
# [
#   {u'locale': u'xx', u'id': 1, u'title': u'Deleting cookies'},
#   {u'locale': u'xx', u'id': 2, u'title': u'What is a cookie?'},
#   {u'locale': u'xx', u'id': 3, u'title': u'Websites say cookies are blocked - Unblock them'}
#   {u'locale': u'xx', u'id': 4, u'title': u'Awesome Bar'},
# ]


# Now try and filter by locale=None, to find the Flash document.
print get_s().filter(locale=None).values_dict()
# This doesn't work. It raises a big scary error.
#
# Traceback (most recent call last):
#   File "<console>", line 1, in <module>
#   File "/home/mythmon/src/kitsune/null_filter.py", line 87, in <module>
#     print get_s().filter(locale=None).values_dict()
#   File "/home/mythmon/src/kitsune/vendor/src/elasticutils/elasticutils/__init__.py", line 256, in __repr__
#     data = list(self)[:REPR_OUTPUT_SIZE + 1]
#   File "/home/mythmon/src/kitsune/vendor/src/elasticutils/elasticutils/__init__.py", line 706, in __iter__
#     return iter(self._do_search())
#   File "/home/mythmon/src/kitsune/vendor/src/elasticutils/elasticutils/__init__.py", line 645, in _do_search
#     hits = self.raw()
#   File "/home/mythmon/src/kitsune/vendor/src/elasticutils/elasticutils/__init__.py", line 700, in raw
#     hits = es.search(qs, self.get_indexes(), self.get_doctypes())
#   File "/home/mythmon/src/kitsune/vendor/packages/pyes/pyes/es.py", line 841, in search
#     return self._query_call("_search", body, indexes, doc_types, **query_params)
#   File "/home/mythmon/src/kitsune/vendor/packages/pyes/pyes/es.py", line 251, in _query_call
#     response = self._send_request('GET', path, body, querystring_args)
#   File "/home/mythmon/src/kitsune/vendor/packages/pyes/pyes/es.py", line 223, in _send_request
#     raise_if_error(response.status, decoded)
#   File "/home/mythmon/src/kitsune/vendor/packages/pyes/pyes/convert_errors.py", line 68, in raise_if_error
#     raise excClass(msg, status, result)
# SearchPhaseExecutionException: Failed to execute phase [query], total failure; shardFailures {[b9ZLwu8WTKqUoDJ7V8UDZw][fooindex][0]: SearchParseException[[fooindex][0]: from[-1],size[-1]: Parse Failure [Failed to parse source [{"filter": {"term": {"locale": null}}}]]]; nested: QueryParsingException[[fooindex] No field specified for term filter]; }{[b9ZLwu8WTKqUoDJ7V8UDZw][fooindex][4]: SearchParseException[[fooindex][4]: from[-1],size[-1]: Parse Failure [Failed to parse source [{"filter": {"term": {"locale": null}}}]]]; nested: QueryParsingException[[fooindex] No field specified for term filter]; }