Skip to content

Instantly share code, notes, and snippets.

@fthzkrtn
Last active December 17, 2015 21:59
Show Gist options
  • Save fthzkrtn/5678663 to your computer and use it in GitHub Desktop.
Save fthzkrtn/5678663 to your computer and use it in GitHub Desktop.
A Thrift interface to insert indexes into Elasticsearch with Python client Pyes
#!/usr/bin/python
# coding=utf-8
"""
* A sample to implement index performance ES with Thrift and Pyes
* Copyright (C) 2013 Fatih Karatana
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
"""
__author__ = 'fatih'
import pyes
def main(r_type):
"""
Insert 4k index at one instance with Thrift and Elasticsearch client Pyes
with one instance ES and 256MB heap size
It takes about 20-25 second to index 4k data seen below. I think it is not that bad
:rtype : none
"""
LIMIT = 1000000
INDEX = "lbr-test-log1m-%s" % r_type
TYPE = ["ERROR", "INFO", "DEBUG", "FATAL", "CRITICAL"]
conn = pyes.ES(
[("thrift", "127.0.0.1", "9500"), ("thrift", "localhost", "9500")],
timeout=3.0)
if r_type is "bulk":
is_bulk = True
else:
is_bulk = False
try:
conn.indices.create_index(INDEX)
except BaseException as exception:
print exception.message
try:
for lim in range(0, LIMIT):
conn.index(
{"name": "Joe Tester", "parsedtext": "Joe Testere nice guy",
"uuid": "11111", "position": lim}
, INDEX, TYPE[0], lim, bulk=is_bulk)
conn.index(
{"name": "Bill Baloney", "parsedtext": "Joe Testere nice guy",
"uuid": "22222", "position": lim + 1}
, INDEX, TYPE[1], lim + 1, bulk=is_bulk)
conn.index(
{"name": "Joe Tester", "parsedtext": "Joe Testere nice guy",
"uuid": "11111", "position": lim + 2}
, INDEX, TYPE[2], lim + 2, bulk=is_bulk)
conn.index(
{"name": "Bill Baloney", "parsedtext": "Joe Testere nice guy",
"uuid": "22222", "position": lim + 3}
, INDEX, TYPE[3], lim + 3, bulk=is_bulk)
print conn.refresh()
print "succeed"
except BaseException as exception:
print exception.message
if __name__ == "__main__":
# give a parameter either bulk or nothing to make index as bulk or single
main('bulk')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment