ElasticSearch is a distributed search engine based on Lucene.
- Data is stored as Json objects
- Data is automatically indexed
- Interact via REST Api
- search
- analysis
Relational DB ElasticSearch
------------- -------------
Database Index
Table Type
Row Document
Column Field
Index Everything is indexed
SQL Query DSL
http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html
PUT nfl-2013
GET nfl-2013/_search
POST nfl-2013/quarterback/
{
"name": "Peyton Manning",
"team": "Denver Broncos",
"college": "Tennessee",
"height": 77,
"weight": 230,
"born": 1976,
"completion": 450,
"attempts": 659,
"yards": 5477,
"longest-pass": 78,
"interceptions": 10,
"rating": 115.1
}
POST nfl-2013/quarterback/
{
"name": "Drew Brees",
"team": "New Orleans Saints",
"college": "Purdue",
"height": 72,
"weight": 209,
"born": 1979,
"completion": 446,
"attempts": 650,
"yards": 5162,
"longest-pass": 76,
"interceptions": 12,
"rating": 104.7
}
POST nfl-2013/quarterback/
{
"name": "Matthew Stafford",
"team": "Detroit Lions",
"college": "Georgia",
"height": 75,
"weight": 232,
"born": 1988,
"completion": 371,
"attempts": 634,
"yards": 4650,
"longest-pass": 87,
"interceptions": 19,
"rating": 84.2
}
POST nfl-2013/quarterback/
{
"name": "Matt Ryan",
"team": "Atlanta Falcons",
"college": "Boston College",
"height": 76,
"weight": 217,
"born": 1985,
"completion": 439,
"attempts": 651,
"yards": 4515,
"longest-pass": 81,
"interceptions": 17,
"rating": 89.6
}
POST nfl-2013/quarterback/
{
"name": "Philip Rivers",
"team": "San Diego Chargers",
"college": "North Carolina State",
"height": 77,
"weight": 228,
"born": 1981,
"completion": 378,
"attempts": 544,
"yards": 4478,
"longest-pass": 60,
"interceptions": 11,
"rating": 71.7
}
###All Indexes GET _search
###One Index / All Types GET nfl-2013/_search
###One Index / One Type GET nfl-2013/quarterback/_search
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10
}
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"sort": [
{
"name": {
"order": "asc"
}
}
]
}
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"_source": [ "name", "team", "rating"]
}
###On fields
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"query": {
"query_string": {
"default_field": "name",
"query": "matt"
}
}
}
###Wildcards
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"query": {
"query_string": {
"default_field": "name",
"query": "mat*"
}
}
}
###Booleans
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"query": {
"query_string": {
"default_field": "name",
"query": "matt OR drew"
}
}
}
###Multiple Fields
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"query": {
"query_string": {
"fields": [ "name", "team", "college" ],
"query": "br*"
}
}
}
###Multiple Fields with Boosting
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"query": {
"query_string": {
"fields": [ "name^2", "team", "college" ],
"query": "br*"
}
}
}
###[usecase] Implementing "type ahead" search
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"query": {
"query_string": {
"default_field": "name",
"query": "br*"
}
}
}
###Multiple Queries (think SQL WHERE Clause)
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"query": {
"bool": {
"must": [
{
"query_string": {
"fields": [ "name^2", "team", "college" ],
"query": "*"
}
},
{
"range": {
"rating": {
"from": 80
}
}
},
{
"range": {
"interceptions": {
"to": 10
}
}
},
{
"range": {
"attempts": {
"from": 300
}
}
}
]
}
}
}
###Terms (think SQL Distinct)
POST nfl-2013/quarterback/_search
{
"size": 0,
"aggs": {
"teams": {
"terms": {
"field": "team",
"size": 20
}
}
}
}
POST nfl-2013/quarterback/_search
{
"size": 0,
"aggs": {
"teams": {
"terms": {
"script": "_source.team",
"size": 20
}
}
}
}
###Ranges (think SQL BETWEEN)
POST nfl-2013/quarterback/_search
{
"size": 0,
"aggs": {
"rating": {
"range": {
"field": "rating",
"ranges": [
{
"to": 80
},
{
"from": 80,
"to": 90
},
{
"from": 90,
"to": 100
},
{
"from": 100
}
]
}
}
}
}
###Histograms
POST nfl-2013/quarterback/_search
{
"size": 0,
"aggs": {
"ages": {
"histogram": {
"field": "born",
"interval": 10
}
}
}
}
###Percentiles
POST nfl-2013/quarterback/_search
{
"size": 0,
"aggs": {
"ratings": {
"percentiles": {
"field": "rating"
}
}
}
}
POST nfl-2013/quarterback/_search
{
"size": 0,
"aggs": {
"ratings": {
"percentiles": {
"field": "rating",
"percents" : [68.27, 95.45, 99.73, 99.99]
}
}
}
}
###Multiple Aggregates
POST nfl-2013/quarterback/_search
{
"size": 0,
"aggs": {
"teams": {
"terms": {
"script": "_source.team",
"size": 20
}
},
"weights": {
"histogram": {
"field": "weight",
"interval": 10
}
}
}
}
###[usecase] Integrating Analysis with Search
POST nfl-2013/quarterback/_search
{
"from": 0,
"size": 10,
"query": {
"bool": {
"must": [
{
"query_string": {
"fields": [ "name^2", "team", "college" ],
"query": "*"
}
},
{
"range": {
"rating": {
"from": 80,
"to": 90
}
}
},
{
"range": {
"interceptions": {
"to": 17
}
}
}
]
}
},
"aggs": {
"rating": {
"range": {
"field": "rating",
"ranges": [
{
"to": 80
},
{
"from": 80,
"to": 90
},
{
"from": 90,
"to": 100
},
{
"from": 100
}
]
}
},
"interceptions": {
"stats": {
"field": "interceptions"
}
}
}
}
###Elasticsearch Configuration
Thread pool settings:
# Search pool
threadpool.search.type: fixed
threadpool.search.size: 20
threadpool.search.queue_size: 100
# Bulk pool
threadpool.bulk.type: fixed
threadpool.bulk.size: 60
threadpool.bulk.queue_size: 300
# Index pool
threadpool.index.type: fixed
threadpool.index.size: 20
threadpool.index.queue_size: 100
###OS /Java settings: Set the open file limits high – 32K or 64K Set min and max memory to the same value to prevent allocations after the service starts. Use 50% to 60% of total machine memory for ES, leave rest for the OS Run on the most recent version of Java possible. 32 bit vs. 64 bit