WebExpo demo

-README.rst

Elasticsearch tutorial

Requirements

• https://gist.github.com/HonzaKral/35451c6322a78d4de8f5 to load stack overflow data. Clone into load_stack subdirectory within this repo.
• Programmers StackExchange dump from http://www.clearbits.net/creators/146-stack-exchange-data-dump - the relevant files (Posts.xml, Users.xml and Comments.xml) should be placed in the data subdirectory in this repo.
• elasticat (https://gist.github.com/karmi/5670573) for pretty-printing es results
• kibana 3 with dashboard loaded from 07-kibana.json

Preparation

Run:

BULK_FILE=$PWD/bulk.json ES_INDEX=programmers.stackexchange.com  load_stack/load_stack.sh $PWD/data

From this repo. This should generate a bulk.json that can be re-used during the presentation (second run of this script won't have to generate it from the xml).

Clear all your indices:

curl -X DELETE http://localhost:9200

At this moment it's good idea to get kibana up and running and load the dashboard, verify that it's sane and displays correct data.

You should be set to go, enjoy!

00-setup.sh

# 1. Download, extract and launch Elasticsearch
# ---------------------------------------------

curl -# http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.5.tar.gz | \
  tar xz && \
  ./elasticsearch-0.90.5/bin/elasticsearch -f

01-index.sh

curl -XDELETE http://localhost:9200/demo?pretty

curl -XPUT http://localhost:9200/demo/question/1?pretty -d '{
  "title": "How does Elasticsearch work?",
  "author": "John",
  "date": "2013-05-30"
}'
echo

curl -XPUT http://localhost:9200/demo/question/2?pretty -d '{
  "title": "How does Lucene work?",
  "author": "John",
  "date": "2013-04-01"
}'
echo

curl -XPUT http://localhost:9200/demo/question/3?pretty -d '{
  "title": "Does Elasticsearch support fuzziness?",
  "author": "Mary",
  "date": "2013-05-28"
}'
echo

curl -XPOST http://localhost:9200/demo/_refresh?pretty

02a-plain-search.sh

curl -XGET http://localhost:9200/demo/_search?pretty

02b-keyword-search.sh

curl -XGET 'http://localhost:9200/demo/_search?pretty&q=elasticsearch'

02c-query-string.sh

curl -XGET 'http://localhost:9200/demo/_search?pretty&q=(title:elasticsearch^10+OR+author:john)+AND+date:>=2013-05-25'

03-simple-dsl.sh

curl -XGET http://localhost:9200/demo/_search?pretty -d '{
  "query": {
    "filtered": {
      "query": {
        "bool": {
          "should": [
            {"match": {"title": {"query": "elasticsearch", "boost": 10}}},
            {"match": {"author": "john"}}
          ]
        }
      },
      "filter": {
        "range": {
          "date": {"from": "2013-05-25"}
        }
      }
    }
  }
}'

04-load-data.sh

# Use the loader from <https://gist.github.com/HonzaKral/35451c6322a78d4de8f5> to
# load data from Stackexchange XML dump files.

ES_INDEX=programmers.stackexchange.com load_stack/load_stack.sh $PWD/data

05-search-stack.sh

# curl -X GET 'http://localhost:9200/programmers.stackexchange.com/_search?pretty'
# echo

curl -X GET 'http://localhost:9200/programmers.stackexchange.com/question/_search?pretty&q=(+php%20+dependency%20+injection)'

06a-search-query.sh

curl -XGET http://localhost:9200/programmers.stackexchange.com/question,answer/_search?pretty -d '{
  "query": {
    "multi_match": {"fields": ["title^10", "body"], "query": "php dependency injection"}
  }
}'

06b-search-recent-questions.sh

curl -XGET http://localhost:9200/programmers.stackexchange.com/question,answer/_search?pretty -d '{
  "query": {
    "filtered": {
      "query": {
        "multi_match": {"fields": ["title^10", "body"], "query": "php dependency injection"}
      },
      "filter": {
        "range": {"creation_date": {"from": "2013-01-01"}}
      }
    }
  }
}'

06c-search-limit-fields.sh

curl -XGET http://localhost:9200/programmers.stackexchange.com/question,answer/_search?pretty -d '{
  "query": {
    "filtered": {
      "query": {
        "multi_match": {"fields": ["title^10", "body"], "query": "php dependency injection"}
      },
      "filter": {
        "range": {"creation_date": {"from": "2012-01-01"}}
      }
    }
  },
  "fields": ["title", "owner.location", "owner.display_name", "tags", "creation_date", "rating"]
}'

06d-search-highlight.sh

curl -XGET http://localhost:9200/programmers.stackexchange.com/question,answer/_search?pretty -d '{
  "query": {
    "filtered": {
      "query": {
        "multi_match": {"fields": ["title^10", "body"], "query": "php dependency injection"}
      },
      "filter": {
        "range": {"creation_date": {"from": "2012-01-01"}}
      }
    }
  },
  "fields": ["title", "owner.location", "owner.display_name", "tags", "creation_date", "rating"],
  "highlight": {
    "fields": {
      "title": {
        "fragment_size" : 50
      },
      "body": {
        "fragment_size" : 50
      }
    }
  }
}'

06e-search-trolls.sh

curl -XGET http://localhost:9200/programmers.stackexchange.com/question,answer/_search?pretty -d '{
  "query": {
    "filtered": {
      "query": {
        "bool": {
          "must": [
            {"multi_match": {"fields": ["title^10", "body"], "query": "service locator"}},

            {
              "has_child": {
                "child_type": "answer",
                "query": {"match": {"body": "dependency injection"}}
              }
            }
          ],
          "must_not": [
            {"multi_match": {"fields": ["title", "body"], "query": "dependency injection"}}
          ]
        }
      },
      "filter": {
        "range": {"creation_date": {"from": "2012-01-01"}}
      }
    }
  },
  "fields": ["title", "owner.location", "owner.display_name", "tags", "creation_date", "rating"],
  "highlight": {
    "fields": {
      "title": {
        "fragment_size" : 50
      },
      "body": {
        "fragment_size" : 50
      }
    }
  }
}'

06f-search-custom-score.sh

curl -XGET http://localhost:9200/programmers.stackexchange.com/question,answer/_search?pretty -d '{
  "query": {
    "custom_score": {
      "query": {
        "filtered": {
          "query": {
            "bool": {
              "must": [
                {"multi_match": {"fields": ["title^10", "body"], "query": "php"}},

                {
                  "has_child": {
                    "child_type": "answer",
                    "query": {"match": {"body": "ruby python"}}
                  }
                }
              ],
              "must_not": [
                {"multi_match": {"fields": ["title", "body"], "query": "ruby python"}}
              ]
            }
          },
          "filter": {
            "range": {"creation_date": {"from": "2012-01-01"}}
          }
        }
      },
      "script": "_score * doc[\"rating\"].value"
    }
  },
  "fields": ["title", "owner.location", "owner.display_name", "tags", "creation_date", "rating"],
  "highlight": {
    "fields": {
      "title": {
        "fragment_size" : 50
      },
      "body": {
        "fragment_size" : 50
      }
    }
  }
}'

06g-search-facets.sh

curl -XGET http://localhost:9200/programmers.stackexchange.com/question,answer/_search?pretty -d '{
  "query": {
    "custom_score": {
      "query": {
        "filtered": {
          "query": {
            "bool": {
              "must": [
                {"multi_match": {"fields": ["title^10", "body"], "query": "php"}},

                {
                  "has_child": {
                    "child_type": "answer",
                    "query": {"match": {"body": "ruby python"}}
                  }
                }
              ],
              "must_not": [
                {"multi_match": {"fields": ["title", "body"], "query": "ruby python"}}
              ]
            }
          },
          "filter": {
            "range": {"creation_date": {"from": "2012-01-01"}}
          }
        }
      },
      "script": "_score * doc[\"rating\"].value"
    }
  },
  "fields": ["title", "owner.location", "owner.display_name", "tags", "creation_date", "rating"],
  "highlight": {
    "fields": {
      "title": {
        "fragment_size" : 50
      },
      "body": {
        "fragment_size" : 50
      }
    }
  },
  "facets": {
    "tags": {"terms": {"field": "tags"}},
    "frequency": {"date_histogram": {"field": "creation_date", "interval": "month"}},
    "comment_stats": {"statistical": {"field": "comment_count"}}
  }
}'

07-kibana.json

{
  "title": "Programmers StackExchange",
  "rows": [
    {
      "title": "Options",
      "height": "50px",
      "editable": true,
      "collapse": false,
      "collapsable": true,
      "panels": [
        {
          "loading": false,
          "error": "",
          "span": 5,
          "editable": true,
          "group": [
            "default"
          ],
          "type": "timepicker",
          "mode": "absolute",
          "time_options": [
            "5m",
            "15m",
            "1h",
            "6h",
            "12h",
            "24h",
            "2d",
            "5d"
          ],
          "timespan": "5d",
          "timefield": "creation_date",
          "index": "dba-stack/question",
          "defaultindex": "stack",
          "index_interval": "none",
          "refresh": {
            "enable": false,
            "interval": 30,
            "min": 3
          },
          "timeformat": "",
          "time": {
            "from": "09/07/2012 13:21:30",
            "to": "03/22/2013 19:19:00"
          },
          "status": "Stable",
          "filter_id": 0
        },
        {
          "span": 7,
          "editable": true,
          "type": "trends",
          "loadingEditor": false,
          "queries": {
            "mode": "all",
            "ids": [
              0,
              1,
              2,
              3
            ]
          },
          "style": {
            "font-size": "14pt"
          },
          "ago": "1w",
          "arrangement": "vertical",
          "spyable": true,
          "title": "Trends"
        }
      ]
    },
    {
      "title": "Query",
      "height": "50px",
      "editable": true,
      "collapse": false,
      "collapsable": true,
      "panels": [
        {
          "error": false,
          "span": 12,
          "editable": true,
          "type": "query",
          "label": "Search",
          "query": "*",
          "pinned": true,
          "history": [
            "php",
            "javascript",
            "ruby",
            "python",
            "comment_count:[4 TO 100]",
            "comment_count:[2 TO 4}",
            "tags:python",
            "comment_count:[0 TO 2}",
            "comment_count:[5 TO ]",
            "comment_count:[3 TO 4]"
          ],
          "remember": 10,
          "title": "Query"
        }
      ]
    },
    {
      "title": "Graph",
      "height": "350px",
      "editable": true,
      "collapse": false,
      "collapsable": true,
      "panels": [
        {
          "loading": false,
          "span": 8,
          "editable": true,
          "group": [
            "default"
          ],
          "type": "histogram",
          "query": [
            {
              "query": "*",
              "label": "*"
            }
          ],
          "interval": "1w",
          "show": [
            "points",
            "lines",
            "legend",
            "x-axis",
            "y-axis"
          ],
          "timezone": "browser",
          "spyable": true,
          "zoomlinks": true,
          "fill": 0,
          "linewidth": 2,
          "bars": true,
          "stack": true,
          "points": false,
          "lines": false,
          "legend": true,
          "x-axis": true,
          "y-axis": true,
          "mode": "count",
          "value_field": null,
          "auto_int": false,
          "resolution": 100,
          "percentage": true,
          "title": "Frequency",
          "interactive": true,
          "time_field": "creation_date",
          "queries": {
            "mode": "all",
            "ids": [
              0,
              1,
              2,
              3
            ]
          },
          "tooltip": {
            "value_type": "cumulative",
            "query_as_alias": false
          },
          "intervals": [
            "auto",
            "1s",
            "1m",
            "5m",
            "10m",
            "30m",
            "1h",
            "3h",
            "12h",
            "1d",
            "1w",
            "1M",
            "1y"
          ],
          "options": true
        },
        {
          "loading": false,
          "error": false,
          "span": 2,
          "editable": true,
          "group": [
            "default"
          ],
          "type": "pie",
          "query": {
            "query": "*",
            "field": "tags"
          },
          "size": 10,
          "exclude": [],
          "donut": true,
          "tilt": false,
          "legend": true,
          "labels": true,
          "mode": "terms",
          "default_field": "DEFAULT",
          "spyable": true,
          "title": "Tags",
          "editorTabs": [
            {
              "title": "Queries",
              "src": "partials/querySelect.html"
            }
          ],
          "queries": {
            "mode": "all",
            "ids": [
              0,
              1,
              2,
              3
            ]
          }
        },
        {
          "loading": false,
          "span": 2,
          "editable": true,
          "group": [
            "default"
          ],
          "type": "hits",
          "mode": "markdown",
          "content": "",
          "style": {},
          "query": [
            {
              "query": "comment_count:[0 TO 2]",
              "label": "0-2"
            },
            {
              "query": "comment_count:[2 TO 4]",
              "label": "2-4"
            },
            {
              "query": "comment_count:[4 TO *]",
              "label": "4 -"
            }
          ],
          "arrangement": "horizontal",
          "chart": "bar",
          "counter_pos": "above",
          "donut": false,
          "tilt": false,
          "labels": true,
          "title": "Totals",
          "queries": {
            "mode": "all",
            "ids": [
              0,
              1,
              2,
              3
            ]
          },
          "spyable": true
        }
      ]
    },
    {
      "title": "Events",
      "height": "350px",
      "editable": true,
      "collapse": false,
      "collapsable": true,
      "panels": [
        {
          "loading": false,
          "error": false,
          "span": 12,
          "editable": true,
          "group": [
            "default"
          ],
          "type": "table",
          "query": "*",
          "interval": "1y",
          "show": [
            "bars",
            "y-axis",
            "x-axis",
            "legend"
          ],
          "fill": 3,
          "overflow": "min-height",
          "timezone": "browser",
          "spyable": true,
          "zoomlinks": true,
          "size": 50,
          "pages": 10,
          "offset": 0,
          "sort": [
            "creation_date",
            "desc"
          ],
          "style": {
            "font-size": "9pt"
          },
          "fields": [
            "title",
            "owner.display_name",
            "tags",
            "comment_count",
            "answer_count"
          ],
          "sortable": true,
          "highlight": [],
          "header": true,
          "paging": true,
          "status": "Stable",
          "queries": {
            "mode": "all",
            "ids": [
              0,
              1,
              2,
              3
            ]
          },
          "field_list": false,
          "trimFactor": 300,
          "normTimes": true
        }
      ]
    }
  ],
  "editable": true,
  "last": {
    "title": "Logstash Search",
    "rows": [
      {
        "title": "Options",
        "height": "50px",
        "editable": true,
        "collapse": false,
        "collapsable": true,
        "panels": [
          {
            "loading": false,
            "error": "Could not match index pattern to any ElasticSearch indices",
            "span": 5,
            "editable": true,
            "group": [
              "default"
            ],
            "type": "timepicker",
            "mode": "relative",
            "time_options": [
              "5m",
              "15m",
              "1h",
              "6h",
              "12h",
              "24h",
              "2d",
              "5d"
            ],
            "timespan": "6h",
            "timefield": "@timestamp",
            "index": "[logstash-]YYYY.MM.DD",
            "defaultindex": "NOINDEX",
            "index_interval": "day",
            "refresh": {
              "enable": false,
              "interval": 30,
              "min": 3
            },
            "timeformat": ""
          },
          {
            "loading": false,
            "error": false,
            "span": 3,
            "editable": true,
            "group": [
              "default"
            ],
            "type": "dashcontrol",
            "save": {
              "gist": false,
              "elasticsearch": true,
              "local": true,
              "default": true
            },
            "load": {
              "gist": true,
              "elasticsearch": true,
              "local": true
            },
            "hide_control": false,
            "elasticsearch_size": 20,
            "elasticsearch_saveto": "kibana-int",
            "temp": true,
            "temp_ttl": "30d"
          }
        ]
      },
      {
        "title": "Query",
        "height": "50px",
        "editable": true,
        "collapse": false,
        "collapsable": true,
        "panels": [
          {
            "loading": false,
            "error": false,
            "span": 12,
            "editable": true,
            "group": [
              "default"
            ],
            "type": "stringquery",
            "label": "Search",
            "query": "*",
            "size": 100,
            "sort": [
              "_score",
              "desc"
            ],
            "multi": false,
            "multi_arrange": "horizontal"
          }
        ]
      },
      {
        "title": "Graph",
        "height": "350px",
        "editable": true,
        "collapse": false,
        "collapsable": true,
        "panels": [
          {
            "loading": false,
            "span": 12,
            "editable": true,
            "group": [
              "default"
            ],
            "type": "histogram",
            "query": [
              {
                "query": "*",
                "label": "*"
              }
            ],
            "interval": "5m",
            "show": [
              "points",
              "lines",
              "legend",
              "x-axis",
              "y-axis"
            ],
            "timezone": "browser",
            "spyable": true,
            "zoomlinks": true,
            "fill": 0,
            "linewidth": 2,
            "bars": true,
            "stack": true,
            "points": false,
            "lines": false,
            "legend": true,
            "x-axis": true,
            "y-axis": true,
            "mode": "count",
            "value_field": null,
            "auto_int": true,
            "resolution": 100,
            "percentage": false,
            "interactive": true
          },
          {
            "loading": true,
            "span": 0,
            "editable": true,
            "group": [
              "default"
            ],
            "type": "hits",
            "query": [
              {
                "query": "*",
                "label": "*"
              }
            ],
            "style": {
              "font-size": "9pt"
            },
            "aggregate": false,
            "arrangement": "horizontal",
            "chart": true,
            "counters": true,
            "count_pos": "above",
            "counter_pos": "above",
            "donut": false,
            "tilt": false,
            "labels": true
          }
        ]
      },
      {
        "title": "Events",
        "height": "350px",
        "editable": true,
        "collapse": false,
        "collapsable": true,
        "panels": [
          {
            "loading": false,
            "error": false,
            "span": 2,
            "editable": true,
            "group": [
              "default"
            ],
            "type": "fields",
            "style": {},
            "arrange": "vertical",
            "micropanel_position": "right",
            "sort": [
              "@timestamp",
              "desc"
            ],
            "query": "*"
          },
          {
            "loading": false,
            "error": false,
            "span": 10,
            "editable": true,
            "group": [
              "default"
            ],
            "type": "table",
            "query": "*",
            "interval": "1y",
            "show": [
              "bars",
              "y-axis",
              "x-axis",
              "legend"
            ],
            "fill": 3,
            "overflow": "min-height",
            "timezone": "browser",
            "spyable": true,
            "zoomlinks": true,
            "size": 50,
            "pages": 10,
            "offset": 0,
            "sort": [
              "@timestamp",
              "desc"
            ],
            "style": {
              "font-size": "9pt"
            },
            "fields": [
              "@timestamp",
              "@message"
            ],
            "sortable": true,
            "highlight": [],
            "header": true,
            "paging": true
          }
        ]
      }
    ],
    "editable": true,
    "last": null
  },
  "style": "dark",
  "failover": false,
  "services": {
    "query": {
      "idQueue": [],
      "list": {
        "0": {
          "id": 0,
          "color": "#7EB26D",
          "query": "python",
          "alias": "",
          "pin": false,
          "type": "lucene"
        },
        "1": {
          "id": 1,
          "color": "#EAB839",
          "query": "ruby",
          "alias": "",
          "pin": false,
          "type": "lucene"
        },
        "2": {
          "id": 2,
          "color": "#6ED0E0",
          "query": "javascript",
          "alias": "",
          "pin": false,
          "type": "lucene"
        },
        "3": {
          "id": 3,
          "color": "#EF843C",
          "query": "php",
          "alias": "",
          "pin": false,
          "type": "lucene"
        }
      },
      "ids": [
        0,
        1,
        2,
        3
      ]
    },
    "filter": {
      "idQueue": [
        1
      ],
      "list": {
        "0": {
          "from": "2012-09-07T11:21:30.000Z",
          "to": "2013-03-22T18:19:00.000Z",
          "field": "creation_date",
          "type": "time",
          "mandate": "must",
          "active": true,
          "alias": "",
          "id": 0
        }
      },
      "ids": [
        0
      ]
    }
  },
  "loader": {
    "save_gist": false,
    "save_elasticsearch": true,
    "save_local": true,
    "save_default": true,
    "save_temp": true,
    "save_temp_ttl_enable": true,
    "save_temp_ttl": "30d",
    "load_gist": true,
    "load_elasticsearch": true,
    "load_elasticsearch_size": 20,
    "load_local": true,
    "hide": false
  },
  "index": {
    "interval": "none",
    "pattern": "_all",
    "default": "prog"
  }
}