Skip to content

Instantly share code, notes, and snippets.

@spinscale
Created March 17, 2021 10:56
Show Gist options
  • Save spinscale/b4a2a6d117c67e1ffe500a84aa0d7c90 to your computer and use it in GitHub Desktop.
Save spinscale/b4a2a6d117c67e1ffe500a84aa0d7c90 to your computer and use it in GitHub Desktop.
Elastic Bytes - Runtime fields
##################
# Runtime fields #
##################
GET /
DELETE orders
PUT orders
{
"mappings": {
"properties": {
"@timestamp" : { "type" : "date" },
"items" : { "type" :"nested"}
}
}
}
PUT orders/_doc/1
{
"@timestamp" : "2021-03-11T12:34:56.789Z",
"total" : 168.43,
"items" : [
{
"name": "Hoodie Foo",
"price" : 99.00
},
{
"name" : "Jeans",
"price" : 56.43
},
{
"name" : "shipping express",
"price" : 11.00
}
]
}
PUT orders/_doc/2
{
"@timestamp" : "2021-03-15T12:59:56.789Z",
"total" : 99,
"items" : [
{
"name": "Hoodie Foo",
"price" : 99.00
}
]
}
PUT orders/_doc/3
{
"@timestamp" : "2021-03-16T18:59:56.789CET",
"total" : 56.43,
"items" : [
{
"name": "Jeans",
"price" : 56.43
}
]
}
# good old aggs
GET orders/_search
{
"size": 0,
"aggs": {
"by_day": {
"date_histogram": {
"field": "@timestamp",
"interval": "day",
"min_doc_count": 1
}
}
}
}
# how can we figure out the most common oder hour of day
# and also use this in queries as a range filter?
# with scripts in the agg as well as a script filter
# let's try something better and create a runtime field
POST orders/_mapping
{
"runtime": {
"hour_of_day": {
"type": "long",
"script": {
"source": "emit(doc['@timestamp'].value.hour)"
}
}
}
}
# let's take a look
GET orders/_search?filter_path=**.hour_of_day,**.@timestamp
{
"fields" : ["_source", "hour_of_day"]
}
# aggs
GET orders/_search
{
"size": 0,
"aggs": {
"by_hour": {
"histogram": {
"field": "hour_of_day",
"interval": 1,
"min_doc_count": 1
}
}
}
}
# search against that field
GET orders/_search
{
"query": {
"range": {
"hour_of_day": {
"lt": 13
}
}
}
}
# also there is no need to change the mapping
# you can also specify the field in the search request
# let's search for the month instead
GET orders/_search
{
"runtime_mappings": {
"month": {
"type": "long",
"script": {
"source": "emit(doc['@timestamp'].value.monthValue)"
}
}
},
"query": {
"range": {
"month": {
"lt": 4
}
}
}
}
# emitting more than a single field - via grok
DELETE logs
PUT logs/_doc/1
{
"message": "2021-01-18T14:03:54.870920Z INFO - User has logged out -- user_id: 1, user_name: \"Alex\""
}
PUT logs/_doc/2
{
"message": "2021-01-18T14:24:11.254881Z ERROR - Error occured"
}
POST logs/_mapping
{
"runtime": {
"@timestamp": {
"type": "date",
"script": {
"source": """
def dateAsString = grok('%{DATA:date} %{GREEDYDATA:input}').extract(doc["message.keyword"].value)?.date;
emit(Instant.parse(dateAsString).toEpochMilli());
"""
}
}
}
}
# let's take a look
GET logs/_search?filter_path=**.@timestamp
{
"fields" : ["_source", "@timestamp"]
}
GET logs/_search
{
"query": {
"range": {
"@timestamp": {
"gte": "2021-01-18"
}
}
}
}
# you may want to use dissect where possible for performance
POST logs/_mapping
{
"runtime": {
"@timestamp": {
"type": "date",
"script": {
"source": """
def dateAsString = dissect('%{date} %{input}').extract(doc["message.keyword"].value)?.date;
emit(Instant.parse(dateAsString).toEpochMilli());
"""
}
}
}
}
GET logs/_mapping
GET logs/_search
{
"query": {
"range": {
"@timestamp": {
"gte": "2021-01-18"
}
}
}
}
# production hint: switch to asynchronous search in production
POST logs/_async_search
{
"query": {
"range": {
"@timestamp": {
"gte": "2021-01-18"
}
}
}
}
# you can also search indices mixing this as a real/runtime field
# changing the mapping in your index template from indexed field to runtime field or vice versa is no problem
DELETE logs-2
PUT logs-2
PUT logs-2/_doc/1
{
"@timestamp" : "2021-01-18T14:03:54.870920Z",
"message": "2021-01-18T14:03:54.870920Z INFO - User has logged out -- user_id: 1, user_name: \"Alex\""
}
PUT logs-2/_doc/2
{
"@timestamp" : "2021-01-18T14:24:11.254881Z",
"message": "2021-01-18T14:24:11.254881Z ERROR - Error occured"
}
GET logs*/_search
{
"query": {
"range": {
"@timestamp": {
"gte": "2021-01-18"
}
}
}
}
# future, potentially emit several fields at once
# future, runtime field editor at index patterns
# please leave feedback, create issues
# Kahoot: https://kahoot.it/challenge/09238643
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment