Skip to content

Instantly share code, notes, and snippets.

@cdahlqvist

cdahlqvist/README.md

Created April 23, 2017 10:38
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save cdahlqvist/2bd87c83300b803f119cf5c684293d6c to your computer and use it in GitHub Desktop.
Save cdahlqvist/2bd87c83300b803f119cf5c684293d6c to your computer and use it in GitHub Desktop.
Download ZIP
Access log index size test
Raw
README.md

Access log size test

This gist contains supporting files for evaluating Elasticsearch index sizes for web access logs.

Prerequisites

  • Machine with Linux or Mac OS X.
  • Local Elasticsearch 5.3.x instance accessible via 127.0.0.1:9200
  • The local Elasticsearch 5.3.x instance must have the geoip and useragent ingest plugins installed
  • Local installation of Filebeat 5.3.x with environment variable FILEBEAT_HOME pointing to the directory containing the filebeat binary.

Running the test

The test will index data into the following indices, which must not previously exist in the Elasticsearch instance:

  • filebeat-test
  • filebeat_es_defaults
  • filebeat_reduced
  • filebeat_best
  • filebeat_noall
  • filebeat_allquery
  • filebeat_combined

Start the test by running the run_test.sh script.

Raw
filebeat.yml
# Filebeat Apache module
filebeat.modules:
- module: apache2
access:
var.paths: ["${PWD}/sample_access_log"]
# Output all access logs directly to Elasticsearch
output.elasticsearch:
hosts: ["127.0.0.1:9200"]
index: "filebeat-test"
Raw
index_size.txt
green open filebeat_allquery ebWsM1cBRyKUEOU-QV7fRQ 1 0 280000 0 72187110 72187110
green open filebeat_best qkt1To1eSE2iK1SVEnd2MQ 1 0 280000 0 83489473 83489473
green open filebeat_combined XPvS4qTRTmSaC-tA-vjFwQ 1 0 280000 0 48964530 48964530
green open filebeat_es_defaults A9LzNGG_TYSOCb4N0kI25A 1 0 280000 0 123056208 123056208
green open filebeat_noall pFVOc03sQMiZazx1RYrRUQ 1 0 280000 0 65274259 65274259
green open filebeat_reduced Kkp3tMSdT5OwWN9AfSayXQ 1 0 280000 0 116472316 116472316
green open filebeat-test poHNL-n9STCFyOwgOvZAmQ 1 0 280000 0 97949223 97949223
Raw
load_pipelines.sh
#!/bin/sh
curl -s -XPUT 127.0.0.1:9200/_ingest/pipeline/convert_filebeat_fields -d '{
"description": "Pipeline for coverting select fields prior to indexing document with default mappings.",
"processors": [{
"convert": {
"field" : "apache2.access.response_code",
"type": "integer"
}
}, {
"convert": {
"field" : "apache2.access.body_sent.bytes",
"type": "integer"
}
}, {
"convert": {
"field" : "apache2.access.user_agent.major",
"type": "integer"
}
}, {
"convert": {
"field" : "apache2.access.user_agent.minor",
"type": "integer"
}
}, {
"convert": {
"field" : "apache2.access.user_agent.patch",
"type": "integer"
}
}, {
"convert": {
"field" : "apache2.access.user_agent.os_major",
"type": "integer"
}
}, {
"convert": {
"field" : "apache2.access.user_agent.os_minor",
"type": "integer"
}
}, {
"convert": {
"field" : "offset",
"type": "integer"
}
}],
"on_failure" : [{
"set" : {
"field" : "error",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
}'
curl -s -XPUT 127.0.0.1:9200/_ingest/pipeline/remove_fields -d '{
"description": "Pipeline for removing fields not deemed necessary.",
"processors": [{
"remove": {
"field" : "apache2.access.geoip.region_name"
}
}, {
"remove": {
"field" : "apache2.access.geoip.city_name"
}
}, {
"remove": {
"field" : "apache2.access.user_agent.major"
}
}, {
"remove": {
"field" : "apache2.access.user_agent.minor"
}
}, {
"remove": {
"field" : "apache2.access.user_agent.patch"
}
}, {
"remove": {
"field" : "apache2.access.user_agent.os_major"
}
}, {
"remove": {
"field" : "apache2.access.user_agent.os_minor"
}
}, {
"remove": {
"field" : "read_timestamp"
}
}],
"on_failure" : [{
"set" : {
"field" : "error",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
}'
Raw
load_templates.sh
#!/bin/sh
curl -s -XPOST 127.0.0.1:9200/_template/filebeat_test_shards -d '{
"order": 1,
"template": "filebeat-test",
"settings": {
"index": {
"number_of_shards": 1,
"number_of_replicas": 0
}
}
}'
curl -s -XPOST 127.0.0.1:9200/_template/filebeat_shards -d '{
"order": 1,
"template": "filebeat_*",
"settings": {
"index": {
"number_of_shards": 1,
"number_of_replicas": 0
}
}
}'
curl -s -XPOST 127.0.0.1:9200/_template/filebeat_es_defaults -d '{
"order": 0,
"template": "filebeat_es_defaults",
"mappings": {
"_default_": {
"properties": {
"@timestamp": {
"type": "date"
},
"apache2": {
"properties": {
"access": {
"properties": {
"geoip": {
"properties": {
"location": {
"type": "geo_point"
}
}
}
}
}
}
}
}
}
}
}'
curl -s -XPOST 127.0.0.1:9200/_template/filebeat_best -d '{
"aliases": {},
"template": "filebeat_best",
"mappings": {
"_default_": {
"properties": {
"error": {
"ignore_above": 1024,
"type": "keyword"
},
"fields": {
"properties": {}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"beat": {
"properties": {
"name": {
"type": "keyword",
"ignore_above": 1024
},
"version": {
"type": "keyword",
"ignore_above": 1024
},
"hostname": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"@timestamp": {
"type": "date"
},
"source": {
"ignore_above": 1024,
"type": "keyword"
},
"fileset": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"meta": {
"properties": {
"cloud": {
"properties": {
"region": {
"type": "keyword",
"ignore_above": 1024
},
"machine_type": {
"type": "keyword",
"ignore_above": 1024
},
"instance_id": {
"type": "keyword",
"ignore_above": 1024
},
"provider": {
"ignore_above": 1024,
"type": "keyword"
},
"availability_zone": {
"ignore_above": 1024,
"type": "keyword"
},
"project_id": {
"type": "keyword",
"ignore_above": 1024
}
}
}
}
},
"input_type": {
"type": "keyword",
"ignore_above": 1024
},
"read_timestamp": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"type": "keyword",
"ignore_above": 1024
},
"message": {
"type": "text",
"norms": false
},
"apache2": {
"properties": {
"access": {
"properties": {
"user_agent": {
"properties": {
"os_major": {
"type": "long"
},
"device": {
"ignore_above": 1024,
"type": "keyword"
},
"os": {
"ignore_above": 1024,
"type": "keyword"
},
"os_minor": {
"type": "long"
},
"major": {
"type": "long"
},
"minor": {
"type": "long"
},
"patch": {
"type": "long"
},
"os_name": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"user_name": {
"type": "keyword",
"ignore_above": 1024
},
"geoip": {
"properties": {
"location": {
"type": "geo_point"
},
"continent_name": {
"type": "keyword",
"ignore_above": 1024
},
"country_iso_code": {
"type": "keyword",
"ignore_above": 1024
}
}
},
"remote_ip": {
"ignore_above": 1024,
"type": "keyword"
},
"http_version": {
"type": "keyword",
"ignore_above": 1024
},
"url": {
"ignore_above": 1024,
"type": "keyword"
},
"body_sent": {
"properties": {
"bytes": {
"type": "long"
}
}
},
"response_code": {
"type": "long"
},
"agent": {
"type": "text",
"norms": false
},
"method": {
"type": "keyword",
"ignore_above": 1024
},
"referrer": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"properties": {
"message": {
"norms": false,
"type": "text"
},
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"client": {
"type": "keyword",
"ignore_above": 1024
},
"module": {
"type": "keyword",
"ignore_above": 1024
},
"pid": {
"type": "long"
},
"tid": {
"type": "long"
}
}
}
}
},
"offset": {
"type": "long"
}
},
"dynamic_templates": [{
"strings_as_keyword": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword",
"ignore_above": 1024
}
}
}],
"_all": {
"norms": false
},
"date_detection": false,
"_meta": {
"version": "5.3.0"
}
}
},
"order": 0,
"settings": {
"index": {
"codec": "best_compression",
"mapping": {
"total_fields": {
"limit": "10000"
}
},
"refresh_interval": "5s"
}
}
}'
curl -s -XPOST 127.0.0.1:9200/_template/filebeat_noall -d '{
"aliases": {},
"template": "filebeat_noall",
"mappings": {
"_default_": {
"properties": {
"error": {
"ignore_above": 1024,
"type": "keyword"
},
"fields": {
"properties": {}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"beat": {
"properties": {
"name": {
"type": "keyword",
"ignore_above": 1024
},
"version": {
"type": "keyword",
"ignore_above": 1024
},
"hostname": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"@timestamp": {
"type": "date"
},
"source": {
"ignore_above": 1024,
"type": "keyword"
},
"fileset": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"meta": {
"properties": {
"cloud": {
"properties": {
"region": {
"type": "keyword",
"ignore_above": 1024
},
"machine_type": {
"type": "keyword",
"ignore_above": 1024
},
"instance_id": {
"type": "keyword",
"ignore_above": 1024
},
"provider": {
"ignore_above": 1024,
"type": "keyword"
},
"availability_zone": {
"ignore_above": 1024,
"type": "keyword"
},
"project_id": {
"type": "keyword",
"ignore_above": 1024
}
}
}
}
},
"input_type": {
"type": "keyword",
"ignore_above": 1024
},
"read_timestamp": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"type": "keyword",
"ignore_above": 1024
},
"message": {
"type": "text",
"norms": false
},
"apache2": {
"properties": {
"access": {
"properties": {
"user_agent": {
"properties": {
"os_major": {
"type": "long"
},
"device": {
"ignore_above": 1024,
"type": "keyword"
},
"os": {
"ignore_above": 1024,
"type": "keyword"
},
"os_minor": {
"type": "long"
},
"major": {
"type": "long"
},
"minor": {
"type": "long"
},
"patch": {
"type": "long"
},
"os_name": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"user_name": {
"type": "keyword",
"ignore_above": 1024
},
"geoip": {
"properties": {
"location": {
"type": "geo_point"
},
"continent_name": {
"type": "keyword",
"ignore_above": 1024
},
"country_iso_code": {
"type": "keyword",
"ignore_above": 1024
}
}
},
"remote_ip": {
"ignore_above": 1024,
"type": "keyword"
},
"http_version": {
"type": "keyword",
"ignore_above": 1024
},
"url": {
"ignore_above": 1024,
"type": "keyword"
},
"body_sent": {
"properties": {
"bytes": {
"type": "long"
}
}
},
"response_code": {
"type": "long"
},
"agent": {
"type": "text",
"norms": false
},
"method": {
"type": "keyword",
"ignore_above": 1024
},
"referrer": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"properties": {
"message": {
"norms": false,
"type": "text"
},
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"client": {
"type": "keyword",
"ignore_above": 1024
},
"module": {
"type": "keyword",
"ignore_above": 1024
},
"pid": {
"type": "long"
},
"tid": {
"type": "long"
}
}
}
}
},
"offset": {
"type": "long"
}
},
"dynamic_templates": [{
"strings_as_keyword": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword",
"ignore_above": 1024
}
}
}],
"_all": {
"enabled": false
},
"date_detection": false,
"_meta": {
"version": "5.3.0"
}
}
},
"order": 0,
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "10000"
}
},
"refresh_interval": "5s"
}
}
}'
curl -s -XPOST 127.0.0.1:9200/_template/filebeat_allquery -d '{
"aliases": {},
"template": "filebeat_allquery",
"mappings": {
"_default_": {
"properties": {
"error": {
"ignore_above": 1024,
"type": "keyword"
},
"fields": {
"properties": {}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"beat": {
"properties": {
"name": {
"type": "keyword",
"ignore_above": 1024
},
"version": {
"type": "keyword",
"ignore_above": 1024
},
"hostname": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"@timestamp": {
"type": "date"
},
"source": {
"type": "keyword",
"ignore_above": 1024
},
"fileset": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"meta": {
"properties": {
"cloud": {
"properties": {
"region": {
"type": "keyword",
"ignore_above": 1024
},
"machine_type": {
"type": "keyword",
"ignore_above": 1024
},
"instance_id": {
"type": "keyword",
"ignore_above": 1024
},
"provider": {
"ignore_above": 1024,
"type": "keyword"
},
"availability_zone": {
"ignore_above": 1024,
"type": "keyword"
},
"project_id": {
"type": "keyword",
"ignore_above": 1024
}
}
}
}
},
"input_type": {
"type": "keyword",
"ignore_above": 1024
},
"read_timestamp": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"type": "keyword",
"ignore_above": 1024
},
"message": {
"type": "text",
"norms": false
},
"apache2": {
"properties": {
"access": {
"properties": {
"user_agent": {
"properties": {
"os_major": {
"type": "long"
},
"device": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
},
"os": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
},
"os_minor": {
"type": "long"
},
"major": {
"type": "long"
},
"minor": {
"type": "long"
},
"patch": {
"type": "long"
},
"os_name": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
},
"name": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
}
}
},
"user_name": {
"type": "keyword",
"ignore_above": 1024
},
"geoip": {
"properties": {
"location": {
"type": "geo_point"
},
"continent_name": {
"type": "keyword",
"ignore_above": 1024
},
"country_iso_code": {
"type": "keyword",
"ignore_above": 1024
},
"city_name": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
}
}
},
"remote_ip": {
"ignore_above": 1024,
"type": "keyword"
},
"http_version": {
"type": "keyword",
"ignore_above": 1024
},
"url": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
},
"body_sent": {
"properties": {
"bytes": {
"type": "long"
}
}
},
"response_code": {
"type": "long"
},
"agent": {
"type": "text",
"norms": false
},
"method": {
"type": "keyword",
"ignore_above": 1024
},
"referrer": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
}
}
},
"error": {
"properties": {
"message": {
"norms": false,
"type": "text"
},
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"client": {
"type": "keyword",
"ignore_above": 1024
},
"module": {
"type": "keyword",
"ignore_above": 1024
},
"pid": {
"type": "long"
},
"tid": {
"type": "long"
}
}
}
}
},
"offset": {
"type": "long"
}
},
"dynamic_templates": [{
"strings_as_keyword": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword",
"ignore_above": 1024
}
}
}],
"_all": {
"enabled": false
},
"date_detection": false,
"_meta": {
"version": "5.3.0"
}
}
},
"order": 0,
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "10000"
}
},
"refresh_interval": "5s"
}
}
}'
curl -s -XPOST 127.0.0.1:9200/_template/filebeat_combined -d '{
"aliases": {},
"template": "filebeat_combined",
"mappings": {
"_default_": {
"properties": {
"error": {
"ignore_above": 1024,
"type": "keyword"
},
"fields": {
"properties": {}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"beat": {
"properties": {
"name": {
"type": "keyword",
"ignore_above": 1024
},
"version": {
"type": "keyword",
"ignore_above": 1024
},
"hostname": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"@timestamp": {
"type": "date"
},
"source": {
"type": "keyword",
"ignore_above": 1024
},
"fileset": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"meta": {
"properties": {
"cloud": {
"properties": {
"region": {
"type": "keyword",
"ignore_above": 1024
},
"machine_type": {
"type": "keyword",
"ignore_above": 1024
},
"instance_id": {
"type": "keyword",
"ignore_above": 1024
},
"provider": {
"ignore_above": 1024,
"type": "keyword"
},
"availability_zone": {
"ignore_above": 1024,
"type": "keyword"
},
"project_id": {
"type": "keyword",
"ignore_above": 1024
}
}
}
}
},
"input_type": {
"type": "keyword",
"ignore_above": 1024
},
"read_timestamp": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"type": "keyword",
"ignore_above": 1024
},
"message": {
"type": "text",
"norms": false
},
"apache2": {
"properties": {
"access": {
"properties": {
"user_agent": {
"properties": {
"os_major": {
"type": "long"
},
"device": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
},
"os": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
},
"os_minor": {
"type": "long"
},
"major": {
"type": "long"
},
"minor": {
"type": "long"
},
"patch": {
"type": "long"
},
"os_name": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
},
"name": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
}
}
},
"user_name": {
"type": "keyword",
"ignore_above": 1024
},
"geoip": {
"properties": {
"location": {
"type": "geo_point"
},
"continent_name": {
"type": "keyword",
"ignore_above": 1024
},
"country_iso_code": {
"type": "keyword",
"ignore_above": 1024
},
"city_name": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
}
}
},
"remote_ip": {
"ignore_above": 1024,
"type": "keyword"
},
"http_version": {
"type": "keyword",
"ignore_above": 1024
},
"url": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
},
"body_sent": {
"properties": {
"bytes": {
"type": "long"
}
}
},
"response_code": {
"type": "long"
},
"agent": {
"type": "text",
"norms": false
},
"method": {
"type": "keyword",
"ignore_above": 1024
},
"referrer": {
"fields" : {
"keyword" : {
"ignore_above" : 1024,
"type" : "keyword"
}
},
"norms": false,
"type" : "text"
}
}
},
"error": {
"properties": {
"message": {
"norms": false,
"type": "text"
},
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"client": {
"type": "keyword",
"ignore_above": 1024
},
"module": {
"type": "keyword",
"ignore_above": 1024
},
"pid": {
"type": "long"
},
"tid": {
"type": "long"
}
}
}
}
},
"offset": {
"type": "long"
}
},
"dynamic_templates": [{
"strings_as_keyword": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword",
"ignore_above": 1024
}
}
}],
"_all": {
"enabled": false
},
"date_detection": false,
"_meta": {
"version": "5.3.0"
}
}
},
"order": 0,
"settings": {
"index": {
"codec": "best_compression",
"mapping": {
"total_fields": {
"limit": "10000"
}
},
"refresh_interval": "5s"
}
}
}'
Raw
run_test.sh
#!/bin/bash
if [[ -z "${FILEBEAT_HOME}" ]]; then
echo 'ERROR: Environment variable FILEBEAT_HOME not defined, aborting.'
exit
fi
echo "Verify test indices do not already exist..."
COUNT=$(curl 127.0.0.1:9200/_cat/indices | grep -E "filebeat-test|filebeat_es_defaults|filebeat_reduced|filebeat_best|filebeat_noall|filebeat_allquery|filebeat_combined" | wc -l)
if [[ $COUNT -ne 0 ]]; then
echo 'ERROR: One or more indices used in this test already exists. Aborting.'
exit
fi
echo "Download sample access log..."
curl -s -O https://s3.amazonaws.com/users.elasticsearch.org/cdahlqvist/files/sample_access_log.gz >> test.log
echo "Unzip sample access log..."
if [ ! -f ./sample_access_log ]; then
gunzip ./sample_access_log.gz
fi
echo "Load index templates..."
source load_templates.sh >> test.log
echo "Load ingest pipelines..."
source load_pipelines.sh >> test.log
echo "Load data through Filebeat"
$FILEBEAT_HOME/filebeat -e -modules=apache2 -setup -c ./filebeat.yml &
echo "Wait until all 280000 log entries have been indexed"
COUNT=0
while [ $COUNT == 0 ]
do
sleep 10
COUNT=$(curl -s 127.0.0.1:9200/_cat/indices | grep "filebeat-test" | grep 280000 | wc -l)
done
echo "Terminate Filebeat..."
ps -ef | grep filebeat.yml | grep -v grep | awk '{print $2}' | xargs kill
echo "Reindex data with default Elasticsearch mappings..."
curl -s -XPOST 127.0.0.1:9200/_reindex -d '{
"source": {
"index": "filebeat-test"
},
"dest": {
"index": "filebeat_es_defaults",
"pipeline": "convert_filebeat_fields"
}
}' >> test.log
echo "Reindex data with reduced number of fields..."
curl -s -XPOST 127.0.0.1:9200/_reindex -d '{
"source": {
"index": "filebeat-test"
},
"dest": {
"index": "filebeat_reduced",
"pipeline": "remove_fields"
}
}' >> test.log
echo "Reindex data with best_compression enabled..."
curl -s -XPOST 127.0.0.1:9200/_reindex -d '{
"source": {
"index": "filebeat-test"
},
"dest": {
"index": "filebeat_best"
}
}' >> test.log
echo "Reindex data with _all field disabled..."
curl -s -XPOST 127.0.0.1:9200/_reindex -d '{
"source": {
"index": "filebeat-test"
},
"dest": {
"index": "filebeat_noall"
}
}' >> test.log
echo "Reindex data with data enhanced for _all query instead of _all field..."
curl -s -XPOST 127.0.0.1:9200/_reindex -d '{
"source": {
"index": "filebeat-test"
},
"dest": {
"index": "filebeat_allquery"
}
}' >> test.log
echo "Reindex data with reduced number of fields, best_compression enabled and _all query instead of _all field..."
curl -s -XPOST 127.0.0.1:9200/_reindex -d '{
"source": {
"index": "filebeat-test"
},
"dest": {
"index": "filebeat_combined",
"pipeline": "remove_fields"
}
}' >> test.log
echo "Wait until all records processed..."
COUNT=0
while [ $COUNT -ne 7 ]
do
sleep 10
COUNT=$(curl -s 127.0.0.1:9200/_cat/indices | grep filebeat | grep 280000 | wc -l)
done
echo "Perform _forcemerge..."
curl -s -XPOST 127.0.0.1:9200/filebeat*/_forcemerge -d '{ "max_num_segments": 1 }' >> test.log
sleep 60
echo "Processing completed. Record results..."
curl -s 127.0.0.1:9200/_cat/indices?bytes=b | grep filebeat | sort > ./index_size.txt
Raw
sample_json_log.json
{"input_type": "log","@timestamp": "2014-05-30T14:31:13.000Z","read_timestamp": "2017-04-20T20:11:30.976Z","source": "/home/ec2-user/access_log_size_test/sample_access_log","type": "log","apache2": {"access": {"body_sent": {"bytes": "4465"},"remote_ip": "85.214.196.224","response_code": "200","method": "GET","url": "/files/xdotool/docs/html/files.html","geoip": {"region_name": "Land Berlin","country_iso_code": "DE","city_name": "Berlin","location": {"lat": 52.5167,"lon": 13.4},"continent_name": "Europe"},"http_version": "1.1","referrer": "http://semicomplete.com/files/xdotool/docs/html/osx__hacks_8h_source.html","user_agent": {"patch": "1","os_minor": "8","os": "Mac OS X 10.8.5","os_name": "Mac OS X","device": "Other","major": "6","minor": "1","name": "Safari","os_major": "10"},"user_name": "-"}},"offset": 2961191,"beat": {"version": "5.3.0","name": "ip-10-0-0-128","hostname": "ip-10-0-0-128"},"fileset": {"module": "apache2","name": "access"}}
Raw
sample_raw_log.txt
85.214.196.224 - - [30/May/2014:09:31:13 -0500] "GET /files/xdotool/docs/html/files.html HTTP/1.1" 200 4465 "http://semicomplete.com/files/xdotool/docs/html/osx__hacks_8h_source.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/6.1.1 Safari/537.73.11"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment