Last active
June 1, 2021 16:01
-
-
Save FrankHassanabad/55013baee6a31aaab47ccd1fd252e309 to your computer and use it in GitHub Desktop.
Perf testing across const keyword and regular keyword and aliases
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ```json | |
| # Adds an index for testing where we can test across: | |
| # * keyword | |
| # * const keyword | |
| # * And a corner case where someone reindexes non-compatible const data into const data from _source. | |
| DELETE const-logs-frank-delme-2 | |
| PUT const-logs-frank-delme-2 | |
| { | |
| "mappings": { | |
| "dynamic": "false", | |
| "properties": { | |
| "@timestamp": { | |
| "type": "date" | |
| }, | |
| "message": { | |
| "type": "text" | |
| }, | |
| "host": { | |
| "properties": { | |
| "name": { | |
| "type": "keyword" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| # Re-index two auditbeat indexes from estc to get us close to 1 million records for a profile run | |
| POST _reindex?wait_for_completion=false | |
| { | |
| "source": { | |
| "index": "auditbeat-8.0.0-2021.04.26-000007,auditbeat-8.0.0-2021.05.26-000008" | |
| }, | |
| "dest": { | |
| "index": "const-logs-frank-delme-2" | |
| } | |
| } | |
| # Count should around 941,317 once indexing is complete. | |
| GET const-logs-frank-delme-2/_count | |
| # We add a constant keyword to the mapping of "event.module" | |
| # to profile against other indexes where "event.module" is a keyword instead | |
| # of a constant and to test queries between these two. | |
| PUT const-logs-frank-delme-2/_mapping | |
| { | |
| "properties": { | |
| "event": { | |
| "properties": { | |
| "module": { | |
| "type": "constant_keyword", | |
| "value": "endpoint" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| # Aggs should always return "endpoint" from "const-logs-frank-delme-2" and then others | |
| # from "logs-*" and mix them together in the aggs query | |
| GET const-logs-frank-delme-2,logs-*/_search?request_cache=false&human=true | |
| { | |
| "size": 0, | |
| "profile": true, | |
| "track_total_hits": false, | |
| "aggs": { | |
| "event_module": { | |
| "terms": { | |
| "field": "event.module" | |
| } | |
| } | |
| } | |
| } | |
| # Adds the slower runtime into the mix to ensure performance for the specific const is not reduced | |
| GET const-logs-frank-delme-1,const-logs-frank-delme-2,auditbeat-8.0.0-2021.04.26-000007/_search?request_cache=false&human=true | |
| { | |
| "size": 0, | |
| "profile": true, | |
| "track_total_hits": false, | |
| "aggs": { | |
| "event_module": { | |
| "terms": { | |
| "field": "event.module" | |
| } | |
| } | |
| } | |
| } | |
| ``` |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
{ "took" : 444, "timed_out" : false, "_shards" : { "total" : 3, "successful" : 3, "skipped" : 0, "failed" : 0 }, "hits" : { "max_score" : null, "hits" : [ ] }, "aggregations" : { "event_module" : { "doc_count_error_upper_bound" : 0, "sum_other_doc_count" : 0, "buckets" : [ { "key" : "endpoint", "doc_count" : 943406 }, { "key" : "nginx", "doc_count" : 941637 }, { "key" : "auditd", "doc_count" : 848901 }, { "key" : "file_integrity", "doc_count" : 6105 }, { "key" : "system", "doc_count" : 1675 } ] } }, "profile" : { "shards" : [ { "id" : "[6zKvMLm3RQOlBuzH_74A1g][const-logs-frank-delme-1][0]", "searches" : [ { "query" : [ { "type" : "MatchAllDocsQuery", "description" : "*:*", "time" : "30.1ms", "time_in_nanos" : 30115113, "breakdown" : { "set_min_competitive_score_count" : 0, "match_count" : 0, "shallow_advance_count" : 0, "set_min_competitive_score" : 0, "next_doc" : 30073172, "match" : 0, "next_doc_count" : 941637, "score_count" : 0, "compute_max_score_count" : 0, "compute_max_score" : 0, "advance" : 2965, "advance_count" : 12, "score" : 0, "build_scorer_count" : 24, "create_weight" : 533, "shallow_advance" : 0, "create_weight_count" : 1, "build_scorer" : 38443 } } ], "rewrite_time" : 1721, "collector" : [ { "name" : "MultiCollector", "reason" : "search_multi", "time" : "404.6ms", "time_in_nanos" : 404643706, "children" : [ { "name" : "EarlyTerminatingCollector", "reason" : "search_count", "time" : "472.9micros", "time_in_nanos" : 472916 }, { "name" : "MultiBucketCollector: [org.elasticsearch.search.aggregations.MultiBucketCollector$1@202e4cb0]", "reason" : "aggregation", "time" : "343.6ms", "time_in_nanos" : 343616763 } ] } ] } ], "aggregations" : [ { "type" : "MapStringTermsAggregator", "description" : "event_module", "time" : "343ms", "time_in_nanos" : 343039148, "breakdown" : { "reduce" : 0, "post_collection_count" : 1, "build_leaf_collector" : 194473, "build_aggregation" : 7384, "build_aggregation_count" : 1, "build_leaf_collector_count" : 12, "post_collection" : 1209, "initialize" : 2137, "initialize_count" : 1, "reduce_count" : 0, "collect" : 342833945, "collect_count" : 941637 }, "debug" : { "total_buckets" : 1, "result_strategy" : "terms" } } ] }, { "id" : "[UmFIfD_iQfepBl6sUyEk2g][auditbeat-8.0.0-2021.04.26-000007][0]", "searches" : [ { "query" : [ { "type" : "MatchAllDocsQuery", "description" : "*:*", "time" : "177nanos", "time_in_nanos" : 177, "breakdown" : { "set_min_competitive_score_count" : 0, "match_count" : 0, "shallow_advance_count" : 0, "set_min_competitive_score" : 0, "next_doc" : 0, "match" : 0, "next_doc_count" : 0, "score_count" : 0, "compute_max_score_count" : 0, "compute_max_score" : 0, "advance" : 0, "advance_count" : 0, "score" : 0, "build_scorer_count" : 0, "create_weight" : 177, "shallow_advance" : 0, "create_weight_count" : 1, "build_scorer" : 0 } } ], "rewrite_time" : 1810, "collector" : [ { "name" : "MultiCollector", "reason" : "search_multi", "time" : "339.4micros", "time_in_nanos" : 339428, "children" : [ { "name" : "EarlyTerminatingCollector", "reason" : "search_count", "time" : "80.5micros", "time_in_nanos" : 80540 }, { "name" : "MultiBucketCollector: [org.elasticsearch.search.aggregations.MultiBucketCollector$1@7d255575]", "reason" : "aggregation", "time" : "200.3micros", "time_in_nanos" : 200375 } ] } ] } ], "aggregations" : [ { "type" : "StringTermsAggregatorFromFilters", "description" : "event_module", "time" : "147.7micros", "time_in_nanos" : 147799, "breakdown" : { "reduce" : 0, "post_collection_count" : 1, "build_leaf_collector" : 138196, "build_aggregation" : 7041, "build_aggregation_count" : 1, "build_leaf_collector_count" : 19, "post_collection" : 465, "initialize" : 2097, "initialize_count" : 1, "reduce_count" : 0, "collect" : 0, "collect_count" : 0 }, "debug" : { "delegate" : "FiltersAggregator.FilterByFilter", "delegate_debug" : { "segments_with_deleted_docs" : 0, "filters" : [ { "scorers_prepared_while_estimating_cost" : 0, "specialized_for" : "term", "results_from_metadata" : 19, "query" : "event.module:auditd" }, { "scorers_prepared_while_estimating_cost" : 0, "specialized_for" : "term", "results_from_metadata" : 19, "query" : "event.module:file_integrity" }, { "scorers_prepared_while_estimating_cost" : 0, "specialized_for" : "term", "results_from_metadata" : 19, "query" : "event.module:system" } ], "segments_counted" : 19, "segments_with_doc_count_field" : 0, "segments_collected" : 0 } } } ] }, { "id" : "[m88rV4HIS7uEbH2ZTe4G1g][const-logs-frank-delme-2][0]", "searches" : [ { "query" : [ { "type" : "MatchAllDocsQuery", "description" : "*:*", "time" : "413nanos", "time_in_nanos" : 413, "breakdown" : { "set_min_competitive_score_count" : 0, "match_count" : 0, "shallow_advance_count" : 0, "set_min_competitive_score" : 0, "next_doc" : 0, "match" : 0, "next_doc_count" : 0, "score_count" : 0, "compute_max_score_count" : 0, "compute_max_score" : 0, "advance" : 0, "advance_count" : 0, "score" : 0, "build_scorer_count" : 0, "create_weight" : 413, "shallow_advance" : 0, "create_weight_count" : 1, "build_scorer" : 0 } }, { "type" : "MatchAllDocsQuery", "description" : "*:*", "time" : "63.5ms", "time_in_nanos" : 63563894, "breakdown" : { "set_min_competitive_score_count" : 0, "match_count" : 0, "shallow_advance_count" : 0, "set_min_competitive_score" : 0, "next_doc" : 63555125, "match" : 0, "next_doc_count" : 1040875, "score_count" : 0, "compute_max_score_count" : 0, "compute_max_score" : 0, "advance" : 0, "advance_count" : 0, "score" : 0, "build_scorer_count" : 4, "create_weight" : 104, "shallow_advance" : 0, "create_weight_count" : 1, "build_scorer" : 8665 } } ], "rewrite_time" : 3511, "collector" : [ { "name" : "MultiCollector", "reason" : "search_multi", "time" : "29.1ms", "time_in_nanos" : 29141021, "children" : [ { "name" : "EarlyTerminatingCollector", "reason" : "search_count", "time" : "101.4micros", "time_in_nanos" : 101441 }, { "name" : "MultiBucketCollector: [org.elasticsearch.search.aggregations.MultiBucketCollector$1@7704a8a5]", "reason" : "aggregation", "time" : "28.9ms", "time_in_nanos" : 28996143 } ] } ] } ], "aggregations" : [ { "type" : "StringTermsAggregatorFromFilters", "description" : "event_module", "time" : "28.8ms", "time_in_nanos" : 28842952, "breakdown" : { "reduce" : 0, "post_collection_count" : 1, "build_leaf_collector" : 28832495, "build_aggregation" : 7404, "build_aggregation_count" : 1, "build_leaf_collector_count" : 10, "post_collection" : 946, "initialize" : 2107, "initialize_count" : 1, "reduce_count" : 0, "collect" : 0, "collect_count" : 0 }, "debug" : { "delegate" : "FiltersAggregator.FilterByFilter", "delegate_debug" : { "segments_with_deleted_docs" : 0, "filters" : [ { "scorers_prepared_while_estimating_cost" : 0, "specialized_for" : "match_all", "results_from_metadata" : 8, "query" : "*:*" } ], "segments_counted" : 10, "segments_with_doc_count_field" : 0, "segments_collected" : 0 } } } ] } ] } }