Skip to content

Instantly share code, notes, and snippets.

@lior-fortifieddata
Created May 14, 2021 22:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save lior-fortifieddata/d51c203fd716b10d715311a2af661262 to your computer and use it in GitHub Desktop.
Save lior-fortifieddata/d51c203fd716b10d715311a2af661262 to your computer and use it in GitHub Desktop.
query:
***************************************************
GET index/_search
{
"profile": "true",
"size": 0,
"query": {
"range": {
"crawled_date": {
"gte": "now-48h"
}
}
},
"aggs": {
"NAME": {
"date_histogram": {
"field": "crawled_timestamp",
"interval": "hour",
"min_doc_count": 1
}
}
}
}
****************************************************************************************************************************
old profiler
****************************************************************************************************************************
{
"took" : 184,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 10000,
"relation" : "gte"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"NAME" : {
"buckets" : [
{
"key_as_string" : "1620864000",
"key" : 1620864000000,
"doc_count" : 121403
},
{
"key_as_string" : "1620867600",
"key" : 1620867600000,
"doc_count" : 116774
},
{
"key_as_string" : "1620871200",
"key" : 1620871200000,
"doc_count" : 112874
},
{
"key_as_string" : "1620874800",
"key" : 1620874800000,
"doc_count" : 101930
},
{
"key_as_string" : "1620878400",
"key" : 1620878400000,
"doc_count" : 96076
},
{
"key_as_string" : "1620882000",
"key" : 1620882000000,
"doc_count" : 97034
},
{
"key_as_string" : "1620885600",
"key" : 1620885600000,
"doc_count" : 94439
},
{
"key_as_string" : "1620889200",
"key" : 1620889200000,
"doc_count" : 72237
},
{
"key_as_string" : "1620892800",
"key" : 1620892800000,
"doc_count" : 104262
},
{
"key_as_string" : "1620896400",
"key" : 1620896400000,
"doc_count" : 70444
},
{
"key_as_string" : "1620900000",
"key" : 1620900000000,
"doc_count" : 54389
},
{
"key_as_string" : "1620903600",
"key" : 1620903600000,
"doc_count" : 51355
},
{
"key_as_string" : "1620907200",
"key" : 1620907200000,
"doc_count" : 49726
},
{
"key_as_string" : "1620910800",
"key" : 1620910800000,
"doc_count" : 11771
},
{
"key_as_string" : "1620921600",
"key" : 1620921600000,
"doc_count" : 83328
},
{
"key_as_string" : "1620925200",
"key" : 1620925200000,
"doc_count" : 61037
},
{
"key_as_string" : "1620928800",
"key" : 1620928800000,
"doc_count" : 54432
},
{
"key_as_string" : "1620932400",
"key" : 1620932400000,
"doc_count" : 56257
},
{
"key_as_string" : "1620936000",
"key" : 1620936000000,
"doc_count" : 49881
},
{
"key_as_string" : "1620939600",
"key" : 1620939600000,
"doc_count" : 23724
},
{
"key_as_string" : "1620943200",
"key" : 1620943200000,
"doc_count" : 150
},
{
"key_as_string" : "1620950400",
"key" : 1620950400000,
"doc_count" : 118243
},
{
"key_as_string" : "1620954000",
"key" : 1620954000000,
"doc_count" : 114528
},
{
"key_as_string" : "1620957600",
"key" : 1620957600000,
"doc_count" : 104722
},
{
"key_as_string" : "1620961200",
"key" : 1620961200000,
"doc_count" : 96103
},
{
"key_as_string" : "1620964800",
"key" : 1620964800000,
"doc_count" : 97885
},
{
"key_as_string" : "1620968400",
"key" : 1620968400000,
"doc_count" : 105795
},
{
"key_as_string" : "1620972000",
"key" : 1620972000000,
"doc_count" : 97413
},
{
"key_as_string" : "1620975600",
"key" : 1620975600000,
"doc_count" : 65816
},
{
"key_as_string" : "1620979200",
"key" : 1620979200000,
"doc_count" : 111160
},
{
"key_as_string" : "1620982800",
"key" : 1620982800000,
"doc_count" : 61888
},
{
"key_as_string" : "1620986400",
"key" : 1620986400000,
"doc_count" : 58916
},
{
"key_as_string" : "1620990000",
"key" : 1620990000000,
"doc_count" : 56110
},
{
"key_as_string" : "1620993600",
"key" : 1620993600000,
"doc_count" : 40541
},
{
"key_as_string" : "1620997200",
"key" : 1620997200000,
"doc_count" : 14
},
{
"key_as_string" : "1621008000",
"key" : 1621008000000,
"doc_count" : 76567
},
{
"key_as_string" : "1621011600",
"key" : 1621011600000,
"doc_count" : 61720
},
{
"key_as_string" : "1621015200",
"key" : 1621015200000,
"doc_count" : 50954
},
{
"key_as_string" : "1621018800",
"key" : 1621018800000,
"doc_count" : 49844
},
{
"key_as_string" : "1621022400",
"key" : 1621022400000,
"doc_count" : 44128
},
{
"key_as_string" : "1621026000",
"key" : 1621026000000,
"doc_count" : 16899
}
]
}
},
"profile" : {
"shards" : [
{
"id" : "[G4s9aNOeTViX4v43Bt7EWQ][old_index][0]",
"searches" : [
{
"query" : [
{
"type" : "IndexOrDocValuesQuery",
"description" : "ingest_date:[1620857539293 TO 9223372036854775807]",
"time_in_nanos" : 24781847,
"breakdown" : {
"set_min_competitive_score_count" : 0,
"match_count" : 0,
"shallow_advance_count" : 0,
"set_min_competitive_score" : 0,
"next_doc" : 22015709,
"match" : 0,
"next_doc_count" : 583588,
"score_count" : 0,
"compute_max_score_count" : 0,
"compute_max_score" : 0,
"advance" : 19194,
"advance_count" : 46,
"score" : 0,
"build_scorer_count" : 92,
"create_weight" : 2669,
"shallow_advance" : 0,
"create_weight_count" : 1,
"build_scorer" : 2160548
}
}
],
"rewrite_time" : 1841,
"collector" : [
{
"name" : "CancellableCollector",
"reason" : "search_cancelled",
"time_in_nanos" : 151901954,
"children" : [
{
"name" : "MultiCollector",
"reason" : "search_multi",
"time_in_nanos" : 115484190,
"children" : [
{
"name" : "EarlyTerminatingCollector",
"reason" : "search_count",
"time_in_nanos" : 737555
},
{
"name" : "ProfilingAggregator: [NAME]",
"reason" : "aggregation",
"time_in_nanos" : 71451996
}
]
}
]
}
]
}
],
"aggregations" : [
{
"type" : "DateHistogramAggregator",
"description" : "NAME",
"time_in_nanos" : 54773645,
"breakdown" : {
"reduce" : 0,
"build_aggregation" : 36107,
"build_aggregation_count" : 1,
"initialize" : 1605,
"initialize_count" : 1,
"reduce_count" : 0,
"collect" : 54152350,
"collect_count" : 583581
}
}
]
},
{
"id" : "[G4s9aNOeTViX4v43Bt7EWQ][old_index][4]",
"searches" : [
{
"query" : [
{
"type" : "IndexOrDocValuesQuery",
"description" : "ingest_date:[1620857539293 TO 9223372036854775807]",
"time_in_nanos" : 24377210,
"breakdown" : {
"set_min_competitive_score_count" : 0,
"match_count" : 0,
"shallow_advance_count" : 0,
"set_min_competitive_score" : 0,
"next_doc" : 21538512,
"match" : 0,
"next_doc_count" : 582292,
"score_count" : 0,
"compute_max_score_count" : 0,
"compute_max_score" : 0,
"advance" : 5381,
"advance_count" : 48,
"score" : 0,
"build_scorer_count" : 96,
"create_weight" : 2334,
"shallow_advance" : 0,
"create_weight_count" : 1,
"build_scorer" : 2248546
}
}
],
"rewrite_time" : 1986,
"collector" : [
{
"name" : "CancellableCollector",
"reason" : "search_cancelled",
"time_in_nanos" : 148920679,
"children" : [
{
"name" : "MultiCollector",
"reason" : "search_multi",
"time_in_nanos" : 113301094,
"children" : [
{
"name" : "EarlyTerminatingCollector",
"reason" : "search_count",
"time_in_nanos" : 630539
},
{
"name" : "ProfilingAggregator: [NAME]",
"reason" : "aggregation",
"time_in_nanos" : 70356268
}
]
}
]
}
]
}
],
"aggregations" : [
{
"type" : "DateHistogramAggregator",
"description" : "NAME",
"time_in_nanos" : 52562411,
"breakdown" : {
"reduce" : 0,
"build_aggregation" : 39965,
"build_aggregation_count" : 1,
"initialize" : 1366,
"initialize_count" : 1,
"reduce_count" : 0,
"collect" : 51938808,
"collect_count" : 582270
}
}
]
},
{
"id" : "[XCmvOOBiQ66VhhY_CE4WaQ][old_index][1]",
"searches" : [
{
"query" : [
{
"type" : "IndexOrDocValuesQuery",
"description" : "ingest_date:[1620857539293 TO 9223372036854775807]",
"time_in_nanos" : 25115281,
"breakdown" : {
"set_min_competitive_score_count" : 0,
"match_count" : 0,
"shallow_advance_count" : 0,
"set_min_competitive_score" : 0,
"next_doc" : 21120760,
"match" : 0,
"next_doc_count" : 580782,
"score_count" : 0,
"compute_max_score_count" : 0,
"compute_max_score" : 0,
"advance" : 9905,
"advance_count" : 45,
"score" : 0,
"build_scorer_count" : 90,
"create_weight" : 2272,
"shallow_advance" : 0,
"create_weight_count" : 1,
"build_scorer" : 3401426
}
}
],
"rewrite_time" : 1941,
"collector" : [
{
"name" : "CancellableCollector",
"reason" : "search_cancelled",
"time_in_nanos" : 135261475,
"children" : [
{
"name" : "MultiCollector",
"reason" : "search_multi",
"time_in_nanos" : 101439469,
"children" : [
{
"name" : "EarlyTerminatingCollector",
"reason" : "search_count",
"time_in_nanos" : 567590
},
{
"name" : "ProfilingAggregator: [NAME]",
"reason" : "aggregation",
"time_in_nanos" : 61574050
}
]
}
]
}
]
}
],
"aggregations" : [
{
"type" : "DateHistogramAggregator",
"description" : "NAME",
"time_in_nanos" : 51577034,
"breakdown" : {
"reduce" : 0,
"build_aggregation" : 55794,
"build_aggregation_count" : 1,
"initialize" : 1324,
"initialize_count" : 1,
"reduce_count" : 0,
"collect" : 50939135,
"collect_count" : 580779
}
}
]
},
{
"id" : "[XCmvOOBiQ66VhhY_CE4WaQ][old_index][3]",
"searches" : [
{
"query" : [
{
"type" : "IndexOrDocValuesQuery",
"description" : "ingest_date:[1620857539293 TO 9223372036854775807]",
"time_in_nanos" : 22424478,
"breakdown" : {
"set_min_competitive_score_count" : 0,
"match_count" : 0,
"shallow_advance_count" : 0,
"set_min_competitive_score" : 0,
"next_doc" : 19083739,
"match" : 0,
"next_doc_count" : 582961,
"score_count" : 0,
"compute_max_score_count" : 0,
"compute_max_score" : 0,
"advance" : 4380,
"advance_count" : 50,
"score" : 0,
"build_scorer_count" : 100,
"create_weight" : 2538,
"shallow_advance" : 0,
"create_weight_count" : 1,
"build_scorer" : 2750709
}
}
],
"rewrite_time" : 1645,
"collector" : [
{
"name" : "CancellableCollector",
"reason" : "search_cancelled",
"time_in_nanos" : 136779773,
"children" : [
{
"name" : "MultiCollector",
"reason" : "search_multi",
"time_in_nanos" : 102719655,
"children" : [
{
"name" : "EarlyTerminatingCollector",
"reason" : "search_count",
"time_in_nanos" : 587322
},
{
"name" : "ProfilingAggregator: [NAME]",
"reason" : "aggregation",
"time_in_nanos" : 62139635
}
]
}
]
}
]
}
],
"aggregations" : [
{
"type" : "DateHistogramAggregator",
"description" : "NAME",
"time_in_nanos" : 48861889,
"breakdown" : {
"reduce" : 0,
"build_aggregation" : 52969,
"build_aggregation_count" : 1,
"initialize" : 911,
"initialize_count" : 1,
"reduce_count" : 0,
"collect" : 48225076,
"collect_count" : 582931
}
}
]
},
{
"id" : "[q8UHmWaXSOCIYLwstiZkgA][old_index][2]",
"searches" : [
{
"query" : [
{
"type" : "IndexOrDocValuesQuery",
"description" : "ingest_date:[1620857539293 TO 9223372036854775807]",
"time_in_nanos" : 28398977,
"breakdown" : {
"set_min_competitive_score_count" : 0,
"match_count" : 0,
"shallow_advance_count" : 0,
"set_min_competitive_score" : 0,
"next_doc" : 22477003,
"match" : 0,
"next_doc_count" : 583214,
"score_count" : 0,
"compute_max_score_count" : 0,
"compute_max_score" : 0,
"advance" : 5173,
"advance_count" : 52,
"score" : 0,
"build_scorer_count" : 104,
"create_weight" : 2608,
"shallow_advance" : 0,
"create_weight_count" : 1,
"build_scorer" : 5330822
}
}
],
"rewrite_time" : 6783,
"collector" : [
{
"name" : "CancellableCollector",
"reason" : "search_cancelled",
"time_in_nanos" : 148921369,
"children" : [
{
"name" : "MultiCollector",
"reason" : "search_multi",
"time_in_nanos" : 114497237,
"children" : [
{
"name" : "EarlyTerminatingCollector",
"reason" : "search_count",
"time_in_nanos" : 556631
},
{
"name" : "ProfilingAggregator: [NAME]",
"reason" : "aggregation",
"time_in_nanos" : 73842679
}
]
}
]
}
]
}
],
"aggregations" : [
{
"type" : "DateHistogramAggregator",
"description" : "NAME",
"time_in_nanos" : 59475782,
"breakdown" : {
"reduce" : 0,
"build_aggregation" : 100666,
"build_aggregation_count" : 1,
"initialize" : 1727,
"initialize_count" : 1,
"reduce_count" : 0,
"collect" : 58790179,
"collect_count" : 583208
}
}
]
}
]
}
}
****************************************************************************************************************************
new profiler
****************************************************************************************************************************
{
"took" : 15097,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 5,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 10000,
"relation" : "gte"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"NAME" : {
"buckets" : [
{
"key_as_string" : "1620864000",
"key" : 1620864000000,
"doc_count" : 121403
},
{
"key_as_string" : "1620867600",
"key" : 1620867600000,
"doc_count" : 116774
},
{
"key_as_string" : "1620871200",
"key" : 1620871200000,
"doc_count" : 112874
},
{
"key_as_string" : "1620874800",
"key" : 1620874800000,
"doc_count" : 101930
},
{
"key_as_string" : "1620878400",
"key" : 1620878400000,
"doc_count" : 96076
},
{
"key_as_string" : "1620882000",
"key" : 1620882000000,
"doc_count" : 97034
},
{
"key_as_string" : "1620885600",
"key" : 1620885600000,
"doc_count" : 94439
},
{
"key_as_string" : "1620889200",
"key" : 1620889200000,
"doc_count" : 72237
},
{
"key_as_string" : "1620892800",
"key" : 1620892800000,
"doc_count" : 104262
},
{
"key_as_string" : "1620896400",
"key" : 1620896400000,
"doc_count" : 70444
},
{
"key_as_string" : "1620900000",
"key" : 1620900000000,
"doc_count" : 54389
},
{
"key_as_string" : "1620903600",
"key" : 1620903600000,
"doc_count" : 51355
},
{
"key_as_string" : "1620907200",
"key" : 1620907200000,
"doc_count" : 49726
},
{
"key_as_string" : "1620910800",
"key" : 1620910800000,
"doc_count" : 11771
},
{
"key_as_string" : "1620921600",
"key" : 1620921600000,
"doc_count" : 83328
},
{
"key_as_string" : "1620925200",
"key" : 1620925200000,
"doc_count" : 61037
},
{
"key_as_string" : "1620928800",
"key" : 1620928800000,
"doc_count" : 54432
},
{
"key_as_string" : "1620932400",
"key" : 1620932400000,
"doc_count" : 56257
},
{
"key_as_string" : "1620936000",
"key" : 1620936000000,
"doc_count" : 49881
},
{
"key_as_string" : "1620939600",
"key" : 1620939600000,
"doc_count" : 23724
},
{
"key_as_string" : "1620943200",
"key" : 1620943200000,
"doc_count" : 150
},
{
"key_as_string" : "1620950400",
"key" : 1620950400000,
"doc_count" : 118243
},
{
"key_as_string" : "1620954000",
"key" : 1620954000000,
"doc_count" : 114528
},
{
"key_as_string" : "1620957600",
"key" : 1620957600000,
"doc_count" : 104722
},
{
"key_as_string" : "1620961200",
"key" : 1620961200000,
"doc_count" : 96103
},
{
"key_as_string" : "1620964800",
"key" : 1620964800000,
"doc_count" : 97885
},
{
"key_as_string" : "1620968400",
"key" : 1620968400000,
"doc_count" : 105795
},
{
"key_as_string" : "1620972000",
"key" : 1620972000000,
"doc_count" : 97413
},
{
"key_as_string" : "1620975600",
"key" : 1620975600000,
"doc_count" : 65816
},
{
"key_as_string" : "1620979200",
"key" : 1620979200000,
"doc_count" : 111160
},
{
"key_as_string" : "1620982800",
"key" : 1620982800000,
"doc_count" : 61888
},
{
"key_as_string" : "1620986400",
"key" : 1620986400000,
"doc_count" : 58916
},
{
"key_as_string" : "1620990000",
"key" : 1620990000000,
"doc_count" : 56110
},
{
"key_as_string" : "1620993600",
"key" : 1620993600000,
"doc_count" : 40541
},
{
"key_as_string" : "1620997200",
"key" : 1620997200000,
"doc_count" : 14
},
{
"key_as_string" : "1621008000",
"key" : 1621008000000,
"doc_count" : 76567
},
{
"key_as_string" : "1621011600",
"key" : 1621011600000,
"doc_count" : 61720
},
{
"key_as_string" : "1621015200",
"key" : 1621015200000,
"doc_count" : 51155
},
{
"key_as_string" : "1621018800",
"key" : 1621018800000,
"doc_count" : 49844
},
{
"key_as_string" : "1621022400",
"key" : 1621022400000,
"doc_count" : 44128
},
{
"key_as_string" : "1621026000",
"key" : 1621026000000,
"doc_count" : 17301
}
]
}
},
"profile" : {
"shards" : [
{
"id" : "[LVROaG6fTfO1Db3lEeKtOw][new_index-000006][0]",
"searches" : [
{
"query" : [
{
"type" : "IndexOrDocValuesQuery",
"description" : "ingest_date:[1620857557890 TO 9223372036854775807]",
"time_in_nanos" : 3633657266,
"breakdown" : {
"set_min_competitive_score_count" : 0,
"match_count" : 0,
"shallow_advance_count" : 0,
"set_min_competitive_score" : 0,
"next_doc" : 3603665691,
"match" : 0,
"next_doc_count" : 2913372,
"score_count" : 0,
"compute_max_score_count" : 0,
"compute_max_score" : 0,
"advance" : 72724,
"advance_count" : 34,
"score" : 0,
"build_scorer_count" : 68,
"create_weight" : 2583,
"shallow_advance" : 0,
"create_weight_count" : 1,
"build_scorer" : 29916268
}
}
],
"rewrite_time" : 9726,
"collector" : [
{
"name" : "MultiCollector",
"reason" : "search_multi",
"time_in_nanos" : 11354639978,
"children" : [
{
"name" : "EarlyTerminatingCollector",
"reason" : "search_count",
"time_in_nanos" : 12968115
},
{
"name" : "ProfilingAggregator: [NAME]",
"reason" : "aggregation",
"time_in_nanos" : 4015480800
}
]
}
]
}
],
"aggregations" : [
{
"type" : "DateHistogramAggregator",
"description" : "NAME",
"time_in_nanos" : 3882309145,
"breakdown" : {
"reduce" : 0,
"post_collection_count" : 1,
"build_leaf_collector" : 275856,
"build_aggregation" : 30657,
"build_aggregation_count" : 1,
"build_leaf_collector_count" : 34,
"post_collection" : 2928,
"initialize" : 5387,
"initialize_count" : 1,
"reduce_count" : 0,
"collect" : 3881994317,
"collect_count" : 2913372
},
"debug" : {
"total_buckets" : 41
}
}
]
}
]
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment