foo.diff

comparing metrics from 2023-09-20T15:56:11Z to 2023-09-20T16:06:11Z

[search-xkgallatin0revert-13428-rever, 923 series]
debug: http://prometheus.search-infra.etsy-searchinfra-gke-dev.etsycloud.com/api/v1/series?match%5B%5D=%7Bnamespace%3D%22search-xkgallatin0revert-13428-rever%22,service%3D%22mmx-recsys-service%22%7D&start=2023-09-20T15:56:11Z&end=2023-09-20T16:06:11Z

[search-xkgallatin1main, 913 series]
debug: http://prometheus.search-infra.etsy-searchinfra-gke-dev.etsycloud.com/api/v1/series?match%5B%5D=%7Bnamespace%3D%22search-xkgallatin1main%22,service%3D%22mmx-recsys-service%22%7D&start=2023-09-20T15:56:11Z&end=2023-09-20T16:06:11Z

command: git diff --no-index /tmp/search-xkgallatin0revert-13428-rever-14651925583344607341.tmp /tmp/search-xkgallatin1main-16647560124810188357.tmp

diff --git a/tmp/search-xkgallatin0revert-13428-rever-14651925583344607341.tmp b/tmp/search-xkgallatin1main-16647560124810188357.tmp
index 9327665..0a486c0 100644
--- a/tmp/search-xkgallatin0revert-13428-rever-14651925583344607341.tmp
+++ b/tmp/search-xkgallatin1main-16647560124810188357.tmp
@@ -67,7 +67,6 @@ dtab_local_size_histogram_sum{client,client_type,container,endpoint,instance,job
 failed_connect_latency_ms_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 failed_connect_latency_ms_histogram_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 failed_connect_latency_ms_histogram_sum{client,client_type,container,endpoint,instance,job,namespace,pod,service}
-failures_by_type_m1_rate{bigtable_instance_id,bigtable_project_id,client,container,dataset,endpoint,error_type,instance,job,namespace,pod,service}
 failures_by_type_m1_rate{client,client_type,container,endpoint,error_type,instance,job,namespace,pod,service,subrequest}
 failures_by_type_m1_rate{client,client_type,container,endpoint,error_type,instance,job,namespace,pod,service}
 failures_by_type_m1_rate{container,endpoint,error_type,instance,job,namespace,pod,service,subrequest}
@@ -214,6 +213,7 @@ http_status_2XX_m1_rate{client,client_type,container,endpoint,instance,job,names
 http_status_404_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 http_status_4XX_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 http_status_500_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
+http_status_502_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 http_status_5XX_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 http_success_m1_rate{container,endpoint,instance,job,namespace,pod,service}
 http_tcp_retransmits_m1_rate{container,endpoint,instance,job,namespace,pod,service}
@@ -260,6 +260,10 @@ http_time_500_count{client,client_type,container,endpoint,instance,job,namespace
 http_time_500_histogram_bucket{client,client_type,container,endpoint,instance,job,le,namespace,pod,service}
 http_time_500_histogram_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 http_time_500_histogram_sum{client,client_type,container,endpoint,instance,job,namespace,pod,service}
+http_time_502_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
+http_time_502_histogram_bucket{client,client_type,container,endpoint,instance,job,le,namespace,pod,service}
+http_time_502_histogram_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
+http_time_502_histogram_sum{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 http_time_5XX_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 http_time_5XX_histogram_bucket{client,client_type,container,endpoint,instance,job,le,namespace,pod,service}
 http_time_5XX_histogram_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
@@ -462,7 +466,6 @@ read_timeout_m1_rate{client,client_type,container,endpoint,instance,job,namespac
 received_bytes_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 removals_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 removed_for_ms_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
-request_failure_m1_rate{bigtable_instance_id,bigtable_project_id,client,container,dataset,endpoint,instance,job,namespace,pod,service}
 request_failure_m1_rate{client,container,endpoint,instance,job,namespace,pod,rivulet_endpoint,service}
 request_latency_ms_count{bigtable_instance_id,bigtable_project_id,client,container,dataset,endpoint,instance,job,namespace,pod,service}
 request_latency_ms_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
@@ -517,7 +520,6 @@ route_requests_m1_rate{container,endpoint,http_method,instance,job,namespace,pod
 route_status_200_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_status_2XX_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_status_500_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
-route_status_504_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_status_5XX_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_success_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_200_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
@@ -532,10 +534,6 @@ route_time_500_count{container,endpoint,http_method,instance,job,namespace,pod,r
 route_time_500_histogram_bucket{container,endpoint,http_method,instance,job,le,namespace,pod,route,service}
 route_time_500_histogram_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_500_histogram_sum{container,endpoint,http_method,instance,job,namespace,pod,route,service}
-route_time_504_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
-route_time_504_histogram_bucket{container,endpoint,http_method,instance,job,le,namespace,pod,route,service}
-route_time_504_histogram_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
-route_time_504_histogram_sum{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_5XX_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_5XX_histogram_bucket{container,endpoint,http_method,instance,job,le,namespace,pod,route,service}
 route_time_5XX_histogram_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
@@ -566,8 +564,6 @@ singletonpool_connects_dead_m1_rate{client,client_type,container,endpoint,instan
 singletonpool_connects_fail_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 socket_unwritable_ms_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 socket_writable_ms_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
-sourcedfailures_com_twitter_finagle_CancelledRequestException_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
-sourcedfailures_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 srv_mmx_recsys_v2_ann_embedding_fetch_bigtable_BigtableFeatureStore_hit_m1_rate{candidateSource,container,dataset,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service,type}
 srv_mmx_recsys_v2_ann_embedding_fetch_bigtable_BigtableFeatureStore_invalid_m1_rate{candidateSource,container,dataset,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service,type}
 srv_mmx_recsys_v2_ann_embedding_fetch_bigtable_BigtableFeatureStore_miss_m1_rate{candidateSource,container,dataset,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service,type}
@@ -634,7 +630,6 @@ srv_mmx_recsys_v2_ann_validation_per_seed_num_success_histogram_bucket{candidate
 srv_mmx_recsys_v2_ann_validation_per_seed_num_success_histogram_count{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_ann_validation_per_seed_num_success_histogram_sum{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_ann_validation_per_seed_success_m1_rate{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
-srv_mmx_recsys_v2_arizona_cand_fetch_error_m1_rate{container,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_arizona_service_empty_keys_m1_rate{container,dataset,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_arizona_service_failure_bigtable_io_m1_rate{container,dataset,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_arizona_service_failure_no_bigtable_cell_m1_rate{container,dataset,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
@@ -852,7 +847,6 @@ standard_service_metric_v1_srv_success_m1_rate{container,endpoint,instance,job,n
 status_200_m1_rate{container,endpoint,instance,job,namespace,pod,service}
 status_2XX_m1_rate{container,endpoint,instance,job,namespace,pod,service}
 status_500_m1_rate{container,endpoint,instance,job,namespace,pod,service}
-status_504_m1_rate{container,endpoint,instance,job,namespace,pod,service}
 status_5XX_m1_rate{container,endpoint,instance,job,namespace,pod,service}
 success_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 target_info{container,container_image_name,endpoint,instance,job,k8s_namespace_name,k8s_pod_name,namespace,pod,service,service_instance_id,service_name,telemetry_sdk_language,telemetry_sdk_name,telemetry_sdk_version}
@@ -872,10 +866,6 @@ time_500_count{container,endpoint,instance,job,namespace,pod,service}
 time_500_histogram_bucket{container,endpoint,instance,job,le,namespace,pod,service}
 time_500_histogram_count{container,endpoint,instance,job,namespace,pod,service}
 time_500_histogram_sum{container,endpoint,instance,job,namespace,pod,service}
-time_504_count{container,endpoint,instance,job,namespace,pod,service}
-time_504_histogram_bucket{container,endpoint,instance,job,le,namespace,pod,service}
-time_504_histogram_count{container,endpoint,instance,job,namespace,pod,service}
-time_504_histogram_sum{container,endpoint,instance,job,namespace,pod,service}
 time_5XX_count{container,endpoint,instance,job,namespace,pod,service}
 time_5XX_histogram_bucket{container,endpoint,instance,job,le,namespace,pod,service}
 time_5XX_histogram_count{container,endpoint,instance,job,namespace,pod,service}