foo.diff

comparing metrics from 2023-09-20T15:52:48Z to 2023-09-20T16:02:48Z

[search-xkgallatin0revert-13428-rever, 851 series]
debug: http://prometheus.search-infra.etsy-searchinfra-gke-dev.etsycloud.com/api/v1/series?match%5B%5D=%7Bnamespace%3D%22search-xkgallatin0revert-13428-rever%22,service%3D%22mmx-recsys-service%22%7D&start=2023-09-20T15:52:48Z&end=2023-09-20T16:02:48Z

[search-xkgallatin1main, 840 series]
debug: http://prometheus.search-infra.etsy-searchinfra-gke-dev.etsycloud.com/api/v1/series?match%5B%5D=%7Bnamespace%3D%22search-xkgallatin1main%22,service%3D%22mmx-recsys-service%22%7D&start=2023-09-20T15:52:48Z&end=2023-09-20T16:02:48Z

command: git diff --no-index /tmp/search-xkgallatin0revert-13428-rever-17960227474054244828.tmp /tmp/search-xkgallatin1main-6809842399054133810.tmp

diff --git a/tmp/search-xkgallatin0revert-13428-rever-17960227474054244828.tmp b/tmp/search-xkgallatin1main-6809842399054133810.tmp
index 403db84..7df957d 100644
--- a/tmp/search-xkgallatin0revert-13428-rever-17960227474054244828.tmp
+++ b/tmp/search-xkgallatin1main-6809842399054133810.tmp
@@ -62,7 +62,6 @@ dtab_local_size_histogram_sum{client,client_type,container,endpoint,instance,job
 failed_connect_latency_ms_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 failed_connect_latency_ms_histogram_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 failed_connect_latency_ms_histogram_sum{client,client_type,container,endpoint,instance,job,namespace,pod,service}
-failures_by_type_m1_rate{bigtable_instance_id,bigtable_project_id,client,container,dataset,endpoint,error_type,instance,job,namespace,pod,service}
 failures_by_type_m1_rate{client,client_type,container,endpoint,error_type,instance,job,namespace,pod,service,subrequest}
 failures_by_type_m1_rate{client,client_type,container,endpoint,error_type,instance,job,namespace,pod,service}
 failures_by_type_m1_rate{container,endpoint,error_type,instance,job,namespace,pod,service,subrequest}
@@ -446,7 +445,6 @@ read_timeout_m1_rate{client,client_type,container,endpoint,instance,job,namespac
 received_bytes_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 removals_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 removed_for_ms_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
-request_failure_m1_rate{bigtable_instance_id,bigtable_project_id,client,container,dataset,endpoint,instance,job,namespace,pod,service}
 request_failure_m1_rate{client,container,endpoint,instance,job,namespace,pod,rivulet_endpoint,service}
 request_latency_ms_count{bigtable_instance_id,bigtable_project_id,client,container,dataset,endpoint,instance,job,namespace,pod,service}
 request_latency_ms_count{client,client_type,container,endpoint,instance,job,namespace,pod,service}
@@ -496,7 +494,6 @@ route_requests_m1_rate{container,endpoint,http_method,instance,job,namespace,pod
 route_status_200_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_status_2XX_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_status_500_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
-route_status_504_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_status_5XX_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_success_m1_rate{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_200_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
@@ -510,9 +507,6 @@ route_time_2XX_histogram_sum{container,endpoint,http_method,instance,job,namespa
 route_time_500_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_500_histogram_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_500_histogram_sum{container,endpoint,http_method,instance,job,namespace,pod,route,service}
-route_time_504_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
-route_time_504_histogram_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
-route_time_504_histogram_sum{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_5XX_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_5XX_histogram_count{container,endpoint,http_method,instance,job,namespace,pod,route,service}
 route_time_5XX_histogram_sum{container,endpoint,http_method,instance,job,namespace,pod,route,service}
@@ -568,6 +562,7 @@ srv_mmx_recsys_v2_ann_embedding_fetch_request_latency_ms_histogram_sum{candidate
 srv_mmx_recsys_v2_ann_embedding_fetch_requests_m1_rate{candidateSource,client,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_ann_embedding_fetch_rivulet_hit_m1_rate{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_ann_embedding_fetch_rivulet_miss_m1_rate{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
+srv_mmx_recsys_v2_ann_embedding_fetch_rivulet_request_failure_m1_rate{candidateSource,container,deploymentId,endpoint,entity_type,error,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_ann_embedding_fetch_rivulet_request_notfound_m1_rate{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_ann_embedding_fetch_rivulet_request_success_m1_rate{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_ann_embedding_fetch_rivulet_signed_in_user_total_m1_rate{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
@@ -598,7 +593,6 @@ srv_mmx_recsys_v2_ann_validation_per_seed_num_success_count{candidateSource,cont
 srv_mmx_recsys_v2_ann_validation_per_seed_num_success_histogram_count{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_ann_validation_per_seed_num_success_histogram_sum{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_ann_validation_per_seed_success_m1_rate{candidateSource,container,deploymentId,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
-srv_mmx_recsys_v2_arizona_cand_fetch_error_m1_rate{container,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_arizona_service_empty_keys_m1_rate{container,dataset,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_arizona_service_failure_bigtable_io_m1_rate{container,dataset,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_arizona_service_failure_no_bigtable_cell_m1_rate{container,dataset,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
@@ -607,7 +601,6 @@ srv_mmx_recsys_v2_arizona_service_requests_m1_rate{container,dataset,endpoint,en
 srv_mmx_recsys_v2_arizona_service_rows_per_key_count{container,dataset,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_arizona_service_rows_per_key_histogram_count{container,dataset,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
 srv_mmx_recsys_v2_arizona_service_rows_per_key_histogram_sum{container,dataset,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
-srv_mmx_recsys_v2_batch_failure_default_m1_rate{container,endpoint,entity_type,instance,job,model_name,module,namespace,pod,query_type,recset,request_type,service}
 srv_mmx_recsys_v2_batch_success_m1_rate{container,endpoint,entity_type,instance,job,model_name,module,namespace,pod,query_type,recset,request_type,service}
 srv_mmx_recsys_v2_batch_total_m1_rate{container,endpoint,entity_type,instance,job,model_name,module,namespace,pod,query_type,recset,request_type,service}
 srv_mmx_recsys_v2_blending_cand_num_loops_m1_rate{container,endpoint,entity_type,instance,job,module,namespace,pod,recset,request_type,service}
@@ -784,7 +777,6 @@ standard_service_metric_v1_srv_success_m1_rate{container,endpoint,instance,job,n
 status_200_m1_rate{container,endpoint,instance,job,namespace,pod,service}
 status_2XX_m1_rate{container,endpoint,instance,job,namespace,pod,service}
 status_500_m1_rate{container,endpoint,instance,job,namespace,pod,service}
-status_504_m1_rate{container,endpoint,instance,job,namespace,pod,service}
 status_5XX_m1_rate{container,endpoint,instance,job,namespace,pod,service}
 success_m1_rate{client,client_type,container,endpoint,instance,job,namespace,pod,service}
 target_info{container,container_image_name,endpoint,instance,job,k8s_namespace_name,k8s_pod_name,namespace,pod,service,service_instance_id,service_name,telemetry_sdk_language,telemetry_sdk_name,telemetry_sdk_version}
@@ -803,9 +795,6 @@ time_2XX_histogram_sum{container,endpoint,instance,job,namespace,pod,service}
 time_500_count{container,endpoint,instance,job,namespace,pod,service}
 time_500_histogram_count{container,endpoint,instance,job,namespace,pod,service}
 time_500_histogram_sum{container,endpoint,instance,job,namespace,pod,service}
-time_504_count{container,endpoint,instance,job,namespace,pod,service}
-time_504_histogram_count{container,endpoint,instance,job,namespace,pod,service}
-time_504_histogram_sum{container,endpoint,instance,job,namespace,pod,service}
 time_5XX_count{container,endpoint,instance,job,namespace,pod,service}
 time_5XX_histogram_count{container,endpoint,instance,job,namespace,pod,service}
 time_5XX_histogram_sum{container,endpoint,instance,job,namespace,pod,service}