Skip to content

Instantly share code, notes, and snippets.

@AstratechPe
Last active September 25, 2026 12:01
Show Gist options
  • Select an option

  • Save AstratechPe/2e61b6cd2e9e9829ce707c0bfdeee6ac to your computer and use it in GitHub Desktop.

Select an option

Save AstratechPe/2e61b6cd2e9e9829ce707c0bfdeee6ac to your computer and use it in GitHub Desktop.
TITAN / Theus: preregistered zero-inference benchmark protocols

TITAN / Theus — preregistro de evaluación sin inferencias

Protocolos publicados antes de generar decisiones o calcular resultados de estos experimentos.

  • RouterBench 0-shot completo: referencia histórica del algoritmo TITAN-U, once modelos históricos con arms nuevos, pesos y fórmula de producción sin ajustes. No representa el pool actual.
  • RouterArena: auditoría de las decisiones del selector y estado de producción congelado sobre las 8.400 preguntas de la caché oficial completa. No mide calidad de respuestas ni coste de inferencia.

No se utilizan resultados, respuestas ni costes por pregunta como entradas del selector. No se hacen inferencias, llamadas a jueces o embeddings. No se entrena ni cambia el router con estos corpus. Se conservarán los resultados aunque una configuración elija siempre la misma ruta.

Evaluación interna de Theus; no implica aceptación en un ranking ni validación independiente. Los hashes de fuentes y snapshot identifican el estado privado; no publicamos credenciales, prompts de usuarios ni código privado. Los artefactos finales se publicarán en https://theus.pe/benchmarks/.

{
"artifacts_sha256": {
"decisions.jsonl": "e698b03baa6d3253f1525f8f0ceb1a21f12997a15d2f31035260e6c091b9e5b1",
"execution-started.json": "f20a89386517d0458aa747686985ed65561bff480dd8775267de032326a289ad",
"inputs.json": "7a694f173d349fcd250e5514d732e73ce85feaf0fe85d80520529bb7cedc6678",
"pool.json": "9c3d84fb16c819032293d20ed8ff63ff4cbab88951e75949b6c93c68d55cc27f",
"protocol.json": "a02c2f8bb1e2e3786d8a702b9282b87729bfb12452aa9bbbf09f96ac8302e9b3",
"protocol.sha256": "e08bce2ccecfea0623cfe4b88e2dbe3c126905254bf2f1c4b18654184140ab7d",
"summary.json": "1a7cb9b5609600e357cbed36719c0792c9f91d0ca311eef75c726880cc3e08c7"
},
"complete": true,
"completed_at": "2026-09-25T11:39:51.043572+00:00",
"inferences": 0,
"protocol_sha256": "a02c2f8bb1e2e3786d8a702b9282b87729bfb12452aa9bbbf09f96ac8302e9b3"
}
{
"corpus_repository_revision": "cff9659dd6a3a07815f4842fcdadea5a597a1df1",
"corpus_source_url": "https://github.com/RouteWorks/RouterArena/tree/cff9659dd6a3a07815f4842fcdadea5a597a1df1",
"created_at": "2026-09-25T11:31:01.202251+00:00",
"description": "Auditoría de selección offline del pool configurado; no mide respuestas, calidad ni coste de IA",
"expected_rows": 8400,
"gate_audit": {
"account_checks_not_simulated": [
"authentication",
"actual_subscription",
"Atlas_grant",
"balance",
"rate_limit",
"live_availability"
],
"candidate_gate": "hard tier_rank(model.tier) <= tier_rank(effective_plan)",
"candidate_tier_counts": {
"free": 4,
"pro-code": 136
},
"catalog_vs_http_rank_mismatches": [],
"effective_plan": "pro-code",
"guards": {
"execution_attempts": 0,
"network_attempts": 0,
"store_access_attempts": 0,
"write_attempts": 0
},
"http_gate": "_preflight_gate(stage=tier) using model_tier and same tier_rank",
"http_tier_mode_default": "on",
"plan_rank": 1,
"pool_count": 140,
"violations": []
},
"input_cache_sha256": "84c83a87977dc1bab556aee6b288ad8a8f17bee4ede80bba29f235c8d29a2e4e",
"input_fields_decoded": [
"global_index",
"question"
],
"input_inventory_sha256": "ebd19ab9dfc33f502dd362498f88e86b978dd3b691e0fbd0190a437ede268c0e",
"kind": "production_pool_selection_audit",
"limitations": [
"No generated answers or evaluation labels are decoded",
"No quality or model inference cost is measured",
"Effective plan is fixed, not an authenticated account entitlement test",
"Circuit breakers and quarantine use a fresh-process snapshot assumption",
"Targets are configured model IDs, not evidence that those models served responses"
],
"metrics": [
"decision_coverage_all_8400",
"model_distribution",
"target_distribution",
"eligibility_counts",
"context_exclusions",
"errors_in_denominator",
"local_selection_latency_p50_p95",
"state_integrity"
],
"pool_metadata_sha256": "c3d2a8bd8bebe2f5ef211b33f668fecce6129c1be09205f95e589b3c9d19a2b0",
"router_integrity_before": {
"allowed_model_ids": [],
"availability_assumption": "fresh_process_breaker_closed_and_no_quarantine; not_historical_availability",
"capability_gates_applied": true,
"frozen_hour": 10,
"guards": {
"execution_attempts": 0,
"network_attempts": 0,
"store_access_attempts": 0,
"write_attempts": 0
},
"identity_injection": "production_default_without_user_profile",
"learning_mode_preserved": "on",
"network_disabled": true,
"no_labels_or_outcomes": true,
"objective": "balanced",
"offline_only": true,
"output_limit": 4096,
"plan_id": "pro-code",
"reference_arm_matching": null,
"reference_pool_sha256": null,
"selector_scope": "production_pool",
"snapshot_sha256": "dd7ce8026b1a8b31a9edd339a7c98f046e7b85779b6103956fbb68b3297aa3a8",
"source_sha256": {
"theus_oauth_server.py": "fb318a5f8c4bacfec40e15b0057c49fc01e8e9245924ee6ffc1d19f7ca8e0a13",
"titan_contract.py": "6499a5a8bd102eec530ff5c93fbf5e47419f81bd150a58b4c02867c7b30c9004",
"titan_evidence.py": "997d5deb99cf43ea8d2ef29e7477fe50f8c61cee8c5196aacc9f82457877b742",
"titan_evidence_http.py": "814b3b53b8357c8cc0375d0d83d7c2e07374b9ab83b259f285b37df8f63e4fad",
"titan_gateway_adapter.py": "b13dc2e80f29bb8f15d3e8999df2714926adad0b9fd8b837a83804e34fa97c74"
},
"state_sha256_after": "2f3a60bc5b47ac5c16539213ad0cb2367b9efc9bd760db9bd723a23a596256b6",
"state_sha256_before": "2f3a60bc5b47ac5c16539213ad0cb2367b9efc9bd760db9bd723a23a596256b6",
"strategy": "single",
"training_disabled": true,
"unchanged": true,
"writes_disabled": true
},
"schema_version": 1,
"settings": {
"allow_multi": false,
"allowlist": null,
"frozen_hour": 10,
"inferences": 0,
"objective": "balanced",
"plan_id": "pro-code",
"strategy": "single"
},
"snapshot_sha256": "dd7ce8026b1a8b31a9edd339a7c98f046e7b85779b6103956fbb68b3297aa3a8",
"source_sha256": {
"scripts/benchmarks/titan_cached/__init__.py": "6d96d481096aa7b68840f5014cecf1e1d6521d5d1c6bf596ef55959c7a582d9d",
"scripts/benchmarks/titan_cached/frozen.py": "d06c1247d3ec28f2d8a3463b75ecd0352bea080eb32961293e1015f2baa10f75",
"scripts/benchmarks/titan_routing_audit/__init__.py": "6eb8ea5976a48e11b73afa0cc1e8727bf97497255262e38ef14906f8228cb27c",
"scripts/benchmarks/titan_routing_audit/run.py": "8679a7787aab044b1e5f61a4ecb04ff2f72344d6ddebdfadfa174864f4d4845a",
"server/native_approvals.py": "f35d727d074419d0ef5c21802b9ae82ca746c71364be31f5930813042e4022e3",
"server/theus_oauth_server.py": "fb318a5f8c4bacfec40e15b0057c49fc01e8e9245924ee6ffc1d19f7ca8e0a13",
"server/titan_contract.py": "6499a5a8bd102eec530ff5c93fbf5e47419f81bd150a58b4c02867c7b30c9004",
"server/titan_evidence.py": "997d5deb99cf43ea8d2ef29e7477fe50f8c61cee8c5196aacc9f82457877b742",
"server/titan_evidence_http.py": "814b3b53b8357c8cc0375d0d83d7c2e07374b9ab83b259f285b37df8f63e4fad",
"server/titan_gateway_adapter.py": "b13dc2e80f29bb8f15d3e8999df2714926adad0b9fd8b837a83804e34fa97c74"
}
}
{"created_at":"2026-09-25T11:59:51.569987+00:00","environment":{"machine":"arm64","python":"3.9.6","system":"Darwin"},"integrity_after":{"allowed_model_ids":["claude-3-haiku-20240307","gemini-2.0-flash-001","gpt-4o-mini"],"availability_assumption":"reference_membership_only; no_current_availability_claim","capability_gates_applied":false,"frozen_hour":10,"guards":{"execution_attempts":0,"network_attempts":0,"store_access_attempts":0,"write_attempts":0},"identity_injection":"production_default_without_user_profile","learning_mode_preserved":"on","network_disabled":true,"no_labels_or_outcomes":true,"objective":"balanced","offline_only":true,"output_limit":null,"plan_id":"pro-code","reference_arm_matching":"exact_id_only; no_family_mapping","reference_pool_sha256":"58487ec54880fb2a2b5391f348130c20756cc37262113d2bee830bbf28d53e3f","selector_scope":"historical_reference_pool_pure_formula","snapshot_sha256":"dd7ce8026b1a8b31a9edd339a7c98f046e7b85779b6103956fbb68b3297aa3a8","source_sha256":{"theus_oauth_server.py":"fb318a5f8c4bacfec40e15b0057c49fc01e8e9245924ee6ffc1d19f7ca8e0a13","titan_contract.py":"6499a5a8bd102eec530ff5c93fbf5e47419f81bd150a58b4c02867c7b30c9004","titan_evidence.py":"997d5deb99cf43ea8d2ef29e7477fe50f8c61cee8c5196aacc9f82457877b742","titan_evidence_http.py":"814b3b53b8357c8cc0375d0d83d7c2e07374b9ab83b259f285b37df8f63e4fad","titan_gateway_adapter.py":"b13dc2e80f29bb8f15d3e8999df2714926adad0b9fd8b837a83804e34fa97c74"},"state_sha256_after":"e731b6ae2085db58b5f71e338a4a51c222a75658918ae0e7611baec930ae173c","state_sha256_before":"e731b6ae2085db58b5f71e338a4a51c222a75658918ae0e7611baec930ae173c","strategy":"single","training_disabled":true,"unchanged":true,"writes_disabled":true},"integrity_before":{"allowed_model_ids":["claude-3-haiku-20240307","gemini-2.0-flash-001","gpt-4o-mini"],"availability_assumption":"reference_membership_only; no_current_availability_claim","capability_gates_applied":false,"frozen_hour":10,"guards":{"execution_attempts":0,"network_attempts":0,"store_access_attempts":0,"write_attempts":0},"identity_injection":"production_default_without_user_profile","learning_mode_preserved":"on","network_disabled":true,"no_labels_or_outcomes":true,"objective":"balanced","offline_only":true,"output_limit":null,"plan_id":"pro-code","reference_arm_matching":"exact_id_only; no_family_mapping","reference_pool_sha256":"58487ec54880fb2a2b5391f348130c20756cc37262113d2bee830bbf28d53e3f","selector_scope":"historical_reference_pool_pure_formula","snapshot_sha256":"dd7ce8026b1a8b31a9edd339a7c98f046e7b85779b6103956fbb68b3297aa3a8","source_sha256":{"theus_oauth_server.py":"fb318a5f8c4bacfec40e15b0057c49fc01e8e9245924ee6ffc1d19f7ca8e0a13","titan_contract.py":"6499a5a8bd102eec530ff5c93fbf5e47419f81bd150a58b4c02867c7b30c9004","titan_evidence.py":"997d5deb99cf43ea8d2ef29e7477fe50f8c61cee8c5196aacc9f82457877b742","titan_evidence_http.py":"814b3b53b8357c8cc0375d0d83d7c2e07374b9ab83b259f285b37df8f63e4fad","titan_gateway_adapter.py":"b13dc2e80f29bb8f15d3e8999df2714926adad0b9fd8b837a83804e34fa97c74"},"state_sha256_after":"e731b6ae2085db58b5f71e338a4a51c222a75658918ae0e7611baec930ae173c","state_sha256_before":"e731b6ae2085db58b5f71e338a4a51c222a75658918ae0e7611baec930ae173c","strategy":"single","training_disabled":true,"unchanged":true,"writes_disabled":true},"latency":{"median_ms":0.542208000000155,"p95_ms":1.105875000000145,"samples":809,"scope":"local_selector_only"},"phase":"predictions_sealed_before_evaluation","pool_metadata":[{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"claude-3-haiku-20240307","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.25,"output_usd_per_million_tokens":1.25},"target_model":"claude-3-haiku-20240307","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"gemini-2.0-flash-001","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.1,"output_usd_per_million_tokens":0.4},"target_model":"gemini-2.0-flash-001","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"gpt-4o-mini","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.15,"output_usd_per_million_tokens":0.6},"target_model":"gpt-4o-mini","tools":null,"vision":null,"weight_tt":1.0}],"predictions_sha256":"5886d8c34f3ca337cb0bfd03ba99380fe964a9a1dbb8a94e954712095a64ff77","protocol_sha256":"9475e52b838ae6dd7c5dbb69ab9134f8de75adcc18503c12676828e891a5e078","rows":809,"started_at":"2026-09-25T11:59:46.490528+00:00"}
{"arms":"fresh_prior_exact_id_only_no_overlap_no_feedback","availability":"Pool histórico de referencia; no afirma disponibilidad o elegibilidad del pool productivo actual","baselines":["claude-3-haiku-20240307","gemini-2.0-flash-001","gpt-4o-mini"],"cache_sha256":{"claude-3-haiku-20240307":"9f5bed666a524d74e629091b57ef2fdf644181dcc1039ef44b44460f254d150a","gemini-2.0-flash-001":"5c2364f65a4c309bc531338672445894957a6465be71e31a21a6727028404c20","gpt-4o-mini":"84c83a87977dc1bab556aee6b288ad8a8f17bee4ede80bba29f235c8d29a2e4e"},"configuration":{"frozen_hour":10,"objective":"balanced","plan_id":"pro-code"},"corpus":{"name":"RouterArena · sub_10 integrado","revision":"cff9659dd6a3a07815f4842fcdadea5a597a1df1","selection":"Todos los 809 IDs de Claude, iguales a Gemini; GPT mini se filtra a esos IDs; orden de Claude; igualdad exacta de preguntas","source_url":"https://github.com/RouteWorks/RouterArena/tree/cff9659dd6a3a07815f4842fcdadea5a597a1df1","total":809},"cost":"Suma de evaluation_result.inference_cost cacheado en USD; cero gasto nuevo; TT sólo es peso interno, no precio USD","created_at":"2026-09-25T11:53:36.444460+00:00","expected_weights_tt":{"claude-3-haiku-20240307":1,"gemini-2.0-flash-001":1,"gpt-4o-mini":1},"experiment":"titan-u-routerarena-sub10-historical-v1","inferences":0,"inputs_sha256":"88444b587767b8806dfdc249ac8bb8b695a95d24774209b4d37556379f17efcb","learning":"frozen_no_updates","limitations":["Subconjunto integrado sub_10 de 809 casos; no es score oficial completo ni ranking de RouterArena","Los tres arms son frescos; sin entrenamiento o ajuste con el conjunto","Costes y calidad corresponden a respuestas históricas cacheadas, no a inferencias de TITAN","Latencia mide sólo el selector local"],"missing_data":"Cualquier ausencia, duplicado, pregunta diferente, coste inválido o score inválido hace fallar la ejecución completa","model_ids":["claude-3-haiku-20240307","gemini-2.0-flash-001","gpt-4o-mini"],"oracle":null,"phases":["prepare sin etiquetas","predict autorizado tras prerregistro","evaluate autorizado tras sello completo de predicciones"],"pool_sha256":"58487ec54880fb2a2b5391f348130c20756cc37262113d2bee830bbf28d53e3f","publication":"Sólo protocolo, IDs/hashes, decisiones y agregados; ninguna pregunta/respuesta/caché redistribuida","quality":"Media aritmética de evaluation_result.score cacheado en [0,1], sin transformar; correct=null; no se presupone clasificación binaria","router_input":"Sólo question: sin IDs, etiquetas, respuestas, costes por caso, eval_name ni resultados","runtime":{"machine":"arm64","python":"3.9.6","system":"Darwin"},"sampling":"none","schema_version":1,"scope":"internal_offline_historical_reference_not_official_arena_score","snapshot_sha256":"dd7ce8026b1a8b31a9edd339a7c98f046e7b85779b6103956fbb68b3297aa3a8","source_sha256":{"scripts/benchmarks/titan_cached/__init__.py":"6d96d481096aa7b68840f5014cecf1e1d6521d5d1c6bf596ef55959c7a582d9d","scripts/benchmarks/titan_cached/frozen.py":"d06c1247d3ec28f2d8a3463b75ecd0352bea080eb32961293e1015f2baa10f75","scripts/benchmarks/titan_routerarena/__init__.py":"8fb635e6afa8ff2bfa17ab03105d1762f0b69a66d4e6fd3f6f0a5c033fe5965d","scripts/benchmarks/titan_routerarena/run.py":"d8547e602a5931483c3ab63693febe932b51c4ecb178eb9263072a427b93e755","scripts/benchmarks/titan_routing_audit/__init__.py":"6eb8ea5976a48e11b73afa0cc1e8727bf97497255262e38ef14906f8228cb27c","scripts/benchmarks/titan_routing_audit/run.py":"8679a7787aab044b1e5f61a4ecb04ff2f72344d6ddebdfadfa174864f4d4845a","server/native_approvals.py":"f35d727d074419d0ef5c21802b9ae82ca746c71364be31f5930813042e4022e3","server/theus_oauth_server.py":"fb318a5f8c4bacfec40e15b0057c49fc01e8e9245924ee6ffc1d19f7ca8e0a13","server/titan_contract.py":"6499a5a8bd102eec530ff5c93fbf5e47419f81bd150a58b4c02867c7b30c9004","server/titan_evidence.py":"997d5deb99cf43ea8d2ef29e7477fe50f8c61cee8c5196aacc9f82457877b742","server/titan_evidence_http.py":"814b3b53b8357c8cc0375d0d83d7c2e07374b9ab83b259f285b37df8f63e4fad","server/titan_gateway_adapter.py":"b13dc2e80f29bb8f15d3e8999df2714926adad0b9fd8b837a83804e34fa97c74"},"static_prices_sha256":"99cc29b3e37eef0906ce1c5ff682149ba9d306692fe4a4693d49784fcbd778e0","training":"disabled"}
{"created_at":"2026-09-25T11:38:18Z","environment":{"machine":"arm64","packages":{"numpy":"1.26.4","pandas":"2.2.3"},"python":"3.11.15","system":"Darwin"},"integrity_after":{"allowed_model_ids":["gpt-3.5-turbo-1106","claude-instant-v1","claude-v1","claude-v2","gpt-4-1106-preview","meta/llama-2-70b-chat","mistralai/mixtral-8x7b-chat","zero-one-ai/Yi-34B-Chat","WizardLM/WizardLM-13B-V1.2","meta/code-llama-instruct-34b-chat","mistralai/mistral-7b-chat"],"availability_assumption":"reference_membership_only; no_current_availability_claim","capability_gates_applied":false,"frozen_hour":10,"guards":{"execution_attempts":0,"network_attempts":0,"store_access_attempts":0,"write_attempts":0},"identity_injection":"production_default_without_user_profile","learning_mode_preserved":"on","network_disabled":true,"no_labels_or_outcomes":true,"objective":"balanced","offline_only":true,"output_limit":null,"plan_id":"pro-code","reference_arm_matching":"exact_id_only; no_family_mapping","reference_pool_sha256":"014b1bed27fef27ea8dcf411b3005811861089fb4f80771235ef7989be346417","selector_scope":"historical_reference_pool_pure_formula","snapshot_sha256":"dd7ce8026b1a8b31a9edd339a7c98f046e7b85779b6103956fbb68b3297aa3a8","source_sha256":{"theus_oauth_server.py":"fb318a5f8c4bacfec40e15b0057c49fc01e8e9245924ee6ffc1d19f7ca8e0a13","titan_contract.py":"6499a5a8bd102eec530ff5c93fbf5e47419f81bd150a58b4c02867c7b30c9004","titan_evidence.py":"997d5deb99cf43ea8d2ef29e7477fe50f8c61cee8c5196aacc9f82457877b742","titan_evidence_http.py":"814b3b53b8357c8cc0375d0d83d7c2e07374b9ab83b259f285b37df8f63e4fad","titan_gateway_adapter.py":"b13dc2e80f29bb8f15d3e8999df2714926adad0b9fd8b837a83804e34fa97c74"},"state_sha256_after":"7e21a2603f29a02b0301485031f328c4e0f04ec722b6828d50050eec6b8eb893","state_sha256_before":"7e21a2603f29a02b0301485031f328c4e0f04ec722b6828d50050eec6b8eb893","strategy":"single","training_disabled":true,"unchanged":true,"writes_disabled":true},"integrity_before":{"allowed_model_ids":["gpt-3.5-turbo-1106","claude-instant-v1","claude-v1","claude-v2","gpt-4-1106-preview","meta/llama-2-70b-chat","mistralai/mixtral-8x7b-chat","zero-one-ai/Yi-34B-Chat","WizardLM/WizardLM-13B-V1.2","meta/code-llama-instruct-34b-chat","mistralai/mistral-7b-chat"],"availability_assumption":"reference_membership_only; no_current_availability_claim","capability_gates_applied":false,"frozen_hour":10,"guards":{"execution_attempts":0,"network_attempts":0,"store_access_attempts":0,"write_attempts":0},"identity_injection":"production_default_without_user_profile","learning_mode_preserved":"on","network_disabled":true,"no_labels_or_outcomes":true,"objective":"balanced","offline_only":true,"output_limit":null,"plan_id":"pro-code","reference_arm_matching":"exact_id_only; no_family_mapping","reference_pool_sha256":"014b1bed27fef27ea8dcf411b3005811861089fb4f80771235ef7989be346417","selector_scope":"historical_reference_pool_pure_formula","snapshot_sha256":"dd7ce8026b1a8b31a9edd339a7c98f046e7b85779b6103956fbb68b3297aa3a8","source_sha256":{"theus_oauth_server.py":"fb318a5f8c4bacfec40e15b0057c49fc01e8e9245924ee6ffc1d19f7ca8e0a13","titan_contract.py":"6499a5a8bd102eec530ff5c93fbf5e47419f81bd150a58b4c02867c7b30c9004","titan_evidence.py":"997d5deb99cf43ea8d2ef29e7477fe50f8c61cee8c5196aacc9f82457877b742","titan_evidence_http.py":"814b3b53b8357c8cc0375d0d83d7c2e07374b9ab83b259f285b37df8f63e4fad","titan_gateway_adapter.py":"b13dc2e80f29bb8f15d3e8999df2714926adad0b9fd8b837a83804e34fa97c74"},"state_sha256_after":"7e21a2603f29a02b0301485031f328c4e0f04ec722b6828d50050eec6b8eb893","state_sha256_before":"7e21a2603f29a02b0301485031f328c4e0f04ec722b6828d50050eec6b8eb893","strategy":"single","training_disabled":true,"unchanged":true,"writes_disabled":true},"latency":{"median_ms":0.6899579893797636,"p95_ms":1.2897920096293092,"percentile_method":"nearest_rank","samples":36497,"scope":"local_router_only"},"model_ids":["gpt-3.5-turbo-1106","claude-instant-v1","claude-v1","claude-v2","gpt-4-1106-preview","meta/llama-2-70b-chat","mistralai/mixtral-8x7b-chat","zero-one-ai/Yi-34B-Chat","WizardLM/WizardLM-13B-V1.2","meta/code-llama-instruct-34b-chat","mistralai/mistral-7b-chat"],"phase":"predictions_sealed_before_evaluation","pool_metadata":[{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"WizardLM/WizardLM-13B-V1.2","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.03,"output_usd_per_million_tokens":0.03},"target_model":"WizardLM/WizardLM-13B-V1.2","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"claude-instant-v1","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.08,"output_usd_per_million_tokens":0.24},"target_model":"claude-instant-v1","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"claude-v1","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.7999999999999999,"output_usd_per_million_tokens":2.4},"target_model":"claude-v1","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"claude-v2","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.7999999999999999,"output_usd_per_million_tokens":2.4},"target_model":"claude-v2","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"gpt-3.5-turbo-1106","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.09999999999999999,"output_usd_per_million_tokens":0.19999999999999998},"target_model":"gpt-3.5-turbo-1106","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"gpt-4-1106-preview","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":1.0,"output_usd_per_million_tokens":3.0},"target_model":"gpt-4-1106-preview","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"meta/code-llama-instruct-34b-chat","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.0776,"output_usd_per_million_tokens":0.0776},"target_model":"meta/code-llama-instruct-34b-chat","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"meta/llama-2-70b-chat","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.09,"output_usd_per_million_tokens":0.09},"target_model":"meta/llama-2-70b-chat","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"mistralai/mistral-7b-chat","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.02,"output_usd_per_million_tokens":0.02},"target_model":"mistralai/mistral-7b-chat","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"mistralai/mixtral-8x7b-chat","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.06,"output_usd_per_million_tokens":0.06},"target_model":"mistralai/mixtral-8x7b-chat","tools":null,"vision":null,"weight_tt":1.0},{"arm_source":"fresh_prior","context_tokens":null,"eligible_for_probe":true,"json":null,"model_id":"zero-one-ai/Yi-34B-Chat","model_provenance":"historical_reference_no_inference","samples":0,"static_reference_price":{"input_usd_per_million_tokens":0.08,"output_usd_per_million_tokens":0.08},"target_model":"zero-one-ai/Yi-34B-Chat","tools":null,"vision":null,"weight_tt":1.0}],"predictions_sha256":"d0c9e66e935a7fee78942cca78e16d9a10cd7a907b0f23ad9ee1b0a84d741155","prompts_sha256":"18fe0bf8d5c544ed53aabceb92ab05875832dc2b55800470a221dcfaf249a918","protocol_sha256":"41a77e3adaf480ab8762049978bcbde9e3e34124df3eeeb13258bdf4afb839bd","rows":36497,"schema_version":1}
{"aiq":null,"arms":"fresh defaults for every historical ID; no feedback, training or family matching","baselines":"all eleven fixed models over the identical full set","configuration":{"frozen_hour":10,"objective":"balanced","plan_id":"pro-code"},"cost":"sum of selected cached USD values; no incurred spend or TT conversion","created_at":"2026-09-25T11:32:26Z","dataset":{"bytes":99567659,"file":"routerbench_0shot.pkl","revision":"784021482c3f320c6619ed4b3bb3b41a21424fcb","rows":36497,"sampling":"all_rows_in_original_order_no_exclusions","sha256":"ba4f77f19517610a707c374e99322d7750c30fc4ae7ff5527888595a1e65d36d"},"expected_weights_tt":{"WizardLM/WizardLM-13B-V1.2":1,"claude-instant-v1":1,"claude-v1":1,"claude-v2":1,"gpt-3.5-turbo-1106":1,"gpt-4-1106-preview":1,"meta/code-llama-instruct-34b-chat":1,"meta/llama-2-70b-chat":1,"mistralai/mistral-7b-chat":1,"mistralai/mixtral-8x7b-chat":1,"zero-one-ai/Yi-34B-Chat":1},"experiment":"titan-u-routerbench-0shot-historical-v1","missing_data":"fail the complete evaluation on any missing/nonfinite/out-of-range quality or cost; never drop or impute","model_ids":["gpt-3.5-turbo-1106","claude-instant-v1","claude-v1","claude-v2","gpt-4-1106-preview","meta/llama-2-70b-chat","mistralai/mixtral-8x7b-chat","zero-one-ai/Yi-34B-Chat","WizardLM/WizardLM-13B-V1.2","meta/code-llama-instruct-34b-chat","mistralai/mistral-7b-chat"],"official_submission":"not_submitted","oracle":"retrospective maximum cached quality, then minimum cached cost, then frozen pool order; never router input","phases":["extract prompt-only JSONL","seal every prediction","separate evaluation process"],"price_basis":"upstream static TOKEN_COSTS literals; divisor 10000000 retained; derive_tt_weight unchanged","prices_sha256":"014b1bed27fef27ea8dcf411b3005811861089fb4f80771235ef7989be346417","publication":"protocol, IDs, hashes, model selections and aggregates only; no prompts/responses/corpus redistribution","quality":"arithmetic mean of cached per-case normalized quality [0,1]; not binary accuracy","router_input":"prompt only; no sample_id, eval_name, answers, quality, cached costs or oracle","runtime":{"machine":"arm64","packages":{"numpy":"1.26.4","pandas":"2.2.3","python-dateutil":"2.9.0.post0","pytz":"2025.2","six":"1.17.0","tzdata":"2025.2"},"python":"3.11.15","system":"Darwin"},"schema_version":1,"scope":"historical_reference_no_inference","snapshot_sha256":"dd7ce8026b1a8b31a9edd339a7c98f046e7b85779b6103956fbb68b3297aa3a8","source_sha256":{"scripts/benchmarks/titan_cached/__init__.py":"6d96d481096aa7b68840f5014cecf1e1d6521d5d1c6bf596ef55959c7a582d9d","scripts/benchmarks/titan_cached/frozen.py":"d06c1247d3ec28f2d8a3463b75ecd0352bea080eb32961293e1015f2baa10f75","scripts/benchmarks/titan_routerbench/__init__.py":"eb1c7d2461b215b7572933d159931ba7a7bb47aae300fd8bd012eb65236a8459","scripts/benchmarks/titan_routerbench/corpus.py":"ffe5bd1768fd198f6a4bbebfb08c7b8d7c0ee7b51497f68b7fa5c26b07cae9f1","scripts/benchmarks/titan_routerbench/protocol.py":"73c4629cc6e6dd7949395fe85f8fff414c5ae144a195d10860d3b25dcb80b158","scripts/benchmarks/titan_routerbench/requirements.txt":"423381f393fc3092a9ce20dec6b539d337e2ed1f2fbd627c03943e4004e9579d","scripts/benchmarks/titan_routerbench/run.py":"50d94d14c38c867cb8726b1347fc21ca0cfb896ea376cc9b88cae3e34bd9eec8","server/native_approvals.py":"f35d727d074419d0ef5c21802b9ae82ca746c71364be31f5930813042e4022e3","server/theus_oauth_server.py":"fb318a5f8c4bacfec40e15b0057c49fc01e8e9245924ee6ffc1d19f7ca8e0a13","server/titan_contract.py":"6499a5a8bd102eec530ff5c93fbf5e47419f81bd150a58b4c02867c7b30c9004","server/titan_evidence.py":"997d5deb99cf43ea8d2ef29e7477fe50f8c61cee8c5196aacc9f82457877b742","server/titan_evidence_http.py":"814b3b53b8357c8cc0375d0d83d7c2e07374b9ab83b259f285b37df8f63e4fad","server/titan_gateway_adapter.py":"b13dc2e80f29bb8f15d3e8999df2714926adad0b9fd8b837a83804e34fa97c74"},"timing":"ReferenceTitan local selector latency; no provider latency, warmup or inference"}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment