{ "version": "pisama-banking77-extension-report-v2", "source_sha256": "3b41e33295c60f5b80807797c9446aed1ff644c022af72442246dbcb770f91ca", "prepared_at": "2026-10-13T19:47:57Z", "incumbent_arm_id": "incumbent-o4", "primary": [ { "arm_id": "incumbent-o4", "model": "openai/o4-mini", "metrics": { "planned": 500, "observed": 500, "valid": 500, "missing": 0, "unresolved": 0, "confirmed_errors": 126, "wrong_reference": 126, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 500, "paired_improvements": 0, "paired_regressions": 0, "both_wrong": 126, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 500, "both_correct": 374, "improvements": 0, "regressions": 0, "shared_wrong_agreement": 126, "correct_incumbent_retention": 1, "priced": 500, "cost_coverage_complete": true, "reported_cost_total_usd": 0.8805807999999997, "mean_cost_usd": 0.0017611615999999994, "cost_per_1000_usd": 1.7611615999999994, "timed": 500, "p50_seconds": 2.069488749606535, "p95_seconds": 5.8729631889611476 }, "eligible": true, "reasons": [], "components": { "accuracy": 0.748, "agreement": 1, "cost": 0.5, "latency": 0.5 }, "scores": { "default": 72.4, "quality": 72.36, "economy": 67.4, "latency": 67.4 }, "pareto": true, "name": "o4-mini", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 374, "confirmed_errors": 126, "accuracy": { "numerator": 374, "denominator": 500, "unresolved": 0, "value": 0.748, "lower": 0.748, "upper": 0.748 }, "end_to_end": { "numerator": 500, "denominator": 500, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_output": { "numerator": 500, "denominator": 500, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 1, "both_valid_pairs": 500, "valid_answer_agreement_bounds": { "lower": 1, "upper": 1 } } }, { "arm_id": "deepseek", "model": "deepseek/deepseek-v4-flash-0731", "metrics": { "planned": 500, "observed": 500, "valid": 498, "missing": 0, "unresolved": 1, "confirmed_errors": 125, "wrong_reference": 124, "invalid_output": 1, "provider_error": 0, "wrong_model": 0, "paired_valid": 498, "paired_improvements": 20, "paired_regressions": 20, "both_wrong": 104, "both_wrong_different": 21, "unpaired_unresolved_or_missing": 1, "unpaired_failure": 1, "agreement_count": 437, "both_correct": 354, "improvements": 20, "regressions": 20, "shared_wrong_agreement": 83, "correct_incumbent_retention": 0.946524064171123, "priced": 499, "cost_coverage_complete": false, "reported_cost_total_usd": 0.07319557919999997, "mean_cost_usd": null, "cost_per_1000_usd": null, "timed": 500, "p50_seconds": 5.6534198962617666, "p95_seconds": 24.156308278767387 }, "eligible": false, "reasons": [ "unresolved_outputs", "incomplete_all_attempt_pricing", "full_run_unresolved_or_missing", "full_run_incomplete_all_attempt_pricing" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "DeepSeek V4 Flash", "coverage": { "planned": 500, "attempted": 500, "resolved": 499, "unresolved": 1, "correct": 374, "confirmed_errors": 125, "accuracy": { "numerator": 374, "denominator": 500, "unresolved": 1, "value": null, "lower": 0.748, "upper": 0.75 }, "end_to_end": { "numerator": 437, "denominator": 500, "unresolved": 1, "value": null, "lower": 0.874, "upper": 0.876 }, "valid_output": { "numerator": 498, "denominator": 500, "unresolved": 1, "value": null, "lower": 0.996, "upper": 0.998 }, "valid_answer_agreement": 0.8775100401606426, "both_valid_pairs": 498, "valid_answer_agreement_bounds": { "lower": 0.875751503006012, "upper": 0.8777555110220441 } } }, { "arm_id": "qwen", "model": "qwen/qwen3.7-flash", "metrics": { "planned": 500, "observed": 500, "valid": 499, "missing": 0, "unresolved": 0, "confirmed_errors": 123, "wrong_reference": 122, "invalid_output": 1, "provider_error": 0, "wrong_model": 0, "paired_valid": 499, "paired_improvements": 27, "paired_regressions": 23, "both_wrong": 99, "both_wrong_different": 27, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 1, "agreement_count": 422, "both_correct": 350, "improvements": 27, "regressions": 24, "shared_wrong_agreement": 72, "correct_incumbent_retention": 0.9358288770053476, "priced": 500, "cost_coverage_complete": true, "reported_cost_total_usd": 0.036745517599999986, "mean_cost_usd": 0.00007349103519999997, "cost_per_1000_usd": 0.07349103519999997, "timed": 500, "p50_seconds": 4.468798020621762, "p95_seconds": 14.573553621117018 }, "eligible": true, "reasons": [], "components": { "accuracy": 0.754, "agreement": 0.844, "cost": 0.9599428067253785, "latency": 0.2872353879887425 }, "scores": { "default": 76.651210014395, "quality": 73.69178194714121, "economy": 77.81063808164879, "latency": 64.35648970691605 }, "pareto": false, "name": "Qwen3.7-Flash", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 377, "confirmed_errors": 123, "accuracy": { "numerator": 377, "denominator": 500, "unresolved": 0, "value": 0.754, "lower": 0.754, "upper": 0.754 }, "end_to_end": { "numerator": 422, "denominator": 500, "unresolved": 0, "value": 0.844, "lower": 0.844, "upper": 0.844 }, "valid_output": { "numerator": 499, "denominator": 500, "unresolved": 0, "value": 0.998, "lower": 0.998, "upper": 0.998 }, "valid_answer_agreement": 0.845691382765531, "both_valid_pairs": 499, "valid_answer_agreement_bounds": { "lower": 0.845691382765531, "upper": 0.845691382765531 } } }, { "arm_id": "nemotron", "model": "nvidia/nemotron-3.5-lightning", "metrics": { "planned": 500, "observed": 500, "valid": 497, "missing": 0, "unresolved": 0, "confirmed_errors": 120, "wrong_reference": 117, "invalid_output": 1, "provider_error": 2, "wrong_model": 0, "paired_valid": 497, "paired_improvements": 28, "paired_regressions": 21, "both_wrong": 96, "both_wrong_different": 29, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 3, "agreement_count": 419, "both_correct": 352, "improvements": 28, "regressions": 22, "shared_wrong_agreement": 67, "correct_incumbent_retention": 0.9411764705882353, "priced": 499, "cost_coverage_complete": false, "reported_cost_total_usd": 0.12556890000000004, "mean_cost_usd": null, "cost_per_1000_usd": null, "timed": 500, "p50_seconds": 3.6871269163675606, "p95_seconds": 13.225167969940227 }, "eligible": false, "reasons": [ "incomplete_all_attempt_pricing", "full_run_incomplete_all_attempt_pricing" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Nemotron 3.5 Lightning", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 380, "confirmed_errors": 120, "accuracy": { "numerator": 380, "denominator": 500, "unresolved": 0, "value": 0.76, "lower": 0.76, "upper": 0.76 }, "end_to_end": { "numerator": 419, "denominator": 500, "unresolved": 0, "value": 0.838, "lower": 0.838, "upper": 0.838 }, "valid_output": { "numerator": 497, "denominator": 500, "unresolved": 0, "value": 0.994, "lower": 0.994, "upper": 0.994 }, "valid_answer_agreement": 0.8430583501006036, "both_valid_pairs": 497, "valid_answer_agreement_bounds": { "lower": 0.8430583501006036, "upper": 0.8430583501006036 } } }, { "arm_id": "glm", "model": "z-ai/glm-5.3-flash", "metrics": { "planned": 500, "observed": 500, "valid": 500, "missing": 0, "unresolved": 0, "confirmed_errors": 111, "wrong_reference": 111, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 500, "paired_improvements": 35, "paired_regressions": 20, "both_wrong": 91, "both_wrong_different": 23, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 422, "both_correct": 354, "improvements": 35, "regressions": 20, "shared_wrong_agreement": 68, "correct_incumbent_retention": 0.946524064171123, "priced": 500, "cost_coverage_complete": true, "reported_cost_total_usd": 0.014091429999999993, "mean_cost_usd": 0.000028182859999999988, "cost_per_1000_usd": 0.028182859999999987, "timed": 500, "p50_seconds": 1.3195262704975903, "p95_seconds": 3.0302008485421537 }, "eligible": true, "reasons": [], "components": { "accuracy": 0.778, "agreement": 0.844, "cost": 0.984249617315159, "latency": 0.6596489926752033 }, "scores": { "default": 82.06148227305523, "quality": 79.33898609990364, "economy": 83.46397844620681, "latency": 76.97196595340769 }, "pareto": true, "name": "GLM-5.3-Flash", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 389, "confirmed_errors": 111, "accuracy": { "numerator": 389, "denominator": 500, "unresolved": 0, "value": 0.778, "lower": 0.778, "upper": 0.778 }, "end_to_end": { "numerator": 422, "denominator": 500, "unresolved": 0, "value": 0.844, "lower": 0.844, "upper": 0.844 }, "valid_output": { "numerator": 500, "denominator": 500, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.844, "both_valid_pairs": 500, "valid_answer_agreement_bounds": { "lower": 0.844, "upper": 0.844 } } }, { "arm_id": "gemini-flash-lite", "model": "google/gemini-3.5-flash-lite", "metrics": { "planned": 500, "observed": 500, "valid": 500, "missing": 0, "unresolved": 0, "confirmed_errors": 117, "wrong_reference": 117, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 500, "paired_improvements": 28, "paired_regressions": 19, "both_wrong": 98, "both_wrong_different": 28, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 425, "both_correct": 355, "improvements": 28, "regressions": 19, "shared_wrong_agreement": 70, "correct_incumbent_retention": 0.9491978609625669, "priced": 500, "cost_coverage_complete": true, "reported_cost_total_usd": 0.1159338, "mean_cost_usd": 0.0002318676, "cost_per_1000_usd": 0.2318676, "timed": 500, "p50_seconds": 0.6529350830242038, "p95_seconds": 0.9703614336671301 }, "eligible": true, "reasons": [], "components": { "accuracy": 0.766, "agreement": 0.85, "cost": 0.8836607110422667, "latency": 0.8582032144933599 }, "scores": { "default": 81.55524636577893, "quality": 79.53863925535627, "economy": 81.8918534762016, "latency": 81.38270354522345 }, "pareto": true, "name": "Gemini 3.5 Flash Lite", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 383, "confirmed_errors": 117, "accuracy": { "numerator": 383, "denominator": 500, "unresolved": 0, "value": 0.766, "lower": 0.766, "upper": 0.766 }, "end_to_end": { "numerator": 425, "denominator": 500, "unresolved": 0, "value": 0.85, "lower": 0.85, "upper": 0.85 }, "valid_output": { "numerator": 500, "denominator": 500, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.85, "both_valid_pairs": 500, "valid_answer_agreement_bounds": { "lower": 0.85, "upper": 0.85 } } }, { "arm_id": "gemini-flash", "model": "google/gemini-3.8-flash", "metrics": { "planned": 500, "observed": 500, "valid": 500, "missing": 0, "unresolved": 0, "confirmed_errors": 82, "wrong_reference": 82, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 500, "paired_improvements": 48, "paired_regressions": 4, "both_wrong": 78, "both_wrong_different": 28, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 420, "both_correct": 370, "improvements": 48, "regressions": 4, "shared_wrong_agreement": 50, "correct_incumbent_retention": 0.9893048128342246, "priced": 500, "cost_coverage_complete": true, "reported_cost_total_usd": 0.5416470000000002, "mean_cost_usd": 0.0010832940000000005, "cost_per_1000_usd": 1.0832940000000004, "timed": 500, "p50_seconds": 2.0906749374698848, "p95_seconds": 4.379972016881219 }, "eligible": true, "reasons": [], "components": { "accuracy": 0.836, "agreement": 0.84, "cost": 0.6191559467477712, "latency": 0.5728079882543872 }, "scores": { "default": 76.7111988174993, "quality": 78.83963935002157, "economy": 74.502758284977, "latency": 73.57579911510933 }, "pareto": true, "name": "Gemini 3.8 Flash", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 418, "confirmed_errors": 82, "accuracy": { "numerator": 418, "denominator": 500, "unresolved": 0, "value": 0.836, "lower": 0.836, "upper": 0.836 }, "end_to_end": { "numerator": 420, "denominator": 500, "unresolved": 0, "value": 0.84, "lower": 0.84, "upper": 0.84 }, "valid_output": { "numerator": 500, "denominator": 500, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.84, "both_valid_pairs": 500, "valid_answer_agreement_bounds": { "lower": 0.84, "upper": 0.84 } } }, { "arm_id": "gpt-luna", "model": "openai/gpt-6-luna", "metrics": { "planned": 500, "observed": 500, "valid": 500, "missing": 0, "unresolved": 0, "confirmed_errors": 95, "wrong_reference": 95, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 500, "paired_improvements": 43, "paired_regressions": 12, "both_wrong": 83, "both_wrong_different": 21, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 424, "both_correct": 362, "improvements": 43, "regressions": 12, "shared_wrong_agreement": 62, "correct_incumbent_retention": 0.9679144385026738, "priced": 500, "cost_coverage_complete": true, "reported_cost_total_usd": 0.04211820000000002, "mean_cost_usd": 0.00008423640000000004, "cost_per_1000_usd": 0.08423640000000004, "timed": 500, "p50_seconds": 2.5197227497119457, "p95_seconds": 4.075243445718661 }, "eligible": true, "reasons": [], "components": { "accuracy": 0.81, "agreement": 0.848, "cost": 0.9543532614644644, "latency": 0.590353960731755 }, "scores": { "default": 82.45060483660684, "quality": 80.6270722219622, "economy": 83.51413745125149, "latency": 76.2341514365973 }, "pareto": true, "name": "GPT-6 Luna", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 405, "confirmed_errors": 95, "accuracy": { "numerator": 405, "denominator": 500, "unresolved": 0, "value": 0.81, "lower": 0.81, "upper": 0.81 }, "end_to_end": { "numerator": 424, "denominator": 500, "unresolved": 0, "value": 0.848, "lower": 0.848, "upper": 0.848 }, "valid_output": { "numerator": 500, "denominator": 500, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.848, "both_valid_pairs": 500, "valid_answer_agreement_bounds": { "lower": 0.848, "upper": 0.848 } } }, { "arm_id": "gpt-sol", "model": "openai/gpt-6.1-sol", "metrics": { "planned": 500, "observed": 500, "valid": 500, "missing": 0, "unresolved": 0, "confirmed_errors": 86, "wrong_reference": 86, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 500, "paired_improvements": 49, "paired_regressions": 9, "both_wrong": 77, "both_wrong_different": 25, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 417, "both_correct": 365, "improvements": 49, "regressions": 9, "shared_wrong_agreement": 52, "correct_incumbent_retention": 0.9759358288770054, "priced": 500, "cost_coverage_complete": true, "reported_cost_total_usd": 0.6588039999999996, "mean_cost_usd": 0.0013176079999999992, "cost_per_1000_usd": 1.3176079999999992, "timed": 500, "p50_seconds": 2.070026708068326, "p95_seconds": 4.525022078934126 }, "eligible": true, "reasons": [], "components": { "accuracy": 0.828, "agreement": 0.834, "cost": 0.5720342308174019, "latency": 0.5648174177640409 }, "scores": { "default": 75.16885879398845, "quality": 77.66851648581442, "economy": 72.54920110216247, "latency": 72.40486484109525 }, "pareto": false, "name": "GPT-6.1 Sol", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 414, "confirmed_errors": 86, "accuracy": { "numerator": 414, "denominator": 500, "unresolved": 0, "value": 0.828, "lower": 0.828, "upper": 0.828 }, "end_to_end": { "numerator": 417, "denominator": 500, "unresolved": 0, "value": 0.834, "lower": 0.834, "upper": 0.834 }, "valid_output": { "numerator": 500, "denominator": 500, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.834, "both_valid_pairs": 500, "valid_answer_agreement_bounds": { "lower": 0.834, "upper": 0.834 } } }, { "arm_id": "claude-haiku", "model": "anthropic/claude-haiku-5.5", "metrics": { "planned": 500, "observed": 500, "valid": 489, "missing": 0, "unresolved": 0, "confirmed_errors": 115, "wrong_reference": 104, "invalid_output": 7, "provider_error": 4, "wrong_model": 0, "paired_valid": 489, "paired_improvements": 38, "paired_regressions": 22, "both_wrong": 82, "both_wrong_different": 20, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 11, "agreement_count": 409, "both_correct": 347, "improvements": 38, "regressions": 27, "shared_wrong_agreement": 62, "correct_incumbent_retention": 0.9278074866310161, "priced": 496, "cost_coverage_complete": false, "reported_cost_total_usd": 0.05431589999999997, "mean_cost_usd": null, "cost_per_1000_usd": null, "timed": 500, "p50_seconds": 0.9808570209424943, "p95_seconds": 2.8269309934461444 }, "eligible": false, "reasons": [ "incomplete_all_attempt_pricing", "full_run_incomplete_all_attempt_pricing" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Claude Haiku 5.5", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 385, "confirmed_errors": 115, "accuracy": { "numerator": 385, "denominator": 500, "unresolved": 0, "value": 0.77, "lower": 0.77, "upper": 0.77 }, "end_to_end": { "numerator": 409, "denominator": 500, "unresolved": 0, "value": 0.818, "lower": 0.818, "upper": 0.818 }, "valid_output": { "numerator": 489, "denominator": 500, "unresolved": 0, "value": 0.978, "lower": 0.978, "upper": 0.978 }, "valid_answer_agreement": 0.83640081799591, "both_valid_pairs": 489, "valid_answer_agreement_bounds": { "lower": 0.83640081799591, "upper": 0.83640081799591 } } }, { "arm_id": "claude-sonnet", "model": "anthropic/claude-sonnet-5.5", "metrics": { "planned": 500, "observed": 500, "valid": 391, "missing": 0, "unresolved": 0, "confirmed_errors": 154, "wrong_reference": 45, "invalid_output": 109, "provider_error": 0, "wrong_model": 0, "paired_valid": 391, "paired_improvements": 16, "paired_regressions": 2, "both_wrong": 43, "both_wrong_different": 5, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 109, "agreement_count": 368, "both_correct": 330, "improvements": 16, "regressions": 44, "shared_wrong_agreement": 38, "correct_incumbent_retention": 0.8823529411764706, "priced": 500, "cost_coverage_complete": true, "reported_cost_total_usd": 1.1384220000000003, "mean_cost_usd": 0.0022768440000000005, "cost_per_1000_usd": 2.2768440000000005, "timed": 500, "p50_seconds": 1.6794083123095334, "p95_seconds": 3.724851422803472 }, "eligible": false, "reasons": [ "error_rate_worse_than_incumbent" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Claude Sonnet 5.5", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 346, "confirmed_errors": 154, "accuracy": { "numerator": 346, "denominator": 500, "unresolved": 0, "value": 0.692, "lower": 0.692, "upper": 0.692 }, "end_to_end": { "numerator": 368, "denominator": 500, "unresolved": 0, "value": 0.736, "lower": 0.736, "upper": 0.736 }, "valid_output": { "numerator": 391, "denominator": 500, "unresolved": 0, "value": 0.782, "lower": 0.782, "upper": 0.782 }, "valid_answer_agreement": 0.9411764705882353, "both_valid_pairs": 391, "valid_answer_agreement_bounds": { "lower": 0.9411764705882353, "upper": 0.9411764705882353 } } }, { "arm_id": "grok", "model": "x-ai/grok-4.7", "metrics": { "planned": 500, "observed": 500, "valid": 500, "missing": 0, "unresolved": 0, "confirmed_errors": 98, "wrong_reference": 98, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 500, "paired_improvements": 34, "paired_regressions": 6, "both_wrong": 92, "both_wrong_different": 20, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 440, "both_correct": 368, "improvements": 34, "regressions": 6, "shared_wrong_agreement": 72, "correct_incumbent_retention": 0.983957219251337, "priced": 500, "cost_coverage_complete": true, "reported_cost_total_usd": 1.6157680000000012, "mean_cost_usd": 0.0032315360000000023, "cost_per_1000_usd": 3.2315360000000024, "timed": 500, "p50_seconds": 2.8051708538550884, "p95_seconds": 9.590933987405151 }, "eligible": true, "reasons": [], "components": { "accuracy": 0.804, "agreement": 0.88, "cost": 0.3527475006697779, "latency": 0.3797854526565841 }, "scores": { "default": 68.65280453996141, "quality": 72.40532953326363, "economy": 63.38027954665919, "latency": 63.921038586395305 }, "pareto": true, "name": "Grok 4.7", "coverage": { "planned": 500, "attempted": 500, "resolved": 500, "unresolved": 0, "correct": 402, "confirmed_errors": 98, "accuracy": { "numerator": 402, "denominator": 500, "unresolved": 0, "value": 0.804, "lower": 0.804, "upper": 0.804 }, "end_to_end": { "numerator": 440, "denominator": 500, "unresolved": 0, "value": 0.88, "lower": 0.88, "upper": 0.88 }, "valid_output": { "numerator": 500, "denominator": 500, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.88, "both_valid_pairs": 500, "valid_answer_agreement_bounds": { "lower": 0.88, "upper": 0.88 } } } ], "challenge": [ { "arm_id": "incumbent-o4", "model": "openai/o4-mini", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 5, "wrong_reference": 5, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 0, "paired_regressions": 0, "both_wrong": 5, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 30, "both_correct": 25, "improvements": 0, "regressions": 0, "shared_wrong_agreement": 5, "correct_incumbent_retention": 1, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.0390434, "mean_cost_usd": 0.0013014466666666667, "cost_per_1000_usd": 1.3014466666666666, "timed": 30, "p50_seconds": 1.4551106039434671, "p95_seconds": 4.676944721047764 }, "eligible": false, "reasons": [ "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "o4-mini", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 25, "confirmed_errors": 5, "accuracy": { "numerator": 25, "denominator": 30, "unresolved": 0, "value": 0.8333333333333334, "lower": 0.8333333333333334, "upper": 0.8333333333333334 }, "end_to_end": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 1, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 1, "upper": 1 } } }, { "arm_id": "deepseek", "model": "deepseek/deepseek-v4-flash-0731", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 5, "wrong_reference": 5, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 1, "paired_regressions": 1, "both_wrong": 4, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 28, "both_correct": 24, "improvements": 1, "regressions": 1, "shared_wrong_agreement": 4, "correct_incumbent_retention": 0.96, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.0055894656, "mean_cost_usd": 0.00018631552, "cost_per_1000_usd": 0.18631552, "timed": 30, "p50_seconds": 4.610919750062749, "p95_seconds": 28.01468085038939 }, "eligible": false, "reasons": [ "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "DeepSeek V4 Flash", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 25, "confirmed_errors": 5, "accuracy": { "numerator": 25, "denominator": 30, "unresolved": 0, "value": 0.8333333333333334, "lower": 0.8333333333333334, "upper": 0.8333333333333334 }, "end_to_end": { "numerator": 28, "denominator": 30, "unresolved": 0, "value": 0.9333333333333333, "lower": 0.9333333333333333, "upper": 0.9333333333333333 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.9333333333333333, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 0.9333333333333333, "upper": 0.9333333333333333 } } }, { "arm_id": "qwen", "model": "qwen/qwen3.7-flash", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 6, "wrong_reference": 6, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 0, "paired_regressions": 1, "both_wrong": 5, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 29, "both_correct": 24, "improvements": 0, "regressions": 1, "shared_wrong_agreement": 5, "correct_incumbent_retention": 0.96, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.0017090724999999998, "mean_cost_usd": 0.000056969083333333326, "cost_per_1000_usd": 0.05696908333333332, "timed": 30, "p50_seconds": 3.6155716460198164, "p95_seconds": 7.685869106464084 }, "eligible": false, "reasons": [ "error_rate_worse_than_incumbent", "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Qwen3.7-Flash", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 24, "confirmed_errors": 6, "accuracy": { "numerator": 24, "denominator": 30, "unresolved": 0, "value": 0.8, "lower": 0.8, "upper": 0.8 }, "end_to_end": { "numerator": 29, "denominator": 30, "unresolved": 0, "value": 0.9666666666666667, "lower": 0.9666666666666667, "upper": 0.9666666666666667 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.9666666666666667, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 0.9666666666666667, "upper": 0.9666666666666667 } } }, { "arm_id": "nemotron", "model": "nvidia/nemotron-3.5-lightning", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 5, "wrong_reference": 5, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 1, "paired_regressions": 1, "both_wrong": 4, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 28, "both_correct": 24, "improvements": 1, "regressions": 1, "shared_wrong_agreement": 4, "correct_incumbent_retention": 0.96, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.00704816, "mean_cost_usd": 0.0002349386666666667, "cost_per_1000_usd": 0.23493866666666668, "timed": 30, "p50_seconds": 4.420968979597092, "p95_seconds": 9.632389781461097 }, "eligible": false, "reasons": [ "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Nemotron 3.5 Lightning", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 25, "confirmed_errors": 5, "accuracy": { "numerator": 25, "denominator": 30, "unresolved": 0, "value": 0.8333333333333334, "lower": 0.8333333333333334, "upper": 0.8333333333333334 }, "end_to_end": { "numerator": 28, "denominator": 30, "unresolved": 0, "value": 0.9333333333333333, "lower": 0.9333333333333333, "upper": 0.9333333333333333 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.9333333333333333, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 0.9333333333333333, "upper": 0.9333333333333333 } } }, { "arm_id": "glm", "model": "z-ai/glm-5.3-flash", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 7, "wrong_reference": 7, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 0, "paired_regressions": 2, "both_wrong": 5, "both_wrong_different": 1, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 27, "both_correct": 23, "improvements": 0, "regressions": 2, "shared_wrong_agreement": 4, "correct_incumbent_retention": 0.92, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.0009234899999999999, "mean_cost_usd": 0.000030783, "cost_per_1000_usd": 0.030783, "timed": 30, "p50_seconds": 1.3050088544841856, "p95_seconds": 2.8308222580002615 }, "eligible": false, "reasons": [ "error_rate_worse_than_incumbent", "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "GLM-5.3-Flash", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 23, "confirmed_errors": 7, "accuracy": { "numerator": 23, "denominator": 30, "unresolved": 0, "value": 0.7666666666666667, "lower": 0.7666666666666667, "upper": 0.7666666666666667 }, "end_to_end": { "numerator": 27, "denominator": 30, "unresolved": 0, "value": 0.9, "lower": 0.9, "upper": 0.9 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.9, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 0.9, "upper": 0.9 } } }, { "arm_id": "gemini-flash-lite", "model": "google/gemini-3.5-flash-lite", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 3, "wrong_reference": 3, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 2, "paired_regressions": 0, "both_wrong": 3, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 28, "both_correct": 25, "improvements": 2, "regressions": 0, "shared_wrong_agreement": 3, "correct_incumbent_retention": 1, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.0071921, "mean_cost_usd": 0.00023973666666666666, "cost_per_1000_usd": 0.23973666666666665, "timed": 30, "p50_seconds": 0.6322966460138559, "p95_seconds": 1.7396338080754452 }, "eligible": false, "reasons": [ "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Gemini 3.5 Flash Lite", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 27, "confirmed_errors": 3, "accuracy": { "numerator": 27, "denominator": 30, "unresolved": 0, "value": 0.9, "lower": 0.9, "upper": 0.9 }, "end_to_end": { "numerator": 28, "denominator": 30, "unresolved": 0, "value": 0.9333333333333333, "lower": 0.9333333333333333, "upper": 0.9333333333333333 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.9333333333333333, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 0.9333333333333333, "upper": 0.9333333333333333 } } }, { "arm_id": "gemini-flash", "model": "google/gemini-3.8-flash", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 3, "wrong_reference": 3, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 2, "paired_regressions": 0, "both_wrong": 3, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 28, "both_correct": 25, "improvements": 2, "regressions": 0, "shared_wrong_agreement": 3, "correct_incumbent_retention": 1, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.028992749999999998, "mean_cost_usd": 0.0009664249999999999, "cost_per_1000_usd": 0.9664249999999999, "timed": 30, "p50_seconds": 2.0139874373562634, "p95_seconds": 4.471087801852262 }, "eligible": false, "reasons": [ "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Gemini 3.8 Flash", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 27, "confirmed_errors": 3, "accuracy": { "numerator": 27, "denominator": 30, "unresolved": 0, "value": 0.9, "lower": 0.9, "upper": 0.9 }, "end_to_end": { "numerator": 28, "denominator": 30, "unresolved": 0, "value": 0.9333333333333333, "lower": 0.9333333333333333, "upper": 0.9333333333333333 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.9333333333333333, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 0.9333333333333333, "upper": 0.9333333333333333 } } }, { "arm_id": "gpt-luna", "model": "openai/gpt-6-luna", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 2, "wrong_reference": 2, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 3, "paired_regressions": 0, "both_wrong": 2, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 27, "both_correct": 25, "improvements": 3, "regressions": 0, "shared_wrong_agreement": 2, "correct_incumbent_retention": 1, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.0023167999999999995, "mean_cost_usd": 0.00007722666666666665, "cost_per_1000_usd": 0.07722666666666665, "timed": 30, "p50_seconds": 2.1352203127462417, "p95_seconds": 3.842972933268174 }, "eligible": false, "reasons": [ "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "GPT-6 Luna", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 28, "confirmed_errors": 2, "accuracy": { "numerator": 28, "denominator": 30, "unresolved": 0, "value": 0.9333333333333333, "lower": 0.9333333333333333, "upper": 0.9333333333333333 }, "end_to_end": { "numerator": 27, "denominator": 30, "unresolved": 0, "value": 0.9, "lower": 0.9, "upper": 0.9 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.9, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 0.9, "upper": 0.9 } } }, { "arm_id": "gpt-sol", "model": "openai/gpt-6.1-sol", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 3, "wrong_reference": 3, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 3, "paired_regressions": 1, "both_wrong": 2, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 26, "both_correct": 24, "improvements": 3, "regressions": 1, "shared_wrong_agreement": 2, "correct_incumbent_retention": 0.96, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.038076, "mean_cost_usd": 0.0012692, "cost_per_1000_usd": 1.2692, "timed": 30, "p50_seconds": 2.645183624699712, "p95_seconds": 4.993220497993751 }, "eligible": false, "reasons": [ "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "GPT-6.1 Sol", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 27, "confirmed_errors": 3, "accuracy": { "numerator": 27, "denominator": 30, "unresolved": 0, "value": 0.9, "lower": 0.9, "upper": 0.9 }, "end_to_end": { "numerator": 26, "denominator": 30, "unresolved": 0, "value": 0.8666666666666667, "lower": 0.8666666666666667, "upper": 0.8666666666666667 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.8666666666666667, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 0.8666666666666667, "upper": 0.8666666666666667 } } }, { "arm_id": "claude-haiku", "model": "anthropic/claude-haiku-5.5", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 6, "wrong_reference": 6, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 2, "paired_regressions": 3, "both_wrong": 3, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 25, "both_correct": 22, "improvements": 2, "regressions": 3, "shared_wrong_agreement": 3, "correct_incumbent_retention": 0.88, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.0033897000000000003, "mean_cost_usd": 0.00011299000000000001, "cost_per_1000_usd": 0.11299000000000002, "timed": 30, "p50_seconds": 0.975729521131143, "p95_seconds": 2.023525377293117 }, "eligible": false, "reasons": [ "error_rate_worse_than_incumbent", "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Claude Haiku 5.5", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 24, "confirmed_errors": 6, "accuracy": { "numerator": 24, "denominator": 30, "unresolved": 0, "value": 0.8, "lower": 0.8, "upper": 0.8 }, "end_to_end": { "numerator": 25, "denominator": 30, "unresolved": 0, "value": 0.8333333333333334, "lower": 0.8333333333333334, "upper": 0.8333333333333334 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 0.8333333333333334, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 0.8333333333333334, "upper": 0.8333333333333334 } } }, { "arm_id": "claude-sonnet", "model": "anthropic/claude-sonnet-5.5", "metrics": { "planned": 30, "observed": 30, "valid": 25, "missing": 0, "unresolved": 0, "confirmed_errors": 7, "wrong_reference": 2, "invalid_output": 5, "provider_error": 0, "wrong_model": 0, "paired_valid": 25, "paired_improvements": 0, "paired_regressions": 0, "both_wrong": 2, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 5, "agreement_count": 25, "both_correct": 23, "improvements": 0, "regressions": 2, "shared_wrong_agreement": 2, "correct_incumbent_retention": 0.92, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.06839399999999998, "mean_cost_usd": 0.0022797999999999994, "cost_per_1000_usd": 2.2797999999999994, "timed": 30, "p50_seconds": 1.7221014581155032, "p95_seconds": 5.83882648560684 }, "eligible": false, "reasons": [ "error_rate_worse_than_incumbent", "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Claude Sonnet 5.5", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 23, "confirmed_errors": 7, "accuracy": { "numerator": 23, "denominator": 30, "unresolved": 0, "value": 0.7666666666666667, "lower": 0.7666666666666667, "upper": 0.7666666666666667 }, "end_to_end": { "numerator": 25, "denominator": 30, "unresolved": 0, "value": 0.8333333333333334, "lower": 0.8333333333333334, "upper": 0.8333333333333334 }, "valid_output": { "numerator": 25, "denominator": 30, "unresolved": 0, "value": 0.8333333333333334, "lower": 0.8333333333333334, "upper": 0.8333333333333334 }, "valid_answer_agreement": 1, "both_valid_pairs": 25, "valid_answer_agreement_bounds": { "lower": 1, "upper": 1 } } }, { "arm_id": "grok", "model": "x-ai/grok-4.7", "metrics": { "planned": 30, "observed": 30, "valid": 30, "missing": 0, "unresolved": 0, "confirmed_errors": 5, "wrong_reference": 5, "invalid_output": 0, "provider_error": 0, "wrong_model": 0, "paired_valid": 30, "paired_improvements": 0, "paired_regressions": 0, "both_wrong": 5, "both_wrong_different": 0, "unpaired_unresolved_or_missing": 0, "unpaired_failure": 0, "agreement_count": 30, "both_correct": 25, "improvements": 0, "regressions": 0, "shared_wrong_agreement": 5, "correct_incumbent_retention": 1, "priced": 30, "cost_coverage_complete": true, "reported_cost_total_usd": 0.09761999999999998, "mean_cost_usd": 0.0032539999999999995, "cost_per_1000_usd": 3.2539999999999996, "timed": 30, "p50_seconds": 3.146895229117945, "p95_seconds": 7.326924749882889 }, "eligible": false, "reasons": [ "separate_challenge_not_ranked" ], "components": null, "scores": { "default": null, "quality": null, "economy": null, "latency": null }, "pareto": false, "name": "Grok 4.7", "coverage": { "planned": 30, "attempted": 30, "resolved": 30, "unresolved": 0, "correct": 25, "confirmed_errors": 5, "accuracy": { "numerator": 25, "denominator": 30, "unresolved": 0, "value": 0.8333333333333334, "lower": 0.8333333333333334, "upper": 0.8333333333333334 }, "end_to_end": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_output": { "numerator": 30, "denominator": 30, "unresolved": 0, "value": 1, "lower": 1, "upper": 1 }, "valid_answer_agreement": 1, "both_valid_pairs": 30, "valid_answer_agreement_bounds": { "lower": 1, "upper": 1 } } } ], "winners": { "default": [ "gpt-luna" ], "quality": [ "gpt-luna" ], "economy": [ "gpt-luna" ], "latency": [ "gemini-flash-lite" ] }, "dates": { "first": "2026-10-13T01:45:24.835240+00:00", "last": "2026-10-13T19:11:49.317240+00:00" }, "configurations": [ { "id": "incumbent-o4", "model": "openai/o4-mini", "provider": "openai", "reasoning": "Enabled", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "retained_history", "first": "2026-10-13T01:45:24.835240+00:00", "last": "2026-10-13T06:29:05.506119+00:00" }, { "id": "deepseek", "model": "deepseek/deepseek-v4-flash-0731", "provider": "streamlake/fp8", "reasoning": "Low effort", "output_tokens": 8192, "retry_output_tokens": 16384, "evidence_origin": "retained_history", "first": "2026-10-13T01:45:24.835240+00:00", "last": "2026-10-13T06:29:05.506119+00:00" }, { "id": "qwen", "model": "qwen/qwen3.7-flash", "provider": "alibaba/us-east-1", "reasoning": "1,024 reasoning tokens", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "retained_history", "first": "2026-10-13T01:45:24.835240+00:00", "last": "2026-10-13T06:29:05.506119+00:00" }, { "id": "nemotron", "model": "nvidia/nemotron-3.5-lightning", "provider": "deepinfra/bf16", "reasoning": "Enabled", "output_tokens": 8192, "retry_output_tokens": 16384, "evidence_origin": "retained_history", "first": "2026-10-13T01:45:24.835240+00:00", "last": "2026-10-13T06:29:05.506119+00:00" }, { "id": "glm", "model": "z-ai/glm-5.3-flash", "provider": "z-ai/fp8", "reasoning": "Low effort", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "retained_history", "first": "2026-10-13T01:45:24.835240+00:00", "last": "2026-10-13T06:29:05.506119+00:00" }, { "id": "gemini-flash-lite", "model": "google/gemini-3.5-flash-lite", "provider": "google-ai-studio", "reasoning": "Low effort", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "new_extension", "first": "2026-10-13T16:22:28.628360+00:00", "last": "2026-10-13T19:11:44.264209+00:00" }, { "id": "gemini-flash", "model": "google/gemini-3.8-flash", "provider": "google-ai-studio", "reasoning": "Low effort", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "new_extension", "first": "2026-10-13T16:22:29.840950+00:00", "last": "2026-10-13T19:11:46.230353+00:00" }, { "id": "gpt-luna", "model": "openai/gpt-6-luna", "provider": "openai", "reasoning": "Low effort", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "new_extension", "first": "2026-10-13T16:22:31.126511+00:00", "last": "2026-10-13T19:11:47.172082+00:00" }, { "id": "gpt-sol", "model": "openai/gpt-6.1-sol", "provider": "openai", "reasoning": "Low effort", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "new_extension", "first": "2026-10-13T16:22:33.781777+00:00", "last": "2026-10-13T19:11:49.317240+00:00" }, { "id": "claude-haiku", "model": "anthropic/claude-haiku-5.5", "provider": "anthropic", "reasoning": "Low effort", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "new_extension", "first": "2026-10-13T16:22:36.166302+00:00", "last": "2026-10-13T19:11:37.461764+00:00" }, { "id": "claude-sonnet", "model": "anthropic/claude-sonnet-5.5", "provider": "anthropic", "reasoning": "Low effort", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "new_extension", "first": "2026-10-13T16:22:37.117881+00:00", "last": "2026-10-13T19:11:39.109406+00:00" }, { "id": "grok", "model": "x-ai/grok-4.7", "provider": "xai", "reasoning": "Low effort", "output_tokens": 2048, "retry_output_tokens": 4096, "evidence_origin": "new_extension", "first": "2026-10-13T16:22:40.012924+00:00", "last": "2026-10-13T19:11:43.697441+00:00" } ], "recovery": [ { "planned": 500, "first_attempted": 500, "first_valid": 499, "first_correct": 374, "first_cap_failures": 1, "first_other_failures": 0, "first_unresolved": 0, "retry_required": 1, "retry_attempted": 1, "recovered_valid": 1, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 501, "arm_id": "incumbent-o4", "name": "o4-mini", "phase": "primary", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "planned": 500, "first_attempted": 500, "first_valid": 498, "first_correct": 374, "first_cap_failures": 0, "first_other_failures": 1, "first_unresolved": 1, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 500, "arm_id": "deepseek", "name": "DeepSeek V4 Flash", "phase": "primary", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "planned": 500, "first_attempted": 500, "first_valid": 499, "first_correct": 377, "first_cap_failures": 0, "first_other_failures": 1, "first_unresolved": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 500, "arm_id": "qwen", "name": "Qwen3.7-Flash", "phase": "primary", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "planned": 500, "first_attempted": 500, "first_valid": 497, "first_correct": 380, "first_cap_failures": 0, "first_other_failures": 3, "first_unresolved": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 500, "arm_id": "nemotron", "name": "Nemotron 3.5 Lightning", "phase": "primary", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "planned": 500, "first_attempted": 500, "first_valid": 500, "first_correct": 389, "first_cap_failures": 0, "first_other_failures": 0, "first_unresolved": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 500, "arm_id": "glm", "name": "GLM-5.3-Flash", "phase": "primary", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "planned": 30, "first_attempted": 30, "first_valid": 30, "first_correct": 25, "first_cap_failures": 0, "first_other_failures": 0, "first_unresolved": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 30, "arm_id": "incumbent-o4", "name": "o4-mini", "phase": "challenge", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "planned": 30, "first_attempted": 30, "first_valid": 30, "first_correct": 25, "first_cap_failures": 0, "first_other_failures": 0, "first_unresolved": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 30, "arm_id": "deepseek", "name": "DeepSeek V4 Flash", "phase": "challenge", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "planned": 30, "first_attempted": 30, "first_valid": 30, "first_correct": 24, "first_cap_failures": 0, "first_other_failures": 0, "first_unresolved": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 30, "arm_id": "qwen", "name": "Qwen3.7-Flash", "phase": "challenge", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "planned": 30, "first_attempted": 30, "first_valid": 30, "first_correct": 25, "first_cap_failures": 0, "first_other_failures": 0, "first_unresolved": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 30, "arm_id": "nemotron", "name": "Nemotron 3.5 Lightning", "phase": "challenge", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "planned": 30, "first_attempted": 30, "first_valid": 30, "first_correct": 23, "first_cap_failures": 0, "first_other_failures": 0, "first_unresolved": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "retry_exhausted_failures": 0, "retry_unresolved": 0, "total_http_attempts": 30, "arm_id": "glm", "name": "GLM-5.3-Flash", "phase": "challenge", "retry_second_length": 0, "retry_unknown_charges": 0 }, { "arm_id": "gemini-flash-lite", "phase": "primary", "planned": 500, "first_attempted": 500, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 500 }, { "arm_id": "gemini-flash-lite", "phase": "challenge", "planned": 30, "first_attempted": 30, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 30 }, { "arm_id": "gemini-flash", "phase": "primary", "planned": 500, "first_attempted": 500, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 500 }, { "arm_id": "gemini-flash", "phase": "challenge", "planned": 30, "first_attempted": 30, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 30 }, { "arm_id": "gpt-luna", "phase": "primary", "planned": 500, "first_attempted": 500, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 500 }, { "arm_id": "gpt-luna", "phase": "challenge", "planned": 30, "first_attempted": 30, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 30 }, { "arm_id": "gpt-sol", "phase": "primary", "planned": 500, "first_attempted": 500, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 500 }, { "arm_id": "gpt-sol", "phase": "challenge", "planned": 30, "first_attempted": 30, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 30 }, { "arm_id": "claude-haiku", "phase": "primary", "planned": 500, "first_attempted": 500, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 500 }, { "arm_id": "claude-haiku", "phase": "challenge", "planned": 30, "first_attempted": 30, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 30 }, { "arm_id": "claude-sonnet", "phase": "primary", "planned": 500, "first_attempted": 500, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 500 }, { "arm_id": "claude-sonnet", "phase": "challenge", "planned": 30, "first_attempted": 30, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 30 }, { "arm_id": "grok", "phase": "primary", "planned": 500, "first_attempted": 500, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 500 }, { "arm_id": "grok", "phase": "challenge", "planned": 30, "first_attempted": 30, "first_cap_failures": 0, "retry_required": 0, "retry_attempted": 0, "recovered_valid": 0, "recovered_correct": 0, "total_http_attempts": 30 } ], "cases": { "version": "pisama-banking77-case-examples-v2", "source_summary_sha256": "3b41e33295c60f5b80807797c9446aed1ff644c022af72442246dbcb770f91ca", "source_dataset": { "name": "Banking77", "revision": "sha256:d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d", "split": "test", "sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d", "source_url": "https://github.com/PolyAI-LDN/task-specific-datasets/tree/master/banking_data", "license_url": "https://github.com/PolyAI-LDN/task-specific-datasets/blob/master/LICENSE" }, "selection": { "phase": "primary", "rule": "lowest_numeric_case_id_per_candidate_and_category", "illustrative_only": true, "excludes": [ "invalid_outputs", "provider_errors", "unresolved_outputs", "challenge_cases" ], "match_definition": "exact_parsed_intent_label", "csv_text_policy": "Formula-like cells are prefixed with an apostrophe; JSON preserves exact visible text." }, "candidates": [ { "arm_id": "deepseek", "model": "deepseek/deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash", "planned": 500, "valid_pairs": 498, "excluded_pairs": 2, "categories": { "both_correct": 354, "shared_wrong": 83, "correction": 20, "regression": 20, "different_wrong": 21 } }, { "arm_id": "qwen", "model": "qwen/qwen3.7-flash", "name": "Qwen3.7-Flash", "planned": 500, "valid_pairs": 499, "excluded_pairs": 1, "categories": { "both_correct": 350, "shared_wrong": 72, "correction": 27, "regression": 23, "different_wrong": 27 } }, { "arm_id": "nemotron", "model": "nvidia/nemotron-3.5-lightning", "name": "Nemotron 3.5 Lightning", "planned": 500, "valid_pairs": 497, "excluded_pairs": 3, "categories": { "both_correct": 352, "shared_wrong": 67, "correction": 28, "regression": 21, "different_wrong": 29 } }, { "arm_id": "glm", "model": "z-ai/glm-5.3-flash", "name": "GLM-5.3-Flash", "planned": 500, "valid_pairs": 500, "excluded_pairs": 0, "categories": { "both_correct": 354, "shared_wrong": 68, "correction": 35, "regression": 20, "different_wrong": 23 } }, { "arm_id": "gemini-flash-lite", "model": "google/gemini-3.5-flash-lite", "name": "Gemini 3.5 Flash Lite", "planned": 500, "valid_pairs": 500, "excluded_pairs": 0, "categories": { "both_correct": 355, "shared_wrong": 70, "correction": 28, "regression": 19, "different_wrong": 28 } }, { "arm_id": "gemini-flash", "model": "google/gemini-3.8-flash", "name": "Gemini 3.8 Flash", "planned": 500, "valid_pairs": 500, "excluded_pairs": 0, "categories": { "both_correct": 370, "shared_wrong": 50, "correction": 48, "regression": 4, "different_wrong": 28 } }, { "arm_id": "gpt-luna", "model": "openai/gpt-6-luna", "name": "GPT-6 Luna", "planned": 500, "valid_pairs": 500, "excluded_pairs": 0, "categories": { "both_correct": 362, "shared_wrong": 62, "correction": 43, "regression": 12, "different_wrong": 21 } }, { "arm_id": "gpt-sol", "model": "openai/gpt-6.1-sol", "name": "GPT-6.1 Sol", "planned": 500, "valid_pairs": 500, "excluded_pairs": 0, "categories": { "both_correct": 365, "shared_wrong": 52, "correction": 49, "regression": 9, "different_wrong": 25 } }, { "arm_id": "claude-haiku", "model": "anthropic/claude-haiku-5.5", "name": "Claude Haiku 5.5", "planned": 500, "valid_pairs": 489, "excluded_pairs": 11, "categories": { "both_correct": 347, "shared_wrong": 62, "correction": 38, "regression": 22, "different_wrong": 20 } }, { "arm_id": "claude-sonnet", "model": "anthropic/claude-sonnet-5.5", "name": "Claude Sonnet 5.5", "planned": 500, "valid_pairs": 391, "excluded_pairs": 109, "categories": { "both_correct": 330, "shared_wrong": 38, "correction": 16, "regression": 2, "different_wrong": 5 } }, { "arm_id": "grok", "model": "x-ai/grok-4.7", "name": "Grok 4.7", "planned": 500, "valid_pairs": 500, "excluded_pairs": 0, "categories": { "both_correct": 368, "shared_wrong": 72, "correction": 34, "regression": 6, "different_wrong": 20 } } ], "examples": [ { "id": "deepseek-both_correct", "category": "both_correct", "candidate_arm_id": "deepseek", "case_id": "banking77:test:20", "input": "I did not get my card yet, is it lost?", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "d1bce59281a16178539697f483d695ef2a6a946022804355c749c1653e625813", "request_sha256": "20592f3e61ad0adc43cc4cc2f344e98242bace1ec37a07ca7680a4bc40919e41", "response_sha256": "ff0e1e664f76be0456acd932a661e0753b4eff025cf6f85f01fe682d18203379", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "deepseek/deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "streamlake/fp8", "receipt_sha256": "e8060b286aa301a25e58d58445446c916e25226ea7680111ecae0e109c0ca60e", "request_sha256": "bcc685731cf6774d1fd871e0babcf7587f879a2be626d4a09d03c53ab93abb3b", "response_sha256": "efd6d9aaae44461f2687b6d5402178494d6b4a9419927472e8845a172ff27509", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "deepseek-shared_wrong", "category": "shared_wrong", "candidate_arm_id": "deepseek", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "deepseek/deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "streamlake/fp8", "receipt_sha256": "e0320f01699377ec6375eda65b332ee3ae7851d251422f3c08b196e3cfa58587", "request_sha256": "e6b64d67df2ce6b60343c97fb96d2743e5d6655df6ed01a9b85c57753e42096a", "response_sha256": "7f06901d1f4967c6613dd0234395adec94d5bad4bd262ace113736f0aeeca012", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "deepseek-correction", "category": "correction", "candidate_arm_id": "deepseek", "case_id": "banking77:test:43", "input": "I have received my card, can you help me put it in the app?", "reference_intent": "card_linking", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"activate_my_card\"}", "parsed_intent": "activate_my_card", "correct": false, "provider_route": "openai", "receipt_sha256": "8640c8124dc6be707f8aaedce6341c1219e9d25d119ae768b215b72a6056bb05", "request_sha256": "6cf7eaef41e2ea7ddbb576d0066d57e403fe5396ab4758b8e3b1c2a6145bb261", "response_sha256": "fd94bb065062f047f11ef7fa8ad74ad3bf744c9b2ecd205b24807b91344b7461", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "deepseek/deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash", "visible_answer": "{\"intent\":\"card_linking\"}", "parsed_intent": "card_linking", "correct": true, "provider_route": "streamlake/fp8", "receipt_sha256": "d2c600f59d6c9e11d64c13fa028d5c3a71f78c1d99300f0f10a55aa0d9e035b2", "request_sha256": "1554422bc843cacfed264891b987436f14ffcf36c341ec8605d8331f9b5f61ed", "response_sha256": "c15f744c8dc25e263d6bb9ea64cae37734520d542c835e2731dbb1790491be77", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "deepseek-regression", "category": "regression", "candidate_arm_id": "deepseek", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "deepseek/deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "streamlake/fp8", "receipt_sha256": "ceec6cd457545d2fa9dcf2a41fb90c28527cd70759a12378d932aa45eddcfe10", "request_sha256": "4f51ab011c8dbbc4f6cc44f06fcddaa37c51084242efda2f7440051d158d4382", "response_sha256": "de15aee6f8c1a63f991fc15e6eb14c65f53e11ce9ffcaeef7d766cf864ef7f80", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "deepseek-different_wrong", "category": "different_wrong", "candidate_arm_id": "deepseek", "case_id": "banking77:test:382", "input": "I broke my card", "reference_intent": "card_not_working", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"getting_spare_card\"}", "parsed_intent": "getting_spare_card", "correct": false, "provider_route": "openai", "receipt_sha256": "8759d1974ee8ba2a4cc159afd88502fb1c89936454b46445c6108e4562e114fd", "request_sha256": "837d08959660607e93449ab2580e5fc25f19e59eac7e8fa6a3224d9d82517433", "response_sha256": "a21d386c3aaef37383da63f6821780fda3eca5e838f8a2ae43040af497c32df6", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "deepseek/deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash", "visible_answer": "{\"intent\":\"order_physical_card\"}", "parsed_intent": "order_physical_card", "correct": false, "provider_route": "streamlake/fp8", "receipt_sha256": "993fc20ee8c7353586ea0ea60fcfa8f7ecd1d950a269672138e6c5becaacc83e", "request_sha256": "21e72b9002326a30912eb6fc46dc67896753e2b73e86e4c31f2ac3c4f31e5ea8", "response_sha256": "e96dbb6ecf979d1f6f570c486f0a6aa65ac5578076a560bfbe996a66c961fd68", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "qwen-both_correct", "category": "both_correct", "candidate_arm_id": "qwen", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "qwen/qwen3.7-flash", "name": "Qwen3.7-Flash", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "alibaba/us-east-1", "receipt_sha256": "db47507f21a8b21ca9ca8f2d7b6d80b5e609a24999c9cc47154f42b140ddd1f1", "request_sha256": "7d215c3a3fd8910356177e6ffe2a89ac1691c68eef6b23776ff45f860b0a1cef", "response_sha256": "772025cacbd5135cd15e108301e5a2473ace029d96eb5f72127575d067319c10", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "qwen-shared_wrong", "category": "shared_wrong", "candidate_arm_id": "qwen", "case_id": "banking77:test:27", "input": "I'm still waiting for delivery of my new card, why is it taking so long?", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a", "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce", "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "qwen/qwen3.7-flash", "name": "Qwen3.7-Flash", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "alibaba/us-east-1", "receipt_sha256": "da53d9e4ec4f6658a99720ed6deb4b542dbc4373bdfc432744abbc45b9910852", "request_sha256": "0088a2755744970a4b91a8c556176dcbdf8cd645f190a22cb32ec0d0e3aefcd8", "response_sha256": "ab154837f5032dd17eb9284009bec20359e7ce1b497bdec488b8e6bd900a8185", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "qwen-correction", "category": "correction", "candidate_arm_id": "qwen", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "qwen/qwen3.7-flash", "name": "Qwen3.7-Flash", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "alibaba/us-east-1", "receipt_sha256": "390ccd8789cf43e3d36c96915e7f6fc6fda2a24282d1e23f37aa2d2188619e0a", "request_sha256": "03dcaf4654c1f020fb9e2556a45e7f76fe8cb10aa581e8914f2d03bf7c52f5ea", "response_sha256": "fc6528909bb349f5071a4545d84dd801f7aca6f4d269f51f867bbed297f5443d", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "qwen-regression", "category": "regression", "candidate_arm_id": "qwen", "case_id": "banking77:test:243", "input": "Does your system support multiple currency.", "reference_intent": "fiat_currency_support", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"fiat_currency_support\"}", "parsed_intent": "fiat_currency_support", "correct": true, "provider_route": "openai", "receipt_sha256": "f9bde41d4ddd045b4306208561a400803ada3ba3581eaf5bed0d041a28cd01c1", "request_sha256": "80e1242173c5b00f98a6505fe3d030c5482481b9de5d50bd7bcb7442072041c4", "response_sha256": "bf4736656a29d02db620793e57e9d8edc793fc19389d3f90db72b0afdc07806c", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "qwen/qwen3.7-flash", "name": "Qwen3.7-Flash", "visible_answer": "{\"intent\":\"supported_cards_and_currencies\"}", "parsed_intent": "supported_cards_and_currencies", "correct": false, "provider_route": "alibaba/us-east-1", "receipt_sha256": "3ddc0b362c30d49990db885c9d9b7f167038b51551a7ceac94993da3ee5acae6", "request_sha256": "99a08436c90af3383597e431e9905c7411ad917a868decba478c356f5a84b459", "response_sha256": "8fc1883b45e4da61f73d0303bd4dfcf827b016a74fd6228475d990f7962eacc2", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "qwen-different_wrong", "category": "different_wrong", "candidate_arm_id": "qwen", "case_id": "banking77:test:586", "input": "what is the charges for new card?", "reference_intent": "contactless_not_working", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"order_physical_card\"}", "parsed_intent": "order_physical_card", "correct": false, "provider_route": "openai", "receipt_sha256": "c79857a77ae76513646f34c2de54fce6c1d7832152d2f5504cc921dfceee0df7", "request_sha256": "6a9b57c9ec33c78e03984be6c5a8b048a3446aaf93814affb17f595aaa253657", "response_sha256": "7945743c24f4d1d00a7e4655b881809863349af6a378cb8a671ac5431357d5a3", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "qwen/qwen3.7-flash", "name": "Qwen3.7-Flash", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": false, "provider_route": "alibaba/us-east-1", "receipt_sha256": "6e433f1faade0c3424e231bbe1a354ee652bde11b97166f6e26cb6ec7371661d", "request_sha256": "5572d201cfbc468bb0f4e7950586478cfbb8ebb87d995f20c0346be4f947e7d5", "response_sha256": "2228813284071559e2d11229add311b237432b89139908e051b097be63bb1541", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "nemotron-both_correct", "category": "both_correct", "candidate_arm_id": "nemotron", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "nvidia/nemotron-3.5-lightning", "name": "Nemotron 3.5 Lightning", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "deepinfra/bf16", "receipt_sha256": "eeea4984dfd6b94c6a2e5c9e0288e7853d49fa56aabaf9ef8b15deed097e55f2", "request_sha256": "cb780bf146ce78e86f4ae3870fb016b286400f82f9578e5f3870e10e8c7c6694", "response_sha256": "c53f3c5ea68c072ac60ea150bf5532c20f16838de78fdd23c20e15bdfc617950", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "nemotron-shared_wrong", "category": "shared_wrong", "candidate_arm_id": "nemotron", "case_id": "banking77:test:27", "input": "I'm still waiting for delivery of my new card, why is it taking so long?", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a", "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce", "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "nvidia/nemotron-3.5-lightning", "name": "Nemotron 3.5 Lightning", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "deepinfra/bf16", "receipt_sha256": "2b2b872b307083fe210b5af168a3eedae11561cc41e0acdbba854e07203c60bc", "request_sha256": "2c078794688e345c6e7eaa04761eb9e0ac7f482e591cddd50545a6acb99a92cf", "response_sha256": "ee9443f78770fada27318b4686e0615924599bef30dd32c220d1e6d2a2e03a15", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "nemotron-correction", "category": "correction", "candidate_arm_id": "nemotron", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "nvidia/nemotron-3.5-lightning", "name": "Nemotron 3.5 Lightning", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "deepinfra/bf16", "receipt_sha256": "04538115745a0ac6299c5497baea87f68e29c554d517736bd57bd4b6f0e06873", "request_sha256": "21426605aafbeede8070ce75283f40ea17777481f7bcdb1c7aadf1b23d581b42", "response_sha256": "8da1fec56f13346ade06261abcfa35654c6b376e304f8c4a326ba711c30384f3", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "nemotron-regression", "category": "regression", "candidate_arm_id": "nemotron", "case_id": "banking77:test:137", "input": "Why is your exchange rate for my card payment wrong", "reference_intent": "card_payment_wrong_exchange_rate", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_payment_wrong_exchange_rate\"}", "parsed_intent": "card_payment_wrong_exchange_rate", "correct": true, "provider_route": "openai", "receipt_sha256": "737627a5a5fffc74ef8645c72e2946a91b1e9177bed26474253b28bb0f456483", "request_sha256": "595a080bc5d3aa283d530588c44f5f8984fc9fbac557ea62736ecaa3ddad61de", "response_sha256": "487c96fc864cce71c15a31fb2313b9dad968d023c41d1287dfefc321f430f358", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "nvidia/nemotron-3.5-lightning", "name": "Nemotron 3.5 Lightning", "visible_answer": "{\"intent\":\"exchange_rate\"}", "parsed_intent": "exchange_rate", "correct": false, "provider_route": "deepinfra/bf16", "receipt_sha256": "7484defc85d0c00603876729e88217cdc1a6528cafda014d7951209e745df7c9", "request_sha256": "e38a1df3e84e0b7dcbb7c6649ec7adb7a2d3035f63f407e02192930fe27c518d", "response_sha256": "435d743346fea8cf9332dc260291f737f970c7a574f5bb15127c6c302a5f9e82", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "nemotron-different_wrong", "category": "different_wrong", "candidate_arm_id": "nemotron", "case_id": "banking77:test:255", "input": "How do I change currencies to euros?", "reference_intent": "fiat_currency_support", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"exchange_via_app\"}", "parsed_intent": "exchange_via_app", "correct": false, "provider_route": "openai", "receipt_sha256": "5b26a5fcb17418bbe4da0a2673d72ebe1d15e46022df8f8a201a777a8715d0f8", "request_sha256": "e8c1f42d5ba0fda2a401a797334bc3c329b4e0e1b4cd8d18e709f615e07995c9", "response_sha256": "c1be4d8a8f3bfaf1f7b9c913b952b5d95d8e0071c5a038193243130bb0e709ac", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "nvidia/nemotron-3.5-lightning", "name": "Nemotron 3.5 Lightning", "visible_answer": "{\"intent\":\"exchange_rate\"}", "parsed_intent": "exchange_rate", "correct": false, "provider_route": "deepinfra/bf16", "receipt_sha256": "c95dc6f96f96439ea41de8b6098e3b68595425b3c330f23595fa62515b78b0dd", "request_sha256": "b827cc11d6cbd890580c1dba0a1bf7bf3ed9546f7b11d4c7d8f266f3b3e94aa5", "response_sha256": "be40bf20b83761152c8466c845984b6a56139b1badce639071be56a1d7f1638b", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "glm-both_correct", "category": "both_correct", "candidate_arm_id": "glm", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "z-ai/glm-5.3-flash", "name": "GLM-5.3-Flash", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "z-ai/fp8", "receipt_sha256": "1afa9d9402d2de096042cd13edd3dae9f6498eefc0ec336f308a23b7b667e9bb", "request_sha256": "ab957f02e8177994c6e9aeb184399a3b7c091d454b9332ed1d6f2139d2983f82", "response_sha256": "1a015e9362c8bd1494970b2190e95f833e72e11370e946740cd45a0aaf526565", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "glm-shared_wrong", "category": "shared_wrong", "candidate_arm_id": "glm", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "z-ai/glm-5.3-flash", "name": "GLM-5.3-Flash", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "z-ai/fp8", "receipt_sha256": "3b8c9a3e07e406fc6af6d87510f585e9b3d3e93c8c6d2bd2ebd019d77c058d68", "request_sha256": "86adf316affd7b5ebc5473517c3e02a201ac4929d8f947f319607258f3a0c0f0", "response_sha256": "fca6f1b0df2ffee3c0d322109febb279195dc35008fd17f6c3b1b3e656116249", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "glm-correction", "category": "correction", "candidate_arm_id": "glm", "case_id": "banking77:test:27", "input": "I'm still waiting for delivery of my new card, why is it taking so long?", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a", "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce", "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "z-ai/glm-5.3-flash", "name": "GLM-5.3-Flash", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "z-ai/fp8", "receipt_sha256": "c40fb0b47af378f58429fd54f282d0fd8f094fd777cf015d044c9a19aa53aeda", "request_sha256": "156dc4b2877ee1f7dc0bb331b136f3786d7573bee743212eb7a206d533ecb656", "response_sha256": "023113ecc61de05014e56874f0149a8daa513090b626add1211e005f7c915e3e", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "glm-regression", "category": "regression", "candidate_arm_id": "glm", "case_id": "banking77:test:243", "input": "Does your system support multiple currency.", "reference_intent": "fiat_currency_support", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"fiat_currency_support\"}", "parsed_intent": "fiat_currency_support", "correct": true, "provider_route": "openai", "receipt_sha256": "f9bde41d4ddd045b4306208561a400803ada3ba3581eaf5bed0d041a28cd01c1", "request_sha256": "80e1242173c5b00f98a6505fe3d030c5482481b9de5d50bd7bcb7442072041c4", "response_sha256": "bf4736656a29d02db620793e57e9d8edc793fc19389d3f90db72b0afdc07806c", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "z-ai/glm-5.3-flash", "name": "GLM-5.3-Flash", "visible_answer": "{\"intent\":\"supported_cards_and_currencies\"}", "parsed_intent": "supported_cards_and_currencies", "correct": false, "provider_route": "z-ai/fp8", "receipt_sha256": "05899c906c9135137262701682d89369823c53c1d5cb1df6c791071b45eb92f4", "request_sha256": "b1ef17edb6cabbffc7e7eeb25d5b81f9e711144eec904c89026e4a3c9135984d", "response_sha256": "5ed71ac632e2f68d199231f82c1df70bb5d7aad8fc1e10855c75181bca0464ee", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "glm-different_wrong", "category": "different_wrong", "candidate_arm_id": "glm", "case_id": "banking77:test:586", "input": "what is the charges for new card?", "reference_intent": "contactless_not_working", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"order_physical_card\"}", "parsed_intent": "order_physical_card", "correct": false, "provider_route": "openai", "receipt_sha256": "c79857a77ae76513646f34c2de54fce6c1d7832152d2f5504cc921dfceee0df7", "request_sha256": "6a9b57c9ec33c78e03984be6c5a8b048a3446aaf93814affb17f595aaa253657", "response_sha256": "7945743c24f4d1d00a7e4655b881809863349af6a378cb8a671ac5431357d5a3", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "z-ai/glm-5.3-flash", "name": "GLM-5.3-Flash", "visible_answer": "{\"intent\":\"getting_spare_card\"}", "parsed_intent": "getting_spare_card", "correct": false, "provider_route": "z-ai/fp8", "receipt_sha256": "fe337a4cca35bd6a4c23e64d2dd52d60306c284f8635bc465559244800a954fd", "request_sha256": "41cadadf4f548b874fe76d77524f9c26142a7743d9573af56ac9cac22d4998b8", "response_sha256": "489a789f7ad06011eb7528153a1fbd426f95e1b4f2115799efa170102d3b89e6", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "provenance": { "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash-lite:both_correct:banking77:test:2", "category": "both_correct", "candidate_arm_id": "gemini-flash-lite", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "google/gemini-3.5-flash-lite", "name": "Gemini 3.5 Flash Lite", "visible_answer": "```json\n{\"intent\":\"card_arrival\"}\n```", "parsed_intent": "card_arrival", "correct": true, "provider_route": "google-ai-studio", "receipt_sha256": "f4dae471b79c99ce36aa8551f873f7b78935946a817cdbb473d3cdcbc093e5cf", "response_sha256": "08ab9dd75119f5cf317f3ceea3965a03ba7f6b15cb61fb2e1e9779bbddd0028c", "request_sha256": "8581f6c5104e1e1663743260c95d724b5086dcc5cbbfc16a9f640d86fda73124", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash-lite:shared_wrong:banking77:test:27", "category": "shared_wrong", "candidate_arm_id": "gemini-flash-lite", "case_id": "banking77:test:27", "input": "I'm still waiting for delivery of my new card, why is it taking so long?", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a", "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01", "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "google/gemini-3.5-flash-lite", "name": "Gemini 3.5 Flash Lite", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "google-ai-studio", "receipt_sha256": "419a0e5778303e38be6eb6a2a8288c2bebe5679f48f0375a04e42c298882a936", "response_sha256": "1fbc4afc5d4d8f73be57fa80e505a0143a2b78a12acebc5e27212485a2e624c0", "request_sha256": "ce04cb67a63117fa4fba145360a0f5c2239362746e3311e8a7263203da0a920c", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash-lite:correction:banking77:test:21", "category": "correction", "candidate_arm_id": "gemini-flash-lite", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "google/gemini-3.5-flash-lite", "name": "Gemini 3.5 Flash Lite", "visible_answer": "```json\n{\"intent\":\"card_arrival\"}\n```", "parsed_intent": "card_arrival", "correct": true, "provider_route": "google-ai-studio", "receipt_sha256": "1765422c24d07182b5aab6050f5c1cdd3e33bf64328751cdc414fc1e9818fb60", "response_sha256": "56e48506e6b1d40ed75392225947c0611e33bc550a5a8a92b0628197b8b7b566", "request_sha256": "2cc4746bde72e447b6144c07e6e9516b2ac15a7d09f6c22fb5898d1d85aa6dd2", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash-lite:regression:banking77:test:560", "category": "regression", "candidate_arm_id": "gemini-flash-lite", "case_id": "banking77:test:560", "input": "Are contactless payments enabled on my new card?", "reference_intent": "contactless_not_working", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"contactless_not_working\"}", "parsed_intent": "contactless_not_working", "correct": true, "provider_route": "openai", "receipt_sha256": "863513fa6a22fa8bfd9dc73ba8fe97c8041efdf607820fbc170ad60e1e7270f7", "response_sha256": "78a4f90b190b685fb7315c74c324b8a242debf725a6f28da9068a0118fb229a7", "request_sha256": "cefcb43247553c8eb7b66ffe6bbfa5ba7717b23d60a9ad3496074a77d6ee2c10", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "google/gemini-3.5-flash-lite", "name": "Gemini 3.5 Flash Lite", "visible_answer": "{\"intent\":\"activate_my_card\"}", "parsed_intent": "activate_my_card", "correct": false, "provider_route": "google-ai-studio", "receipt_sha256": "3962a824bc8d2a47dac281d444f3595702c2419d40f0c488c5a047f2a0da2b50", "response_sha256": "6ab17d7f71be6624b53e1afbc5336e01ccd6f6f0d4abdb931ba4ba48cdda7c73", "request_sha256": "cad8b9525dd5069f527010b3433e7b6197b10f24bfca51510146e3d31c922763", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash-lite:different_wrong:banking77:test:382", "category": "different_wrong", "candidate_arm_id": "gemini-flash-lite", "case_id": "banking77:test:382", "input": "I broke my card", "reference_intent": "card_not_working", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"getting_spare_card\"}", "parsed_intent": "getting_spare_card", "correct": false, "provider_route": "openai", "receipt_sha256": "8759d1974ee8ba2a4cc159afd88502fb1c89936454b46445c6108e4562e114fd", "response_sha256": "a21d386c3aaef37383da63f6821780fda3eca5e838f8a2ae43040af497c32df6", "request_sha256": "837d08959660607e93449ab2580e5fc25f19e59eac7e8fa6a3224d9d82517433", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "google/gemini-3.5-flash-lite", "name": "Gemini 3.5 Flash Lite", "visible_answer": "{\"intent\":\"lost_or_stolen_card\"}", "parsed_intent": "lost_or_stolen_card", "correct": false, "provider_route": "google-ai-studio", "receipt_sha256": "f9174447931a896f194b6a02da077efe037a3521d4e6ab915b45e3741223dd0f", "response_sha256": "f0cc85b11e64723d6277d7929d81c3d597e9fc88961b55efcdeb4ef68414f76e", "request_sha256": "2c08f14c0e7fb733aa687631600eb88665e347facd1bcf9744ce1183d95586af", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash:both_correct:banking77:test:2", "category": "both_correct", "candidate_arm_id": "gemini-flash", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "google/gemini-3.8-flash", "name": "Gemini 3.8 Flash", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "google-ai-studio", "receipt_sha256": "b6b3fc1854d373a0712be0ff24518f99beb6533e4bb90fb74dc92f56959d4076", "response_sha256": "5a4c80940fab1e0214426f6a0b583833afafc054da4f058eb1fb2cf3490d210a", "request_sha256": "d1ac78514e2207673b8a308833fef2df83459895f4ebdae67350ca9899076902", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash:shared_wrong:banking77:test:121", "category": "shared_wrong", "candidate_arm_id": "gemini-flash", "case_id": "banking77:test:121", "input": "I made a currency exchange and think I was charged more than I should of been.", "reference_intent": "card_payment_wrong_exchange_rate", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"exchange_charge\"}", "parsed_intent": "exchange_charge", "correct": false, "provider_route": "openai", "receipt_sha256": "96df5f011f7f5b627f6390cea52a6770ff51c2d840821bddac0429cf2c9a22dd", "response_sha256": "4a7c922ca1b8df8c51579c4b072e50f0e75398a93bbb692b5c4c088fbde7b1da", "request_sha256": "b68aa904f86e9278a5aaee57f536fcb79af44fb583194271d106ca8d739486b3", "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193" }, "candidate": { "model": "google/gemini-3.8-flash", "name": "Gemini 3.8 Flash", "visible_answer": "{\"intent\":\"exchange_charge\"}", "parsed_intent": "exchange_charge", "correct": false, "provider_route": "google-ai-studio", "receipt_sha256": "56625ea3d8d7d5a112fcaa70b149de6e634ab69e9f696e6028498e1fce430bf1", "response_sha256": "1cfdb5181a9e1ea32f0d4218fc889a25cdeb968797a8d0f663c88718d7ed97d3", "request_sha256": "7e19f84eb3b718a21f5f059ae66f9b577665494b67dd419bb14a04207cb73238", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash:correction:banking77:test:21", "category": "correction", "candidate_arm_id": "gemini-flash", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "google/gemini-3.8-flash", "name": "Gemini 3.8 Flash", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "google-ai-studio", "receipt_sha256": "36924a80adbb55b30bd1235337e1b7efc59bb1cc60b307dd3ceb2b20ee642531", "response_sha256": "e7663f1ddbc80dc455cd5e12442fd8b8e8b1e8e801e1e82eee96e73e986a74e1", "request_sha256": "bae06e35d45d42610d24c5629ad373880269f9e30df14e98181d6f327efb1ba8", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash:regression:banking77:test:901", "category": "regression", "candidate_arm_id": "gemini-flash", "case_id": "banking77:test:901", "input": "Which forms of currency are accepted?", "reference_intent": "supported_cards_and_currencies", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"supported_cards_and_currencies\"}", "parsed_intent": "supported_cards_and_currencies", "correct": true, "provider_route": "openai", "receipt_sha256": "aaa4d0e40d2d96dc6e502390aeb17edfbe6e0d498cb89b001207722e980b6fd9", "response_sha256": "7e8e2eb4502876a787acd4f421e1f19261a8e27f36a1c68a123c93c7e6aa6357", "request_sha256": "5e07b720c40eb70b10b59bbef22d498d1430c492aaa0aeebd6daacac49b1211b", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "google/gemini-3.8-flash", "name": "Gemini 3.8 Flash", "visible_answer": "{\"intent\":\"fiat_currency_support\"}", "parsed_intent": "fiat_currency_support", "correct": false, "provider_route": "google-ai-studio", "receipt_sha256": "407937293e025cff47b43681d0e456f75ded030602935e0994e522da1116de74", "response_sha256": "465e8f9ae8fba42f9f1c5d71ee56e4e51edc6c480ba3f25b345d71da3d1d72f9", "request_sha256": "a52b19418a092d065136b57d0e9e37a217ab390953510300ee36ea268f1ca353", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gemini-flash:different_wrong:banking77:test:416", "category": "different_wrong", "candidate_arm_id": "gemini-flash", "case_id": "banking77:test:416", "input": "How would I use my money in a different country?", "reference_intent": "exchange_via_app", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"country_support\"}", "parsed_intent": "country_support", "correct": false, "provider_route": "openai", "receipt_sha256": "6f91aaff5c61d975ccaa0c6571df3ee78a3414fd6a6bcf6ed7c9e8215aaee428", "response_sha256": "9a8284aa0e3a1fe588a74a46a840923bce158d638ee9a4474fed6ec1c6f7d37d", "request_sha256": "04a0b49576ef174d3383ead16fbb96d36b04fddda346077c4e8a00d1fd4773c6", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "google/gemini-3.8-flash", "name": "Gemini 3.8 Flash", "visible_answer": "{\"intent\":\"card_acceptance\"}", "parsed_intent": "card_acceptance", "correct": false, "provider_route": "google-ai-studio", "receipt_sha256": "5ee8beb61625fca716fe63464c4ac4e45f062d9c4a6f08c6ebd88e701cdb5afd", "response_sha256": "b359d844a3b650a0696948dea23087634a19dca9ea6535635dafb07df3487803", "request_sha256": "ace96a66f002ad3718e67b25837d1f65b15f6feffbd9cc7360d9f871c575760c", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-luna:both_correct:banking77:test:2", "category": "both_correct", "candidate_arm_id": "gpt-luna", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "openai/gpt-6-luna", "name": "GPT-6 Luna", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "ed2bf7e2e3d3d762a8c9976010a2f1f7b944c2d93c0547b82333a6a665caa462", "response_sha256": "e30e1ba95f5041cb07a4e106614f7fb5343abbc87d3d1a912e32849184510a12", "request_sha256": "79aa4316d77d815bf3854cc234c42aeca8f5d883cd2b24ce6e734273fbbbd84e", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-luna:shared_wrong:banking77:test:27", "category": "shared_wrong", "candidate_arm_id": "gpt-luna", "case_id": "banking77:test:27", "input": "I'm still waiting for delivery of my new card, why is it taking so long?", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a", "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01", "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "openai/gpt-6-luna", "name": "GPT-6 Luna", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "fe9f0c69ad82926a0ebb27b8bc3d5585953d843f33b493292d0680dce30f1818", "response_sha256": "8a8bd2b4f122b6c69fb4c2263b7ad09b9f5a5944f2870f14b3d946de2457958b", "request_sha256": "361c4fb5eaededcf398f9a8676b947c00b024fbe5437ec5a4bf33a495599b924", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-luna:correction:banking77:test:21", "category": "correction", "candidate_arm_id": "gpt-luna", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "openai/gpt-6-luna", "name": "GPT-6 Luna", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "35333ef88adbdfb4d5d1ea62972d69b2845918fb8c3b08ddc49d13576e70813a", "response_sha256": "551502e7359e0e13849fa55147ddf4eaac391d68c193d098eefff81d18a5548a", "request_sha256": "682dfdf856a187bb92d6a4d606b3e5085e0e36d6f2493d7a5a624e5f55fc2b27", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-luna:regression:banking77:test:274", "category": "regression", "candidate_arm_id": "gpt-luna", "case_id": "banking77:test:274", "input": "How man currencies can I hold?", "reference_intent": "fiat_currency_support", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"fiat_currency_support\"}", "parsed_intent": "fiat_currency_support", "correct": true, "provider_route": "openai", "receipt_sha256": "b4127726699f600fce2293fbc3eba31bfb310e9297d5cd9941f43e92aa6bb32e", "response_sha256": "9e41bede87331961d72f1347957944fec649df688659134d10a2e9d01a3fbcfd", "request_sha256": "d1170e7c7bca8bf66ba0fdfd34128594bc1356f8aef5bb88704391c87472cc15", "plan_sha256": "9233745dc9c1a76050b1d2922aac48dcc4454ab2831b20fc29dec5c11b4c91b5" }, "candidate": { "model": "openai/gpt-6-luna", "name": "GPT-6 Luna", "visible_answer": "{\"intent\":\"supported_cards_and_currencies\"}", "parsed_intent": "supported_cards_and_currencies", "correct": false, "provider_route": "openai", "receipt_sha256": "ca3de7c03d33d000a3d44e7bd537c2ce80b9d73f713cba133f90622a65acf701", "response_sha256": "b925aeed29b3be9c4c93ab987ed9ed76fdf76b8d55eef8b0e808b671faf3a76f", "request_sha256": "524b1e6445560665248884dbcd304d92ccdcbed25f9372dc09666c34f9fbb4a6", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-luna:different_wrong:banking77:test:586", "category": "different_wrong", "candidate_arm_id": "gpt-luna", "case_id": "banking77:test:586", "input": "what is the charges for new card?", "reference_intent": "contactless_not_working", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"order_physical_card\"}", "parsed_intent": "order_physical_card", "correct": false, "provider_route": "openai", "receipt_sha256": "c79857a77ae76513646f34c2de54fce6c1d7832152d2f5504cc921dfceee0df7", "response_sha256": "7945743c24f4d1d00a7e4655b881809863349af6a378cb8a671ac5431357d5a3", "request_sha256": "6a9b57c9ec33c78e03984be6c5a8b048a3446aaf93814affb17f595aaa253657", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "openai/gpt-6-luna", "name": "GPT-6 Luna", "visible_answer": "{\"intent\":\"get_physical_card\"}", "parsed_intent": "get_physical_card", "correct": false, "provider_route": "openai", "receipt_sha256": "940a1a308b370dd7cb7520efc0692fa8b562683a845caf17c1ceecfcca724569", "response_sha256": "17d8f0486e8742b791ed7b882edf20b72f81c095e95425e9631cfb6f973021e9", "request_sha256": "6f5ddd8a1ce300638ccbc5f8136360971467d8e3cc21b6ade111decc66b6908b", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-sol:both_correct:banking77:test:2", "category": "both_correct", "candidate_arm_id": "gpt-sol", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "openai/gpt-6.1-sol", "name": "GPT-6.1 Sol", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "b761007b3dc11d096134ed0a7e37c1b7ec00d4c388724210f1fe9344281ef9e4", "response_sha256": "76cb5a5c944754bfc5c2a06e8755b3addf81cb19f8e31de8059d18b57fbdb653", "request_sha256": "7c0392b8e20c29846be6eb2aefa06b738faac0ff4029665e3983ee65a2452123", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-sol:shared_wrong:banking77:test:121", "category": "shared_wrong", "candidate_arm_id": "gpt-sol", "case_id": "banking77:test:121", "input": "I made a currency exchange and think I was charged more than I should of been.", "reference_intent": "card_payment_wrong_exchange_rate", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"exchange_charge\"}", "parsed_intent": "exchange_charge", "correct": false, "provider_route": "openai", "receipt_sha256": "96df5f011f7f5b627f6390cea52a6770ff51c2d840821bddac0429cf2c9a22dd", "response_sha256": "4a7c922ca1b8df8c51579c4b072e50f0e75398a93bbb692b5c4c088fbde7b1da", "request_sha256": "b68aa904f86e9278a5aaee57f536fcb79af44fb583194271d106ca8d739486b3", "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193" }, "candidate": { "model": "openai/gpt-6.1-sol", "name": "GPT-6.1 Sol", "visible_answer": "{\"intent\":\"exchange_charge\"}", "parsed_intent": "exchange_charge", "correct": false, "provider_route": "openai", "receipt_sha256": "fa921ad5a4ee213ce46a682d28b01960085e4df503ed37b3e9df599439c26249", "response_sha256": "16a3534da16f286881db12693fe9855ccf3390d6ec8a6665206cdcb6b5779113", "request_sha256": "32a03ba514b13535f3c161ca77d5f020df729477d94f63ccaa766bb309b664d5", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-sol:correction:banking77:test:21", "category": "correction", "candidate_arm_id": "gpt-sol", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "openai/gpt-6.1-sol", "name": "GPT-6.1 Sol", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "9f4ca033792cfa5f3fe72a8efea0b7f114970a63c285737081b56635ac24d9f7", "response_sha256": "95475700c85eff5a29ef0dc454ea3a2a30bbba088cfdd3263d85b6f82973559c", "request_sha256": "d93ebb0f4a7612dc2f3e78bc1c0bc4c0f9c66bc29eb7f101b851f4f39ae1213b", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-sol:regression:banking77:test:696", "category": "regression", "candidate_arm_id": "gpt-sol", "case_id": "banking77:test:696", "input": "I want to reverse my transaction from earlier", "reference_intent": "cancel_transfer", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"cancel_transfer\"}", "parsed_intent": "cancel_transfer", "correct": true, "provider_route": "openai", "receipt_sha256": "4d3c2a04a0a8e1f5cf745d6310f291c1395d5fa3f6c42ec415c38e1956d0b2d9", "response_sha256": "046dc6cb939b32491d7a10815dec6a4dc69215d721607f9a51895a359654de65", "request_sha256": "0fc2a12bfcb1390cef14afd99350f084db0ba1e9b4eb78afbf9ecc9314993b5a", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "openai/gpt-6.1-sol", "name": "GPT-6.1 Sol", "visible_answer": "{\"intent\":\"request_refund\"}", "parsed_intent": "request_refund", "correct": false, "provider_route": "openai", "receipt_sha256": "6fea75f4481d1ea06a1a3d4c456ed91a0c77a9e90c63549b20fc72116569db73", "response_sha256": "90ce57a54731755f752e05df7a186af7d72d9cf279035f587114465ad0d24f87", "request_sha256": "6ea903a15a08a4c07512d59d7eb2570da7d0e86aee796939f8655ed21c4c9ea5", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "gpt-sol:different_wrong:banking77:test:416", "category": "different_wrong", "candidate_arm_id": "gpt-sol", "case_id": "banking77:test:416", "input": "How would I use my money in a different country?", "reference_intent": "exchange_via_app", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"country_support\"}", "parsed_intent": "country_support", "correct": false, "provider_route": "openai", "receipt_sha256": "6f91aaff5c61d975ccaa0c6571df3ee78a3414fd6a6bcf6ed7c9e8215aaee428", "response_sha256": "9a8284aa0e3a1fe588a74a46a840923bce158d638ee9a4474fed6ec1c6f7d37d", "request_sha256": "04a0b49576ef174d3383ead16fbb96d36b04fddda346077c4e8a00d1fd4773c6", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "openai/gpt-6.1-sol", "name": "GPT-6.1 Sol", "visible_answer": "{\"intent\":\"card_acceptance\"}", "parsed_intent": "card_acceptance", "correct": false, "provider_route": "openai", "receipt_sha256": "e56602c51e1d79fad6b004130258e3ab0b51418e21d6d0247027d92336fbfbf9", "response_sha256": "fd389a250f6900cb7e774e6ff1f6ea25a11384e3c2fd4afd957247c3c3e78b5a", "request_sha256": "5af191e1fed9c31a9eb8004c43ba3e3fb23966307b1e643ab14df5e7ee6659d1", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-haiku:both_correct:banking77:test:2", "category": "both_correct", "candidate_arm_id": "claude-haiku", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "anthropic/claude-haiku-5.5", "name": "Claude Haiku 5.5", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "anthropic", "receipt_sha256": "fc60ee3f0b8f6fd8514cea86c194772cbf4a750f5f9ceec4b7c994f64b46a279", "response_sha256": "91b6ea8ca1cb66f9c6af15aa96dfc201e6a72433af8914a49ee02eb102d598ad", "request_sha256": "67b10277f0c58d8e36ceead8fcf3af359c6801cd413632eac3a72314484f6143", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-haiku:shared_wrong:banking77:test:27", "category": "shared_wrong", "candidate_arm_id": "claude-haiku", "case_id": "banking77:test:27", "input": "I'm still waiting for delivery of my new card, why is it taking so long?", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a", "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01", "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "anthropic/claude-haiku-5.5", "name": "Claude Haiku 5.5", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "anthropic", "receipt_sha256": "4d6cd264ec380a5d30c3ad33efe77edb98822d279b36eadcd4bef972aa8f687a", "response_sha256": "53829714ac5b16cac4dd8435317049d2ffb8b3dd08f526433d7e09e70f6a0fa4", "request_sha256": "a1ee3c6994bc264923e11c165ed962fc5b0aff79539a185e28a21949208f7bf7", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-haiku:correction:banking77:test:21", "category": "correction", "candidate_arm_id": "claude-haiku", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "anthropic/claude-haiku-5.5", "name": "Claude Haiku 5.5", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "anthropic", "receipt_sha256": "942367f7c8e0d62fa036a0fa787e768b98714e2b9aa781c920d071d9c39e5e87", "response_sha256": "7b17edcfba11adf93c1039dde912167a87cb5e4dad347941d58309f1eaa42266", "request_sha256": "4f158639595c1e4eb13fb8f69c38d5142ddd0e1139b381bf26365f1a6bef304f", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-haiku:regression:banking77:test:543", "category": "regression", "candidate_arm_id": "claude-haiku", "case_id": "banking77:test:543", "input": "Why did I get blocked?", "reference_intent": "pin_blocked", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"pin_blocked\"}", "parsed_intent": "pin_blocked", "correct": true, "provider_route": "openai", "receipt_sha256": "8d0b4bc5bcbd9ca4f60d49b79f4abbd6bdb48ef2205a751ea478ca906023f3c1", "response_sha256": "611b2ad952b26b271dacdb065b118c764e8c167ac14239b5f3a9d9999897ffc2", "request_sha256": "1521bc44a0da11dcae5d5758d255383c6d7b3690019728e779ee6eec2d327582", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "anthropic/claude-haiku-5.5", "name": "Claude Haiku 5.5", "visible_answer": "{\"intent\":\"card_not_working\"}", "parsed_intent": "card_not_working", "correct": false, "provider_route": "anthropic", "receipt_sha256": "fb0153ced320919ab623a2cdbaaaf0ad241c0818edf75cf47a558c0ad54ddda7", "response_sha256": "cf6b4fcf314e538f6aad269e590f7db5a9e55cb255116b664e27ec9edec63367", "request_sha256": "1bff81cfdf9810c11532206ee8328cc1e28f7305445838e5b190bcba0b101a99", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-haiku:different_wrong:banking77:test:586", "category": "different_wrong", "candidate_arm_id": "claude-haiku", "case_id": "banking77:test:586", "input": "what is the charges for new card?", "reference_intent": "contactless_not_working", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"order_physical_card\"}", "parsed_intent": "order_physical_card", "correct": false, "provider_route": "openai", "receipt_sha256": "c79857a77ae76513646f34c2de54fce6c1d7832152d2f5504cc921dfceee0df7", "response_sha256": "7945743c24f4d1d00a7e4655b881809863349af6a378cb8a671ac5431357d5a3", "request_sha256": "6a9b57c9ec33c78e03984be6c5a8b048a3446aaf93814affb17f595aaa253657", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "anthropic/claude-haiku-5.5", "name": "Claude Haiku 5.5", "visible_answer": "{\"intent\":\"card_payment_fee_charged\"}", "parsed_intent": "card_payment_fee_charged", "correct": false, "provider_route": "anthropic", "receipt_sha256": "9223d201e96cfaf1bd1c6de5c7c4aeabee14f9b00d108484a2d526db55924d1a", "response_sha256": "5635672057767b96a413e0ac6b977dbe38c668cb92f99da34ee39ed4b82327eb", "request_sha256": "4ce5c3760586a43e276e01257a7815c1cac0b7cf6fafa16bf65b87ad657c582f", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-sonnet:both_correct:banking77:test:2", "category": "both_correct", "candidate_arm_id": "claude-sonnet", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "anthropic/claude-sonnet-5.5", "name": "Claude Sonnet 5.5", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "anthropic", "receipt_sha256": "a2e96a3c7fbed754937b61f506540c8f481d27f67e16f5a6a03a329f720d66fe", "response_sha256": "31013269965893e5b1c3ec096f8080a6c78147aea4b10a606b9fef38ca2bd2eb", "request_sha256": "e15d8a6df5cba03b25333ee781cc07e2b2fac1e9ff493d6ccc5957cc56976f2e", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-sonnet:shared_wrong:banking77:test:121", "category": "shared_wrong", "candidate_arm_id": "claude-sonnet", "case_id": "banking77:test:121", "input": "I made a currency exchange and think I was charged more than I should of been.", "reference_intent": "card_payment_wrong_exchange_rate", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"exchange_charge\"}", "parsed_intent": "exchange_charge", "correct": false, "provider_route": "openai", "receipt_sha256": "96df5f011f7f5b627f6390cea52a6770ff51c2d840821bddac0429cf2c9a22dd", "response_sha256": "4a7c922ca1b8df8c51579c4b072e50f0e75398a93bbb692b5c4c088fbde7b1da", "request_sha256": "b68aa904f86e9278a5aaee57f536fcb79af44fb583194271d106ca8d739486b3", "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193" }, "candidate": { "model": "anthropic/claude-sonnet-5.5", "name": "Claude Sonnet 5.5", "visible_answer": "{\"intent\":\"exchange_charge\"}", "parsed_intent": "exchange_charge", "correct": false, "provider_route": "anthropic", "receipt_sha256": "b923c44e11a25d3b9f75cf2597cfbb2467bac10ac64c6e2cd21ec35301570bb6", "response_sha256": "e697c14c9acf3f3c982e5fa92d43c12f2017a4c3a618aa1a6e6c343069b372b3", "request_sha256": "48a2be204180b86d4cff08c2dcb81e4613a9062e8c7b90d826812ef9b13a93ab", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-sonnet:correction:banking77:test:21", "category": "correction", "candidate_arm_id": "claude-sonnet", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "anthropic/claude-sonnet-5.5", "name": "Claude Sonnet 5.5", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "anthropic", "receipt_sha256": "c6341afc1ec8f8cb5eb3bb3f6c00152a578a0dffa7785c9f82b75845c2ca3578", "response_sha256": "8c582fae07d659d68f9adb74ff7f6fdd78f7f43e732bee8f8c004effb2d9b58b", "request_sha256": "931eb8917e657c4d054571739ae889b48621f5def301b017886b112451d2485c", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-sonnet:regression:banking77:test:406", "category": "regression", "candidate_arm_id": "claude-sonnet", "case_id": "banking77:test:406", "input": "How are currencies exchanged?", "reference_intent": "exchange_via_app", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"exchange_via_app\"}", "parsed_intent": "exchange_via_app", "correct": true, "provider_route": "openai", "receipt_sha256": "ebe328c7bc1057cc775b5643f2a62a96c35ac317ecbe8093b24384477e205d94", "response_sha256": "9ac07a3318c06426ad49d0b47a8015ccb78f7e7522599bda9e6235a489a92d02", "request_sha256": "bc878ddd0c054245bb4fe13f80efc673959b32d8b200599c447a233e0dd6a20a", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "anthropic/claude-sonnet-5.5", "name": "Claude Sonnet 5.5", "visible_answer": "{\"intent\":\"exchange_rate\"}", "parsed_intent": "exchange_rate", "correct": false, "provider_route": "anthropic", "receipt_sha256": "07d6dc9a26d1f2a82cc930de37244741b3e617934edeafe8293aa76f29c71db4", "response_sha256": "bd650d3ac15ce05ff20cf272b32ab94a372a4756a2c18388a5c5b0e635821daa", "request_sha256": "bd4c9f051bfc7f49f6a4ae185f651dba55061b130c3790abed1da0b70cb9e2e3", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "claude-sonnet:different_wrong:banking77:test:1329", "category": "different_wrong", "candidate_arm_id": "claude-sonnet", "case_id": "banking77:test:1329", "input": "How do my friends top up my account", "reference_intent": "topping_up_by_card", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"transfer_into_account\"}", "parsed_intent": "transfer_into_account", "correct": false, "provider_route": "openai", "receipt_sha256": "0803c197841569d58be217c3ba2053b2c920b50eda3bbb8db2c98a15119e0f25", "response_sha256": "0404950add231126550b6c8b48ba15ecdc108f954559ce403b916a24414ef4d7", "request_sha256": "6606239c33e9edbd00010d467a13183cd20501a95b6b33a8f107dd5755f73ba9", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "anthropic/claude-sonnet-5.5", "name": "Claude Sonnet 5.5", "visible_answer": "{\"intent\":\"receiving_money\"}", "parsed_intent": "receiving_money", "correct": false, "provider_route": "anthropic", "receipt_sha256": "a8de945cb2ee56ce073be66f4dd544b5d25a5bf961adbfd0d261790286244007", "response_sha256": "6742d6a71dac45ed15fe6553646d397f36572e943177c9cc357ccbd3cffaf1a1", "request_sha256": "3ada6be7ad3767a6aa6829c865936d4a6aa61cb42bbd3791de028558945a2ac7", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "grok:both_correct:banking77:test:2", "category": "both_correct", "candidate_arm_id": "grok", "case_id": "banking77:test:2", "input": "I ordered a card but it has not arrived. Help please!", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "openai", "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028", "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2", "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "x-ai/grok-4.7", "name": "Grok 4.7", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "xai", "receipt_sha256": "55d864dbe19c5c2088467d30c5ccc4890823821f0bb7f8e213c96aa676b8b161", "response_sha256": "322c98cbddba162f0e7294c893e834bb42d1e24c862cb011936234f38f1c8e9d", "request_sha256": "0b0c254fcc33d57a5ab238abc985b8db9d7f7ceba28517cadabde0cc5164bf39", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "grok:shared_wrong:banking77:test:27", "category": "shared_wrong", "candidate_arm_id": "grok", "case_id": "banking77:test:27", "input": "I'm still waiting for delivery of my new card, why is it taking so long?", "reference_intent": "card_arrival", "outputs_match": true, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a", "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01", "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "x-ai/grok-4.7", "name": "Grok 4.7", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "xai", "receipt_sha256": "654ec53aa34ee81ce0d4bea68dd14440a5e4604a7a9a2299847e16f3bd7acaff", "response_sha256": "f3236fa850dec73278175a8097913d6e60a6fd4597cfbe76efb97e38eb92596e", "request_sha256": "2c6475745ed38d09d32afc96642f9a7b79ebd9b4cbe4f7e5a88b8e2143c5dd19", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "grok:correction:banking77:test:21", "category": "correction", "candidate_arm_id": "grok", "case_id": "banking77:test:21", "input": "Status of the card I ordered.", "reference_intent": "card_arrival", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"card_delivery_estimate\"}", "parsed_intent": "card_delivery_estimate", "correct": false, "provider_route": "openai", "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8", "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2", "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028", "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1" }, "candidate": { "model": "x-ai/grok-4.7", "name": "Grok 4.7", "visible_answer": "{\"intent\":\"card_arrival\"}", "parsed_intent": "card_arrival", "correct": true, "provider_route": "xai", "receipt_sha256": "23ab54058ce6dbeb01c90cad7b546acaee3f003452d0172bea3226ce186e3625", "response_sha256": "6a414f26e6258b0c20808f386c292155fce2e2d3c2bc8cf2d0a3223e18b83f0c", "request_sha256": "d9c98d75fa1bc83c23df659e0734a4c59ebbd04c84ccc8bc93328072037ce8be", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, "provenance": { "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "grok:regression:banking77:test:596", "category": "regression", "candidate_arm_id": "grok", "case_id": "banking77:test:596", "input": "How do I enable contactless pay?", "reference_intent": "contactless_not_working", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"contactless_not_working\"}", "parsed_intent": "contactless_not_working", "correct": true, "provider_route": "openai", "receipt_sha256": "df17435dda829d0a976f8aa577c516d9dd65964e40f46e4a2c7d870bd0b98097", "response_sha256": "5e73fa568a47bf1f4bfaf79bf22594f1a28d0106b2fada9cb0b28a74f3870fa8", "request_sha256": "a4d66cd758c58af48c35a8942189583f85359e969a2d2fc808ce19b7cc623868", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "x-ai/grok-4.7", "name": "Grok 4.7", "visible_answer": "{\"intent\":\"apple_pay_or_google_pay\"}", "parsed_intent": "apple_pay_or_google_pay", "correct": false, "provider_route": "xai", "receipt_sha256": "4c2243fa8f21a6deffc39155e2f0e0ce52b23bff38e2e99bf9edf6f8621d8d99", "response_sha256": "b1bee60835cedbb521fc136fa67de585ec262f0c2c52e415038672626980eca5", "request_sha256": "55abbbe228d5b7eb7baf825d1c94dd5b491e86be91f0db8c718aff0baf6fceb6", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } }, { "id": "grok:different_wrong:banking77:test:586", "category": "different_wrong", "candidate_arm_id": "grok", "case_id": "banking77:test:586", "input": "what is the charges for new card?", "reference_intent": "contactless_not_working", "outputs_match": false, "incumbent": { "model": "openai/o4-mini", "name": "o4-mini", "visible_answer": "{\"intent\":\"order_physical_card\"}", "parsed_intent": "order_physical_card", "correct": false, "provider_route": "openai", "receipt_sha256": "c79857a77ae76513646f34c2de54fce6c1d7832152d2f5504cc921dfceee0df7", "response_sha256": "7945743c24f4d1d00a7e4655b881809863349af6a378cb8a671ac5431357d5a3", "request_sha256": "6a9b57c9ec33c78e03984be6c5a8b048a3446aaf93814affb17f595aaa253657", "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33" }, "candidate": { "model": "x-ai/grok-4.7", "name": "Grok 4.7", "visible_answer": "{\"intent\":\"getting_spare_card\"}", "parsed_intent": "getting_spare_card", "correct": false, "provider_route": "xai", "receipt_sha256": "8d7e229294a4d4fcf32fe39777db5ac61e42ad4295cf7cb3286fe17e7d241c60", "response_sha256": "7976ad566a42a69894aa5ea7d48057d9d85f807ac03512da675d9093f8ead662", "request_sha256": "b5bbad4ee15bc0fc290ef245aa4cea3750e5fe2eaf01428e4405389368fabc8a", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, "provenance": { "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d" } } ], "source_summaries": [ { "role": "retained_history", "sha256": "31d2f1a4ddc87eb9a6682c2c69f51a679d0c51f91bac9279772aa51c3d209b1d" }, { "role": "extension", "sha256": "3b41e33295c60f5b80807797c9446aed1ff644c022af72442246dbcb770f91ca" } ] }, "collection_history": { "amendment_sha256": "7b9e5e5933782f87c18368a75c52d341e778b5a1ab27ba630a159d4483007014", "segments": [ { "role": "operational_prefix", "status": "readiness_failed", "initial_attempts": 35, "retry_attempts": 0, "first": "2026-10-13T16:22:28.628360+00:00", "last": "2026-10-13T16:23:47.795105+00:00", "plan_sha256": "7d754c636ae5406ccd0ed86dee815bcc7b1e112167c68dedc9df0d05ae8368f4", "journal_sha256": "9a6a559f8bfe814bacb318498cfc7b81a5c6b6bec80436ef7dfb9feb6743c85d", "closure": { "kind": "prefix_review", "sha256": "5b6889ddf760f78012cf73f374ecc96e88823651fec1733a6b1ad98fb7580ddf" }, "known_cost_usd": "0.04103170", "held_unknown_reservation_usd": "0" }, { "role": "stopped_continuation", "status": "stopped", "initial_attempts": 1740, "retry_attempts": 0, "first": "2026-10-13T16:35:11.311412+00:00", "last": "2026-10-13T17:40:22.287501+00:00", "plan_sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b", "journal_sha256": "a2069c72f60b76f91dd1badf8b46e6900cd2522823e7b3b1fe75766efef65ad8", "closure": { "kind": "stopped_result", "sha256": "2a19b2b06747ff660401ac72a54cb257ed4194c6391eb8c36b9ed84db2f2ec20" }, "known_cost_usd": "2.08393090", "held_unknown_reservation_usd": "0.001792" }, { "role": "completed_continuation", "status": "completed_with_billing_unknown", "initial_attempts": 1935, "retry_attempts": 0, "first": "2026-10-13T18:02:59.300269+00:00", "last": "2026-10-13T19:11:49.317240+00:00", "plan_sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c", "journal_sha256": "e2e0b650a36824e09db0d4544ad45d7dc8c42149aea7ec89906baf3421e68a1b", "closure": { "kind": "completed_result", "sha256": "9629be29651ded2dc859fbbc6efb2ca0dd65e9a79d91831066da94286cb20563" }, "known_cost_usd": "2.28802765", "held_unknown_reservation_usd": "0.005376" } ], "rejections": [ { "arm_id": "claude-haiku", "model_name": "Claude Haiku 5.5", "case_id": "banking77:test:2602", "http_status": 429, "classification": "known_no_answer", "cost_usd": null, "reserved_usd": "0.001792", "receipt_sha256": "3e779bd93881982832879d4175c760b5de39ab51ea75225b781857d073480ce5", "raw_response_sha256": "659d11b962d28a9e94f6a85b3e36706522f07c7e3ade1100f0996b8baf95b1a4" }, { "arm_id": "claude-haiku", "model_name": "Claude Haiku 5.5", "case_id": "banking77:test:2606", "http_status": 429, "classification": "known_no_answer", "cost_usd": null, "reserved_usd": "0.001792", "receipt_sha256": "e80f6b7da176ea0b1806f0a325e8653c74bf164164bf5fe9f8e9ba3d2bee5c77", "raw_response_sha256": "f2f5403d6211249b3d22d3c459e0df158979d608ba6979f8a664c2bf8659c2ad" }, { "arm_id": "claude-haiku", "model_name": "Claude Haiku 5.5", "case_id": "banking77:test:2608", "http_status": 429, "classification": "known_no_answer", "cost_usd": null, "reserved_usd": "0.001792", "receipt_sha256": "8c3796ec2ee0bdd2947fc8ef542631fa4b8533f504c1444823feea9e591144f5", "raw_response_sha256": "c937e2ba8a5d6d4a29bfc6e0d75951b0547f828048b344c85836eee640156405" }, { "arm_id": "claude-haiku", "model_name": "Claude Haiku 5.5", "case_id": "banking77:test:2612", "http_status": 429, "classification": "known_no_answer", "cost_usd": null, "reserved_usd": "0.001792", "receipt_sha256": "0661b90d0edb9af0033279c028c530a3d7fb72294c001737ce4f798f437e0cc8", "raw_response_sha256": "33f9c90ff70fcf71e001931dcd9da3a862fba9594098d73d785dd3f57946e16c" } ] }, "billing": { "reported_known_total_usd": "5.5974860649", "retained_unknown_reservation_usd": "0.013845", "scope": "All 12 compared model configurations; primary and challenge cases." }, "format_diagnostics": [ { "arm_id": "gemini-flash-lite", "phase": "primary", "strict_format_failures": 132, "fenced_valid_labels": 132 }, { "arm_id": "gemini-flash-lite", "phase": "challenge", "strict_format_failures": 14, "fenced_valid_labels": 14 }, { "arm_id": "gemini-flash", "phase": "primary", "strict_format_failures": 3, "fenced_valid_labels": 3 }, { "arm_id": "gemini-flash", "phase": "challenge", "strict_format_failures": 0, "fenced_valid_labels": 0 }, { "arm_id": "gpt-luna", "phase": "primary", "strict_format_failures": 0, "fenced_valid_labels": 0 }, { "arm_id": "gpt-luna", "phase": "challenge", "strict_format_failures": 0, "fenced_valid_labels": 0 }, { "arm_id": "gpt-sol", "phase": "primary", "strict_format_failures": 0, "fenced_valid_labels": 0 }, { "arm_id": "gpt-sol", "phase": "challenge", "strict_format_failures": 0, "fenced_valid_labels": 0 }, { "arm_id": "claude-haiku", "phase": "primary", "strict_format_failures": 7, "fenced_valid_labels": 0 }, { "arm_id": "claude-haiku", "phase": "challenge", "strict_format_failures": 0, "fenced_valid_labels": 0 }, { "arm_id": "claude-sonnet", "phase": "primary", "strict_format_failures": 109, "fenced_valid_labels": 0 }, { "arm_id": "claude-sonnet", "phase": "challenge", "strict_format_failures": 5, "fenced_valid_labels": 0 }, { "arm_id": "grok", "phase": "primary", "strict_format_failures": 0, "fenced_valid_labels": 0 }, { "arm_id": "grok", "phase": "challenge", "strict_format_failures": 0, "fenced_valid_labels": 0 } ], "source_pins": [ { "label": "Retained five-model summary", "sha256": "31d2f1a4ddc87eb9a6682c2c69f51a679d0c51f91bac9279772aa51c3d209b1d" }, { "label": "Extension plan", "sha256": "7d754c636ae5406ccd0ed86dee815bcc7b1e112167c68dedc9df0d05ae8368f4" }, { "label": "Untouched continuation plan", "sha256": "e1cbd00495afb9c5aa74e2b024c39d0034eb1c70130dbdc55109b1d79661075b" }, { "label": "Readiness amendment", "sha256": "c2e3f6e758920525a56cc1e22882a2aab8b562bda1e39195d7cc2e9c57e94b19" }, { "label": "Rejection continuation plan", "sha256": "d0e53b2ccbc823fb7a960bf5cf31e12c3c95b2cfc7465691725175e81e7e649c" }, { "label": "Rejection amendment", "sha256": "7b9e5e5933782f87c18368a75c52d341e778b5a1ab27ba630a159d4483007014" }, { "label": "Final result", "sha256": "9629be29651ded2dc859fbbc6efb2ca0dd65e9a79d91831066da94286cb20563" }, { "label": "Independent audit", "sha256": "ee9496dae06a559b90191b577fab02107ed411979801b78523d4479175206cdb" }, { "label": "Audited input manifest", "sha256": "9d5b9237701042b222b2ebcccb5754225f7baefaa22e09d828df36343d449b17" } ], "priority_sensitivity": { "version": "pisama-banking77-priority-sensitivity/1", "scope": "illustrative_post_collection_not_acceptance_policy", "metric_order": [ "correctness", "consistency", "cost", "p95_time" ], "scenarios": [ { "id": "default", "label": "Default", "weights": { "accuracy": 0.5, "agreement": 0.2, "cost": 0.2, "latency": 0.1 }, "origin": "existing_default", "ranking": [ { "arm_id": "gpt-luna", "name": "GPT-6 Luna", "score": 82.45060483660684 }, { "arm_id": "glm", "name": "GLM-5.3-Flash", "score": 82.06148227305523 }, { "arm_id": "gemini-flash-lite", "name": "Gemini 3.5 Flash Lite", "score": 81.55524636577893 }, { "arm_id": "gemini-flash", "name": "Gemini 3.8 Flash", "score": 76.7111988174993 }, { "arm_id": "qwen", "name": "Qwen3.7-Flash", "score": 76.651210014395 }, { "arm_id": "gpt-sol", "name": "GPT-6.1 Sol", "score": 75.16885879398845 }, { "arm_id": "incumbent-o4", "name": "o4-mini", "score": 72.4 }, { "arm_id": "grok", "name": "Grok 4.7", "score": 68.65280453996141 } ], "winner_ids": [ "gpt-luna" ], "winner_names": [ "GPT-6 Luna" ], "winner_score": 82.45060483660684, "margin": 0.38912256355160935, "status": "candidate", "change": "default", "caveats": [] }, { "id": "correctness", "label": "Correctness", "weights": { "accuracy": 0.8, "agreement": 0.1, "cost": 0.05, "latency": 0.05 }, "origin": "illustrative_post_collection", "ranking": [ { "arm_id": "gemini-flash", "name": "Gemini 3.8 Flash", "score": 81.23981967501079 }, { "arm_id": "gpt-luna", "name": "GPT-6 Luna", "score": 81.00353611098112 }, { "arm_id": "gpt-sol", "name": "GPT-6.1 Sol", "score": 80.26425824290722 }, { "arm_id": "glm", "name": "GLM-5.3-Flash", "score": 78.89949304995181 }, { "arm_id": "gemini-flash-lite", "name": "Gemini 3.5 Flash Lite", "score": 78.48931962767813 }, { "arm_id": "grok", "name": "Grok 4.7", "score": 76.78266476663183 }, { "arm_id": "qwen", "name": "Qwen3.7-Flash", "score": 74.99589097357061 }, { "arm_id": "incumbent-o4", "name": "o4-mini", "score": 74.84 } ], "winner_ids": [ "gemini-flash" ], "winner_names": [ "Gemini 3.8 Flash" ], "winner_score": 81.23981967501079, "margin": 0.23628356402967654, "status": "candidate", "change": "changed", "caveats": [ "Gemini 3.8 Flash: 3/500 strict-format violations; intent scoring still accepts valid fenced JSON. Validate the output contract before use." ] }, { "id": "consistency", "label": "Consistency", "weights": { "accuracy": 0.3, "agreement": 0.5, "cost": 0.1, "latency": 0.1 }, "origin": "illustrative_post_collection", "ranking": [ { "arm_id": "gemini-flash-lite", "name": "Gemini 3.5 Flash Lite", "score": 82.89863925535627 }, { "arm_id": "incumbent-o4", "name": "o4-mini", "score": 82.44 }, { "arm_id": "gpt-luna", "name": "GPT-6 Luna", "score": 82.1470722219622 }, { "arm_id": "glm", "name": "GLM-5.3-Flash", "score": 81.97898609990362 }, { "arm_id": "gemini-flash", "name": "Gemini 3.8 Flash", "score": 78.99963935002158 }, { "arm_id": "gpt-sol", "name": "GPT-6.1 Sol", "score": 77.90851648581443 }, { "arm_id": "qwen", "name": "Qwen3.7-Flash", "score": 77.29178194714122 }, { "arm_id": "grok", "name": "Grok 4.7", "score": 75.44532953326362 } ], "winner_ids": [ "gemini-flash-lite" ], "winner_names": [ "Gemini 3.5 Flash Lite" ], "winner_score": 82.89863925535627, "margin": 0.4586392553562746, "status": "candidate", "change": "changed", "caveats": [ "Gemini 3.5 Flash Lite: 132/500 strict-format violations; intent scoring still accepts valid fenced JSON. Validate the output contract before use." ] }, { "id": "cost", "label": "Cost", "weights": { "accuracy": 0.3, "agreement": 0.1, "cost": 0.5, "latency": 0.1 }, "origin": "illustrative_post_collection", "ranking": [ { "arm_id": "glm", "name": "GLM-5.3-Flash", "score": 87.58897079250998 }, { "arm_id": "gpt-luna", "name": "GPT-6 Luna", "score": 86.40120268054076 }, { "arm_id": "gemini-flash-lite", "name": "Gemini 3.5 Flash Lite", "score": 84.24506769704693 }, { "arm_id": "qwen", "name": "Qwen3.7-Flash", "score": 81.92949421615634 }, { "arm_id": "gemini-flash", "name": "Gemini 3.8 Flash", "score": 70.16587721993243 }, { "arm_id": "gpt-sol", "name": "GPT-6.1 Sol", "score": 67.42988571851049 }, { "arm_id": "incumbent-o4", "name": "o4-mini", "score": 62.440000000000005 }, { "arm_id": "grok", "name": "Grok 4.7", "score": 54.35522956005474 } ], "winner_ids": [ "glm" ], "winner_names": [ "GLM-5.3-Flash" ], "winner_score": 87.58897079250998, "margin": 1.1877681119692198, "status": "candidate", "change": "changed", "caveats": [] }, { "id": "latency", "label": "Latency", "weights": { "accuracy": 0.3, "agreement": 0.1, "cost": 0.1, "latency": 0.5 }, "origin": "illustrative_post_collection", "ranking": [ { "arm_id": "gemini-flash-lite", "name": "Gemini 3.5 Flash Lite", "score": 83.22676783509067 }, { "arm_id": "glm", "name": "GLM-5.3-Flash", "score": 74.60494580691174 }, { "arm_id": "gpt-luna", "name": "GPT-6 Luna", "score": 71.8412306512324 }, { "arm_id": "gemini-flash", "name": "Gemini 3.8 Flash", "score": 68.31195888019708 }, { "arm_id": "gpt-sol", "name": "GPT-6.1 Sol", "score": 67.14121319637607 }, { "arm_id": "incumbent-o4", "name": "o4-mini", "score": 62.440000000000005 }, { "arm_id": "grok", "name": "Grok 4.7", "score": 55.436747639526985 }, { "arm_id": "qwen", "name": "Qwen3.7-Flash", "score": 55.02119746669091 } ], "winner_ids": [ "gemini-flash-lite" ], "winner_names": [ "Gemini 3.5 Flash Lite" ], "winner_score": 83.22676783509067, "margin": 8.621822028178926, "status": "candidate", "change": "changed", "caveats": [ "Gemini 3.5 Flash Lite: 132/500 strict-format violations; intent scoring still accepts valid fenced JSON. Validate the output contract before use." ] } ] } }