{
  "version": "pisama-banking77-adaptive-collection-presentation-v1",
  "status": "adaptive_collected_with_unresolved",
  "score_release_scope": "preferred_among_fully_evidenced_adaptive_configurations",
  "original_whole_completion_gate_passed": false,
  "source_sha256": "31d2f1a4ddc87eb9a6682c2c69f51a679d0c51f91bac9279772aa51c3d209b1d",
  "prepared_at": "2026-10-13T06:43:08Z",
  "dates": {
    "first": "2026-10-13T01:45:24.835240+00:00",
    "last": "2026-10-13T06:29:05.506119+00:00"
  },
  "primary": [
    {
      "arm_id": "incumbent-o4",
      "name": "o4-mini",
      "model": "openai/o4-mini",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 500,
        "missing": 0,
        "unresolved": 0,
        "confirmed_errors": 126,
        "wrong_reference": 126,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "agreement_count": 500,
        "paired_valid": 500,
        "paired_improvements": 0,
        "paired_regressions": 0,
        "both_wrong": 126,
        "both_wrong_different": 0,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 374,
        "improvements": 0,
        "regressions": 0,
        "shared_wrong_agreement": 126,
        "correct_incumbent_retention": 1,
        "priced": 500,
        "reported_cost_total_usd": 0.8805807999999997,
        "mean_cost_usd": 0.0017611615999999994,
        "cost_per_1000_usd": 1.7611615999999994,
        "timed": 500,
        "p50_seconds": 2.069488749606535,
        "p95_seconds": 5.8729631889611476,
        "cost_coverage_complete": true
      },
      "eligible": true,
      "reasons": [],
      "components": {
        "accuracy": 0.748,
        "agreement": 1,
        "cost": 0.5,
        "latency": 0.5
      },
      "scores": {
        "default": 72.4,
        "quality": 72.36,
        "economy": 67.4,
        "latency": 67.4
      },
      "pareto": true,
      "coverage": {
        "planned": 500,
        "attempted": 500,
        "resolved": 500,
        "unresolved": 0,
        "correct": 374,
        "confirmed_errors": 126,
        "accuracy": {
          "numerator": 374,
          "denominator": 500,
          "unresolved": 0,
          "value": 0.748,
          "lower": 0.748,
          "upper": 0.748
        },
        "end_to_end": {
          "numerator": 500,
          "denominator": 500,
          "unresolved": 0,
          "value": 1,
          "lower": 1,
          "upper": 1
        },
        "valid_output": {
          "numerator": 500,
          "denominator": 500,
          "unresolved": 0,
          "value": 1,
          "lower": 1,
          "upper": 1
        },
        "valid_answer_agreement": 1,
        "both_valid_pairs": 500,
        "valid_answer_agreement_bounds": {
          "lower": 1,
          "upper": 1
        }
      }
    },
    {
      "arm_id": "deepseek",
      "name": "DeepSeek V4 Flash",
      "model": "deepseek/deepseek-v4-flash-0731",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 498,
        "missing": 0,
        "unresolved": 1,
        "confirmed_errors": 125,
        "wrong_reference": 124,
        "invalid_output": 1,
        "provider_error": 0,
        "wrong_model": 0,
        "agreement_count": 437,
        "paired_valid": 498,
        "paired_improvements": 20,
        "paired_regressions": 20,
        "both_wrong": 104,
        "both_wrong_different": 21,
        "unpaired_unresolved_or_missing": 1,
        "unpaired_failure": 1,
        "both_correct": 354,
        "improvements": 20,
        "regressions": 20,
        "shared_wrong_agreement": 83,
        "correct_incumbent_retention": 0.946524064171123,
        "priced": 499,
        "reported_cost_total_usd": 0.07319557919999997,
        "mean_cost_usd": null,
        "cost_per_1000_usd": null,
        "timed": 500,
        "p50_seconds": 5.6534198962617666,
        "p95_seconds": 24.156308278767387,
        "cost_coverage_complete": false
      },
      "eligible": false,
      "reasons": [
        "unresolved_outputs",
        "incomplete_all_attempt_pricing",
        "full_run_unresolved_or_missing",
        "full_run_incomplete_all_attempt_pricing"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false,
      "coverage": {
        "planned": 500,
        "attempted": 500,
        "resolved": 499,
        "unresolved": 1,
        "correct": 374,
        "confirmed_errors": 125,
        "accuracy": {
          "numerator": 374,
          "denominator": 500,
          "unresolved": 1,
          "value": null,
          "lower": 0.748,
          "upper": 0.75
        },
        "end_to_end": {
          "numerator": 437,
          "denominator": 500,
          "unresolved": 1,
          "value": null,
          "lower": 0.874,
          "upper": 0.876
        },
        "valid_output": {
          "numerator": 498,
          "denominator": 500,
          "unresolved": 1,
          "value": null,
          "lower": 0.996,
          "upper": 0.998
        },
        "valid_answer_agreement": 0.8775100401606426,
        "both_valid_pairs": 498,
        "valid_answer_agreement_bounds": {
          "lower": 0.875751503006012,
          "upper": 0.8777555110220441
        }
      }
    },
    {
      "arm_id": "qwen",
      "name": "Qwen3.7-Flash",
      "model": "qwen/qwen3.7-flash",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 499,
        "missing": 0,
        "unresolved": 0,
        "confirmed_errors": 123,
        "wrong_reference": 122,
        "invalid_output": 1,
        "provider_error": 0,
        "wrong_model": 0,
        "agreement_count": 422,
        "paired_valid": 499,
        "paired_improvements": 27,
        "paired_regressions": 23,
        "both_wrong": 99,
        "both_wrong_different": 27,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 1,
        "both_correct": 350,
        "improvements": 27,
        "regressions": 24,
        "shared_wrong_agreement": 72,
        "correct_incumbent_retention": 0.9358288770053476,
        "priced": 500,
        "reported_cost_total_usd": 0.036745517599999986,
        "mean_cost_usd": 0.00007349103519999997,
        "cost_per_1000_usd": 0.07349103519999997,
        "timed": 500,
        "p50_seconds": 4.468798020621762,
        "p95_seconds": 14.573553621117018,
        "cost_coverage_complete": true
      },
      "eligible": true,
      "reasons": [],
      "components": {
        "accuracy": 0.754,
        "agreement": 0.844,
        "cost": 0.9599428067253785,
        "latency": 0.2872353879887425
      },
      "scores": {
        "default": 76.651210014395,
        "quality": 73.69178194714121,
        "economy": 77.81063808164879,
        "latency": 64.35648970691605
      },
      "pareto": false,
      "coverage": {
        "planned": 500,
        "attempted": 500,
        "resolved": 500,
        "unresolved": 0,
        "correct": 377,
        "confirmed_errors": 123,
        "accuracy": {
          "numerator": 377,
          "denominator": 500,
          "unresolved": 0,
          "value": 0.754,
          "lower": 0.754,
          "upper": 0.754
        },
        "end_to_end": {
          "numerator": 422,
          "denominator": 500,
          "unresolved": 0,
          "value": 0.844,
          "lower": 0.844,
          "upper": 0.844
        },
        "valid_output": {
          "numerator": 499,
          "denominator": 500,
          "unresolved": 0,
          "value": 0.998,
          "lower": 0.998,
          "upper": 0.998
        },
        "valid_answer_agreement": 0.845691382765531,
        "both_valid_pairs": 499,
        "valid_answer_agreement_bounds": {
          "lower": 0.845691382765531,
          "upper": 0.845691382765531
        }
      }
    },
    {
      "arm_id": "nemotron",
      "name": "Nemotron 3.5 Lightning",
      "model": "nvidia/nemotron-3.5-lightning",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 497,
        "missing": 0,
        "unresolved": 0,
        "confirmed_errors": 120,
        "wrong_reference": 117,
        "invalid_output": 1,
        "provider_error": 2,
        "wrong_model": 0,
        "agreement_count": 419,
        "paired_valid": 497,
        "paired_improvements": 28,
        "paired_regressions": 21,
        "both_wrong": 96,
        "both_wrong_different": 29,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 3,
        "both_correct": 352,
        "improvements": 28,
        "regressions": 22,
        "shared_wrong_agreement": 67,
        "correct_incumbent_retention": 0.9411764705882353,
        "priced": 499,
        "reported_cost_total_usd": 0.12556890000000004,
        "mean_cost_usd": null,
        "cost_per_1000_usd": null,
        "timed": 500,
        "p50_seconds": 3.6871269163675606,
        "p95_seconds": 13.225167969940227,
        "cost_coverage_complete": false
      },
      "eligible": false,
      "reasons": [
        "incomplete_all_attempt_pricing",
        "full_run_incomplete_all_attempt_pricing"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false,
      "coverage": {
        "planned": 500,
        "attempted": 500,
        "resolved": 500,
        "unresolved": 0,
        "correct": 380,
        "confirmed_errors": 120,
        "accuracy": {
          "numerator": 380,
          "denominator": 500,
          "unresolved": 0,
          "value": 0.76,
          "lower": 0.76,
          "upper": 0.76
        },
        "end_to_end": {
          "numerator": 419,
          "denominator": 500,
          "unresolved": 0,
          "value": 0.838,
          "lower": 0.838,
          "upper": 0.838
        },
        "valid_output": {
          "numerator": 497,
          "denominator": 500,
          "unresolved": 0,
          "value": 0.994,
          "lower": 0.994,
          "upper": 0.994
        },
        "valid_answer_agreement": 0.8430583501006036,
        "both_valid_pairs": 497,
        "valid_answer_agreement_bounds": {
          "lower": 0.8430583501006036,
          "upper": 0.8430583501006036
        }
      }
    },
    {
      "arm_id": "glm",
      "name": "GLM-5.3-Flash",
      "model": "z-ai/glm-5.3-flash",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 500,
        "missing": 0,
        "unresolved": 0,
        "confirmed_errors": 111,
        "wrong_reference": 111,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "agreement_count": 422,
        "paired_valid": 500,
        "paired_improvements": 35,
        "paired_regressions": 20,
        "both_wrong": 91,
        "both_wrong_different": 23,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 354,
        "improvements": 35,
        "regressions": 20,
        "shared_wrong_agreement": 68,
        "correct_incumbent_retention": 0.946524064171123,
        "priced": 500,
        "reported_cost_total_usd": 0.014091429999999993,
        "mean_cost_usd": 0.000028182859999999988,
        "cost_per_1000_usd": 0.028182859999999987,
        "timed": 500,
        "p50_seconds": 1.3195262704975903,
        "p95_seconds": 3.0302008485421537,
        "cost_coverage_complete": true
      },
      "eligible": true,
      "reasons": [],
      "components": {
        "accuracy": 0.778,
        "agreement": 0.844,
        "cost": 0.984249617315159,
        "latency": 0.6596489926752033
      },
      "scores": {
        "default": 82.06148227305523,
        "quality": 79.33898609990364,
        "economy": 83.46397844620681,
        "latency": 76.97196595340769
      },
      "pareto": true,
      "coverage": {
        "planned": 500,
        "attempted": 500,
        "resolved": 500,
        "unresolved": 0,
        "correct": 389,
        "confirmed_errors": 111,
        "accuracy": {
          "numerator": 389,
          "denominator": 500,
          "unresolved": 0,
          "value": 0.778,
          "lower": 0.778,
          "upper": 0.778
        },
        "end_to_end": {
          "numerator": 422,
          "denominator": 500,
          "unresolved": 0,
          "value": 0.844,
          "lower": 0.844,
          "upper": 0.844
        },
        "valid_output": {
          "numerator": 500,
          "denominator": 500,
          "unresolved": 0,
          "value": 1,
          "lower": 1,
          "upper": 1
        },
        "valid_answer_agreement": 0.844,
        "both_valid_pairs": 500,
        "valid_answer_agreement_bounds": {
          "lower": 0.844,
          "upper": 0.844
        }
      }
    }
  ],
  "challenge": [
    {
      "arm_id": "incumbent-o4",
      "name": "o4-mini",
      "model": "openai/o4-mini",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 30,
        "missing": 0,
        "unresolved": 0,
        "confirmed_errors": 5,
        "wrong_reference": 5,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "agreement_count": 30,
        "paired_valid": 30,
        "paired_improvements": 0,
        "paired_regressions": 0,
        "both_wrong": 5,
        "both_wrong_different": 0,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 25,
        "improvements": 0,
        "regressions": 0,
        "shared_wrong_agreement": 5,
        "correct_incumbent_retention": 1,
        "priced": 30,
        "reported_cost_total_usd": 0.0390434,
        "mean_cost_usd": 0.0013014466666666667,
        "cost_per_1000_usd": 1.3014466666666666,
        "timed": 30,
        "p50_seconds": 1.4551106039434671,
        "p95_seconds": 4.676944721047764,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false,
      "coverage": {
        "planned": 30,
        "attempted": 30,
        "resolved": 30,
        "unresolved": 0,
        "correct": 25,
        "confirmed_errors": 5,
        "accuracy": {
          "numerator": 25,
          "denominator": 30,
          "unresolved": 0,
          "value": 0.8333333333333334,
          "lower": 0.8333333333333334,
          "upper": 0.8333333333333334
        },
        "end_to_end": {
          "numerator": 30,
          "denominator": 30,
          "unresolved": 0,
          "value": 1,
          "lower": 1,
          "upper": 1
        },
        "valid_output": {
          "numerator": 30,
          "denominator": 30,
          "unresolved": 0,
          "value": 1,
          "lower": 1,
          "upper": 1
        },
        "valid_answer_agreement": 1,
        "both_valid_pairs": 30,
        "valid_answer_agreement_bounds": {
          "lower": 1,
          "upper": 1
        }
      }
    },
    {
      "arm_id": "deepseek",
      "name": "DeepSeek V4 Flash",
      "model": "deepseek/deepseek-v4-flash-0731",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 30,
        "missing": 0,
        "unresolved": 0,
        "confirmed_errors": 5,
        "wrong_reference": 5,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "agreement_count": 28,
        "paired_valid": 30,
        "paired_improvements": 1,
        "paired_regressions": 1,
        "both_wrong": 4,
        "both_wrong_different": 0,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 24,
        "improvements": 1,
        "regressions": 1,
        "shared_wrong_agreement": 4,
        "correct_incumbent_retention": 0.96,
        "priced": 30,
        "reported_cost_total_usd": 0.0055894656,
        "mean_cost_usd": 0.00018631552,
        "cost_per_1000_usd": 0.18631552,
        "timed": 30,
        "p50_seconds": 4.610919750062749,
        "p95_seconds": 28.01468085038939,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false,
      "coverage": {
        "planned": 30,
        "attempted": 30,
        "resolved": 30,
        "unresolved": 0,
        "correct": 25,
        "confirmed_errors": 5,
        "accuracy": {
          "numerator": 25,
          "denominator": 30,
          "unresolved": 0,
          "value": 0.8333333333333334,
          "lower": 0.8333333333333334,
          "upper": 0.8333333333333334
        },
        "end_to_end": {
          "numerator": 28,
          "denominator": 30,
          "unresolved": 0,
          "value": 0.9333333333333333,
          "lower": 0.9333333333333333,
          "upper": 0.9333333333333333
        },
        "valid_output": {
          "numerator": 30,
          "denominator": 30,
          "unresolved": 0,
          "value": 1,
          "lower": 1,
          "upper": 1
        },
        "valid_answer_agreement": 0.9333333333333333,
        "both_valid_pairs": 30,
        "valid_answer_agreement_bounds": {
          "lower": 0.9333333333333333,
          "upper": 0.9333333333333333
        }
      }
    },
    {
      "arm_id": "qwen",
      "name": "Qwen3.7-Flash",
      "model": "qwen/qwen3.7-flash",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 30,
        "missing": 0,
        "unresolved": 0,
        "confirmed_errors": 6,
        "wrong_reference": 6,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "agreement_count": 29,
        "paired_valid": 30,
        "paired_improvements": 0,
        "paired_regressions": 1,
        "both_wrong": 5,
        "both_wrong_different": 0,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 24,
        "improvements": 0,
        "regressions": 1,
        "shared_wrong_agreement": 5,
        "correct_incumbent_retention": 0.96,
        "priced": 30,
        "reported_cost_total_usd": 0.0017090724999999998,
        "mean_cost_usd": 0.000056969083333333326,
        "cost_per_1000_usd": 0.05696908333333332,
        "timed": 30,
        "p50_seconds": 3.6155716460198164,
        "p95_seconds": 7.685869106464084,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "error_rate_worse_than_incumbent",
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false,
      "coverage": {
        "planned": 30,
        "attempted": 30,
        "resolved": 30,
        "unresolved": 0,
        "correct": 24,
        "confirmed_errors": 6,
        "accuracy": {
          "numerator": 24,
          "denominator": 30,
          "unresolved": 0,
          "value": 0.8,
          "lower": 0.8,
          "upper": 0.8
        },
        "end_to_end": {
          "numerator": 29,
          "denominator": 30,
          "unresolved": 0,
          "value": 0.9666666666666667,
          "lower": 0.9666666666666667,
          "upper": 0.9666666666666667
        },
        "valid_output": {
          "numerator": 30,
          "denominator": 30,
          "unresolved": 0,
          "value": 1,
          "lower": 1,
          "upper": 1
        },
        "valid_answer_agreement": 0.9666666666666667,
        "both_valid_pairs": 30,
        "valid_answer_agreement_bounds": {
          "lower": 0.9666666666666667,
          "upper": 0.9666666666666667
        }
      }
    },
    {
      "arm_id": "nemotron",
      "name": "Nemotron 3.5 Lightning",
      "model": "nvidia/nemotron-3.5-lightning",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 30,
        "missing": 0,
        "unresolved": 0,
        "confirmed_errors": 5,
        "wrong_reference": 5,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "agreement_count": 28,
        "paired_valid": 30,
        "paired_improvements": 1,
        "paired_regressions": 1,
        "both_wrong": 4,
        "both_wrong_different": 0,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 24,
        "improvements": 1,
        "regressions": 1,
        "shared_wrong_agreement": 4,
        "correct_incumbent_retention": 0.96,
        "priced": 30,
        "reported_cost_total_usd": 0.00704816,
        "mean_cost_usd": 0.0002349386666666667,
        "cost_per_1000_usd": 0.23493866666666668,
        "timed": 30,
        "p50_seconds": 4.420968979597092,
        "p95_seconds": 9.632389781461097,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false,
      "coverage": {
        "planned": 30,
        "attempted": 30,
        "resolved": 30,
        "unresolved": 0,
        "correct": 25,
        "confirmed_errors": 5,
        "accuracy": {
          "numerator": 25,
          "denominator": 30,
          "unresolved": 0,
          "value": 0.8333333333333334,
          "lower": 0.8333333333333334,
          "upper": 0.8333333333333334
        },
        "end_to_end": {
          "numerator": 28,
          "denominator": 30,
          "unresolved": 0,
          "value": 0.9333333333333333,
          "lower": 0.9333333333333333,
          "upper": 0.9333333333333333
        },
        "valid_output": {
          "numerator": 30,
          "denominator": 30,
          "unresolved": 0,
          "value": 1,
          "lower": 1,
          "upper": 1
        },
        "valid_answer_agreement": 0.9333333333333333,
        "both_valid_pairs": 30,
        "valid_answer_agreement_bounds": {
          "lower": 0.9333333333333333,
          "upper": 0.9333333333333333
        }
      }
    },
    {
      "arm_id": "glm",
      "name": "GLM-5.3-Flash",
      "model": "z-ai/glm-5.3-flash",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 30,
        "missing": 0,
        "unresolved": 0,
        "confirmed_errors": 7,
        "wrong_reference": 7,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "agreement_count": 27,
        "paired_valid": 30,
        "paired_improvements": 0,
        "paired_regressions": 2,
        "both_wrong": 5,
        "both_wrong_different": 1,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 23,
        "improvements": 0,
        "regressions": 2,
        "shared_wrong_agreement": 4,
        "correct_incumbent_retention": 0.92,
        "priced": 30,
        "reported_cost_total_usd": 0.0009234899999999999,
        "mean_cost_usd": 0.000030783,
        "cost_per_1000_usd": 0.030783,
        "timed": 30,
        "p50_seconds": 1.3050088544841856,
        "p95_seconds": 2.8308222580002615,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "error_rate_worse_than_incumbent",
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false,
      "coverage": {
        "planned": 30,
        "attempted": 30,
        "resolved": 30,
        "unresolved": 0,
        "correct": 23,
        "confirmed_errors": 7,
        "accuracy": {
          "numerator": 23,
          "denominator": 30,
          "unresolved": 0,
          "value": 0.7666666666666667,
          "lower": 0.7666666666666667,
          "upper": 0.7666666666666667
        },
        "end_to_end": {
          "numerator": 27,
          "denominator": 30,
          "unresolved": 0,
          "value": 0.9,
          "lower": 0.9,
          "upper": 0.9
        },
        "valid_output": {
          "numerator": 30,
          "denominator": 30,
          "unresolved": 0,
          "value": 1,
          "lower": 1,
          "upper": 1
        },
        "valid_answer_agreement": 0.9,
        "both_valid_pairs": 30,
        "valid_answer_agreement_bounds": {
          "lower": 0.9,
          "upper": 0.9
        }
      }
    }
  ],
  "winners": {
    "default": [
      "glm"
    ],
    "quality": [
      "glm"
    ],
    "economy": [
      "glm"
    ],
    "latency": [
      "glm"
    ]
  },
  "source_parent_plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
  "source_parent_result_sha256": "011822529936c0e244456e1abce733cac12bd44b70c9337c4e3ad83d75e26c91",
  "source_continuation_plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1",
  "source_continuation_result_sha256": "78cf2288bb9462aa0fb0813c6ce49b3768832c81c271bf77af5e7a0af5a6e1b9",
  "source_reporting_amendment_sha256": "b940a96188f7adfa9affcc629f668e90be015faea136c126847b76ff1b726da9",
  "source_adaptive_plan_sha256": "9233745dc9c1a76050b1d2922aac48dcc4454ab2831b20fc29dec5c11b4c91b5",
  "source_adaptive_result_sha256": "b02ea1b381ba9ed0934c7c66857f42d4b76c9f3aa69f51ce44680b5c2b9e111e",
  "collection": {
    "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33",
    "result_sha256": "3d7f93d43856670f0ca5fb77783384308fab20c25825da2c6e50de11235ce097",
    "amendment_sha256": "6a6da5802d9cdab3a8f08c1e43499fae4436592bba7579394a660daf4cfb46f8"
  },
  "completion": {
    "planned": 2650,
    "attempted": 2650,
    "resolved": 2649,
    "unresolved": 1,
    "unknown_charge_attempts": 2
  },
  "billing": {
    "reported_known_total_usd": "1.1844958149",
    "retained_unknown_reservation_usd": "0.006677",
    "by_arm": [
      {
        "arm_id": "incumbent-o4",
        "priced": 530,
        "reported_known_total_usd": "0.9196242",
        "retained_unknown_reservation_usd": "0",
        "all_attempt_costs_known": true
      },
      {
        "arm_id": "deepseek",
        "priced": 529,
        "reported_known_total_usd": "0.0787850448",
        "retained_unknown_reservation_usd": "0.004731",
        "all_attempt_costs_known": false
      },
      {
        "arm_id": "qwen",
        "priced": 530,
        "reported_known_total_usd": "0.0384545901",
        "retained_unknown_reservation_usd": "0",
        "all_attempt_costs_known": true
      },
      {
        "arm_id": "nemotron",
        "priced": 529,
        "reported_known_total_usd": "0.13261706",
        "retained_unknown_reservation_usd": "0.001946",
        "all_attempt_costs_known": false
      },
      {
        "arm_id": "glm",
        "priced": 530,
        "reported_known_total_usd": "0.01501492",
        "retained_unknown_reservation_usd": "0",
        "all_attempt_costs_known": true
      }
    ]
  },
  "configurations": [
    {
      "id": "incumbent-o4",
      "model": "openai/o4-mini",
      "provider": "openai",
      "reasoning": "Enabled",
      "output_tokens": 2048,
      "retry_output_tokens": 4096,
      "configuration_sha256": "bd4a4701b671062224cbb261f4a3dfde20de5839bef024174ad45e292ba69d42"
    },
    {
      "id": "deepseek",
      "model": "deepseek/deepseek-v4-flash-0731",
      "provider": "streamlake/fp8",
      "reasoning": "Low effort",
      "output_tokens": 8192,
      "retry_output_tokens": 16384,
      "configuration_sha256": "cfc691eb10c1da0e83d293f3b0708d9bcfef60679e909f0b2a129bdc16096252"
    },
    {
      "id": "qwen",
      "model": "qwen/qwen3.7-flash",
      "provider": "alibaba/us-east-1",
      "reasoning": "1,024 reasoning tokens",
      "output_tokens": 2048,
      "retry_output_tokens": 4096,
      "configuration_sha256": "ddf7b9ef3b26defddc52ab1e09d7be88d0a5d4606fa9f05190a2fef5353a1f04"
    },
    {
      "id": "nemotron",
      "model": "nvidia/nemotron-3.5-lightning",
      "provider": "deepinfra/bf16",
      "reasoning": "Enabled",
      "output_tokens": 8192,
      "retry_output_tokens": 16384,
      "configuration_sha256": "efc0ed3d282863324c18c2d0e60cf477de96ebd0b4b981d56b100e17c548aff7"
    },
    {
      "id": "glm",
      "model": "z-ai/glm-5.3-flash",
      "provider": "z-ai/fp8",
      "reasoning": "Low effort",
      "output_tokens": 2048,
      "retry_output_tokens": 4096,
      "configuration_sha256": "83331aff5c4a888f95ae2ab98de2b7810197abdd5e237e506d324ab9a1e87b3e"
    }
  ],
  "segments": [
    {
      "name": "Original prefix",
      "attempted": 248,
      "timeout_seconds": 120,
      "first": "2026-10-13T01:45:24.835240+00:00",
      "last": "2026-10-13T02:07:38.505022+00:00",
      "pause_seconds": null
    },
    {
      "name": "First continuation",
      "attempted": 1091,
      "timeout_seconds": 300,
      "first": "2026-10-13T02:26:08.824426+00:00",
      "last": "2026-10-13T04:01:17.158469+00:00",
      "pause_seconds": 1110.319
    },
    {
      "name": "Adaptive continuation",
      "attempted": 28,
      "timeout_seconds": 300,
      "first": "2026-10-13T04:21:57.683313+00:00",
      "last": "2026-10-13T04:26:58.193289+00:00",
      "pause_seconds": 1240.525
    },
    {
      "name": "Collection continuation",
      "attempted": 1284,
      "timeout_seconds": 300,
      "first": "2026-10-13T04:49:13.649616+00:00",
      "last": "2026-10-13T06:29:05.506119+00:00",
      "pause_seconds": 1335.456
    }
  ],
  "adaptive": {
    "policy_sha256": "b940a96188f7adfa9affcc629f668e90be015faea136c126847b76ff1b726da9",
    "timing_protocol": "adaptive-cumulative-http-attempt-time-v1",
    "physical_attempts": 2651,
    "required_retries": 1,
    "retry_receipts": 1,
    "recovery": [
      {
        "planned": 500,
        "first_attempted": 500,
        "first_valid": 499,
        "first_correct": 374,
        "first_cap_failures": 1,
        "first_other_failures": 0,
        "first_unresolved": 0,
        "retry_required": 1,
        "retry_attempted": 1,
        "recovered_valid": 1,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 501,
        "arm_id": "incumbent-o4",
        "name": "o4-mini",
        "phase": "primary",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      },
      {
        "planned": 500,
        "first_attempted": 500,
        "first_valid": 498,
        "first_correct": 374,
        "first_cap_failures": 0,
        "first_other_failures": 1,
        "first_unresolved": 1,
        "retry_required": 0,
        "retry_attempted": 0,
        "recovered_valid": 0,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 500,
        "arm_id": "deepseek",
        "name": "DeepSeek V4 Flash",
        "phase": "primary",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      },
      {
        "planned": 500,
        "first_attempted": 500,
        "first_valid": 499,
        "first_correct": 377,
        "first_cap_failures": 0,
        "first_other_failures": 1,
        "first_unresolved": 0,
        "retry_required": 0,
        "retry_attempted": 0,
        "recovered_valid": 0,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 500,
        "arm_id": "qwen",
        "name": "Qwen3.7-Flash",
        "phase": "primary",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      },
      {
        "planned": 500,
        "first_attempted": 500,
        "first_valid": 497,
        "first_correct": 380,
        "first_cap_failures": 0,
        "first_other_failures": 3,
        "first_unresolved": 0,
        "retry_required": 0,
        "retry_attempted": 0,
        "recovered_valid": 0,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 500,
        "arm_id": "nemotron",
        "name": "Nemotron 3.5 Lightning",
        "phase": "primary",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      },
      {
        "planned": 500,
        "first_attempted": 500,
        "first_valid": 500,
        "first_correct": 389,
        "first_cap_failures": 0,
        "first_other_failures": 0,
        "first_unresolved": 0,
        "retry_required": 0,
        "retry_attempted": 0,
        "recovered_valid": 0,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 500,
        "arm_id": "glm",
        "name": "GLM-5.3-Flash",
        "phase": "primary",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      },
      {
        "planned": 30,
        "first_attempted": 30,
        "first_valid": 30,
        "first_correct": 25,
        "first_cap_failures": 0,
        "first_other_failures": 0,
        "first_unresolved": 0,
        "retry_required": 0,
        "retry_attempted": 0,
        "recovered_valid": 0,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 30,
        "arm_id": "incumbent-o4",
        "name": "o4-mini",
        "phase": "challenge",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      },
      {
        "planned": 30,
        "first_attempted": 30,
        "first_valid": 30,
        "first_correct": 25,
        "first_cap_failures": 0,
        "first_other_failures": 0,
        "first_unresolved": 0,
        "retry_required": 0,
        "retry_attempted": 0,
        "recovered_valid": 0,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 30,
        "arm_id": "deepseek",
        "name": "DeepSeek V4 Flash",
        "phase": "challenge",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      },
      {
        "planned": 30,
        "first_attempted": 30,
        "first_valid": 30,
        "first_correct": 24,
        "first_cap_failures": 0,
        "first_other_failures": 0,
        "first_unresolved": 0,
        "retry_required": 0,
        "retry_attempted": 0,
        "recovered_valid": 0,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 30,
        "arm_id": "qwen",
        "name": "Qwen3.7-Flash",
        "phase": "challenge",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      },
      {
        "planned": 30,
        "first_attempted": 30,
        "first_valid": 30,
        "first_correct": 25,
        "first_cap_failures": 0,
        "first_other_failures": 0,
        "first_unresolved": 0,
        "retry_required": 0,
        "retry_attempted": 0,
        "recovered_valid": 0,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 30,
        "arm_id": "nemotron",
        "name": "Nemotron 3.5 Lightning",
        "phase": "challenge",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      },
      {
        "planned": 30,
        "first_attempted": 30,
        "first_valid": 30,
        "first_correct": 23,
        "first_cap_failures": 0,
        "first_other_failures": 0,
        "first_unresolved": 0,
        "retry_required": 0,
        "retry_attempted": 0,
        "recovered_valid": 0,
        "recovered_correct": 0,
        "retry_exhausted_failures": 0,
        "retry_unresolved": 0,
        "total_http_attempts": 30,
        "arm_id": "glm",
        "name": "GLM-5.3-Flash",
        "phase": "challenge",
        "retry_second_length": 0,
        "retry_unknown_charges": 0
      }
    ],
    "billing_attempts": [
      {
        "arm_id": "incumbent-o4",
        "physical_attempts": 531,
        "priced_attempts": 531
      },
      {
        "arm_id": "deepseek",
        "physical_attempts": 530,
        "priced_attempts": 529
      },
      {
        "arm_id": "qwen",
        "physical_attempts": 530,
        "priced_attempts": 530
      },
      {
        "arm_id": "nemotron",
        "physical_attempts": 530,
        "priced_attempts": 529
      },
      {
        "arm_id": "glm",
        "physical_attempts": 530,
        "priced_attempts": 530
      }
    ]
  }
}