{
  "version": "pisama-benchmark-supplement/1",
  "source_sha256": "7687c9fbec7063ae1835e8d44ff9611e51f3bc882d315325c917ca6bc3e54f5e",
  "preview_only": false,
  "prepared_at": "2026-10-10T08:13:29.530090+00:00",
  "status": "completed_with_provider_rejections",
  "dates": {
    "first": "2026-10-10T05:37:25.337011+00:00",
    "last": "2026-10-10T08:12:08.754252+00:00"
  },
  "attempted": 2650,
  "never_sent": 0,
  "primary": [
    {
      "arm_id": "incumbent-o4",
      "name": "o4-mini",
      "model": "openai/o4-mini",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 500,
        "missing": 0,
        "unresolved": 0,
        "errors": 121,
        "confirmed_errors": 121,
        "wrong_reference": 121,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "accuracy": 0.758,
        "agreement_count": 500,
        "agreement": 1,
        "paired_valid": 500,
        "paired_improvements": 0,
        "paired_regressions": 0,
        "both_wrong": 121,
        "both_wrong_different": 0,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 379,
        "improvements": 0,
        "regressions": 0,
        "shared_wrong_agreement": 121,
        "correct_incumbent_retention": 1,
        "priced": 500,
        "reported_cost_total_usd": 0.8783103999999995,
        "mean_cost_usd": 0.001756620799999999,
        "cost_per_1000_usd": 1.756620799999999,
        "timed": 500,
        "p50_seconds": 2.255327749531716,
        "p95_seconds": 5.919557741587045,
        "cost_coverage_complete": true
      },
      "eligible": true,
      "reasons": [],
      "components": {
        "accuracy": 0.758,
        "agreement": 1,
        "cost": 0.5,
        "latency": 0.5
      },
      "scores": {
        "default": 72.89999999999999,
        "quality": 73.06,
        "economy": 67.9,
        "latency": 67.9
      },
      "pareto": true
    },
    {
      "arm_id": "deepseek",
      "name": "DeepSeek V4 Flash",
      "model": "deepseek/deepseek-v4-flash-0731",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 177,
        "missing": 0,
        "unresolved": 0,
        "errors": 372,
        "confirmed_errors": 372,
        "wrong_reference": 49,
        "invalid_output": 0,
        "provider_error": 323,
        "wrong_model": 0,
        "accuracy": 0.256,
        "agreement_count": 153,
        "agreement": 0.306,
        "paired_valid": 177,
        "paired_improvements": 8,
        "paired_regressions": 11,
        "both_wrong": 38,
        "both_wrong_different": 5,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 323,
        "both_correct": 120,
        "improvements": 8,
        "regressions": 259,
        "shared_wrong_agreement": 33,
        "correct_incumbent_retention": 0.316622691292876,
        "priced": 177,
        "reported_cost_total_usd": 0.014797919999999994,
        "mean_cost_usd": null,
        "cost_per_1000_usd": null,
        "timed": 500,
        "p50_seconds": 0.16332522872835398,
        "p95_seconds": 15.71977955205365,
        "cost_coverage_complete": false
      },
      "eligible": false,
      "reasons": [
        "incomplete_all_attempt_pricing",
        "full_run_incomplete_all_attempt_pricing",
        "error_rate_worse_than_incumbent"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false
    },
    {
      "arm_id": "qwen",
      "name": "Qwen3.7-Flash",
      "model": "qwen/qwen3.7-flash",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 489,
        "missing": 0,
        "unresolved": 0,
        "errors": 132,
        "confirmed_errors": 132,
        "wrong_reference": 121,
        "invalid_output": 2,
        "provider_error": 9,
        "wrong_model": 0,
        "accuracy": 0.736,
        "agreement_count": 421,
        "agreement": 0.842,
        "paired_valid": 489,
        "paired_improvements": 24,
        "paired_regressions": 25,
        "both_wrong": 96,
        "both_wrong_different": 19,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 11,
        "both_correct": 344,
        "improvements": 24,
        "regressions": 35,
        "shared_wrong_agreement": 77,
        "correct_incumbent_retention": 0.9076517150395779,
        "priced": 491,
        "reported_cost_total_usd": 0.0364854911,
        "mean_cost_usd": null,
        "cost_per_1000_usd": null,
        "timed": 500,
        "p50_seconds": 4.039335916517302,
        "p95_seconds": 13.189200102048924,
        "cost_coverage_complete": false
      },
      "eligible": false,
      "reasons": [
        "incomplete_all_attempt_pricing",
        "full_run_incomplete_all_attempt_pricing",
        "error_rate_worse_than_incumbent"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false
    },
    {
      "arm_id": "nemotron",
      "name": "Nemotron 3.5 Lightning",
      "model": "nvidia/nemotron-3.5-lightning",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 491,
        "missing": 0,
        "unresolved": 0,
        "errors": 127,
        "confirmed_errors": 127,
        "wrong_reference": 118,
        "invalid_output": 7,
        "provider_error": 2,
        "wrong_model": 0,
        "accuracy": 0.746,
        "agreement_count": 411,
        "agreement": 0.822,
        "paired_valid": 491,
        "paired_improvements": 24,
        "paired_regressions": 23,
        "both_wrong": 95,
        "both_wrong_different": 33,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 9,
        "both_correct": 349,
        "improvements": 24,
        "regressions": 30,
        "shared_wrong_agreement": 62,
        "correct_incumbent_retention": 0.920844327176781,
        "priced": 500,
        "reported_cost_total_usd": 0.10098860000000003,
        "mean_cost_usd": 0.00020197720000000006,
        "cost_per_1000_usd": 0.20197720000000005,
        "timed": 500,
        "p50_seconds": 3.2207032500300556,
        "p95_seconds": 5.280728489742613,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "error_rate_worse_than_incumbent"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false
    },
    {
      "arm_id": "glm",
      "name": "GLM-5.3-Flash",
      "model": "z-ai/glm-5.3-flash",
      "metrics": {
        "planned": 500,
        "observed": 500,
        "valid": 500,
        "missing": 0,
        "unresolved": 0,
        "errors": 102,
        "confirmed_errors": 102,
        "wrong_reference": 102,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "accuracy": 0.796,
        "agreement_count": 430,
        "agreement": 0.86,
        "paired_valid": 500,
        "paired_improvements": 33,
        "paired_regressions": 14,
        "both_wrong": 88,
        "both_wrong_different": 23,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 365,
        "improvements": 33,
        "regressions": 14,
        "shared_wrong_agreement": 65,
        "correct_incumbent_retention": 0.9630606860158312,
        "priced": 500,
        "reported_cost_total_usd": 0.01370272999999999,
        "mean_cost_usd": 0.000027405459999999977,
        "cost_per_1000_usd": 0.02740545999999998,
        "timed": 500,
        "p50_seconds": 1.310710541671142,
        "p95_seconds": 3.080266860267147,
        "cost_coverage_complete": true
      },
      "eligible": true,
      "reasons": [],
      "components": {
        "accuracy": 0.796,
        "agreement": 0.86,
        "cost": 0.9846384211855715,
        "latency": 0.6577414564687701
      },
      "scores": {
        "default": 83.27018298839914,
        "quality": 80.74379877654341,
        "economy": 84.51656720025484,
        "latency": 77.97862790591881
      },
      "pareto": true
    }
  ],
  "challenge": [
    {
      "arm_id": "incumbent-o4",
      "name": "o4-mini",
      "model": "openai/o4-mini",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 30,
        "missing": 0,
        "unresolved": 0,
        "errors": 5,
        "confirmed_errors": 5,
        "wrong_reference": 5,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "accuracy": 0.8333333333333334,
        "agreement_count": 30,
        "agreement": 1,
        "paired_valid": 30,
        "paired_improvements": 0,
        "paired_regressions": 0,
        "both_wrong": 5,
        "both_wrong_different": 0,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 25,
        "improvements": 0,
        "regressions": 0,
        "shared_wrong_agreement": 5,
        "correct_incumbent_retention": 1,
        "priced": 30,
        "reported_cost_total_usd": 0.039919,
        "mean_cost_usd": 0.0013306333333333335,
        "cost_per_1000_usd": 1.3306333333333336,
        "timed": 30,
        "p50_seconds": 1.8309602295048535,
        "p95_seconds": 5.038344033295286,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false
    },
    {
      "arm_id": "deepseek",
      "name": "DeepSeek V4 Flash",
      "model": "deepseek/deepseek-v4-flash-0731",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 10,
        "missing": 0,
        "unresolved": 0,
        "errors": 22,
        "confirmed_errors": 22,
        "wrong_reference": 2,
        "invalid_output": 0,
        "provider_error": 20,
        "wrong_model": 0,
        "accuracy": 0.26666666666666666,
        "agreement_count": 9,
        "agreement": 0.3,
        "paired_valid": 10,
        "paired_improvements": 0,
        "paired_regressions": 1,
        "both_wrong": 1,
        "both_wrong_different": 0,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 20,
        "both_correct": 8,
        "improvements": 0,
        "regressions": 17,
        "shared_wrong_agreement": 1,
        "correct_incumbent_retention": 0.32,
        "priced": 10,
        "reported_cost_total_usd": 0.0008608800000000001,
        "mean_cost_usd": null,
        "cost_per_1000_usd": null,
        "timed": 30,
        "p50_seconds": 0.1298541456926614,
        "p95_seconds": 10.60874680809209,
        "cost_coverage_complete": false
      },
      "eligible": false,
      "reasons": [
        "incomplete_all_attempt_pricing",
        "error_rate_worse_than_incumbent",
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false
    },
    {
      "arm_id": "qwen",
      "name": "Qwen3.7-Flash",
      "model": "qwen/qwen3.7-flash",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 30,
        "missing": 0,
        "unresolved": 0,
        "errors": 5,
        "confirmed_errors": 5,
        "wrong_reference": 5,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "accuracy": 0.8333333333333334,
        "agreement_count": 28,
        "agreement": 0.9333333333333333,
        "paired_valid": 30,
        "paired_improvements": 1,
        "paired_regressions": 1,
        "both_wrong": 4,
        "both_wrong_different": 0,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 24,
        "improvements": 1,
        "regressions": 1,
        "shared_wrong_agreement": 4,
        "correct_incumbent_retention": 0.96,
        "priced": 30,
        "reported_cost_total_usd": 0.0017286179,
        "mean_cost_usd": 0.00005762059666666667,
        "cost_per_1000_usd": 0.05762059666666667,
        "timed": 30,
        "p50_seconds": 3.0278067083563656,
        "p95_seconds": 8.916279997979288,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false
    },
    {
      "arm_id": "nemotron",
      "name": "Nemotron 3.5 Lightning",
      "model": "nvidia/nemotron-3.5-lightning",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 30,
        "missing": 0,
        "unresolved": 0,
        "errors": 6,
        "confirmed_errors": 6,
        "wrong_reference": 6,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "accuracy": 0.8,
        "agreement_count": 27,
        "agreement": 0.9,
        "paired_valid": 30,
        "paired_improvements": 0,
        "paired_regressions": 1,
        "both_wrong": 5,
        "both_wrong_different": 2,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 24,
        "improvements": 0,
        "regressions": 1,
        "shared_wrong_agreement": 3,
        "correct_incumbent_retention": 0.96,
        "priced": 30,
        "reported_cost_total_usd": 0.006188960000000001,
        "mean_cost_usd": 0.00020629866666666672,
        "cost_per_1000_usd": 0.2062986666666667,
        "timed": 30,
        "p50_seconds": 3.294255416840315,
        "p95_seconds": 3.9936802775831888,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "error_rate_worse_than_incumbent",
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false
    },
    {
      "arm_id": "glm",
      "name": "GLM-5.3-Flash",
      "model": "z-ai/glm-5.3-flash",
      "metrics": {
        "planned": 30,
        "observed": 30,
        "valid": 30,
        "missing": 0,
        "unresolved": 0,
        "errors": 6,
        "confirmed_errors": 6,
        "wrong_reference": 6,
        "invalid_output": 0,
        "provider_error": 0,
        "wrong_model": 0,
        "accuracy": 0.8,
        "agreement_count": 26,
        "agreement": 0.8666666666666667,
        "paired_valid": 30,
        "paired_improvements": 0,
        "paired_regressions": 1,
        "both_wrong": 5,
        "both_wrong_different": 3,
        "unpaired_unresolved_or_missing": 0,
        "unpaired_failure": 0,
        "both_correct": 24,
        "improvements": 0,
        "regressions": 1,
        "shared_wrong_agreement": 2,
        "correct_incumbent_retention": 0.96,
        "priced": 30,
        "reported_cost_total_usd": 0.0008650499999999999,
        "mean_cost_usd": 0.000028834999999999998,
        "cost_per_1000_usd": 0.028834999999999996,
        "timed": 30,
        "p50_seconds": 1.2247391873970628,
        "p95_seconds": 3.0844985374947997,
        "cost_coverage_complete": true
      },
      "eligible": false,
      "reasons": [
        "error_rate_worse_than_incumbent",
        "separate_challenge_not_ranked"
      ],
      "components": null,
      "scores": {
        "default": null,
        "quality": null,
        "economy": null,
        "latency": null
      },
      "pareto": false
    }
  ],
  "winners": {
    "default": [
      "glm"
    ],
    "quality": [
      "glm"
    ],
    "economy": [
      "glm"
    ],
    "latency": [
      "glm"
    ]
  },
  "configurations": [
    {
      "id": "incumbent-o4",
      "model": "openai/o4-mini",
      "provider": "openai",
      "reasoning": "Enabled"
    },
    {
      "id": "deepseek",
      "model": "deepseek/deepseek-v4-flash-0731",
      "provider": "deepinfra/fp8",
      "reasoning": "Low effort"
    },
    {
      "id": "qwen",
      "model": "qwen/qwen3.7-flash",
      "provider": "alibaba/us-east-1",
      "reasoning": "1,024 reasoning tokens"
    },
    {
      "id": "nemotron",
      "model": "nvidia/nemotron-3.5-lightning",
      "provider": "deepinfra/bf16",
      "reasoning": "Enabled"
    },
    {
      "id": "glm",
      "model": "z-ai/glm-5.3-flash",
      "provider": "z-ai/fp8",
      "reasoning": "Low effort"
    }
  ],
  "reported_known_total_usd": 1.0938476489999984,
  "reserved_unknown_usd": 0.2684639999999987,
  "segments": [
    {
      "name": "parent",
      "first": "2026-10-10T05:37:25.337011+00:00",
      "last": "2026-10-10T05:40:56.254813+00:00",
      "pause_seconds": null
    },
    {
      "name": "continuation",
      "first": "2026-10-10T05:51:50.020751+00:00",
      "last": "2026-10-10T05:57:08.192920+00:00",
      "pause_seconds": 653.765938
    },
    {
      "name": "second continuation",
      "first": "2026-10-10T06:01:16.888746+00:00",
      "last": "2026-10-10T08:12:08.754252+00:00",
      "pause_seconds": 248.695826
    }
  ]
}