{
  "version": "pisama-banking77-case-examples-v1",
  "source_summary_sha256": "31d2f1a4ddc87eb9a6682c2c69f51a679d0c51f91bac9279772aa51c3d209b1d",
  "source_dataset": {
    "name": "Banking77",
    "revision": "sha256:d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d",
    "split": "test",
    "sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d",
    "source_url": "https://github.com/PolyAI-LDN/task-specific-datasets/tree/master/banking_data",
    "license_url": "https://github.com/PolyAI-LDN/task-specific-datasets/blob/master/LICENSE"
  },
  "selection": {
    "phase": "primary",
    "rule": "lowest_numeric_case_id_per_candidate_and_category",
    "illustrative_only": true,
    "excludes": [
      "invalid_outputs",
      "provider_errors",
      "unresolved_outputs",
      "challenge_cases"
    ],
    "match_definition": "exact_parsed_intent_label",
    "csv_text_policy": "Formula-like cells are prefixed with an apostrophe; JSON preserves exact visible text."
  },
  "candidates": [
    {
      "arm_id": "deepseek",
      "model": "deepseek/deepseek-v4-flash-0731",
      "name": "DeepSeek V4 Flash",
      "planned": 500,
      "valid_pairs": 498,
      "excluded_pairs": 2,
      "categories": {
        "both_correct": 354,
        "shared_wrong": 83,
        "correction": 20,
        "regression": 20,
        "different_wrong": 21
      }
    },
    {
      "arm_id": "qwen",
      "model": "qwen/qwen3.7-flash",
      "name": "Qwen3.7-Flash",
      "planned": 500,
      "valid_pairs": 499,
      "excluded_pairs": 1,
      "categories": {
        "both_correct": 350,
        "shared_wrong": 72,
        "correction": 27,
        "regression": 23,
        "different_wrong": 27
      }
    },
    {
      "arm_id": "nemotron",
      "model": "nvidia/nemotron-3.5-lightning",
      "name": "Nemotron 3.5 Lightning",
      "planned": 500,
      "valid_pairs": 497,
      "excluded_pairs": 3,
      "categories": {
        "both_correct": 352,
        "shared_wrong": 67,
        "correction": 28,
        "regression": 21,
        "different_wrong": 29
      }
    },
    {
      "arm_id": "glm",
      "model": "z-ai/glm-5.3-flash",
      "name": "GLM-5.3-Flash",
      "planned": 500,
      "valid_pairs": 500,
      "excluded_pairs": 0,
      "categories": {
        "both_correct": 354,
        "shared_wrong": 68,
        "correction": 35,
        "regression": 20,
        "different_wrong": 23
      }
    }
  ],
  "examples": [
    {
      "id": "deepseek-both_correct",
      "category": "both_correct",
      "candidate_arm_id": "deepseek",
      "case_id": "banking77:test:20",
      "input": "I did not get my card yet, is it lost?",
      "reference_intent": "card_arrival",
      "outputs_match": true,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "openai",
        "receipt_sha256": "d1bce59281a16178539697f483d695ef2a6a946022804355c749c1653e625813",
        "request_sha256": "20592f3e61ad0adc43cc4cc2f344e98242bace1ec37a07ca7680a4bc40919e41",
        "response_sha256": "ff0e1e664f76be0456acd932a661e0753b4eff025cf6f85f01fe682d18203379",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "candidate": {
        "model": "deepseek/deepseek-v4-flash-0731",
        "name": "DeepSeek V4 Flash",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "streamlake/fp8",
        "receipt_sha256": "e8060b286aa301a25e58d58445446c916e25226ea7680111ecae0e109c0ca60e",
        "request_sha256": "bcc685731cf6774d1fd871e0babcf7587f879a2be626d4a09d03c53ab93abb3b",
        "response_sha256": "efd6d9aaae44461f2687b6d5402178494d6b4a9419927472e8845a172ff27509",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "deepseek-shared_wrong",
      "category": "shared_wrong",
      "candidate_arm_id": "deepseek",
      "case_id": "banking77:test:21",
      "input": "Status of the card I ordered.",
      "reference_intent": "card_arrival",
      "outputs_match": true,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8",
        "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028",
        "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "deepseek/deepseek-v4-flash-0731",
        "name": "DeepSeek V4 Flash",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "streamlake/fp8",
        "receipt_sha256": "e0320f01699377ec6375eda65b332ee3ae7851d251422f3c08b196e3cfa58587",
        "request_sha256": "e6b64d67df2ce6b60343c97fb96d2743e5d6655df6ed01a9b85c57753e42096a",
        "response_sha256": "7f06901d1f4967c6613dd0234395adec94d5bad4bd262ace113736f0aeeca012",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "deepseek-correction",
      "category": "correction",
      "candidate_arm_id": "deepseek",
      "case_id": "banking77:test:43",
      "input": "I have received my card, can you help me put it in the app?",
      "reference_intent": "card_linking",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"activate_my_card\"}",
        "parsed_intent": "activate_my_card",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "8640c8124dc6be707f8aaedce6341c1219e9d25d119ae768b215b72a6056bb05",
        "request_sha256": "6cf7eaef41e2ea7ddbb576d0066d57e403fe5396ab4758b8e3b1c2a6145bb261",
        "response_sha256": "fd94bb065062f047f11ef7fa8ad74ad3bf744c9b2ecd205b24807b91344b7461",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "candidate": {
        "model": "deepseek/deepseek-v4-flash-0731",
        "name": "DeepSeek V4 Flash",
        "visible_answer": "{\"intent\":\"card_linking\"}",
        "parsed_intent": "card_linking",
        "correct": true,
        "provider_route": "streamlake/fp8",
        "receipt_sha256": "d2c600f59d6c9e11d64c13fa028d5c3a71f78c1d99300f0f10a55aa0d9e035b2",
        "request_sha256": "1554422bc843cacfed264891b987436f14ffcf36c341ec8605d8331f9b5f61ed",
        "response_sha256": "c15f744c8dc25e263d6bb9ea64cae37734520d542c835e2731dbb1790491be77",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "deepseek-regression",
      "category": "regression",
      "candidate_arm_id": "deepseek",
      "case_id": "banking77:test:2",
      "input": "I ordered a card but it has not arrived. Help please!",
      "reference_intent": "card_arrival",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "openai",
        "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028",
        "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b",
        "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "deepseek/deepseek-v4-flash-0731",
        "name": "DeepSeek V4 Flash",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "streamlake/fp8",
        "receipt_sha256": "ceec6cd457545d2fa9dcf2a41fb90c28527cd70759a12378d932aa45eddcfe10",
        "request_sha256": "4f51ab011c8dbbc4f6cc44f06fcddaa37c51084242efda2f7440051d158d4382",
        "response_sha256": "de15aee6f8c1a63f991fc15e6eb14c65f53e11ce9ffcaeef7d766cf864ef7f80",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "deepseek-different_wrong",
      "category": "different_wrong",
      "candidate_arm_id": "deepseek",
      "case_id": "banking77:test:382",
      "input": "I broke my card",
      "reference_intent": "card_not_working",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"getting_spare_card\"}",
        "parsed_intent": "getting_spare_card",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "8759d1974ee8ba2a4cc159afd88502fb1c89936454b46445c6108e4562e114fd",
        "request_sha256": "837d08959660607e93449ab2580e5fc25f19e59eac7e8fa6a3224d9d82517433",
        "response_sha256": "a21d386c3aaef37383da63f6821780fda3eca5e838f8a2ae43040af497c32df6",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "candidate": {
        "model": "deepseek/deepseek-v4-flash-0731",
        "name": "DeepSeek V4 Flash",
        "visible_answer": "{\"intent\":\"order_physical_card\"}",
        "parsed_intent": "order_physical_card",
        "correct": false,
        "provider_route": "streamlake/fp8",
        "receipt_sha256": "993fc20ee8c7353586ea0ea60fcfa8f7ecd1d950a269672138e6c5becaacc83e",
        "request_sha256": "21e72b9002326a30912eb6fc46dc67896753e2b73e86e4c31f2ac3c4f31e5ea8",
        "response_sha256": "e96dbb6ecf979d1f6f570c486f0a6aa65ac5578076a560bfbe996a66c961fd68",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "qwen-both_correct",
      "category": "both_correct",
      "candidate_arm_id": "qwen",
      "case_id": "banking77:test:2",
      "input": "I ordered a card but it has not arrived. Help please!",
      "reference_intent": "card_arrival",
      "outputs_match": true,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "openai",
        "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028",
        "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b",
        "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "qwen/qwen3.7-flash",
        "name": "Qwen3.7-Flash",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "alibaba/us-east-1",
        "receipt_sha256": "db47507f21a8b21ca9ca8f2d7b6d80b5e609a24999c9cc47154f42b140ddd1f1",
        "request_sha256": "7d215c3a3fd8910356177e6ffe2a89ac1691c68eef6b23776ff45f860b0a1cef",
        "response_sha256": "772025cacbd5135cd15e108301e5a2473ace029d96eb5f72127575d067319c10",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "qwen-shared_wrong",
      "category": "shared_wrong",
      "candidate_arm_id": "qwen",
      "case_id": "banking77:test:27",
      "input": "I'm still waiting for delivery of my new card, why is it taking so long?",
      "reference_intent": "card_arrival",
      "outputs_match": true,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a",
        "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce",
        "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "qwen/qwen3.7-flash",
        "name": "Qwen3.7-Flash",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "alibaba/us-east-1",
        "receipt_sha256": "da53d9e4ec4f6658a99720ed6deb4b542dbc4373bdfc432744abbc45b9910852",
        "request_sha256": "0088a2755744970a4b91a8c556176dcbdf8cd645f190a22cb32ec0d0e3aefcd8",
        "response_sha256": "ab154837f5032dd17eb9284009bec20359e7ce1b497bdec488b8e6bd900a8185",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "qwen-correction",
      "category": "correction",
      "candidate_arm_id": "qwen",
      "case_id": "banking77:test:21",
      "input": "Status of the card I ordered.",
      "reference_intent": "card_arrival",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8",
        "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028",
        "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "qwen/qwen3.7-flash",
        "name": "Qwen3.7-Flash",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "alibaba/us-east-1",
        "receipt_sha256": "390ccd8789cf43e3d36c96915e7f6fc6fda2a24282d1e23f37aa2d2188619e0a",
        "request_sha256": "03dcaf4654c1f020fb9e2556a45e7f76fe8cb10aa581e8914f2d03bf7c52f5ea",
        "response_sha256": "fc6528909bb349f5071a4545d84dd801f7aca6f4d269f51f867bbed297f5443d",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "qwen-regression",
      "category": "regression",
      "candidate_arm_id": "qwen",
      "case_id": "banking77:test:243",
      "input": "Does your system support multiple currency.",
      "reference_intent": "fiat_currency_support",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"fiat_currency_support\"}",
        "parsed_intent": "fiat_currency_support",
        "correct": true,
        "provider_route": "openai",
        "receipt_sha256": "f9bde41d4ddd045b4306208561a400803ada3ba3581eaf5bed0d041a28cd01c1",
        "request_sha256": "80e1242173c5b00f98a6505fe3d030c5482481b9de5d50bd7bcb7442072041c4",
        "response_sha256": "bf4736656a29d02db620793e57e9d8edc793fc19389d3f90db72b0afdc07806c",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "qwen/qwen3.7-flash",
        "name": "Qwen3.7-Flash",
        "visible_answer": "{\"intent\":\"supported_cards_and_currencies\"}",
        "parsed_intent": "supported_cards_and_currencies",
        "correct": false,
        "provider_route": "alibaba/us-east-1",
        "receipt_sha256": "3ddc0b362c30d49990db885c9d9b7f167038b51551a7ceac94993da3ee5acae6",
        "request_sha256": "99a08436c90af3383597e431e9905c7411ad917a868decba478c356f5a84b459",
        "response_sha256": "8fc1883b45e4da61f73d0303bd4dfcf827b016a74fd6228475d990f7962eacc2",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "qwen-different_wrong",
      "category": "different_wrong",
      "candidate_arm_id": "qwen",
      "case_id": "banking77:test:586",
      "input": "what is the charges for new card?",
      "reference_intent": "contactless_not_working",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"order_physical_card\"}",
        "parsed_intent": "order_physical_card",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "c79857a77ae76513646f34c2de54fce6c1d7832152d2f5504cc921dfceee0df7",
        "request_sha256": "6a9b57c9ec33c78e03984be6c5a8b048a3446aaf93814affb17f595aaa253657",
        "response_sha256": "7945743c24f4d1d00a7e4655b881809863349af6a378cb8a671ac5431357d5a3",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "candidate": {
        "model": "qwen/qwen3.7-flash",
        "name": "Qwen3.7-Flash",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": false,
        "provider_route": "alibaba/us-east-1",
        "receipt_sha256": "6e433f1faade0c3424e231bbe1a354ee652bde11b97166f6e26cb6ec7371661d",
        "request_sha256": "5572d201cfbc468bb0f4e7950586478cfbb8ebb87d995f20c0346be4f947e7d5",
        "response_sha256": "2228813284071559e2d11229add311b237432b89139908e051b097be63bb1541",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "nemotron-both_correct",
      "category": "both_correct",
      "candidate_arm_id": "nemotron",
      "case_id": "banking77:test:2",
      "input": "I ordered a card but it has not arrived. Help please!",
      "reference_intent": "card_arrival",
      "outputs_match": true,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "openai",
        "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028",
        "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b",
        "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "nvidia/nemotron-3.5-lightning",
        "name": "Nemotron 3.5 Lightning",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "deepinfra/bf16",
        "receipt_sha256": "eeea4984dfd6b94c6a2e5c9e0288e7853d49fa56aabaf9ef8b15deed097e55f2",
        "request_sha256": "cb780bf146ce78e86f4ae3870fb016b286400f82f9578e5f3870e10e8c7c6694",
        "response_sha256": "c53f3c5ea68c072ac60ea150bf5532c20f16838de78fdd23c20e15bdfc617950",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "nemotron-shared_wrong",
      "category": "shared_wrong",
      "candidate_arm_id": "nemotron",
      "case_id": "banking77:test:27",
      "input": "I'm still waiting for delivery of my new card, why is it taking so long?",
      "reference_intent": "card_arrival",
      "outputs_match": true,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a",
        "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce",
        "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "nvidia/nemotron-3.5-lightning",
        "name": "Nemotron 3.5 Lightning",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "deepinfra/bf16",
        "receipt_sha256": "2b2b872b307083fe210b5af168a3eedae11561cc41e0acdbba854e07203c60bc",
        "request_sha256": "2c078794688e345c6e7eaa04761eb9e0ac7f482e591cddd50545a6acb99a92cf",
        "response_sha256": "ee9443f78770fada27318b4686e0615924599bef30dd32c220d1e6d2a2e03a15",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "nemotron-correction",
      "category": "correction",
      "candidate_arm_id": "nemotron",
      "case_id": "banking77:test:21",
      "input": "Status of the card I ordered.",
      "reference_intent": "card_arrival",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8",
        "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028",
        "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "nvidia/nemotron-3.5-lightning",
        "name": "Nemotron 3.5 Lightning",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "deepinfra/bf16",
        "receipt_sha256": "04538115745a0ac6299c5497baea87f68e29c554d517736bd57bd4b6f0e06873",
        "request_sha256": "21426605aafbeede8070ce75283f40ea17777481f7bcdb1c7aadf1b23d581b42",
        "response_sha256": "8da1fec56f13346ade06261abcfa35654c6b376e304f8c4a326ba711c30384f3",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "nemotron-regression",
      "category": "regression",
      "candidate_arm_id": "nemotron",
      "case_id": "banking77:test:137",
      "input": "Why is your exchange rate for my card payment wrong",
      "reference_intent": "card_payment_wrong_exchange_rate",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_payment_wrong_exchange_rate\"}",
        "parsed_intent": "card_payment_wrong_exchange_rate",
        "correct": true,
        "provider_route": "openai",
        "receipt_sha256": "737627a5a5fffc74ef8645c72e2946a91b1e9177bed26474253b28bb0f456483",
        "request_sha256": "595a080bc5d3aa283d530588c44f5f8984fc9fbac557ea62736ecaa3ddad61de",
        "response_sha256": "487c96fc864cce71c15a31fb2313b9dad968d023c41d1287dfefc321f430f358",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "nvidia/nemotron-3.5-lightning",
        "name": "Nemotron 3.5 Lightning",
        "visible_answer": "{\"intent\":\"exchange_rate\"}",
        "parsed_intent": "exchange_rate",
        "correct": false,
        "provider_route": "deepinfra/bf16",
        "receipt_sha256": "7484defc85d0c00603876729e88217cdc1a6528cafda014d7951209e745df7c9",
        "request_sha256": "e38a1df3e84e0b7dcbb7c6649ec7adb7a2d3035f63f407e02192930fe27c518d",
        "response_sha256": "435d743346fea8cf9332dc260291f737f970c7a574f5bb15127c6c302a5f9e82",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "nemotron-different_wrong",
      "category": "different_wrong",
      "candidate_arm_id": "nemotron",
      "case_id": "banking77:test:255",
      "input": "How do I change currencies to euros?",
      "reference_intent": "fiat_currency_support",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"exchange_via_app\"}",
        "parsed_intent": "exchange_via_app",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "5b26a5fcb17418bbe4da0a2673d72ebe1d15e46022df8f8a201a777a8715d0f8",
        "request_sha256": "e8c1f42d5ba0fda2a401a797334bc3c329b4e0e1b4cd8d18e709f615e07995c9",
        "response_sha256": "c1be4d8a8f3bfaf1f7b9c913b952b5d95d8e0071c5a038193243130bb0e709ac",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "nvidia/nemotron-3.5-lightning",
        "name": "Nemotron 3.5 Lightning",
        "visible_answer": "{\"intent\":\"exchange_rate\"}",
        "parsed_intent": "exchange_rate",
        "correct": false,
        "provider_route": "deepinfra/bf16",
        "receipt_sha256": "c95dc6f96f96439ea41de8b6098e3b68595425b3c330f23595fa62515b78b0dd",
        "request_sha256": "b827cc11d6cbd890580c1dba0a1bf7bf3ed9546f7b11d4c7d8f266f3b3e94aa5",
        "response_sha256": "be40bf20b83761152c8466c845984b6a56139b1badce639071be56a1d7f1638b",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "glm-both_correct",
      "category": "both_correct",
      "candidate_arm_id": "glm",
      "case_id": "banking77:test:2",
      "input": "I ordered a card but it has not arrived. Help please!",
      "reference_intent": "card_arrival",
      "outputs_match": true,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "openai",
        "receipt_sha256": "590264eb8918169dad4ea57bb612c421d889ccee094bd11331c241b77190e028",
        "request_sha256": "fac7bb7a024b32009fa2cf5b5274b2815ff84cbe984ea43827cc24299046989b",
        "response_sha256": "c2d7abc7f7c7fb3abd1916ad347115c044c4fd23a29732ef7e2c6d2c05163da2",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "z-ai/glm-5.3-flash",
        "name": "GLM-5.3-Flash",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "z-ai/fp8",
        "receipt_sha256": "1afa9d9402d2de096042cd13edd3dae9f6498eefc0ec336f308a23b7b667e9bb",
        "request_sha256": "ab957f02e8177994c6e9aeb184399a3b7c091d454b9332ed1d6f2139d2983f82",
        "response_sha256": "1a015e9362c8bd1494970b2190e95f833e72e11370e946740cd45a0aaf526565",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "glm-shared_wrong",
      "category": "shared_wrong",
      "candidate_arm_id": "glm",
      "case_id": "banking77:test:21",
      "input": "Status of the card I ordered.",
      "reference_intent": "card_arrival",
      "outputs_match": true,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "45294616436f78c8eaa173097c819f86e59b4e4d00135bb4a1f7f467bffa9ee8",
        "request_sha256": "c3714dedcedd12e484da6029d079b39ccf4d28d907e6e836e61063a186465028",
        "response_sha256": "537c01acb4a20773ed1093a9629bb5b146017b51d49691732ae33b4f3d2b46d2",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "z-ai/glm-5.3-flash",
        "name": "GLM-5.3-Flash",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "z-ai/fp8",
        "receipt_sha256": "3b8c9a3e07e406fc6af6d87510f585e9b3d3e93c8c6d2bd2ebd019d77c058d68",
        "request_sha256": "86adf316affd7b5ebc5473517c3e02a201ac4929d8f947f319607258f3a0c0f0",
        "response_sha256": "fca6f1b0df2ffee3c0d322109febb279195dc35008fd17f6c3b1b3e656116249",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "glm-correction",
      "category": "correction",
      "candidate_arm_id": "glm",
      "case_id": "banking77:test:27",
      "input": "I'm still waiting for delivery of my new card, why is it taking so long?",
      "reference_intent": "card_arrival",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"card_delivery_estimate\"}",
        "parsed_intent": "card_delivery_estimate",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "83b44b057e9c54add2464329895b98e1bfc98bd1a0e9c858a7ea274a564ce15a",
        "request_sha256": "e7514903005193b31092d9facf73f27835faa3b94290ee76f3ea9fb80ed34bce",
        "response_sha256": "4ff29adcfd8cbe92e01c0cca4b08d98199bd8c03d030df8f588f1c1c72346e01",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "z-ai/glm-5.3-flash",
        "name": "GLM-5.3-Flash",
        "visible_answer": "{\"intent\":\"card_arrival\"}",
        "parsed_intent": "card_arrival",
        "correct": true,
        "provider_route": "z-ai/fp8",
        "receipt_sha256": "c40fb0b47af378f58429fd54f282d0fd8f094fd777cf015d044c9a19aa53aeda",
        "request_sha256": "156dc4b2877ee1f7dc0bb331b136f3786d7573bee743212eb7a206d533ecb656",
        "response_sha256": "023113ecc61de05014e56874f0149a8daa513090b626add1211e005f7c915e3e",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "glm-regression",
      "category": "regression",
      "candidate_arm_id": "glm",
      "case_id": "banking77:test:243",
      "input": "Does your system support multiple currency.",
      "reference_intent": "fiat_currency_support",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"fiat_currency_support\"}",
        "parsed_intent": "fiat_currency_support",
        "correct": true,
        "provider_route": "openai",
        "receipt_sha256": "f9bde41d4ddd045b4306208561a400803ada3ba3581eaf5bed0d041a28cd01c1",
        "request_sha256": "80e1242173c5b00f98a6505fe3d030c5482481b9de5d50bd7bcb7442072041c4",
        "response_sha256": "bf4736656a29d02db620793e57e9d8edc793fc19389d3f90db72b0afdc07806c",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "candidate": {
        "model": "z-ai/glm-5.3-flash",
        "name": "GLM-5.3-Flash",
        "visible_answer": "{\"intent\":\"supported_cards_and_currencies\"}",
        "parsed_intent": "supported_cards_and_currencies",
        "correct": false,
        "provider_route": "z-ai/fp8",
        "receipt_sha256": "05899c906c9135137262701682d89369823c53c1d5cb1df6c791071b45eb92f4",
        "request_sha256": "b1ef17edb6cabbffc7e7eeb25d5b81f9e711144eec904c89026e4a3c9135984d",
        "response_sha256": "5ed71ac632e2f68d199231f82c1df70bb5d7aad8fc1e10855c75181bca0464ee",
        "plan_sha256": "fe1274037d6c43988e0f1f8035ec130e731a9df694dd1cbe4a46a4d2d98955a1"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    },
    {
      "id": "glm-different_wrong",
      "category": "different_wrong",
      "candidate_arm_id": "glm",
      "case_id": "banking77:test:586",
      "input": "what is the charges for new card?",
      "reference_intent": "contactless_not_working",
      "outputs_match": false,
      "incumbent": {
        "model": "openai/o4-mini",
        "name": "o4-mini",
        "visible_answer": "{\"intent\":\"order_physical_card\"}",
        "parsed_intent": "order_physical_card",
        "correct": false,
        "provider_route": "openai",
        "receipt_sha256": "c79857a77ae76513646f34c2de54fce6c1d7832152d2f5504cc921dfceee0df7",
        "request_sha256": "6a9b57c9ec33c78e03984be6c5a8b048a3446aaf93814affb17f595aaa253657",
        "response_sha256": "7945743c24f4d1d00a7e4655b881809863349af6a378cb8a671ac5431357d5a3",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "candidate": {
        "model": "z-ai/glm-5.3-flash",
        "name": "GLM-5.3-Flash",
        "visible_answer": "{\"intent\":\"getting_spare_card\"}",
        "parsed_intent": "getting_spare_card",
        "correct": false,
        "provider_route": "z-ai/fp8",
        "receipt_sha256": "fe337a4cca35bd6a4c23e64d2dd52d60306c284f8635bc465559244800a954fd",
        "request_sha256": "41cadadf4f548b874fe76d77524f9c26142a7743d9573af56ac9cac22d4998b8",
        "response_sha256": "489a789f7ad06011eb7528153a1fbd426f95e1b4f2115799efa170102d3b89e6",
        "plan_sha256": "fa28afcfde818f2234f7d94fc946db34b2d7254acca6a8fc1b9ea7a829a12c33"
      },
      "provenance": {
        "plan_sha256": "fc86a515c397671c38d6ed40083938075dcc30df692c771de7b660f66a25d193",
        "dataset_sha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d"
      }
    }
  ]
}