{
 "scope": "the reference API (v1.13.0) and joint-marker-stage-k-t-v1 (stage I + 68,194 rows from 1,705 Bend-oracle families, 12,000 updates) on identical requests, constraints sent to neither. One seed.",
 "families2": {
  "reference": "data/generated/reference_families2_300",
  "candidate": "reports/families2_joint-marker-stage-k-t-v1_cached",
  "requests_sha256": "38ec969751cfb5229b870764166b6b52fa543ea822e42116a35c64ce9bbf8e85",
  "requests": 300,
  "fields": 675,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 675,
   "only_reference_correct": 127,
   "only_candidate_correct": 21,
   "both_wrong": 5,
   "mcnemar_exact_p": 1.1113341249848234e-19,
   "reference": {
    "accuracy": 0.9614814814814815,
    "nll": 0.13403138704289766,
    "brier": 0.06748059259259259
   },
   "candidate": {
    "accuracy": 0.8044444444444444,
    "nll": 0.7750595594773675,
    "brier": 0.32071960827438467
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     -0.19311377245508982,
     -0.1222707423580786
    ],
    "nll": [
     0.5077667422021948,
     0.7782970188038557
    ],
    "brier": [
     0.1995465480794728,
     0.30869293190607205
    ]
   }
  },
  "groups": {
   "data/generated/families_v2/expense_test.jsonl:money_paid_now": {
    "fields": 75,
    "only_reference_correct": 32,
    "only_candidate_correct": 0,
    "both_wrong": 3,
    "mcnemar_exact_p": 4.656612873077393e-10,
    "reference": {
     "accuracy": 0.96,
     "nll": 0.16204260920119007,
     "brier": 0.073104
    },
    "candidate": {
     "accuracy": 0.5333333333333333,
     "nll": 1.2775084276892443,
     "brier": 0.7179088197714005
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5466666666666666,
      -0.32
     ],
     "nll": [
      0.7837965095267,
      1.4741229291281017
     ],
     "brier": [
      0.4733412024080902,
      0.8221826826954421
     ]
    }
   },
   "data/generated/families_v2/expense_test.jsonl:self_approved": {
    "fields": 75,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.05529415743451878,
     "brier": 0.006002666666666673
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.004856722767894913,
     "brier": 5.3321286665926294e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      -0.052884446703726544,
      -0.04787203574646686
     ],
     "brier": [
      -0.006428993235177907,
      -0.00544065792680682
     ]
    }
   },
   "data/generated/families_v2/expense_test.jsonl:verdict": {
    "fields": 75,
    "only_reference_correct": 0,
    "only_candidate_correct": 9,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.00390625,
    "reference": {
     "accuracy": 0.88,
     "nll": 0.2773060535551642,
     "brier": 0.18784533333333334
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.02512210674737811,
     "brier": 0.00881743085342769
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.05333333333333334,
      0.2
     ],
     "nll": [
      -0.41939623804022635,
      -0.12166807164518846
     ],
     "brier": [
      -0.29556594769957345,
      -0.08831862109735528
     ]
    }
   },
   "data/generated/families_v2/rating_test.jsonl:concerns": {
    "fields": 75,
    "only_reference_correct": 34,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.1641532182693481e-10,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.01649957296460515,
     "brier": 0.0007546666666666673
    },
    "candidate": {
     "accuracy": 0.5466666666666666,
     "nll": 2.566611967638847,
     "brier": 0.8371266451974064
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5733333333333334,
      -0.3466666666666667
     ],
     "nll": [
      1.9051695612017705,
      3.2506027393426247
     ],
     "brier": [
      0.634686602791869,
      1.0519160656560476
     ]
    }
   },
   "data/generated/families_v2/rating_test.jsonl:label": {
    "fields": 75,
    "only_reference_correct": 21,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 9.5367431640625e-07,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.04577626975785714,
     "brier": 0.006893333333333334
    },
    "candidate": {
     "accuracy": 0.72,
     "nll": 1.7278486323781306,
     "brier": 0.5432001836906537
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.37333333333333335,
      -0.18666666666666668
     ],
     "nll": [
      1.0543592524169627,
      2.3196303976626176
     ],
     "brier": [
      0.35468366835638443,
      0.7258449450442125
     ]
    }
   },
   "data/generated/families_v2/refill_test.jsonl:action": {
    "fields": 75,
    "only_reference_correct": 0,
    "only_candidate_correct": 5,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0625,
    "reference": {
     "accuracy": 0.9333333333333333,
     "nll": 0.10456732712719895,
     "brier": 0.06365333333333334
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.008639546678336517,
     "brier": 0.00014330382946547776
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.013333333333333334,
      0.13333333333333333
     ],
     "nll": [
      -0.1594925989396436,
      -0.04222894603814309
     ],
     "brier": [
      -0.11279611943948391,
      -0.022221770430558795
     ]
    }
   },
   "data/generated/families_v2/refill_test.jsonl:medicine_handed_over": {
    "fields": 75,
    "only_reference_correct": 6,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.03125,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.1411544716057163,
     "brier": 0.040722666666666664
    },
    "candidate": {
     "accuracy": 0.92,
     "nll": 0.34194519475989016,
     "brier": 0.1554264271266305
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.14666666666666667,
      -0.02666666666666667
     ],
     "nll": [
      -0.016098110024466806,
      0.4625378765898646
     ],
     "brier": [
      0.018779686923643176,
      0.22824044162741935
     ]
    }
   },
   "data/generated/families_v2/uptime_test.jsonl:credit": {
    "fields": 75,
    "only_reference_correct": 20,
    "only_candidate_correct": 7,
    "both_wrong": 2,
    "mcnemar_exact_p": 0.0191572904586792,
    "reference": {
     "accuracy": 0.88,
     "nll": 0.3699541917316345,
     "brier": 0.22592800000000002
    },
    "candidate": {
     "accuracy": 0.7066666666666667,
     "nll": 0.7756151159465158,
     "brier": 0.44486863381027375
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.30666666666666664,
      -0.05333333333333334
     ],
     "nll": [
      0.07270284616566607,
      0.7534996348652915
     ],
     "brier": [
      0.020407715829495562,
      0.4231196443590289
     ]
    }
   },
   "data/generated/families_v2/uptime_test.jsonl:promise_met": {
    "fields": 75,
    "only_reference_correct": 14,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0001220703125,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.03368783000819378,
     "brier": 0.0024213333333333357
    },
    "candidate": {
     "accuracy": 0.8133333333333334,
     "nll": 0.2473883206900704,
     "brier": 0.1789317089035378
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.28,
      -0.10666666666666667
     ],
     "nll": [
      0.12047980987204428,
      0.3161588404956665
     ],
     "brier": [
      0.10053366154309175,
      0.26071644752197254
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 675,
     "confidence_min": 0.5,
     "accuracy": 0.9614814814814815
    },
    {
     "coverage": 0.9,
     "fields": 608,
     "confidence_min": 0.84,
     "accuracy": 0.9769736842105263
    },
    {
     "coverage": 0.75,
     "fields": 506,
     "confidence_min": 0.9299999999999999,
     "accuracy": 0.9881422924901185
    },
    {
     "coverage": 0.5,
     "fields": 338,
     "confidence_min": 0.96,
     "accuracy": 0.9970414201183432
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 675,
     "confidence_min": 0.49746533776784085,
     "accuracy": 0.8044444444444444
    },
    {
     "coverage": 0.9,
     "fields": 608,
     "confidence_min": 0.8171305613104234,
     "accuracy": 0.8519736842105263
    },
    {
     "coverage": 0.75,
     "fields": 506,
     "confidence_min": 0.9742686664826123,
     "accuracy": 0.8972332015810277
    },
    {
     "coverage": 0.5,
     "fields": 338,
     "confidence_min": 0.9928736934501102,
     "accuracy": 0.8757396449704142
    }
   ]
  }
 },
 "families": {
  "reference": "data/generated/reference_families_400",
  "candidate": "reports/families_joint-marker-stage-k-t-v1_cached",
  "requests_sha256": "f0f16c91ae37c76b82cb2af00a1a1b8fef724d8c8db81b585f4f150aec97420a",
  "requests": 400,
  "fields": 900,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 900,
   "only_reference_correct": 77,
   "only_candidate_correct": 34,
   "both_wrong": 19,
   "mcnemar_exact_p": 5.4948162502714474e-05,
   "reference": {
    "accuracy": 0.9411111111111111,
    "nll": 0.14860271857619678,
    "brier": 0.08541288247231009
   },
   "candidate": {
    "accuracy": 0.8933333333333333,
    "nll": 0.48286528815923346,
    "brier": 0.1878305525058098
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     -0.0753045404208195,
     -0.020224719101123594
    ],
    "nll": [
     0.22997530748240333,
     0.44350064820302193
    ],
    "brier": [
     0.0593434838391094,
     0.14511805468643157
    ]
   }
  },
  "groups": {
   "data/generated/families_v1/access_test.jsonl:outcome": {
    "fields": 100,
    "only_reference_correct": 0,
    "only_candidate_correct": 9,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.00390625,
    "reference": {
     "accuracy": 0.91,
     "nll": 0.17436345593873961,
     "brier": 0.113954
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0030220412655286412,
     "brier": 1.2367035213711542e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.04,
      0.15
     ],
     "nll": [
      -0.260201670304431,
      -0.09342597233346404
     ],
     "brier": [
      -0.18305984480243936,
      -0.054854060537231034
     ]
    }
   },
   "data/generated/families_v1/access_test.jsonl:risk": {
    "fields": 100,
    "only_reference_correct": 25,
    "only_candidate_correct": 20,
    "both_wrong": 2,
    "mcnemar_exact_p": 0.5514843298025198,
    "reference": {
     "accuracy": 0.78,
     "nll": 0.4122989552572631,
     "brier": 0.2801659422507907
    },
    "candidate": {
     "accuracy": 0.73,
     "nll": 1.8598483897235918,
     "brier": 0.5376423094221798
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.18,
      0.08
     ],
     "nll": [
      0.8093584096262997,
      2.113347323352463
     ],
     "brier": [
      0.032368889309262766,
      0.48103402749528745
     ]
    }
   },
   "data/generated/families_v1/incident_test.jsonl:page_oncall": {
    "fields": 100,
    "only_reference_correct": 15,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 6.103515625e-05,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.0810447472874216,
     "brier": 0.014967999999999999
    },
    "candidate": {
     "accuracy": 0.85,
     "nll": 0.6434767116459225,
     "brier": 0.2771907068167273
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.22,
      -0.08
     ],
     "nll": [
      0.26595520854245724,
      0.8817542740922738
     ],
     "brier": [
      0.13941646193637638,
      0.39041892227501007
     ]
    }
   },
   "data/generated/families_v1/incident_test.jsonl:severity": {
    "fields": 100,
    "only_reference_correct": 3,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.25,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.0033341040526544304,
     "brier": 0.0001340000000000001
    },
    "candidate": {
     "accuracy": 0.97,
     "nll": 0.13482532943841408,
     "brier": 0.054858486403441056
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.07,
      0.0
     ],
     "nll": [
      0.010752017654594376,
      0.3046570346448529
     ],
     "brier": [
      0.0038157314941574083,
      0.12206476728957467
     ]
    }
   },
   "data/generated/families_v1/invoice_test.jsonl:amount_within_tolerance": {
    "fields": 100,
    "only_reference_correct": 20,
    "only_candidate_correct": 2,
    "both_wrong": 9,
    "mcnemar_exact_p": 0.00012111663818359375,
    "reference": {
     "accuracy": 0.89,
     "nll": 0.3184560917455249,
     "brier": 0.19009399999999999
    },
    "candidate": {
     "accuracy": 0.71,
     "nll": 0.8937252094520513,
     "brier": 0.45917054289509696
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.27,
      -0.1
     ],
     "nll": [
      0.3055457489274144,
      0.8582818729912757
     ],
     "brier": [
      0.14801408958511508,
      0.39881634475734323
     ]
    }
   },
   "data/generated/families_v1/invoice_test.jsonl:disposition": {
    "fields": 100,
    "only_reference_correct": 14,
    "only_candidate_correct": 2,
    "both_wrong": 8,
    "mcnemar_exact_p": 0.004180908203125,
    "reference": {
     "accuracy": 0.9,
     "nll": 0.1976287774013133,
     "brier": 0.12887
    },
    "candidate": {
     "accuracy": 0.78,
     "nll": 0.7898579794430144,
     "brier": 0.36047275679000085
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.19,
      -0.05
     ],
     "nll": [
      0.33710027898651307,
      0.8752465920539837
     ],
     "brier": [
      0.12392631940767358,
      0.3454959747425932
     ]
    }
   },
   "data/generated/families_v1/invoice_test.jsonl:duplicate_invoice": {
    "fields": 100,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.07053526327862135,
     "brier": 0.009998000000000002
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.012316662893502195,
     "brier": 0.0010606431868435521
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      -0.06294263984530885,
      -0.05349260214436388
     ],
     "brier": [
      -0.009958969469488465,
      -0.007889082561661537
     ]
    }
   },
   "data/generated/families_v1/refund_test.jsonl:decision": {
    "fields": 100,
    "only_reference_correct": 0,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.99,
     "nll": 0.045747256583372586,
     "brier": 0.028132
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.005099624679869363,
     "brier": 4.0080065162507154e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.03
     ],
     "nll": [
      -0.07454241989578288,
      -0.014059074314846574
     ],
     "brier": [
      -0.051617552006011695,
      -0.010019462369818903
     ]
    }
   },
   "data/generated/families_v1/refund_test.jsonl:within_refund_window": {
    "fields": 100,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.03401581564086002,
     "brier": 0.0024000000000000037
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0036156448912075144,
     "brier": 2.7079937622238788e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      -0.03222774738352141,
      -0.028571896098044006
     ],
     "brier": [
      -0.002626467594481184,
      -0.0021209281652392565
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 900,
     "confidence_min": 0.5,
     "accuracy": 0.9411111111111111
    },
    {
     "coverage": 0.9,
     "fields": 810,
     "confidence_min": 0.76,
     "accuracy": 0.9728395061728395
    },
    {
     "coverage": 0.75,
     "fields": 675,
     "confidence_min": 0.91,
     "accuracy": 1.0
    },
    {
     "coverage": 0.5,
     "fields": 450,
     "confidence_min": 0.97,
     "accuracy": 1.0
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 900,
     "confidence_min": 0.4865263835048701,
     "accuracy": 0.8933333333333333
    },
    {
     "coverage": 0.9,
     "fields": 810,
     "confidence_min": 0.9730817013122851,
     "accuracy": 0.9259259259259259
    },
    {
     "coverage": 0.75,
     "fields": 675,
     "confidence_min": 0.9933720126645095,
     "accuracy": 0.9437037037037037
    },
    {
     "coverage": 0.5,
     "fields": 450,
     "confidence_min": 0.9956317551428137,
     "accuracy": 0.94
    }
   ]
  }
 },
 "public768": {
  "reference": "data/generated/reference_public_768",
  "candidate": "reports/public768_joint-marker-stage-k-t-v1_cached",
  "requests_sha256": "798d018d89c02c00e51c609463d1216cfcd777a20650cc8b6efbbe993518d27d",
  "requests": 768,
  "fields": 768,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 768,
   "only_reference_correct": 212,
   "only_candidate_correct": 24,
   "both_wrong": 43,
   "mcnemar_exact_p": 8.731280991760007e-39,
   "reference": {
    "accuracy": 0.9127604166666666,
    "nll": 0.5297108014398391,
    "brier": 0.1502735482720726
   },
   "candidate": {
    "accuracy": 0.66796875,
    "nll": 1.3383109308914765,
    "brier": 0.5837933783949593
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     -0.2799479166666667,
     -0.2109375
    ],
    "nll": [
     0.5823668798192932,
     1.0294340372069068
    ],
    "brier": [
     0.3775514767124475,
     0.49445187517921624
    ]
   }
  },
  "groups": {
   "data/public_validation_v1/hellaswag_calibration.jsonl:answer": {
    "fields": 256,
    "only_reference_correct": 30,
    "only_candidate_correct": 9,
    "both_wrong": 10,
    "mcnemar_exact_p": 0.0010650196345523,
    "reference": {
     "accuracy": 0.92578125,
     "nll": 0.3873098396840055,
     "brier": 0.15116439481621774
    },
    "candidate": {
     "accuracy": 0.84375,
     "nll": 0.7004651278663554,
     "brier": 0.2911142408325036
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.12890625,
      -0.03515625
     ],
     "nll": [
      0.033266721829194955,
      0.5641973209838969
     ],
     "brier": [
      0.06375939681881358,
      0.22170807268464973
     ]
    }
   },
   "data/public_validation_v1/mmlu_calibration.jsonl:answer": {
    "fields": 256,
    "only_reference_correct": 133,
    "only_candidate_correct": 4,
    "both_wrong": 25,
    "mcnemar_exact_p": 1.6613665736354485e-34,
    "reference": {
     "accuracy": 0.88671875,
     "nll": 0.8228434591672611,
     "brier": 0.1892578125
    },
    "candidate": {
     "accuracy": 0.3828125,
     "nll": 2.4421279693888387,
     "brier": 1.0521342832893945
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5703125,
      -0.44140625
     ],
     "nll": [
      1.063783697166187,
      2.120966274473022
     ],
     "brier": [
      0.7560411575774169,
      0.9744153855959601
     ]
    }
   },
   "data/public_validation_v1/winogrande_calibration.jsonl:answer": {
    "fields": 256,
    "only_reference_correct": 49,
    "only_candidate_correct": 11,
    "both_wrong": 8,
    "mcnemar_exact_p": 7.561280982396751e-07,
    "reference": {
     "accuracy": 0.92578125,
     "nll": 0.37897910546825053,
     "brier": 0.1103984375
    },
    "candidate": {
     "accuracy": 0.77734375,
     "nll": 0.8723396954192347,
     "brier": 0.4081316110629795
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.20703125,
      -0.09375
     ],
     "nll": [
      0.1535695037567709,
      0.7940068064070515
     ],
     "brier": [
      0.20488366887749382,
      0.3946966317708833
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 768,
     "confidence_min": 0.29,
     "accuracy": 0.9127604166666666
    },
    {
     "coverage": 0.9,
     "fields": 691,
     "confidence_min": 0.66,
     "accuracy": 0.9392185238784371
    },
    {
     "coverage": 0.75,
     "fields": 576,
     "confidence_min": 0.87,
     "accuracy": 0.9583333333333334
    },
    {
     "coverage": 0.5,
     "fields": 384,
     "confidence_min": 0.97,
     "accuracy": 0.9791666666666666
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 768,
     "confidence_min": 0.31870537057148945,
     "accuracy": 0.66796875
    },
    {
     "coverage": 0.9,
     "fields": 691,
     "confidence_min": 0.8303681740512856,
     "accuracy": 0.7076700434153401
    },
    {
     "coverage": 0.75,
     "fields": 576,
     "confidence_min": 0.9660550040074408,
     "accuracy": 0.7725694444444444
    },
    {
     "coverage": 0.5,
     "fields": 384,
     "confidence_min": 0.9881778196182925,
     "accuracy": 0.8723958333333334
    }
   ]
  }
 },
 "unseen2": {
  "reference": "data/generated/reference_unseen2_144",
  "candidate": "reports/unseen2_joint-marker-stage-k-t-v1_cached",
  "requests_sha256": "fc5a3cdacc4cbab287543c69d365664108b04ed4fcf1732e1ce3a63212e617cb",
  "requests": 144,
  "fields": 432,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 432,
   "only_reference_correct": 7,
   "only_candidate_correct": 50,
   "both_wrong": 1,
   "mcnemar_exact_p": 4.236709844063213e-09,
   "reference": {
    "accuracy": 0.8819444444444444,
    "nll": 0.35543278183580723,
    "brier": 0.20653177288220778
   },
   "candidate": {
    "accuracy": 0.9814814814814815,
    "nll": 0.09511095616347721,
    "brier": 0.032640249597932505
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.05787037037037037,
     0.15046296296296297
    ],
    "nll": [
     -0.3997845234270151,
     -0.11648889082925977
    ],
    "brier": [
     -0.24900611376121398,
     -0.10555371958050576
    ]
   }
  },
  "groups": {
   "data/generated/unseen_v2/test.jsonl:action": {
    "fields": 144,
    "only_reference_correct": 5,
    "only_candidate_correct": 23,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0009122341871261597,
    "reference": {
     "accuracy": 0.8402777777777778,
     "nll": 0.39054634438174823,
     "brier": 0.2395847222222222
    },
    "candidate": {
     "accuracy": 0.9652777777777778,
     "nll": 0.14827104804012456,
     "brier": 0.055876294656399504
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.05555555555555555,
      0.19444444444444445
     ],
     "nll": [
      -0.4297912430107813,
      -0.04049432854712759
     ],
     "brier": [
      -0.2844760085624049,
      -0.0876664505285705
     ]
    }
   },
   "data/generated/unseen_v2/test.jsonl:evidence_quality": {
    "fields": 144,
    "only_reference_correct": 0,
    "only_candidate_correct": 18,
    "both_wrong": 0,
    "mcnemar_exact_p": 7.62939453125e-06,
    "reference": {
     "accuracy": 0.875,
     "nll": 0.347194612209041,
     "brier": 0.19897309642440114
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0030807854470530256,
     "brier": 0.00019424066519220442
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0763888888888889,
      0.18055555555555555
     ],
     "nll": [
      -0.501453557296476,
      -0.2118552956581121
     ],
     "brier": [
      -0.2882960394149301,
      -0.12015202601144849
     ]
    }
   },
   "data/generated/unseen_v2/test.jsonl:matches_requirement": {
    "fields": 144,
    "only_reference_correct": 2,
    "only_candidate_correct": 9,
    "both_wrong": 1,
    "mcnemar_exact_p": 0.0654296875,
    "reference": {
     "accuracy": 0.9305555555555556,
     "nll": 0.32855738891663255,
     "brier": 0.1810375
    },
    "candidate": {
     "accuracy": 0.9791666666666666,
     "nll": 0.1339810350032541,
     "brier": 0.04185021347220581
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.006944444444444444,
      0.09722222222222222
     ],
     "nll": [
      -0.3331891815257578,
      -0.023964607575345367
     ],
     "brier": [
      -0.2021960981011507,
      -0.07460279706356274
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 432,
     "confidence_min": 0.43,
     "accuracy": 0.8819444444444444
    },
    {
     "coverage": 0.9,
     "fields": 389,
     "confidence_min": 0.6699999999999999,
     "accuracy": 0.9023136246786633
    },
    {
     "coverage": 0.75,
     "fields": 324,
     "confidence_min": 0.8,
     "accuracy": 0.9135802469135802
    },
    {
     "coverage": 0.5,
     "fields": 216,
     "confidence_min": 0.93,
     "accuracy": 0.9398148148148148
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 432,
     "confidence_min": 0.4406827861929684,
     "accuracy": 0.9814814814814815
    },
    {
     "coverage": 0.9,
     "fields": 389,
     "confidence_min": 0.9979917142143738,
     "accuracy": 0.9948586118251928
    },
    {
     "coverage": 0.75,
     "fields": 324,
     "confidence_min": 0.9981354948513719,
     "accuracy": 0.9938271604938271
    },
    {
     "coverage": 0.5,
     "fields": 216,
     "confidence_min": 0.9982784248890603,
     "accuracy": 0.9907407407407407
    }
   ]
  }
 },
 "blind": {
  "reference": "data/generated/reference_blind_500",
  "candidate": "reports/blind_joint-marker-stage-k-t-v1_cached",
  "requests_sha256": "4647dc3dd7edd87d97dc5e5a2f115eb1454a1006af2db5164b2eb3756f60afd7",
  "requests": 500,
  "fields": 1180,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1180,
   "only_reference_correct": 29,
   "only_candidate_correct": 161,
   "both_wrong": 6,
   "mcnemar_exact_p": 2.234852059582319e-23,
   "reference": {
    "accuracy": 0.8584745762711864,
    "nll": 0.3308882778156683,
    "brier": 0.20319474576271188
   },
   "candidate": {
    "accuracy": 0.9703389830508474,
    "nll": 0.14266457084260828,
    "brier": 0.05221386560651571
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.08647260273972603,
     0.13872340425531915
    ],
    "nll": [
     -0.2629674135438645,
     -0.10651360936709793
    ],
    "brier": [
     -0.18774646647467413,
     -0.11477148756105386
    ]
   }
  },
  "groups": {
   "data/generated/judgment_v1/cases.jsonl:action": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 8,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0078125,
    "reference": {
     "accuracy": 0.8333333333333334,
     "nll": 0.5649138665795856,
     "brier": 0.29636666666666667
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0016684267945961007,
     "brier": 4.191969319722331e-06
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0625,
      0.2916666666666667
     ],
     "nll": [
      -0.9481907347836844,
      -0.2556442286574599
     ],
     "brier": [
      -0.4855665522247548,
      -0.14629176332872912
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:evidence_quality": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 2,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.5,
    "reference": {
     "accuracy": 0.9583333333333334,
     "nll": 0.11543657120998631,
     "brier": 0.0624125
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0019162067538638745,
     "brier": 4.917494385668617e-06
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.10416666666666667
     ],
     "nll": [
      -0.22342580701731204,
      -0.03300133135083253
     ],
     "brier": [
      -0.13769106477963367,
      -0.009682655361493885
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:matches_requirement": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 2,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.5,
    "reference": {
     "accuracy": 0.9583333333333334,
     "nll": 0.19715591766304155,
     "brier": 0.09675416666666668
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0012592795881696199,
     "brier": 3.1906508723129963e-06
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.10416666666666667
     ],
     "nll": [
      -0.2917335679287772,
      -0.12539541940634988
     ],
     "brier": [
      -0.17550094454584622,
      -0.03951341983088297
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:action": {
    "fields": 88,
    "only_reference_correct": 0,
    "only_candidate_correct": 15,
    "both_wrong": 0,
    "mcnemar_exact_p": 6.103515625e-05,
    "reference": {
     "accuracy": 0.8295454545454546,
     "nll": 0.4681900139759881,
     "brier": 0.258
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0029396299643222015,
     "brier": 1.5960065704234303e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.09090909090909091,
      0.25
     ],
     "nll": [
      -0.6624627555938887,
      -0.28072201037165584
     ],
     "brier": [
      -0.37613415383030896,
      -0.14931124239595459
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:department": {
    "fields": 40,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.0002512583963375363,
     "brier": 5.000000000000005e-06
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0032922776940310363,
     "brier": 2.1845333668942152e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      0.002478601100993821,
      0.0033781326143823517
     ],
     "brier": [
      6.0005624983898654e-06,
      2.303032899443895e-05
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:evidence_quality": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 4,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.125,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.19186169388221053,
     "brier": 0.11143333333333333
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0018951252100200804,
     "brier": 4.84412710732334e-06
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.020833333333333332,
      0.16666666666666666
     ],
     "nll": [
      -0.3463325281214704,
      -0.07224154884976648
     ],
     "brier": [
      -0.20972027064290324,
      -0.035516033188369454
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:matches_requirement": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 3,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.25,
    "reference": {
     "accuracy": 0.9375,
     "nll": 0.2240429975360382,
     "brier": 0.110975
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0012530737984813246,
     "brier": 3.186977823336345e-06
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.14583333333333334
     ],
     "nll": [
      -0.34494577945091937,
      -0.12655432180658244
     ],
     "brier": [
      -0.20421776980551784,
      -0.03770532086441342
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:action": {
    "fields": 40,
    "only_reference_correct": 0,
    "only_candidate_correct": 3,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.25,
    "reference": {
     "accuracy": 0.925,
     "nll": 0.3466242154785327,
     "brier": 0.17624
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0043706333376341885,
     "brier": 2.8775298617227318e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.15
     ],
     "nll": [
      -0.5520580307069431,
      -0.18820393844984257
     ],
     "brier": [
      -0.2931370071528673,
      -0.08668087231480429
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:department": {
    "fields": 40,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.001005033585350145,
     "brier": 2.000000000000002e-05
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.003331718301501886,
     "brier": 2.2418413849728136e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      0.0012338893115604586,
      0.003183752584829727
     ],
     "brier": [
      -1.8797446190549023e-05,
      1.8816991503587594e-05
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:action": {
    "fields": 72,
    "only_reference_correct": 14,
    "only_candidate_correct": 4,
    "both_wrong": 2,
    "mcnemar_exact_p": 0.0308837890625,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.2289145902005903,
     "brier": 0.1344972222222222
    },
    "candidate": {
     "accuracy": 0.7777777777777778,
     "nll": 1.087883164208579,
     "brier": 0.40341066298352796
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.25,
      -0.027777777777777776
     ],
     "nll": [
      0.4011389565610364,
      1.373900148728157
     ],
     "brier": [
      0.10564169376009912,
      0.4519434041358193
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:evidence_quality": {
    "fields": 72,
    "only_reference_correct": 9,
    "only_candidate_correct": 5,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.4239501953125,
    "reference": {
     "accuracy": 0.9305555555555556,
     "nll": 0.13080803955282563,
     "brier": 0.07639166666666666
    },
    "candidate": {
     "accuracy": 0.875,
     "nll": 0.744668016961895,
     "brier": 0.23310128320651133
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.16666666666666666,
      0.041666666666666664
     ],
     "nll": [
      0.17969773369839043,
      1.1082726719835905
     ],
     "brier": [
      0.014248881516146077,
      0.3178885519392008
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:matches_requirement": {
    "fields": 72,
    "only_reference_correct": 6,
    "only_candidate_correct": 8,
    "both_wrong": 2,
    "mcnemar_exact_p": 0.79052734375,
    "reference": {
     "accuracy": 0.8611111111111112,
     "nll": 0.36130463756224185,
     "brier": 0.21525
    },
    "candidate": {
     "accuracy": 0.8888888888888888,
     "nll": 0.4206724730491097,
     "brier": 0.19028290270297304
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.06944444444444445,
      0.125
     ],
     "nll": [
      -0.22328626049426795,
      0.40005911786494885
     ],
     "brier": [
      -0.1459396533731285,
      0.11440058538539212
     ]
    }
   },
   "data/rlcd_synth_v3/majority_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 33,
    "both_wrong": 0,
    "mcnemar_exact_p": 2.3283064365386963e-10,
    "reference": {
     "accuracy": 0.5875,
     "nll": 0.7704734274880431,
     "brier": 0.53452
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.00426250875475013,
     "brier": 3.647663153166432e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.3125,
      0.5125
     ],
     "nll": [
      -0.9631470811047901,
      -0.5747601068797468
     ],
     "brier": [
      -0.6750207030909611,
      -0.39854470985251567
     ]
    }
   },
   "data/rlcd_synth_v3/majority_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 11,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0009765625,
    "reference": {
     "accuracy": 0.8625,
     "nll": 0.30030802182472804,
     "brier": 0.17363
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.004396805812291979,
     "brier": 3.8976127183417675e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0625,
      0.2125
     ],
     "nll": [
      -0.3694092698851318,
      -0.2253318463614336
     ],
     "brier": [
      -0.23695737224022828,
      -0.11456830802753719
     ]
    }
   },
   "data/rlcd_synth_v3/override_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 27,
    "both_wrong": 1,
    "mcnemar_exact_p": 1.4901161193847656e-08,
    "reference": {
     "accuracy": 0.65,
     "nll": 0.6607649163294025,
     "brier": 0.47980999999999996
    },
    "candidate": {
     "accuracy": 0.9875,
     "nll": 0.03017686777989581,
     "brier": 0.018869575439590912
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.2375,
      0.4375
     ],
     "nll": [
      -0.8095674435221234,
      -0.4587648241360271
     ],
     "brier": [
      -0.5927573747779625,
      -0.336882125924396
     ]
    }
   },
   "data/rlcd_synth_v3/override_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 7,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.015625,
    "reference": {
     "accuracy": 0.9125,
     "nll": 0.20750992356246,
     "brier": 0.1171775
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.004552783199715717,
     "brier": 4.4456847444702786e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0375,
      0.15
     ],
     "nll": [
      -0.27894947175902135,
      -0.1377495939428366
     ],
     "brier": [
      -0.18131275566008173,
      -0.06212473924037846
     ]
    }
   },
   "data/rlcd_synth_v3/veto_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 27,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.4901161193847656e-08,
    "reference": {
     "accuracy": 0.6625,
     "nll": 0.6325119017462898,
     "brier": 0.4476975
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.004429929569980553,
     "brier": 4.723216471651049e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.2375,
      0.4375
     ],
     "nll": [
      -0.8306972393493025,
      -0.43510747877607575
     ],
     "brier": [
      -0.5894353372939986,
      -0.3106590679506866
     ]
    }
   },
   "data/rlcd_synth_v3/veto_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.13382107094010265,
     "brier": 0.045527500000000005
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.004334924266908555,
     "brier": 3.923888032361817e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      -0.15678407664843025,
      -0.10539649102599716
     ],
     "brier": [
      -0.06232130228869198,
      -0.030860764316534957
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:action": {
    "fields": 12,
    "only_reference_correct": 0,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.14175288215384776,
     "brier": 0.09385
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.003628578329392451,
     "brier": 3.668121923696242e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.25
     ],
     "nll": [
      -0.29600654825011524,
      -0.00040586353172269345
     ],
     "brier": [
      -0.2116284096580858,
      -0.0001151724122602785
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:eligible": {
    "fields": 12,
    "only_reference_correct": 0,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.1436268852546587,
     "brier": 0.07148333333333334
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0033677967406237496,
     "brier": 3.087143964876808e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.25
     ],
     "nll": [
      -0.2729261936362353,
      -0.040813897767076654
     ],
     "brier": [
      -0.1695874573880568,
      -0.0038521586475773126
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:permission": {
    "fields": 12,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 1,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.12323621978999721,
     "brier": 0.07825
    },
    "candidate": {
     "accuracy": 0.9166666666666666,
     "nll": 0.0644778704825675,
     "brier": 0.045911935838162284
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      -0.11602516567196901,
      -0.002344582943360412
     ],
     "brier": [
      -0.07256293163813865,
      -0.00013535909912275413
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1180,
     "confidence_min": 0.47,
     "accuracy": 0.8584745762711864
    },
    {
     "coverage": 0.9,
     "fields": 1062,
     "confidence_min": 0.67,
     "accuracy": 0.8945386064030132
    },
    {
     "coverage": 0.75,
     "fields": 885,
     "confidence_min": 0.82,
     "accuracy": 0.9322033898305084
    },
    {
     "coverage": 0.5,
     "fields": 590,
     "confidence_min": 0.9299999999999999,
     "accuracy": 0.9677966101694915
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1180,
     "confidence_min": 0.5247772474504915,
     "accuracy": 0.9703389830508474
    },
    {
     "coverage": 0.9,
     "fields": 1062,
     "confidence_min": 0.995269374476461,
     "accuracy": 0.9896421845574388
    },
    {
     "coverage": 0.75,
     "fields": 885,
     "confidence_min": 0.9957344694649259,
     "accuracy": 0.9875706214689266
    },
    {
     "coverage": 0.5,
     "fields": 590,
     "confidence_min": 0.9962859523748765,
     "accuracy": 0.9813559322033898
    }
   ]
  }
 },
 "development": {
  "reference": "data/generated/reference_development_640",
  "candidate": "reports/development_joint-marker-stage-k-t-v1_cached",
  "requests_sha256": "d3389dbf1b9dade03c9ce23716a14b3ac2fb9760aa4e88be0ddc43e7d9ee2f97",
  "requests": 640,
  "fields": 1280,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1280,
   "only_reference_correct": 0,
   "only_candidate_correct": 271,
   "both_wrong": 7,
   "mcnemar_exact_p": 5.271098971615262e-82,
   "reference": {
    "accuracy": 0.7828125,
    "nll": 0.5989872310599031,
    "brier": 0.32722859375
   },
   "candidate": {
    "accuracy": 0.99453125,
    "nll": 0.026353637740479302,
    "brier": 0.010042809748663133
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.1890625,
     0.23359375
    ],
    "nll": [
     -0.6628575531110332,
     -0.48348165756457984
    ],
    "brier": [
     -0.3486590640781906,
     -0.28591408342460267
    ]
   }
  },
  "groups": {
   "data/generated/reference_panels_v1/development.jsonl:action": {
    "fields": 640,
    "only_reference_correct": 0,
    "only_candidate_correct": 217,
    "both_wrong": 7,
    "mcnemar_exact_p": 9.495567745759799e-66,
    "reference": {
     "accuracy": 0.65,
     "nll": 0.9388350134625263,
     "brier": 0.5090984375
    },
    "candidate": {
     "accuracy": 0.9890625,
     "nll": 0.048237131495764154,
     "brier": 0.02004456468556111
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.303125,
      0.3734375
     ],
     "nll": [
      -1.0543489808348365,
      -0.7280478738021361
     ],
     "brier": [
      -0.5421440471200338,
      -0.4347384380897532
     ]
    }
   },
   "data/generated/reference_panels_v1/development.jsonl:condition": {
    "fields": 640,
    "only_reference_correct": 0,
    "only_candidate_correct": 54,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.1102230246251565e-16,
    "reference": {
     "accuracy": 0.915625,
     "nll": 0.2591394486572801,
     "brier": 0.14535875
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.004470143985194444,
     "brier": 4.105481176515685e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0640625,
      0.10625
     ],
     "nll": [
      -0.28780053270547634,
      -0.22398863273959085
     ],
     "brier": [
      -0.17100266806095626,
      -0.12114785994567268
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1280,
     "confidence_min": 0.5,
     "accuracy": 0.7828125
    },
    {
     "coverage": 0.9,
     "fields": 1152,
     "confidence_min": 0.66,
     "accuracy": 0.8133680555555556
    },
    {
     "coverage": 0.75,
     "fields": 960,
     "confidence_min": 0.82,
     "accuracy": 0.8427083333333333
    },
    {
     "coverage": 0.5,
     "fields": 640,
     "confidence_min": 0.92,
     "accuracy": 0.8671875
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1280,
     "confidence_min": 0.6893099313837402,
     "accuracy": 0.99453125
    },
    {
     "coverage": 0.9,
     "fields": 1152,
     "confidence_min": 0.9950536915723391,
     "accuracy": 1.0
    },
    {
     "coverage": 0.75,
     "fields": 960,
     "confidence_min": 0.9954703849808458,
     "accuracy": 1.0
    },
    {
     "coverage": 0.5,
     "fields": 640,
     "confidence_min": 0.9957808680951384,
     "accuracy": 1.0
    }
   ]
  }
 },
 "order": {
  "reference": "data/generated/reference_order_336",
  "candidate": "reports/order_joint-marker-stage-k-t-v1_cached",
  "requests_sha256": "17f9088d4fe2c7dd7461e1c04b14fca63cf8d6af90ff59bbd8c895c87743341e",
  "requests": 336,
  "fields": 1008,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1008,
   "only_reference_correct": 162,
   "only_candidate_correct": 51,
   "both_wrong": 10,
   "mcnemar_exact_p": 1.1628800596632321e-14,
   "reference": {
    "accuracy": 0.939484126984127,
    "nll": 0.19824015084544572,
    "brier": 0.10872287784855125
   },
   "candidate": {
    "accuracy": 0.8293650793650794,
    "nll": 0.9796106761213,
    "brier": 0.32685214544104463
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     -0.14583333333333334,
     -0.07341269841269842
    ],
    "nll": [
     0.5916665475987548,
     0.9772013327592998
    ],
    "brier": [
     0.15467898235540828,
     0.28008609435259235
    ]
   }
  },
  "groups": {
   "data/generated/order_v1/test.jsonl:action": {
    "fields": 336,
    "only_reference_correct": 56,
    "only_candidate_correct": 14,
    "both_wrong": 10,
    "mcnemar_exact_p": 4.3028401315359276e-07,
    "reference": {
     "accuracy": 0.9285714285714286,
     "nll": 0.17963740596887126,
     "brier": 0.09942440476190476
    },
    "candidate": {
     "accuracy": 0.8035714285714286,
     "nll": 1.111573268194697,
     "brier": 0.3743092944551274
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.16964285714285715,
      -0.08035714285714286
     ],
     "nll": [
      0.7060842937359022,
      1.1700220627963343
     ],
     "brier": [
      0.20296674095941936,
      0.3475118833534233
     ]
    }
   },
   "data/generated/order_v1/test.jsonl:evidence_quality": {
    "fields": 336,
    "only_reference_correct": 74,
    "only_candidate_correct": 8,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.647020623262653e-14,
    "reference": {
     "accuracy": 0.9761904761904762,
     "nll": 0.08507247933394088,
     "brier": 0.041618038307558516
    },
    "candidate": {
     "accuracy": 0.7797619047619048,
     "nll": 1.3497723347978858,
     "brier": 0.42961802261744614
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.24404761904761904,
      -0.1488095238095238
     ],
     "nll": [
      0.9914055515641987,
      1.5487933797130717
     ],
     "brier": [
      0.3019287151259552,
      0.4771407001785396
     ]
    }
   },
   "data/generated/order_v1/test.jsonl:matches_requirement": {
    "fields": 336,
    "only_reference_correct": 32,
    "only_candidate_correct": 29,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.798152627305718,
    "reference": {
     "accuracy": 0.9136904761904762,
     "nll": 0.330010567233525,
     "brier": 0.18512619047619047
    },
    "candidate": {
     "accuracy": 0.9047619047619048,
     "nll": 0.47748642537131747,
     "brier": 0.1766291192505604
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.05357142857142857,
      0.03571428571428571
     ],
     "nll": [
      -0.001987559235410518,
      0.31918380926385015
     ],
     "brier": [
      -0.06642771897056614,
      0.0563716230966146
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1008,
     "confidence_min": 0.44,
     "accuracy": 0.939484126984127
    },
    {
     "coverage": 0.9,
     "fields": 907,
     "confidence_min": 0.58,
     "accuracy": 0.9746416758544653
    },
    {
     "coverage": 0.75,
     "fields": 756,
     "confidence_min": 0.78,
     "accuracy": 0.9947089947089947
    },
    {
     "coverage": 0.5,
     "fields": 504,
     "confidence_min": 0.93,
     "accuracy": 1.0
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1008,
     "confidence_min": 0.5278424236994175,
     "accuracy": 0.8293650793650794
    },
    {
     "coverage": 0.9,
     "fields": 907,
     "confidence_min": 0.9939234961053397,
     "accuracy": 0.8721058434399118
    },
    {
     "coverage": 0.75,
     "fields": 756,
     "confidence_min": 0.9976473369716695,
     "accuracy": 0.9417989417989417
    },
    {
     "coverage": 0.5,
     "fields": 504,
     "confidence_min": 0.9980713549361174,
     "accuracy": 0.9642857142857143
    }
   ]
  }
 },
 "bendsynth_holdout": {
  "checkpoint": "checkpoints/joint-marker-stage-k-t-v1",
  "checkpoint_sha256": {
   "decision.json": "e102f342195df397baf6cb307a12d83203c1ee66a01f2722c54f22f9dfefc633",
   "joint_head.pt": "1362ed74822195ef94b1f503c88c6fb6ebc2e9bad0b80b3a08ba55b95c8f70eb"
  },
  "seed": 419,
  "scope": "Test split only; checkpoint frozen before evaluation; no reference API calls.",
  "reports": {
   "data/bendsynth_v1/holdout.jsonl": {
    "sha256": "a60021e3a72e62382ea88a66a6e96d1a00ee9962e39f659c3500cc7e976dae05",
    "test_cases": 3840,
    "indices": [
     22,
     31,
     32,
     35,
     41,
     57,
     61,
     63,
     70,
     97,
     101,
     112,
     115,
     143,
     148,
     155,
     158,
     168,
     170,
     210,
     221,
     238,
     250,
     263,
     264,
     277,
     288,
     335,
     340,
     343,
     378,
     389,
     403,
     405,
     417,
     444,
     448,
     451,
     478,
     484,
     490,
     492,
     519,
     520,
     523,
     531,
     535,
     544,
     550,
     559,
     561,
     580,
     581,
     592,
     593,
     626,
     641,
     660,
     695,
     742,
     761,
     772,
     789,
     794,
     812,
     821,
     833,
     838,
     886,
     934,
     953,
     957,
     967,
     999,
     1018,
     1043,
     1072,
     1080,
     1091,
     1102,
     1103,
     1107,
     1150,
     1155,
     1159,
     1165,
     1170,
     1197,
     1203,
     1224,
     1235,
     1256,
     1266,
     1316,
     1318,
     1339,
     1347,
     1370,
     1386,
     1389,
     1391,
     1407,
     1442,
     1452,
     1473,
     1481,
     1507,
     1511,
     1547,
     1555,
     1624,
     1625,
     1626,
     1648,
     1694,
     1706,
     1720,
     1735,
     1742,
     1757,
     1794,
     1796,
     1820,
     1823,
     1832,
     1867,
     1871,
     1898,
     1899,
     1905,
     1922,
     1928,
     1949,
     1962,
     1984,
     1997,
     2017,
     2018,
     2026,
     2038,
     2047,
     2054,
     2076,
     2083,
     2103,
     2112,
     2130,
     2191,
     2219,
     2221,
     2233,
     2285,
     2288,
     2316,
     2333,
     2348,
     2355,
     2362,
     2366,
     2369,
     2371,
     2384,
     2396,
     2399,
     2401,
     2440,
     2442,
     2472,
     2476,
     2486,
     2487,
     2490,
     2507,
     2525,
     2526,
     2539,
     2551,
     2552,
     2555,
     2557,
     2567,
     2593,
     2598,
     2615,
     2636,
     2655,
     2674,
     2724,
     2737,
     2776,
     2779,
     2786,
     2795,
     2800,
     2829,
     2853,
     2854,
     2865,
     2866,
     2869,
     2894,
     2927,
     2934,
     2959,
     2987,
     3005,
     3009,
     3024,
     3025,
     3037,
     3082,
     3088,
     3095,
     3114,
     3117,
     3128,
     3133,
     3153,
     3198,
     3207,
     3214,
     3244,
     3247,
     3278,
     3280,
     3301,
     3311,
     3334,
     3335,
     3382,
     3384,
     3391,
     3412,
     3432,
     3451,
     3459,
     3477,
     3524,
     3538,
     3546,
     3553,
     3557,
     3610,
     3625,
     3634,
     3650,
     3689,
     3726,
     3753,
     3754,
     3772,
     3781,
     3786,
     3802,
     3832,
     3836
    ],
    "case_count": 256,
    "field_count": 786,
    "calibrated": false,
    "metrics": {
     "base": {
      "joint_accuracy": 0.8359375,
      "field_accuracy": 0.9338422391857506,
      "joint_nll": 1.095082830751964,
      "field_brier": 0.1230800378971528,
      "constraint_violation_rate": 0.0
     },
     "constrained": {
      "joint_accuracy": 0.8359375,
      "field_accuracy": 0.9338422391857506,
      "joint_nll": 1.095082830751964,
      "field_brier": 0.1230800378971528,
      "constraint_violation_rate": 0.0
     }
    },
    "probability_metrics": {
     "all": {
      "field_count": 786,
      "temperature": 1.0,
      "accuracy": 0.9325699745547074,
      "nll": 0.3608094459547233,
      "brier": 0.12308003802919192,
      "ece_10_equal_width": 0.059225309913689524,
      "reliability": [
       {
        "lower": 0.0,
        "upper": 0.1,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.1,
        "upper": 0.2,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.2,
        "upper": 0.3,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.3,
        "upper": 0.4,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.4,
        "upper": 0.5,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.48726740165137733
       },
       {
        "lower": 0.5,
        "upper": 0.6,
        "count": 3,
        "accuracy": 0.3333333333333333,
        "mean_probability": 0.5726409056177918
       },
       {
        "lower": 0.6,
        "upper": 0.7,
        "count": 4,
        "accuracy": 0.5,
        "mean_probability": 0.6558516137274716
       },
       {
        "lower": 0.7,
        "upper": 0.8,
        "count": 3,
        "accuracy": 0.0,
        "mean_probability": 0.751070333509907
       },
       {
        "lower": 0.8,
        "upper": 0.9,
        "count": 4,
        "accuracy": 0.5,
        "mean_probability": 0.8642534351147402
       },
       {
        "lower": 0.9,
        "upper": 1.0,
        "count": 771,
        "accuracy": 0.9442282749675746,
        "mean_probability": 0.9974218836287375
       }
      ],
      "selective": [
       {
        "threshold": 0.0,
        "count": 786,
        "coverage": 1.0,
        "accuracy": 0.9325699745547074,
        "error_rate": 0.06743002544529264
       },
       {
        "threshold": 0.5,
        "count": 785,
        "coverage": 0.9987277353689568,
        "accuracy": 0.9337579617834395,
        "error_rate": 0.0662420382165605
       },
       {
        "threshold": 0.7,
        "count": 778,
        "coverage": 0.989821882951654,
        "accuracy": 0.9383033419023136,
        "error_rate": 0.06169665809768643
       },
       {
        "threshold": 0.8,
        "count": 775,
        "coverage": 0.9860050890585241,
        "accuracy": 0.9419354838709677,
        "error_rate": 0.058064516129032295
       },
       {
        "threshold": 0.9,
        "count": 771,
        "coverage": 0.9809160305343512,
        "accuracy": 0.9442282749675746,
        "error_rate": 0.055771725032425445
       },
       {
        "threshold": 0.95,
        "count": 767,
        "coverage": 0.9758269720101781,
        "accuracy": 0.9452411994784876,
        "error_rate": 0.054758800521512385
       },
       {
        "threshold": 0.99,
        "count": 749,
        "coverage": 0.9529262086513995,
        "accuracy": 0.9506008010680908,
        "error_rate": 0.04939919893190925
       },
       {
        "threshold": 1.0,
        "count": 0,
        "coverage": 0.0,
        "accuracy": null,
        "error_rate": null
       }
      ]
     },
     "type:score": {
      "field_count": 251,
      "temperature": 1.0,
      "accuracy": 0.9402390438247012,
      "nll": 0.3445806764748063,
      "brier": 0.11032375656898928,
      "ece_10_equal_width": 0.05491551942406197,
      "reliability": [
       {
        "lower": 0.0,
        "upper": 0.1,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.1,
        "upper": 0.2,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.2,
        "upper": 0.3,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.3,
        "upper": 0.4,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.4,
        "upper": 0.5,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.5,
        "upper": 0.6,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.5589786696048502
       },
       {
        "lower": 0.6,
        "upper": 0.7,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.7,
        "upper": 0.8,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.7871134136487881
       },
       {
        "lower": 0.8,
        "upper": 0.9,
        "count": 1,
        "accuracy": 1.0,
        "mean_probability": 0.8679867866115066
       },
       {
        "lower": 0.9,
        "upper": 1.0,
        "count": 248,
        "accuracy": 0.9475806451612904,
        "mean_probability": 0.9972003632209574
       }
      ],
      "selective": [
       {
        "threshold": 0.0,
        "count": 251,
        "coverage": 1.0,
        "accuracy": 0.9402390438247012,
        "error_rate": 0.05976095617529875
       },
       {
        "threshold": 0.5,
        "count": 251,
        "coverage": 1.0,
        "accuracy": 0.9402390438247012,
        "error_rate": 0.05976095617529875
       },
       {
        "threshold": 0.7,
        "count": 250,
        "coverage": 0.9960159362549801,
        "accuracy": 0.944,
        "error_rate": 0.05600000000000005
       },
       {
        "threshold": 0.8,
        "count": 249,
        "coverage": 0.9920318725099602,
        "accuracy": 0.9477911646586346,
        "error_rate": 0.052208835341365445
       },
       {
        "threshold": 0.9,
        "count": 248,
        "coverage": 0.9880478087649402,
        "accuracy": 0.9475806451612904,
        "error_rate": 0.05241935483870963
       },
       {
        "threshold": 0.95,
        "count": 246,
        "coverage": 0.9800796812749004,
        "accuracy": 0.9512195121951219,
        "error_rate": 0.04878048780487809
       },
       {
        "threshold": 0.99,
        "count": 242,
        "coverage": 0.9641434262948207,
        "accuracy": 0.9504132231404959,
        "error_rate": 0.04958677685950408
       },
       {
        "threshold": 1.0,
        "count": 0,
        "coverage": 0.0,
        "accuracy": null,
        "error_rate": null
       }
      ]
     },
     "choices:5": {
      "field_count": 25,
      "temperature": 1.0,
      "accuracy": 0.92,
      "nll": 0.6277247639091026,
      "brier": 0.1610265116012207,
      "ece_10_equal_width": 0.0835548981065311,
      "reliability": [
       {
        "lower": 0.0,
        "upper": 0.1,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.1,
        "upper": 0.2,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.2,
        "upper": 0.3,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.3,
        "upper": 0.4,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.4,
        "upper": 0.5,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.5,
        "upper": 0.6,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.6,
        "upper": 0.7,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.7,
        "upper": 0.8,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.8,
        "upper": 0.9,
        "count": 1,
        "accuracy": 1.0,
        "mean_probability": 0.8679867866115066
       },
       {
        "lower": 0.9,
        "upper": 1.0,
        "count": 24,
        "accuracy": 0.9166666666666666,
        "mean_probability": 0.998202468303116
       }
      ],
      "selective": [
       {
        "threshold": 0.0,
        "count": 25,
        "coverage": 1.0,
        "accuracy": 0.92,
        "error_rate": 0.07999999999999996
       },
       {
        "threshold": 0.5,
        "count": 25,
        "coverage": 1.0,
        "accuracy": 0.92,
        "error_rate": 0.07999999999999996
       },
       {
        "threshold": 0.7,
        "count": 25,
        "coverage": 1.0,
        "accuracy": 0.92,
        "error_rate": 0.07999999999999996
       },
       {
        "threshold": 0.8,
        "count": 25,
        "coverage": 1.0,
        "accuracy": 0.92,
        "error_rate": 0.07999999999999996
       },
       {
        "threshold": 0.9,
        "count": 24,
        "coverage": 0.96,
        "accuracy": 0.9166666666666666,
        "error_rate": 0.08333333333333337
       },
       {
        "threshold": 0.95,
        "count": 24,
        "coverage": 0.96,
        "accuracy": 0.9166666666666666,
        "error_rate": 0.08333333333333337
       },
       {
        "threshold": 0.99,
        "count": 24,
        "coverage": 0.96,
        "accuracy": 0.9166666666666666,
        "error_rate": 0.08333333333333337
       },
       {
        "threshold": 1.0,
        "count": 0,
        "coverage": 0.0,
        "accuracy": null,
        "error_rate": null
       }
      ]
     },
     "type:choice": {
      "field_count": 256,
      "temperature": 1.0,
      "accuracy": 0.8984375,
      "nll": 0.5267863763339525,
      "brier": 0.1812119618800849,
      "ece_10_equal_width": 0.0877178983408792,
      "reliability": [
       {
        "lower": 0.0,
        "upper": 0.1,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.1,
        "upper": 0.2,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.2,
        "upper": 0.3,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.3,
        "upper": 0.4,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.4,
        "upper": 0.5,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.48726740165137733
       },
       {
        "lower": 0.5,
        "upper": 0.6,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.5925435870800273
       },
       {
        "lower": 0.6,
        "upper": 0.7,
        "count": 4,
        "accuracy": 0.5,
        "mean_probability": 0.6558516137274716
       },
       {
        "lower": 0.7,
        "upper": 0.8,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.7126569526389898
       },
       {
        "lower": 0.8,
        "upper": 0.9,
        "count": 2,
        "accuracy": 0.5,
        "mean_probability": 0.8810097172224285
       },
       {
        "lower": 0.9,
        "upper": 1.0,
        "count": 247,
        "accuracy": 0.9190283400809717,
        "mean_probability": 0.9970764702208095
       }
      ],
      "selective": [
       {
        "threshold": 0.0,
        "count": 256,
        "coverage": 1.0,
        "accuracy": 0.8984375,
        "error_rate": 0.1015625
       },
       {
        "threshold": 0.5,
        "count": 255,
        "coverage": 0.99609375,
        "accuracy": 0.9019607843137255,
        "error_rate": 0.0980392156862745
       },
       {
        "threshold": 0.7,
        "count": 250,
        "coverage": 0.9765625,
        "accuracy": 0.912,
        "error_rate": 0.08799999999999997
       },
       {
        "threshold": 0.8,
        "count": 249,
        "coverage": 0.97265625,
        "accuracy": 0.9156626506024096,
        "error_rate": 0.08433734939759041
       },
       {
        "threshold": 0.9,
        "count": 247,
        "coverage": 0.96484375,
        "accuracy": 0.9190283400809717,
        "error_rate": 0.08097165991902833
       },
       {
        "threshold": 0.95,
        "count": 246,
        "coverage": 0.9609375,
        "accuracy": 0.9186991869918699,
        "error_rate": 0.08130081300813008
       },
       {
        "threshold": 0.99,
        "count": 237,
        "coverage": 0.92578125,
        "accuracy": 0.9324894514767933,
        "error_rate": 0.0675105485232067
       },
       {
        "threshold": 1.0,
        "count": 0,
        "coverage": 0.0,
        "accuracy": null,
        "error_rate": null
       }
      ]
     },
     "choices:4": {
      "field_count": 296,
      "temperature": 1.0,
      "accuracy": 0.9358108108108109,
      "nll": 0.3536564190429364,
      "brier": 0.11755841588404763,
      "ece_10_equal_width": 0.055015019981466975,
      "reliability": [
       {
        "lower": 0.0,
        "upper": 0.1,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.1,
        "upper": 0.2,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.2,
        "upper": 0.3,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.3,
        "upper": 0.4,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.4,
        "upper": 0.5,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.5,
        "upper": 0.6,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.5589786696048502
       },
       {
        "lower": 0.6,
        "upper": 0.7,
        "count": 3,
        "accuracy": 0.6666666666666666,
        "mean_probability": 0.6604476016057238
       },
       {
        "lower": 0.7,
        "upper": 0.8,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.7871134136487881
       },
       {
        "lower": 0.8,
        "upper": 0.9,
        "count": 2,
        "accuracy": 0.5,
        "mean_probability": 0.8810097172224285
       },
       {
        "lower": 0.9,
        "upper": 1.0,
        "count": 289,
        "accuracy": 0.9480968858131488,
        "mean_probability": 0.9970853882409443
       }
      ],
      "selective": [
       {
        "threshold": 0.0,
        "count": 296,
        "coverage": 1.0,
        "accuracy": 0.9358108108108109,
        "error_rate": 0.06418918918918914
       },
       {
        "threshold": 0.5,
        "count": 296,
        "coverage": 1.0,
        "accuracy": 0.9358108108108109,
        "error_rate": 0.06418918918918914
       },
       {
        "threshold": 0.7,
        "count": 292,
        "coverage": 0.9864864864864865,
        "accuracy": 0.9417808219178082,
        "error_rate": 0.05821917808219179
       },
       {
        "threshold": 0.8,
        "count": 291,
        "coverage": 0.9831081081081081,
        "accuracy": 0.9450171821305842,
        "error_rate": 0.054982817869415834
       },
       {
        "threshold": 0.9,
        "count": 289,
        "coverage": 0.9763513513513513,
        "accuracy": 0.9480968858131488,
        "error_rate": 0.05190311418685123
       },
       {
        "threshold": 0.95,
        "count": 286,
        "coverage": 0.9662162162162162,
        "accuracy": 0.951048951048951,
        "error_rate": 0.04895104895104896
       },
       {
        "threshold": 0.99,
        "count": 281,
        "coverage": 0.9493243243243243,
        "accuracy": 0.9537366548042705,
        "error_rate": 0.0462633451957295
       },
       {
        "threshold": 1.0,
        "count": 0,
        "coverage": 0.0,
        "accuracy": null,
        "error_rate": null
       }
      ]
     },
     "type:noul": {
      "field_count": 279,
      "temperature": 1.0,
      "accuracy": 0.956989247311828,
      "nll": 0.22311527736109096,
      "brier": 0.08121643279866253,
      "ece_10_equal_width": 0.041013411282778536,
      "reliability": [
       {
        "lower": 0.0,
        "upper": 0.1,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.1,
        "upper": 0.2,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.2,
        "upper": 0.3,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.3,
        "upper": 0.4,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.4,
        "upper": 0.5,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.5,
        "upper": 0.6,
        "count": 1,
        "accuracy": 1.0,
        "mean_probability": 0.5664004601684981
       },
       {
        "lower": 0.6,
        "upper": 0.7,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.7,
        "upper": 0.8,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.753440634241943
       },
       {
        "lower": 0.8,
        "upper": 0.9,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.8270075194025969
       },
       {
        "lower": 0.9,
        "upper": 1.0,
        "count": 276,
        "accuracy": 0.9637681159420289,
        "mean_probability": 0.9979300509218085
       }
      ],
      "selective": [
       {
        "threshold": 0.0,
        "count": 279,
        "coverage": 1.0,
        "accuracy": 0.956989247311828,
        "error_rate": 0.043010752688172005
       },
       {
        "threshold": 0.5,
        "count": 279,
        "coverage": 1.0,
        "accuracy": 0.956989247311828,
        "error_rate": 0.043010752688172005
       },
       {
        "threshold": 0.7,
        "count": 278,
        "coverage": 0.996415770609319,
        "accuracy": 0.9568345323741008,
        "error_rate": 0.043165467625899234
       },
       {
        "threshold": 0.8,
        "count": 277,
        "coverage": 0.992831541218638,
        "accuracy": 0.9602888086642599,
        "error_rate": 0.03971119133574008
       },
       {
        "threshold": 0.9,
        "count": 276,
        "coverage": 0.989247311827957,
        "accuracy": 0.9637681159420289,
        "error_rate": 0.036231884057971064
       },
       {
        "threshold": 0.95,
        "count": 275,
        "coverage": 0.985663082437276,
        "accuracy": 0.9636363636363636,
        "error_rate": 0.036363636363636376
       },
       {
        "threshold": 0.99,
        "count": 270,
        "coverage": 0.967741935483871,
        "accuracy": 0.9666666666666667,
        "error_rate": 0.033333333333333326
       },
       {
        "threshold": 1.0,
        "count": 0,
        "coverage": 0.0,
        "accuracy": null,
        "error_rate": null
       }
      ]
     },
     "choices:2": {
      "field_count": 279,
      "temperature": 1.0,
      "accuracy": 0.956989247311828,
      "nll": 0.22311527736109096,
      "brier": 0.08121643279866253,
      "ece_10_equal_width": 0.041013411282778536,
      "reliability": [
       {
        "lower": 0.0,
        "upper": 0.1,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.1,
        "upper": 0.2,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.2,
        "upper": 0.3,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.3,
        "upper": 0.4,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.4,
        "upper": 0.5,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.5,
        "upper": 0.6,
        "count": 1,
        "accuracy": 1.0,
        "mean_probability": 0.5664004601684981
       },
       {
        "lower": 0.6,
        "upper": 0.7,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.7,
        "upper": 0.8,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.753440634241943
       },
       {
        "lower": 0.8,
        "upper": 0.9,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.8270075194025969
       },
       {
        "lower": 0.9,
        "upper": 1.0,
        "count": 276,
        "accuracy": 0.9637681159420289,
        "mean_probability": 0.9979300509218085
       }
      ],
      "selective": [
       {
        "threshold": 0.0,
        "count": 279,
        "coverage": 1.0,
        "accuracy": 0.956989247311828,
        "error_rate": 0.043010752688172005
       },
       {
        "threshold": 0.5,
        "count": 279,
        "coverage": 1.0,
        "accuracy": 0.956989247311828,
        "error_rate": 0.043010752688172005
       },
       {
        "threshold": 0.7,
        "count": 278,
        "coverage": 0.996415770609319,
        "accuracy": 0.9568345323741008,
        "error_rate": 0.043165467625899234
       },
       {
        "threshold": 0.8,
        "count": 277,
        "coverage": 0.992831541218638,
        "accuracy": 0.9602888086642599,
        "error_rate": 0.03971119133574008
       },
       {
        "threshold": 0.9,
        "count": 276,
        "coverage": 0.989247311827957,
        "accuracy": 0.9637681159420289,
        "error_rate": 0.036231884057971064
       },
       {
        "threshold": 0.95,
        "count": 275,
        "coverage": 0.985663082437276,
        "accuracy": 0.9636363636363636,
        "error_rate": 0.036363636363636376
       },
       {
        "threshold": 0.99,
        "count": 270,
        "coverage": 0.967741935483871,
        "accuracy": 0.9666666666666667,
        "error_rate": 0.033333333333333326
       },
       {
        "threshold": 1.0,
        "count": 0,
        "coverage": 0.0,
        "accuracy": null,
        "error_rate": null
       }
      ]
     },
     "choices:3": {
      "field_count": 186,
      "temperature": 1.0,
      "accuracy": 0.8924731182795699,
      "nll": 0.5428582957109219,
      "brier": 0.189562211012954,
      "ece_10_equal_width": 0.0962557708273799,
      "reliability": [
       {
        "lower": 0.0,
        "upper": 0.1,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.1,
        "upper": 0.2,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.2,
        "upper": 0.3,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.3,
        "upper": 0.4,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.4,
        "upper": 0.5,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.48726740165137733
       },
       {
        "lower": 0.5,
        "upper": 0.6,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.5925435870800273
       },
       {
        "lower": 0.6,
        "upper": 0.7,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.6420636500927152
       },
       {
        "lower": 0.7,
        "upper": 0.8,
        "count": 1,
        "accuracy": 0.0,
        "mean_probability": 0.7126569526389898
       },
       {
        "lower": 0.8,
        "upper": 0.9,
        "count": 0,
        "accuracy": null,
        "mean_probability": null
       },
       {
        "lower": 0.9,
        "upper": 1.0,
        "count": 182,
        "accuracy": 0.9120879120879121,
        "mean_probability": 0.9970826471562063
       }
      ],
      "selective": [
       {
        "threshold": 0.0,
        "count": 186,
        "coverage": 1.0,
        "accuracy": 0.8924731182795699,
        "error_rate": 0.10752688172043012
       },
       {
        "threshold": 0.5,
        "count": 185,
        "coverage": 0.9946236559139785,
        "accuracy": 0.8972972972972973,
        "error_rate": 0.10270270270270265
       },
       {
        "threshold": 0.7,
        "count": 183,
        "coverage": 0.9838709677419355,
        "accuracy": 0.907103825136612,
        "error_rate": 0.09289617486338797
       },
       {
        "threshold": 0.8,
        "count": 182,
        "coverage": 0.978494623655914,
        "accuracy": 0.9120879120879121,
        "error_rate": 0.08791208791208793
       },
       {
        "threshold": 0.9,
        "count": 182,
        "coverage": 0.978494623655914,
        "accuracy": 0.9120879120879121,
        "error_rate": 0.08791208791208793
       },
       {
        "threshold": 0.95,
        "count": 182,
        "coverage": 0.978494623655914,
        "accuracy": 0.9120879120879121,
        "error_rate": 0.08791208791208793
       },
       {
        "threshold": 0.99,
        "count": 174,
        "coverage": 0.9354838709677419,
        "accuracy": 0.9252873563218391,
        "error_rate": 0.07471264367816088
       },
       {
        "threshold": 1.0,
        "count": 0,
        "coverage": 0.0,
        "accuracy": null,
        "error_rate": null
       }
      ]
     }
    },
    "performance": {
     "latency_ms_p50": 36.48015786893666,
     "latency_ms_p95": 41.09123907983303,
     "requests_per_second": 27.39945030665782,
     "decisions_per_second": 84.12487476966034,
     "input_tokens_per_second": 18854.888916885473,
     "input_tokens": 176166,
     "generated_tokens": 0
    },
    "valid_mass_mean": 1.0,
    "evaluation_wall_seconds": 9.557715606875718,
    "cases": [
     {
      "id": "bendsynth-0a49f979766a387c-0022",
      "group": "bendsynth-0a49f979766a387c",
      "provenance": "bendsynth-v1; family=bendsynth-0a49f979766a387c; domain=mobile phone repair warranty chatbot; parameters=window=107,cap=130; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flags": 2,
       "decision": "discounted repair",
       "no_water": true
      },
      "logits": [
       [
        -5.863150596618652,
        2.104158401489258,
        -5.8073577880859375,
        -5.873562335968018,
        -5.840961456298828
       ],
       [
        -5.523018836975098,
        2.1041030883789062,
        -5.402163028717041,
        -5.649406433105469
       ],
       [
        -5.0201568603515625,
        2.1047635078430176
       ]
      ],
      "joint_logits": [
       -8.064090728759766,
       -7.844750881195068,
       -8.194507598876953,
       -7.851635932922363,
       -7.987760066986084,
       -7.707188129425049,
       -8.130480766296387,
       -7.857215881347656,
       -7.918750762939453,
       -7.232675075531006,
       -7.491165637969971,
       2.1037073135375977,
       -7.8761515617370605,
       -6.99835205078125,
       -8.071626663208008,
       -7.70856237411499,
       -8.049198150634766,
       -7.762759208679199,
       -8.140923500061035,
       -7.713033199310303,
       -7.931450366973877,
       -7.655545711517334,
       -8.110243797302246,
       -7.851244926452637,
       -8.04590892791748,
       -7.820289134979248,
       -8.218596458435059,
       -7.930051803588867,
       -7.990427017211914,
       -7.7272515296936035,
       -8.118898391723633,
       -7.86393404006958,
       -8.029972076416016,
       -7.802333831787109,
       -8.188491821289062,
       -7.885200500488281,
       -7.930530071258545,
       -7.6997270584106445,
       -8.095582962036133,
       -7.823602676391602
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 1,
           "probability": 0.9985914700418312,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003461223185534022
            },
            {
             "value": 1,
             "probability": 0.9985914700418312
            },
            {
             "value": 2,
             "probability": 0.0003659822021658199
            },
            {
             "value": 3,
             "probability": 0.0003425371647479827
            },
            {
             "value": 4,
             "probability": 0.00035388827270163385
            }
           ]
          },
          "decision": {
           "value": "discounted repair",
           "probability": 0.9985362221014157,
           "distribution": [
            {
             "value": "claim denied",
             "probability": 0.0004863478091607091
            },
            {
             "value": "discounted repair",
             "probability": 0.9985362221014157
            },
            {
             "value": "paid repair",
             "probability": 0.0005488248414771548
            },
            {
             "value": "free repair",
             "probability": 0.00042860524794647724
            }
           ]
          },
          "no_water": {
           "value": true,
           "probability": 0.9991958498036135,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008041501963864488
            },
            {
             "value": true,
             "probability": 0.9991958498036135
            }
           ]
          }
         },
         "joint_probability": 0.9981410489195177,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 9.818601193922257,
        "field_brier": 0.6654858711181314
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 1,
           "probability": 0.9985914700418312,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003461223185534022
            },
            {
             "value": 1,
             "probability": 0.9985914700418312
            },
            {
             "value": 2,
             "probability": 0.0003659822021658199
            },
            {
             "value": 3,
             "probability": 0.0003425371647479827
            },
            {
             "value": 4,
             "probability": 0.00035388827270163385
            }
           ]
          },
          "decision": {
           "value": "discounted repair",
           "probability": 0.9985362221014157,
           "distribution": [
            {
             "value": "claim denied",
             "probability": 0.0004863478091607091
            },
            {
             "value": "discounted repair",
             "probability": 0.9985362221014157
            },
            {
             "value": "paid repair",
             "probability": 0.0005488248414771548
            },
            {
             "value": "free repair",
             "probability": 0.00042860524794647724
            }
           ]
          },
          "no_water": {
           "value": true,
           "probability": 0.9991958498036135,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008041501963864488
            },
            {
             "value": true,
             "probability": 0.9991958498036135
            }
           ]
          }
         },
         "joint_probability": 0.9981410489195177,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 9.818601193922257,
        "field_brier": 0.6654858711181314
       }
      },
      "timing_ms": {
       "scoring": 43.24295395053923,
       "base_joint": 0.11924887076020241,
       "constrained_joint": 0.07314910180866718,
       "constrained_total": 43.3161030523479
      },
      "input_tokens": 742,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a49f979766a387c-0031",
      "group": "bendsynth-0a49f979766a387c",
      "provenance": "bendsynth-v1; family=bendsynth-0a49f979766a387c; domain=mobile phone repair warranty chatbot; parameters=window=120,cap=243; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "no_water": true,
       "flags": 0,
       "decision": "free repair"
      },
      "logits": [
       [
        -4.958318710327148,
        2.0875356197357178
       ],
       [
        2.08695650100708,
        -5.847857475280762,
        -5.909247875213623,
        -5.7895426750183105,
        -5.797837257385254
       ],
       [
        -5.552377700805664,
        -5.568777084350586,
        -5.542213439941406,
        2.0869667530059814
       ]
      ],
      "joint_logits": [
       -8.069733619689941,
       -8.096029281616211,
       -8.077669143676758,
       -7.874632358551025,
       -7.970594882965088,
       -7.893753528594971,
       -7.885472297668457,
       -8.149353981018066,
       -8.019305229187012,
       -7.913669109344482,
       -7.893978595733643,
       -8.222262382507324,
       -7.873892784118652,
       -7.837396144866943,
       -7.761966228485107,
       -8.087753295898438,
       -7.910187244415283,
       -7.81997013092041,
       -7.777867317199707,
       -8.109045028686523,
       -7.362863063812256,
       -7.666999340057373,
       -7.678936958312988,
       2.0866000652313232,
       -7.852784156799316,
       -7.920566082000732,
       -7.8932929039001465,
       -7.882359027862549,
       -7.95877742767334,
       -7.967034816741943,
       -7.944214820861816,
       -8.025899887084961,
       -7.861236572265625,
       -7.824716567993164,
       -7.792042255401611,
       -7.948760032653809,
       -7.861548900604248,
       -7.829970359802246,
       -7.798487186431885,
       -7.950430393218994
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "no_water": {
           "value": true,
           "probability": 0.9991297456037671,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008702543962328469
            },
            {
             "value": true,
             "probability": 0.9991297456037671
            }
           ]
          },
          "flags": {
           "value": 0,
           "probability": 0.9985513210955425,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9985513210955425
            },
            {
             "value": 1,
             "probability": 0.0003575398599318901
            },
            {
             "value": 2,
             "probability": 0.0003362505360222777
            },
            {
             "value": 3,
             "probability": 0.0003790096031400726
            },
            {
             "value": 4,
             "probability": 0.00037587890536331016
            }
           ]
          },
          "decision": {
           "value": "free repair",
           "probability": 0.9985615513060023,
           "distribution": [
            {
             "value": "discounted repair",
             "probability": 0.0004804517894957067
            },
            {
             "value": "claim denied",
             "probability": 0.0004726368979975105
            },
            {
             "value": "paid repair",
             "probability": 0.0004853600065044546
            },
            {
             "value": "free repair",
             "probability": 0.9985615513060023
            }
           ]
          }
         },
         "joint_probability": 0.9981953836190355,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018062466627564752,
        "field_brier": 2.2993723140954815e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "no_water": {
           "value": true,
           "probability": 0.9991297456037671,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008702543962328469
            },
            {
             "value": true,
             "probability": 0.9991297456037671
            }
           ]
          },
          "flags": {
           "value": 0,
           "probability": 0.9985513210955425,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9985513210955425
            },
            {
             "value": 1,
             "probability": 0.0003575398599318901
            },
            {
             "value": 2,
             "probability": 0.0003362505360222777
            },
            {
             "value": 3,
             "probability": 0.0003790096031400726
            },
            {
             "value": 4,
             "probability": 0.00037587890536331016
            }
           ]
          },
          "decision": {
           "value": "free repair",
           "probability": 0.9985615513060023,
           "distribution": [
            {
             "value": "discounted repair",
             "probability": 0.0004804517894957067
            },
            {
             "value": "claim denied",
             "probability": 0.0004726368979975105
            },
            {
             "value": "paid repair",
             "probability": 0.0004853600065044546
            },
            {
             "value": "free repair",
             "probability": 0.9985615513060023
            }
           ]
          }
         },
         "joint_probability": 0.9981953836190355,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018062466627564752,
        "field_brier": 2.2993723140954815e-06
       }
      },
      "timing_ms": {
       "scoring": 37.06313110888004,
       "base_joint": 0.08887890726327896,
       "constrained_joint": 0.06698002107441425,
       "constrained_total": 37.13011112995446
      },
      "input_tokens": 712,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a49f979766a387c-0032",
      "group": "bendsynth-0a49f979766a387c",
      "provenance": "bendsynth-v1; family=bendsynth-0a49f979766a387c; domain=mobile phone repair warranty chatbot; parameters=window=179,cap=150; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flags": 3,
       "decision": "paid repair",
       "no_water": true
      },
      "logits": [
       [
        -5.966179847717285,
        -5.736846446990967,
        -5.316452980041504,
        2.1426286697387695,
        -5.716327667236328
       ],
       [
        -5.373977184295654,
        -5.564736366271973,
        -5.603407859802246,
        2.1428444385528564
       ],
       [
        -4.916026592254639,
        2.1434102058410645
       ]
      ],
      "joint_logits": [
       -8.093445777893066,
       -7.947429656982422,
       -8.15447998046875,
       -8.058178901672363,
       -8.205981254577637,
       -8.064984321594238,
       -8.116155624389648,
       -7.787426948547363,
       -7.857947826385498,
       -7.703415393829346,
       -7.9743781089782715,
       -7.811716079711914,
       -8.179566383361816,
       -7.845916271209717,
       -7.871472358703613,
       -7.444414138793945,
       -7.593071937561035,
       -7.387986660003662,
       -7.765335559844971,
       -7.556506633758545,
       -7.81190824508667,
       -7.600374221801758,
       -7.569003105163574,
       -6.557572364807129,
       -8.041359901428223,
       -7.062932014465332,
       -8.232253074645996,
       -7.543481826782227,
       -8.24787425994873,
       -7.6335835456848145,
       -7.4183831214904785,
       2.1422390937805176,
       -7.865995407104492,
       -7.696684837341309,
       -7.981560707092285,
       -7.840597629547119,
       -7.910977363586426,
       -7.80263090133667,
       -7.875417232513428,
       -7.481492519378662
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 3,
           "probability": 0.9983609202332061,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00030038400081569316
            },
            {
             "value": 1,
             "probability": 0.0003778113620288636
            },
            {
             "value": 2,
             "probability": 0.0005752406080196712
            },
            {
             "value": 3,
             "probability": 0.9983609202332061
            },
            {
             "value": 4,
             "probability": 0.0003856437959297213
            }
           ]
          },
          "decision": {
           "value": "paid repair",
           "probability": 0.9985764038207616,
           "distribution": [
            {
             "value": "claim denied",
             "probability": 0.0005430842312524021
            },
            {
             "value": "discounted repair",
             "probability": 0.0004487676068780458
            },
            {
             "value": "free repair",
             "probability": 0.00043174434110800726
            },
            {
             "value": "paid repair",
             "probability": 0.9985764038207616
            }
           ]
          },
          "no_water": {
           "value": true,
           "probability": 0.9991414758363117,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008585241636883536
            },
            {
             "value": true,
             "probability": 0.9991414758363117
            }
           ]
          }
         },
         "joint_probability": 0.9979720312583958,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002030027854561413,
        "field_brier": 2.52755564456785e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 3,
           "probability": 0.9983609202332061,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00030038400081569316
            },
            {
             "value": 1,
             "probability": 0.0003778113620288636
            },
            {
             "value": 2,
             "probability": 0.0005752406080196712
            },
            {
             "value": 3,
             "probability": 0.9983609202332061
            },
            {
             "value": 4,
             "probability": 0.0003856437959297213
            }
           ]
          },
          "decision": {
           "value": "paid repair",
           "probability": 0.9985764038207616,
           "distribution": [
            {
             "value": "claim denied",
             "probability": 0.0005430842312524021
            },
            {
             "value": "discounted repair",
             "probability": 0.0004487676068780458
            },
            {
             "value": "free repair",
             "probability": 0.00043174434110800726
            },
            {
             "value": "paid repair",
             "probability": 0.9985764038207616
            }
           ]
          },
          "no_water": {
           "value": true,
           "probability": 0.9991414758363117,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008585241636883536
            },
            {
             "value": true,
             "probability": 0.9991414758363117
            }
           ]
          }
         },
         "joint_probability": 0.9979720312583958,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002030027854561413,
        "field_brier": 2.52755564456785e-06
       }
      },
      "timing_ms": {
       "scoring": 38.74819213524461,
       "base_joint": 0.09023910388350487,
       "constrained_joint": 0.07294886745512486,
       "constrained_total": 38.82114100269973
      },
      "input_tokens": 736,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a49f979766a387c-0035",
      "group": "bendsynth-0a49f979766a387c",
      "provenance": "bendsynth-v1; family=bendsynth-0a49f979766a387c; domain=mobile phone repair warranty chatbot; parameters=window=61,cap=50; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flags": 3,
       "no_water": true,
       "decision": "paid repair"
      },
      "logits": [
       [
        -6.128008842468262,
        -6.171395778656006,
        -5.771420478820801,
        1.9497839212417603,
        -5.908632278442383
       ],
       [
        -5.180405616760254,
        1.950420618057251
       ],
       [
        -5.760531902313232,
        1.949922800064087,
        -5.6991353034973145,
        -5.941847801208496
       ]
      ],
      "joint_logits": [
       -8.253863334655762,
       -8.260805130004883,
       -8.168214797973633,
       -8.472668647766113,
       -8.179525375366211,
       -8.033119201660156,
       -8.077201843261719,
       -8.277467727661133,
       -8.359641075134277,
       -8.30681324005127,
       -8.255107879638672,
       -8.615416526794434,
       -8.256536483764648,
       -7.898388385772705,
       -8.14533519744873,
       -8.313592910766602,
       -8.087957382202148,
       -7.95979642868042,
       -7.996241092681885,
       -8.25635051727295,
       -7.855589389801025,
       -7.2058329582214355,
       -7.790384292602539,
       -8.050190925598145,
       -8.191388130187988,
       -7.8618035316467285,
       -8.15963363647461,
       -8.343673706054688,
       -7.6430230140686035,
       1.9494367837905884,
       -7.688263416290283,
       -8.058079719543457,
       -8.049010276794434,
       -8.065153121948242,
       -8.008310317993164,
       -8.15613842010498,
       -7.973286151885986,
       -7.768017768859863,
       -7.881837844848633,
       -8.055381774902344
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 3,
           "probability": 0.9985647179322089,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003099099129916324
            },
            {
             "value": 1,
             "probability": 0.0002967513332444135
            },
            {
             "value": 2,
             "probability": 0.0004426900412487569
            },
            {
             "value": 3,
             "probability": 0.9985647179322089
            },
            {
             "value": 4,
             "probability": 0.0003859307803063319
            }
           ]
          },
          "no_water": {
           "value": true,
           "probability": 0.999200581398618,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007994186013820314
            },
            {
             "value": true,
             "probability": 0.999200581398618
            }
           ]
          },
          "decision": {
           "value": "paid repair",
           "probability": 0.9987032660600478,
           "distribution": [
            {
             "value": "claim denied",
             "probability": 0.0004475365852822726
            },
            {
             "value": "paid repair",
             "probability": 0.9987032660600478
            },
            {
             "value": "discounted repair",
             "probability": 0.0004758748933488137
            },
            {
             "value": "free repair",
             "probability": 0.00037332246132104163
            }
           ]
          }
         },
         "joint_probability": 0.9982179704565091,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0017836192470177571,
        "field_brier": 2.038277280930661e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 3,
           "probability": 0.9985647179322089,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003099099129916324
            },
            {
             "value": 1,
             "probability": 0.0002967513332444135
            },
            {
             "value": 2,
             "probability": 0.0004426900412487569
            },
            {
             "value": 3,
             "probability": 0.9985647179322089
            },
            {
             "value": 4,
             "probability": 0.0003859307803063319
            }
           ]
          },
          "no_water": {
           "value": true,
           "probability": 0.999200581398618,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007994186013820314
            },
            {
             "value": true,
             "probability": 0.999200581398618
            }
           ]
          },
          "decision": {
           "value": "paid repair",
           "probability": 0.9987032660600478,
           "distribution": [
            {
             "value": "claim denied",
             "probability": 0.0004475365852822726
            },
            {
             "value": "paid repair",
             "probability": 0.9987032660600478
            },
            {
             "value": "discounted repair",
             "probability": 0.0004758748933488137
            },
            {
             "value": "free repair",
             "probability": 0.00037332246132104163
            }
           ]
          }
         },
         "joint_probability": 0.9982179704565091,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0017836192470177571,
        "field_brier": 2.038277280930661e-06
       }
      },
      "timing_ms": {
       "scoring": 35.605516051873565,
       "base_joint": 0.0870788935571909,
       "constrained_joint": 0.0765589065849781,
       "constrained_total": 35.68207495845854
      },
      "input_tokens": 724,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03c1362acc4580e0-0001",
      "group": "bendsynth-03c1362acc4580e0",
      "provenance": "bendsynth-v1; family=bendsynth-03c1362acc4580e0; domain=chemistry department stockroom requisition chatbot; parameters=cooldown=13; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "wait_met": true,
       "red_flags": 2,
       "decision": "denied"
      },
      "logits": [
       [
        1.093384861946106,
        -4.8710832595825195
       ],
       [
        -6.24553918838501,
        -5.795608043670654,
        1.093577265739441,
        -6.160632133483887
       ],
       [
        1.093737244606018,
        -5.727963447570801,
        -5.695910453796387
       ]
      ],
      "joint_logits": [
       -8.292243003845215,
       -8.12669563293457,
       -8.226848602294922,
       -7.531807899475098,
       -7.808629035949707,
       -7.871468544006348,
       1.0920826196670532,
       -7.840662002563477,
       -7.629560947418213,
       -8.242304801940918,
       -7.992830276489258,
       -8.229756355285645,
       -7.962826728820801,
       -7.864872455596924,
       -7.8403120040893555,
       -7.380097389221191,
       -7.498890399932861,
       -7.523597717285156,
       -5.977410793304443,
       -7.667731761932373,
       -7.459928512573242,
       -7.878075122833252,
       -7.786557197570801,
       -7.7086615562438965
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_met": {
           "value": false,
           "probability": 0.9974381696021807,
           "distribution": [
            {
             "value": false,
             "probability": 0.9974381696021807
            },
            {
             "value": true,
             "probability": 0.0025618303978193176
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.997630070745617,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006480847426342005
            },
            {
             "value": 1,
             "probability": 0.0010163292633118858
            },
            {
             "value": 2,
             "probability": 0.997630070745617
            },
            {
             "value": 3,
             "probability": 0.0007055152484369113
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9977896649657885,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.9977896649657885
            },
            {
             "value": "approved",
             "probability": 0.0010874569451605957
            },
            {
             "value": "deferred",
             "probability": 0.0011228780890509208
            }
           ]
          }
         },
         "joint_probability": 0.9961400959891582,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 7.073360785636879,
        "field_brier": 0.6632602335376375
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_met": {
           "value": false,
           "probability": 0.9974381696021807,
           "distribution": [
            {
             "value": false,
             "probability": 0.9974381696021807
            },
            {
             "value": true,
             "probability": 0.0025618303978193176
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.997630070745617,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006480847426342005
            },
            {
             "value": 1,
             "probability": 0.0010163292633118858
            },
            {
             "value": 2,
             "probability": 0.997630070745617
            },
            {
             "value": 3,
             "probability": 0.0007055152484369113
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9977896649657885,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.9977896649657885
            },
            {
             "value": "approved",
             "probability": 0.0010874569451605957
            },
            {
             "value": "deferred",
             "probability": 0.0011228780890509208
            }
           ]
          }
         },
         "joint_probability": 0.9961400959891582,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 7.073360785636879,
        "field_brier": 0.6632602335376375
       }
      },
      "timing_ms": {
       "scoring": 38.79128093831241,
       "base_joint": 0.06854883395135403,
       "constrained_joint": 0.048029934987425804,
       "constrained_total": 38.83931087329984
      },
      "input_tokens": 781,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03c1362acc4580e0-0017",
      "group": "bendsynth-03c1362acc4580e0",
      "provenance": "bendsynth-v1; family=bendsynth-03c1362acc4580e0; domain=chemistry department stockroom requisition chatbot; parameters=cooldown=11; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "denied",
       "wait_met": true,
       "red_flags": 2
      },
      "logits": [
       [
        -6.032811164855957,
        0.867149293422699,
        -5.931641101837158
       ],
       [
        0.8670400977134705,
        -5.237674713134766
       ],
       [
        -6.468866348266602,
        -6.0171308517456055,
        0.8668295741081238,
        -6.30583381652832
       ]
      ],
      "joint_logits": [
       -8.353300094604492,
       -8.092243194580078,
       -8.303157806396484,
       -8.197712898254395,
       -8.161576271057129,
       -7.769911766052246,
       -8.161707878112793,
       -7.9831342697143555,
       -8.508946418762207,
       -7.717235565185547,
       0.8657497763633728,
       -8.379668235778809,
       -8.156228065490723,
       -7.621044158935547,
       -6.613378524780273,
       -8.007528305053711,
       -8.414654731750488,
       -8.041699409484863,
       -8.013110160827637,
       -8.280184745788574,
       -8.047944068908691,
       -7.702676773071289,
       -7.925093650817871,
       -7.841064929962158
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "denied",
           "probability": 0.9978815483861468,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.0010056906348813453
            },
            {
             "value": "denied",
             "probability": 0.9978815483861468
            },
            {
             "value": "deferred",
             "probability": 0.0011127609789718597
            }
           ]
          },
          "wait_met": {
           "value": false,
           "probability": 0.9977726541427955,
           "distribution": [
            {
             "value": false,
             "probability": 0.9977726541427955
            },
            {
             "value": true,
             "probability": 0.0022273458572044913
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.9975627476548738,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006502614528198384
            },
            {
             "value": 1,
             "probability": 0.0010215842653089268
            },
            {
             "value": 2,
             "probability": 0.9975627476548738
            },
            {
             "value": 3,
             "probability": 0.0007654066269974062
            }
           ]
          }
         },
         "joint_probability": 0.9964860746824281,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 7.482648414797942,
        "field_brier": 0.6637050895729578
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "denied",
           "probability": 0.9978815483861468,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.0010056906348813453
            },
            {
             "value": "denied",
             "probability": 0.9978815483861468
            },
            {
             "value": "deferred",
             "probability": 0.0011127609789718597
            }
           ]
          },
          "wait_met": {
           "value": false,
           "probability": 0.9977726541427955,
           "distribution": [
            {
             "value": false,
             "probability": 0.9977726541427955
            },
            {
             "value": true,
             "probability": 0.0022273458572044913
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.9975627476548738,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006502614528198384
            },
            {
             "value": 1,
             "probability": 0.0010215842653089268
            },
            {
             "value": 2,
             "probability": 0.9975627476548738
            },
            {
             "value": 3,
             "probability": 0.0007654066269974062
            }
           ]
          }
         },
         "joint_probability": 0.9964860746824281,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 7.482648414797942,
        "field_brier": 0.6637050895729578
       }
      },
      "timing_ms": {
       "scoring": 38.900911109521985,
       "base_joint": 0.068610068410635,
       "constrained_joint": 0.04867883399128914,
       "constrained_total": 38.949589943513274
      },
      "input_tokens": 769,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03c1362acc4580e0-0021",
      "group": "bendsynth-03c1362acc4580e0",
      "provenance": "bendsynth-v1; family=bendsynth-03c1362acc4580e0; domain=chemistry department stockroom requisition chatbot; parameters=cooldown=9; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "wait_met": true,
       "red_flags": 2,
       "decision": "denied"
      },
      "logits": [
       [
        -6.069324493408203,
        0.490241140127182
       ],
       [
        -6.815871715545654,
        -6.656499862670898,
        0.48946455121040344,
        -6.726772308349609
       ],
       [
        -6.306454658508301,
        0.48956936597824097,
        -6.447427749633789
       ]
      ],
      "joint_logits": [
       -8.559564590454102,
       -8.821776390075684,
       -8.70152759552002,
       -8.483269691467285,
       -8.667686462402344,
       -8.600164413452148,
       -8.448226928710938,
       -8.064475059509277,
       -8.732287406921387,
       -8.450057029724121,
       -8.757952690124512,
       -8.606033325195312,
       -8.340837478637695,
       -8.65908145904541,
       -8.63035774230957,
       -8.373895645141602,
       -8.175145149230957,
       -8.467331886291504,
       -8.220929145812988,
       0.48868855834007263,
       -8.091011047363281,
       -8.257267951965332,
       -8.56771469116211,
       -8.543594360351562
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_met": {
           "value": true,
           "probability": 0.9985855022395781,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014144977604218995
            },
            {
             "value": true,
             "probability": 0.9985855022395781
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.9978102621490685,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000670472255129213
            },
            {
             "value": 1,
             "probability": 0.0007863124610472895
            },
            {
             "value": 2,
             "probability": 0.9978102621490685
            },
            {
             "value": 3,
             "probability": 0.0007329531347550046
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9979149621186005,
           "distribution": [
            {
             "value": "deferred",
             "probability": 0.0011158811435638073
            },
            {
             "value": "denied",
             "probability": 0.9979149621186005
            },
            {
             "value": "approved",
             "probability": 0.0009691567378356595
            }
           ]
          }
         },
         "joint_probability": 0.9970363680755057,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0029680321775570526,
        "field_brier": 5.6444795964156255e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_met": {
           "value": true,
           "probability": 0.9985855022395781,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014144977604218995
            },
            {
             "value": true,
             "probability": 0.9985855022395781
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.9978102621490685,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000670472255129213
            },
            {
             "value": 1,
             "probability": 0.0007863124610472895
            },
            {
             "value": 2,
             "probability": 0.9978102621490685
            },
            {
             "value": 3,
             "probability": 0.0007329531347550046
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9979149621186005,
           "distribution": [
            {
             "value": "deferred",
             "probability": 0.0011158811435638073
            },
            {
             "value": "denied",
             "probability": 0.9979149621186005
            },
            {
             "value": "approved",
             "probability": 0.0009691567378356595
            }
           ]
          }
         },
         "joint_probability": 0.9970363680755057,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0029680321775570526,
        "field_brier": 5.6444795964156255e-06
       }
      },
      "timing_ms": {
       "scoring": 38.95227098837495,
       "base_joint": 0.07700896821916103,
       "constrained_joint": 0.04792003892362118,
       "constrained_total": 39.00019102729857
      },
      "input_tokens": 796,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03c1362acc4580e0-0023",
      "group": "bendsynth-03c1362acc4580e0",
      "provenance": "bendsynth-v1; family=bendsynth-03c1362acc4580e0; domain=chemistry department stockroom requisition chatbot; parameters=cooldown=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "denied",
       "red_flags": 2,
       "wait_met": false
      },
      "logits": [
       [
        0.23668652772903442,
        -6.730705261230469,
        -6.794974327087402
       ],
       [
        -7.166248321533203,
        -7.057984352111816,
        0.23654204607009888,
        -7.054867267608643
       ],
       [
        0.23693758249282837,
        -6.217258453369141
       ]
      ],
      "joint_logits": [
       -9.219731330871582,
       -8.81273078918457,
       -8.785162925720215,
       -8.708501815795898,
       0.2358289510011673,
       -8.301904678344727,
       -9.12341594696045,
       -8.722850799560547,
       -9.061300277709961,
       -8.809228897094727,
       -8.993884086608887,
       -8.80099868774414,
       -8.63707160949707,
       -8.625847816467285,
       -8.952652931213379,
       -8.695640563964844,
       -9.030769348144531,
       -8.879366874694824,
       -9.054776191711426,
       -8.79976749420166,
       -8.986120223999023,
       -8.661273956298828,
       -8.913467407226562,
       -8.74020004272461
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "denied",
           "probability": 0.9981777562278916,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.9981777562278916
            },
            {
             "value": "deferred",
             "probability": 0.0009403900739406135
            },
            {
             "value": "approved",
             "probability": 0.0008818536981677665
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.998033710693415,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006083510759980378
            },
            {
             "value": 1,
             "probability": 0.0006779110582674823
            },
            {
             "value": 2,
             "probability": 0.998033710693415
            },
            {
             "value": 3,
             "probability": 0.0006800271723196132
            }
           ]
          },
          "wait_met": {
           "value": false,
           "probability": 0.9984285690557944,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984285690557944
            },
            {
             "value": true,
             "probability": 0.0015714309442056299
            }
           ]
          }
         },
         "joint_probability": 0.9973222040360269,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026813876729274914,
        "field_brier": 5.02658241760434e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "denied",
           "probability": 0.9981777562278916,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.9981777562278916
            },
            {
             "value": "deferred",
             "probability": 0.0009403900739406135
            },
            {
             "value": "approved",
             "probability": 0.0008818536981677665
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.998033710693415,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006083510759980378
            },
            {
             "value": 1,
             "probability": 0.0006779110582674823
            },
            {
             "value": 2,
             "probability": 0.998033710693415
            },
            {
             "value": 3,
             "probability": 0.0006800271723196132
            }
           ]
          },
          "wait_met": {
           "value": false,
           "probability": 0.9984285690557944,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984285690557944
            },
            {
             "value": true,
             "probability": 0.0015714309442056299
            }
           ]
          }
         },
         "joint_probability": 0.9973222040360269,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026813876729274914,
        "field_brier": 5.02658241760434e-06
       }
      },
      "timing_ms": {
       "scoring": 37.655673921108246,
       "base_joint": 0.0684990081936121,
       "constrained_joint": 0.04854006692767143,
       "constrained_total": 37.70421398803592
      },
      "input_tokens": 793,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03c1362acc4580e0-0030",
      "group": "bendsynth-03c1362acc4580e0",
      "provenance": "bendsynth-v1; family=bendsynth-03c1362acc4580e0; domain=chemistry department stockroom requisition chatbot; parameters=cooldown=16; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "red_flags": 1,
       "decision": "deferred",
       "wait_met": false
      },
      "logits": [
       [
        -6.237436294555664,
        1.0792875289916992,
        -6.398101806640625,
        -6.311873912811279
       ],
       [
        1.0793033838272095,
        -5.966056823730469,
        -5.869810104370117
       ],
       [
        1.0796527862548828,
        -5.435435771942139
       ]
      ],
      "joint_logits": [
       -8.253972053527832,
       -8.005561828613281,
       -8.122037887573242,
       -7.95029878616333,
       -8.021296501159668,
       -7.867051124572754,
       1.0785744190216064,
       -7.572293281555176,
       -7.933505058288574,
       -7.813477516174316,
       -7.869479179382324,
       -7.73304557800293,
       -8.393278121948242,
       -8.199542045593262,
       -8.291570663452148,
       -8.124013900756836,
       -8.175090789794922,
       -8.001340866088867,
       -8.423200607299805,
       -8.063313484191895,
       -8.227335929870605,
       -7.992842197418213,
       -8.109541893005371,
       -7.890555381774902
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "red_flags": {
           "value": 1,
           "probability": 0.9981566568793322,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006631106856864055
            },
            {
             "value": 1,
             "probability": 0.9981566568793322
            },
            {
             "value": 2,
             "probability": 0.0005646897338349821
            },
            {
             "value": 3,
             "probability": 0.0006155427011464142
            }
           ]
          },
          "decision": {
           "value": "deferred",
           "probability": 0.9981724172139019,
           "distribution": [
            {
             "value": "deferred",
             "probability": 0.9981724172139019
            },
            {
             "value": "approved",
             "probability": 0.0008698506961355619
            },
            {
             "value": "denied",
             "probability": 0.0009577320899624822
            }
           ]
          },
          "wait_met": {
           "value": false,
           "probability": 0.9985212649561039,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985212649561039
            },
            {
             "value": true,
             "probability": 0.0014787350438960427
            }
           ]
          }
         },
         "joint_probability": 0.9974451175384165,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025581517433874317,
        "field_brier": 4.640887148087999e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "red_flags": {
           "value": 1,
           "probability": 0.9981566568793322,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006631106856864055
            },
            {
             "value": 1,
             "probability": 0.9981566568793322
            },
            {
             "value": 2,
             "probability": 0.0005646897338349821
            },
            {
             "value": 3,
             "probability": 0.0006155427011464142
            }
           ]
          },
          "decision": {
           "value": "deferred",
           "probability": 0.9981724172139019,
           "distribution": [
            {
             "value": "deferred",
             "probability": 0.9981724172139019
            },
            {
             "value": "approved",
             "probability": 0.0008698506961355619
            },
            {
             "value": "denied",
             "probability": 0.0009577320899624822
            }
           ]
          },
          "wait_met": {
           "value": false,
           "probability": 0.9985212649561039,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985212649561039
            },
            {
             "value": true,
             "probability": 0.0014787350438960427
            }
           ]
          }
         },
         "joint_probability": 0.9974451175384165,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025581517433874317,
        "field_brier": 4.640887148087999e-06
       }
      },
      "timing_ms": {
       "scoring": 39.83957110904157,
       "base_joint": 0.08022994734346867,
       "constrained_joint": 0.048768939450383186,
       "constrained_total": 39.888340048491955
      },
      "input_tokens": 772,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09ba618ab858f874-0017",
      "group": "bendsynth-09ba618ab858f874",
      "provenance": "bendsynth-v1; family=bendsynth-09ba618ab858f874; domain=art gallery incoming loan approval chatbot; parameters=high_value_line=307; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "board review",
       "over_threshold": true,
       "gap_factors": 2,
       "courier_ok": false
      },
      "logits": [
       [
        1.0467779636383057,
        -7.077955722808838,
        -7.053031921386719,
        -6.826922416687012
       ],
       [
        -6.3575439453125,
        1.047149658203125
       ],
       [
        -7.06646728515625,
        -7.001384735107422,
        1.0468498468399048,
        -7.101096153259277
       ],
       [
        1.0471471548080444,
        -6.353426933288574
       ]
      ],
      "joint_logits": [
       -9.596085548400879,
       -9.949788093566895,
       -9.531595230102539,
       -9.931930541992188,
       -8.49570083618164,
       -9.451445579528809,
       -9.643295288085938,
       -9.986366271972656,
       -9.225534439086914,
       -9.669177055358887,
       -8.925480842590332,
       -9.558944702148438,
       1.0462908744812012,
       -8.346230506896973,
       -9.267903327941895,
       -9.725981712341309,
       -10.023306846618652,
       -10.421607971191406,
       -9.984933853149414,
       -10.399155616760254,
       -9.570149421691895,
       -9.972895622253418,
       -10.025870323181152,
       -10.42091178894043,
       -9.69827938079834,
       -10.047585487365723,
       -9.707250595092773,
       -10.009714126586914,
       -8.98401165008545,
       -9.612053871154785,
       -9.762042999267578,
       -10.079099655151367,
       -10.049956321716309,
       -10.411368370056152,
       -10.034867286682129,
       -10.402647972106934,
       -9.520297050476074,
       -9.982525825500488,
       -10.060458183288574,
       -10.438352584838867,
       -9.704499244689941,
       -10.079940795898438,
       -9.698090553283691,
       -10.043241500854492,
       -8.831500053405762,
       -9.537015914916992,
       -9.751717567443848,
       -10.088517189025879,
       -9.820425987243652,
       -10.215446472167969,
       -9.786250114440918,
       -10.207192420959473,
       -9.305435180664062,
       -9.747791290283203,
       -9.841090202331543,
       -10.228180885314941,
       -9.482128143310547,
       -9.832623481750488,
       -9.498279571533203,
       -9.782241821289062,
       -8.586930274963379,
       -9.325583457946777,
       -9.54545783996582,
       -9.84598445892334
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "board review",
           "probability": 0.9990206162849582,
           "distribution": [
            {
             "value": "board review",
             "probability": 0.9990206162849582
            },
            {
             "value": "reject",
             "probability": 0.0002958335437527597
            },
            {
             "value": "accept",
             "probability": 0.00030329955837528794
            },
            {
             "value": "accept with courier required",
             "probability": 0.00038025061291376895
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.999391979508449,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006080204915509983
            },
            {
             "value": true,
             "probability": 0.999391979508449
            }
           ]
          },
          "gap_factors": {
           "value": 2,
           "probability": 0.9990923060333302,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0002992517854017912
            },
            {
             "value": 1,
             "probability": 0.00031937563726585846
            },
            {
             "value": 2,
             "probability": 0.9990923060333302
            },
            {
             "value": 3,
             "probability": 0.00028906654400209634
            }
           ]
          },
          "courier_ok": {
           "value": false,
           "probability": 0.9993894709304587,
           "distribution": [
            {
             "value": false,
             "probability": 0.9993894709304587
            },
            {
             "value": true,
             "probability": 0.0006105290695413191
            }
           ]
          }
         },
         "joint_probability": 0.9985340639727954,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.001467011562661843,
        "field_brier": 9.66795164041811e-07
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "board review",
           "probability": 0.9990206162849582,
           "distribution": [
            {
             "value": "board review",
             "probability": 0.9990206162849582
            },
            {
             "value": "reject",
             "probability": 0.0002958335437527597
            },
            {
             "value": "accept",
             "probability": 0.00030329955837528794
            },
            {
             "value": "accept with courier required",
             "probability": 0.00038025061291376895
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.999391979508449,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006080204915509983
            },
            {
             "value": true,
             "probability": 0.999391979508449
            }
           ]
          },
          "gap_factors": {
           "value": 2,
           "probability": 0.9990923060333302,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0002992517854017912
            },
            {
             "value": 1,
             "probability": 0.00031937563726585846
            },
            {
             "value": 2,
             "probability": 0.9990923060333302
            },
            {
             "value": 3,
             "probability": 0.00028906654400209634
            }
           ]
          },
          "courier_ok": {
           "value": false,
           "probability": 0.9993894709304587,
           "distribution": [
            {
             "value": false,
             "probability": 0.9993894709304587
            },
            {
             "value": true,
             "probability": 0.0006105290695413191
            }
           ]
          }
         },
         "joint_probability": 0.9985340639727954,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.001467011562661843,
        "field_brier": 9.66795164041811e-07
       }
      },
      "timing_ms": {
       "scoring": 39.50580395758152,
       "base_joint": 0.13081892393529415,
       "constrained_joint": 0.10521779768168926,
       "constrained_total": 39.61102175526321
      },
      "input_tokens": 807,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09ba618ab858f874-0021",
      "group": "bendsynth-09ba618ab858f874",
      "provenance": "bendsynth-v1; family=bendsynth-09ba618ab858f874; domain=art gallery incoming loan approval chatbot; parameters=high_value_line=500; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "board review",
       "gap_factors": 1,
       "courier_ok": false,
       "over_threshold": true
      },
      "logits": [
       [
        -6.3683857917785645,
        -6.906938552856445,
        1.1225610971450806,
        -6.7152862548828125
       ],
       [
        -6.835003852844238,
        1.122869849205017,
        -6.977366924285889,
        -6.936920166015625
       ],
       [
        1.1229760646820068,
        -5.932185649871826
       ],
       [
        -6.113777160644531,
        1.1231194734573364
       ]
      ],
      "joint_logits": [
       -9.807580947875977,
       -9.500506401062012,
       -9.76569652557373,
       -9.748176574707031,
       -9.096488952636719,
       -7.252745151519775,
       -9.463541030883789,
       -8.897811889648438,
       -9.940013885498047,
       -9.665233612060547,
       -9.894119262695312,
       -9.880653381347656,
       -9.88702392578125,
       -9.644993782043457,
       -9.856447219848633,
       -9.837421417236328,
       -9.736991882324219,
       -9.697222709655762,
       -10.009751319885254,
       -9.722578048706055,
       -9.415512084960938,
       -8.94351863861084,
       -9.57518196105957,
       -9.32619571685791,
       -9.875679016113281,
       -9.81692123413086,
       -10.148794174194336,
       -9.841185569763184,
       -9.816768646240234,
       -9.772814750671387,
       -10.094819068908691,
       -9.829540252685547,
       -9.41112232208252,
       -8.923661231994629,
       -9.51745319366455,
       -9.310600280761719,
       -8.335381507873535,
       1.1219035387039185,
       -8.860692977905273,
       -7.437543869018555,
       -9.566448211669922,
       -9.082825660705566,
       -9.626296997070312,
       -9.448518753051758,
       -9.510871887207031,
       -9.02236557006836,
       -9.607741355895996,
       -9.429926872253418,
       -9.848756790161133,
       -9.599157333374023,
       -9.889892578125,
       -9.824068069458008,
       -9.23812484741211,
       -8.083969116210938,
       -9.608793258666992,
       -9.026386260986328,
       -9.972978591918945,
       -9.782855987548828,
       -10.036314010620117,
       -9.987909317016602,
       -9.963078498840332,
       -9.748920440673828,
       -9.962895393371582,
       -9.935615539550781
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "accept with courier required",
           "probability": 0.9987232892505999,
           "distribution": [
            {
             "value": "accept",
             "probability": 0.000557401627163027
            },
            {
             "value": "reject",
             "probability": 0.000325295366410628
            },
            {
             "value": "accept with courier required",
             "probability": 0.9987232892505999
            },
            {
             "value": "board review",
             "probability": 0.0003940137558265006
            }
           ]
          },
          "gap_factors": {
           "value": 1,
           "probability": 0.9990315820549431,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00034955736550732413
            },
            {
             "value": 1,
             "probability": 0.9990315820549431
            },
            {
             "value": 2,
             "probability": 0.0003031734277279172
            },
            {
             "value": 3,
             "probability": 0.0003156871518216334
            }
           ]
          },
          "courier_ok": {
           "value": false,
           "probability": 0.9991378008335822,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991378008335822
            },
            {
             "value": true,
             "probability": 0.0008621991664178468
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9992809759490382,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007190240509617783
            },
            {
             "value": true,
             "probability": 0.9992809759490382
            }
           ]
          }
         },
         "joint_probability": 0.9980666899942128,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 3,
        "joint_nll": 9.207807836176633,
        "field_brier": 0.4991661312756309
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "accept with courier required",
           "probability": 0.9987232892505999,
           "distribution": [
            {
             "value": "accept",
             "probability": 0.000557401627163027
            },
            {
             "value": "reject",
             "probability": 0.000325295366410628
            },
            {
             "value": "accept with courier required",
             "probability": 0.9987232892505999
            },
            {
             "value": "board review",
             "probability": 0.0003940137558265006
            }
           ]
          },
          "gap_factors": {
           "value": 1,
           "probability": 0.9990315820549431,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00034955736550732413
            },
            {
             "value": 1,
             "probability": 0.9990315820549431
            },
            {
             "value": 2,
             "probability": 0.0003031734277279172
            },
            {
             "value": 3,
             "probability": 0.0003156871518216334
            }
           ]
          },
          "courier_ok": {
           "value": false,
           "probability": 0.9991378008335822,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991378008335822
            },
            {
             "value": true,
             "probability": 0.0008621991664178468
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9992809759490382,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007190240509617783
            },
            {
             "value": true,
             "probability": 0.9992809759490382
            }
           ]
          }
         },
         "joint_probability": 0.9980666899942128,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 3,
        "joint_nll": 9.207807836176633,
        "field_brier": 0.4991661312756309
       }
      },
      "timing_ms": {
       "scoring": 39.291976019740105,
       "base_joint": 0.12240908108651638,
       "constrained_joint": 0.10433909483253956,
       "constrained_total": 39.396315114572644
      },
      "input_tokens": 863,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09ba618ab858f874-0032",
      "group": "bendsynth-09ba618ab858f874",
      "provenance": "bendsynth-v1; family=bendsynth-09ba618ab858f874; domain=art gallery incoming loan approval chatbot; parameters=high_value_line=400; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "accept with courier required",
       "over_threshold": true,
       "gap_factors": 1,
       "courier_ok": true
      },
      "logits": [
       [
        -6.140642166137695,
        1.6835792064666748,
        -6.450719356536865,
        -6.470433235168457
       ],
       [
        -5.684915542602539,
        1.6839289665222168
       ],
       [
        -6.362757682800293,
        1.6836334466934204,
        -6.429478645324707,
        -6.406426906585693
       ],
       [
        -5.609201431274414,
        1.6838794946670532
       ]
      ],
      "joint_logits": [
       -9.496419906616211,
       -9.280536651611328,
       -9.076150894165039,
       -8.687943458557129,
       -9.540257453918457,
       -9.319506645202637,
       -9.532550811767578,
       -9.294841766357422,
       -9.265670776367188,
       -8.880393981933594,
       -8.599266052246094,
       -7.547235488891602,
       -9.305420875549316,
       -9.004590034484863,
       -9.26803970336914,
       -9.006244659423828,
       -9.124544143676758,
       -8.815133094787598,
       -8.586225509643555,
       -7.816501617431641,
       -9.17623233795166,
       -8.927294731140137,
       -9.151382446289062,
       -8.927214622497559,
       -8.786033630371094,
       -8.343738555908203,
       -7.394974231719971,
       1.6830120086669922,
       -8.9085054397583,
       -8.429667472839355,
       -8.910573959350586,
       -8.347533226013184,
       -9.668375968933105,
       -9.419912338256836,
       -9.306634902954102,
       -8.932961463928223,
       -9.68295669555664,
       -9.472563743591309,
       -9.694405555725098,
       -9.459061622619629,
       -9.390069961547852,
       -9.093050956726074,
       -8.902898788452148,
       -8.396113395690918,
       -9.437950134277344,
       -9.137954711914062,
       -9.4356107711792,
       -9.12724494934082,
       -9.678790092468262,
       -9.434915542602539,
       -9.233839988708496,
       -8.926346778869629,
       -9.690675735473633,
       -9.489762306213379,
       -9.667781829833984,
       -9.462422370910645,
       -9.405951499938965,
       -9.121060371398926,
       -8.875248908996582,
       -8.559797286987305,
       -9.444177627563477,
       -9.159675598144531,
       -9.436262130737305,
       -9.1159086227417
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "accept with courier required",
           "probability": 0.9990201470490612,
           "distribution": [
            {
             "value": "accept",
             "probability": 0.000399538092192697
            },
            {
             "value": "accept with courier required",
             "probability": 0.9990201470490612
            },
            {
             "value": "board review",
             "probability": 0.00029301742647529325
            },
            {
             "value": "reject",
             "probability": 0.00028729743227074204
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9993698011094606,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006301988905393963
            },
            {
             "value": true,
             "probability": 0.9993698011094606
            }
           ]
          },
          "gap_factors": {
           "value": 1,
           "probability": 0.9990744453767675,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003199592092591818
            },
            {
             "value": 1,
             "probability": 0.9990744453767675
            },
            {
             "value": 2,
             "probability": 0.0002993079075467958
            },
            {
             "value": 3,
             "probability": 0.0003062875064264992
            }
           ]
          },
          "courier_ok": {
           "value": true,
           "probability": 0.9993202334762323,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006797665237676984
            },
            {
             "value": true,
             "probability": 0.9993202334762323
            }
           ]
          }
         },
         "joint_probability": 0.998453803956461,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0015473926382472988,
        "field_brier": 1.0372575931546236e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "accept with courier required",
           "probability": 0.9990201470490612,
           "distribution": [
            {
             "value": "accept",
             "probability": 0.000399538092192697
            },
            {
             "value": "accept with courier required",
             "probability": 0.9990201470490612
            },
            {
             "value": "board review",
             "probability": 0.00029301742647529325
            },
            {
             "value": "reject",
             "probability": 0.00028729743227074204
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9993698011094606,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006301988905393963
            },
            {
             "value": true,
             "probability": 0.9993698011094606
            }
           ]
          },
          "gap_factors": {
           "value": 1,
           "probability": 0.9990744453767675,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003199592092591818
            },
            {
             "value": 1,
             "probability": 0.9990744453767675
            },
            {
             "value": 2,
             "probability": 0.0002993079075467958
            },
            {
             "value": 3,
             "probability": 0.0003062875064264992
            }
           ]
          },
          "courier_ok": {
           "value": true,
           "probability": 0.9993202334762323,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006797665237676984
            },
            {
             "value": true,
             "probability": 0.9993202334762323
            }
           ]
          }
         },
         "joint_probability": 0.998453803956461,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0015473926382472988,
        "field_brier": 1.0372575931546236e-06
       }
      },
      "timing_ms": {
       "scoring": 40.751300053671,
       "base_joint": 0.12345914728939533,
       "constrained_joint": 0.10450882837176323,
       "constrained_total": 40.855808882042766
      },
      "input_tokens": 867,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09ba618ab858f874-0035",
      "group": "bendsynth-09ba618ab858f874",
      "provenance": "bendsynth-v1; family=bendsynth-09ba618ab858f874; domain=art gallery incoming loan approval chatbot; parameters=high_value_line=500; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "accept with courier required",
       "over_threshold": true,
       "courier_ok": true,
       "gap_factors": 1
      },
      "logits": [
       [
        1.723465919494629,
        -6.3456902503967285,
        -6.212162971496582,
        -6.322450637817383
       ],
       [
        -5.594784259796143,
        1.723793625831604
       ],
       [
        -5.580074310302734,
        1.7237838506698608
       ],
       [
        -6.266182899475098,
        1.7234994173049927,
        -6.387072563171387,
        -6.329195976257324
       ]
      ],
      "joint_logits": [
       -9.029162406921387,
       -8.522138595581055,
       -9.084959983825684,
       -9.065417289733887,
       -8.712278366088867,
       -7.6503496170043945,
       -8.824671745300293,
       -8.817636489868164,
       -8.738993644714355,
       -7.41794490814209,
       -8.911659240722656,
       -8.883387565612793,
       -8.169610023498535,
       1.7228766679763794,
       -8.358569145202637,
       -8.233743667602539,
       -9.553505897521973,
       -9.103910446166992,
       -9.688493728637695,
       -9.56624984741211,
       -9.221686363220215,
       -8.818862915039062,
       -9.336666107177734,
       -9.262517929077148,
       -9.256772994995117,
       -8.829386711120605,
       -9.368612289428711,
       -9.305994987487793,
       -8.982983589172363,
       -8.37535285949707,
       -9.09156322479248,
       -8.998967170715332,
       -9.50631046295166,
       -9.174223899841309,
       -9.611978530883789,
       -9.561302185058594,
       -9.288121223449707,
       -8.723431587219238,
       -9.36048698425293,
       -9.313454627990723,
       -9.326201438903809,
       -8.726826667785645,
       -9.416478157043457,
       -9.36390209197998,
       -8.934426307678223,
       -7.6498942375183105,
       -9.065328598022461,
       -9.06052017211914,
       -9.538504600524902,
       -9.212594032287598,
       -9.608288764953613,
       -9.560734748840332,
       -9.217872619628906,
       -8.800247192382812,
       -9.319910049438477,
       -9.273773193359375,
       -9.276910781860352,
       -8.83734130859375,
       -9.360859870910645,
       -9.31435775756836,
       -8.960927963256836,
       -8.189549446105957,
       -9.090807914733887,
       -9.034462928771973
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "accept with courier required",
           "probability": 0.9990097595568815,
           "distribution": [
            {
             "value": "accept with courier required",
             "probability": 0.9990097595568815
            },
            {
             "value": "reject",
             "probability": 0.00031273736436670575
            },
            {
             "value": "accept",
             "probability": 0.0003574126111870619
            },
            {
             "value": "board review",
             "probability": 0.00032009046756470395
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9993373348711525,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006626651288475429
            },
            {
             "value": true,
             "probability": 0.9993373348711525
            }
           ]
          },
          "courier_ok": {
           "value": true,
           "probability": 0.9993275148340327,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006724851659672945
            },
            {
             "value": true,
             "probability": 0.9993275148340327
            }
           ]
          },
          "gap_factors": {
           "value": 1,
           "probability": 0.9990433840715475,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003386175725172832
            },
            {
             "value": 1,
             "probability": 0.9990433840715475
            },
            {
             "value": 2,
             "probability": 0.0003000597808719874
            },
            {
             "value": 3,
             "probability": 0.0003179385750632984
            }
           ]
          }
         },
         "joint_probability": 0.9984213718578643,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0015798754884447609,
        "field_brier": 1.0780504809474415e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "accept with courier required",
           "probability": 0.9990097595568815,
           "distribution": [
            {
             "value": "accept with courier required",
             "probability": 0.9990097595568815
            },
            {
             "value": "reject",
             "probability": 0.00031273736436670575
            },
            {
             "value": "accept",
             "probability": 0.0003574126111870619
            },
            {
             "value": "board review",
             "probability": 0.00032009046756470395
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9993373348711525,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006626651288475429
            },
            {
             "value": true,
             "probability": 0.9993373348711525
            }
           ]
          },
          "courier_ok": {
           "value": true,
           "probability": 0.9993275148340327,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006724851659672945
            },
            {
             "value": true,
             "probability": 0.9993275148340327
            }
           ]
          },
          "gap_factors": {
           "value": 1,
           "probability": 0.9990433840715475,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003386175725172832
            },
            {
             "value": 1,
             "probability": 0.9990433840715475
            },
            {
             "value": 2,
             "probability": 0.0003000597808719874
            },
            {
             "value": 3,
             "probability": 0.0003179385750632984
            }
           ]
          }
         },
         "joint_probability": 0.9984213718578643,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0015798754884447609,
        "field_brier": 1.0780504809474415e-06
       }
      },
      "timing_ms": {
       "scoring": 39.173748111352324,
       "base_joint": 0.1305991318076849,
       "constrained_joint": 0.10199891403317451,
       "constrained_total": 39.2757470253855
      },
      "input_tokens": 835,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aea0120a31b76a4-0023",
      "group": "bendsynth-0aea0120a31b76a4",
      "provenance": "bendsynth-v1; family=bendsynth-0aea0120a31b76a4; domain=harbour berth assignment intake desk; parameters=fishing_len_limit=30,large_len_limit=77,draught_limit=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "berth assigned",
       "oversize": false,
       "issues": 0
      },
      "logits": [
       [
        -5.567234992980957,
        -5.465594291687012,
        1.530576229095459
       ],
       [
        1.5310529470443726,
        -5.141716003417969
       ],
       [
        1.530379056930542,
        -5.671709060668945,
        -5.874074459075928,
        -5.91103458404541
       ]
      ],
      "joint_logits": [
       -7.477492332458496,
       -7.54550313949585,
       -7.75730562210083,
       -7.767762184143066,
       -7.699527740478516,
       -7.5160136222839355,
       -7.7153849601745605,
       -7.745101451873779,
       -7.359840393066406,
       -7.460852146148682,
       -7.634887218475342,
       -7.672897815704346,
       -7.611777305603027,
       -7.423460960388184,
       -7.628796577453613,
       -7.617267608642578,
       1.5298115015029907,
       -7.3208489418029785,
       -7.574528217315674,
       -7.693777561187744,
       -7.660971641540527,
       -7.5325822830200195,
       -7.695273399353027,
       -7.727462291717529
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "berth assigned",
           "probability": 0.9982607362705199,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0008254749473088251
            },
            {
             "value": "deny",
             "probability": 0.0009137887821713512
            },
            {
             "value": "berth assigned",
             "probability": 0.9982607362705199
            }
           ]
          },
          "oversize": {
           "value": false,
           "probability": 0.9987367060012811,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987367060012811
            },
            {
             "value": true,
             "probability": 0.001263293998718989
            }
           ]
          },
          "issues": {
           "value": 0,
           "probability": 0.9980637351215065,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980637351215065
            },
            {
             "value": 1,
             "probability": 0.0007435862386293102
            },
            {
             "value": 2,
             "probability": 0.0006073583748414304
            },
            {
             "value": 3,
             "probability": 0.0005853202650228107
            }
           ]
          }
         },
         "joint_probability": 0.9974976259224807,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002505510248539112,
        "field_brier": 4.2489355947477245e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "berth assigned",
           "probability": 0.9982607362705199,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0008254749473088251
            },
            {
             "value": "deny",
             "probability": 0.0009137887821713512
            },
            {
             "value": "berth assigned",
             "probability": 0.9982607362705199
            }
           ]
          },
          "oversize": {
           "value": false,
           "probability": 0.9987367060012811,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987367060012811
            },
            {
             "value": true,
             "probability": 0.001263293998718989
            }
           ]
          },
          "issues": {
           "value": 0,
           "probability": 0.9980637351215065,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980637351215065
            },
            {
             "value": 1,
             "probability": 0.0007435862386293102
            },
            {
             "value": 2,
             "probability": 0.0006073583748414304
            },
            {
             "value": 3,
             "probability": 0.0005853202650228107
            }
           ]
          }
         },
         "joint_probability": 0.9974976259224807,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002505510248539112,
        "field_brier": 4.2489355947477245e-06
       }
      },
      "timing_ms": {
       "scoring": 33.92664389684796,
       "base_joint": 0.06621889770030975,
       "constrained_joint": 0.04755007103085518,
       "constrained_total": 33.97419396787882
      },
      "input_tokens": 606,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aea0120a31b76a4-0028",
      "group": "bendsynth-0aea0120a31b76a4",
      "provenance": "bendsynth-v1; family=bendsynth-0aea0120a31b76a4; domain=harbour berth assignment intake desk; parameters=fishing_len_limit=40,large_len_limit=70,draught_limit=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "oversize": false,
       "issues": 1,
       "decision": "waitlist"
      },
      "logits": [
       [
        1.1619793176651,
        -5.369777679443359
       ],
       [
        -6.097902297973633,
        1.1612299680709839,
        -5.908703327178955,
        -6.172632217407227
       ],
       [
        -5.636514186859131,
        1.1612677574157715,
        -5.6949143409729
       ]
      ],
      "joint_logits": [
       -7.861904621124268,
       -7.915982723236084,
       -7.824679851531982,
       -7.151642322540283,
       1.1603752374649048,
       -7.441427230834961,
       -7.6670451164245605,
       -7.391416549682617,
       -7.702826976776123,
       -7.938326358795166,
       -7.971381664276123,
       -7.916085243225098,
       -7.896460056304932,
       -7.949506759643555,
       -7.894116401672363,
       -7.742152214050293,
       -7.64602518081665,
       -7.713578224182129,
       -7.7832536697387695,
       -7.8821821212768555,
       -7.860777854919434,
       -7.960700035095215,
       -8.063672065734863,
       -7.942747592926025
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "oversize": {
           "value": false,
           "probability": 0.9985456735535475,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985456735535475
            },
            {
             "value": true,
             "probability": 0.0014543264464525025
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.997797805528928,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000702168510067364
            },
            {
             "value": 1,
             "probability": 0.997797805528928
            },
            {
             "value": 2,
             "probability": 0.0008484173861206562
            },
            {
             "value": 3,
             "probability": 0.0006516085748838983
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9978355205538297,
           "distribution": [
            {
             "value": "berth assigned",
             "probability": 0.0011138323231780494
            },
            {
             "value": "waitlist",
             "probability": 0.9978355205538297
            },
            {
             "value": "deny",
             "probability": 0.0010506471229922024
            }
           ]
          }
         },
         "joint_probability": 0.9969452912070678,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003059383939071251,
        "field_brier": 5.915563606895984e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "oversize": {
           "value": false,
           "probability": 0.9985456735535475,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985456735535475
            },
            {
             "value": true,
             "probability": 0.0014543264464525025
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.997797805528928,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000702168510067364
            },
            {
             "value": 1,
             "probability": 0.997797805528928
            },
            {
             "value": 2,
             "probability": 0.0008484173861206562
            },
            {
             "value": 3,
             "probability": 0.0006516085748838983
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9978355205538297,
           "distribution": [
            {
             "value": "berth assigned",
             "probability": 0.0011138323231780494
            },
            {
             "value": "waitlist",
             "probability": 0.9978355205538297
            },
            {
             "value": "deny",
             "probability": 0.0010506471229922024
            }
           ]
          }
         },
         "joint_probability": 0.9969452912070678,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003059383939071251,
        "field_brier": 5.915563606895984e-06
       }
      },
      "timing_ms": {
       "scoring": 34.16956099681556,
       "base_joint": 0.06765895523130894,
       "constrained_joint": 0.06186007522046566,
       "constrained_total": 34.23142107203603
      },
      "input_tokens": 594,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aea0120a31b76a4-0035",
      "group": "bendsynth-0aea0120a31b76a4",
      "provenance": "bendsynth-v1; family=bendsynth-0aea0120a31b76a4; domain=harbour berth assignment intake desk; parameters=fishing_len_limit=38,large_len_limit=130,draught_limit=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "issues": 1,
       "oversize": false,
       "decision": "waitlist"
      },
      "logits": [
       [
        -5.947018146514893,
        1.2388403415679932,
        -5.63731050491333,
        -6.065698146820068
       ],
       [
        1.239784836769104,
        -5.252190589904785
       ],
       [
        -5.615906238555908,
        -5.322254180908203,
        1.2388334274291992
       ]
      ],
      "joint_logits": [
       -7.6962361335754395,
       -7.519833564758301,
       -7.6037211418151855,
       -7.966568946838379,
       -7.785019397735596,
       -7.942025184631348,
       -7.151686191558838,
       -6.7059245109558105,
       1.2377581596374512,
       -7.694804668426514,
       -7.468414306640625,
       -7.263850688934326,
       -7.564945220947266,
       -7.259454727172852,
       -6.9553351402282715,
       -7.900134563446045,
       -7.470386505126953,
       -7.713613986968994,
       -7.819478511810303,
       -7.6493449211120605,
       -7.731653213500977,
       -8.083623886108398,
       -7.876482963562012,
       -8.059403419494629
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 1,
           "probability": 0.9975442379036364,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007553590453817988
            },
            {
             "value": 1,
             "probability": 0.9975442379036364
            },
            {
             "value": 2,
             "probability": 0.001029574666150275
            },
            {
             "value": 3,
             "probability": 0.0006708283848315064
            }
           ]
          },
          "oversize": {
           "value": false,
           "probability": 0.9984867416490913,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984867416490913
            },
            {
             "value": true,
             "probability": 0.001513258350908624
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9975372870201489,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0010518495480458562
            },
            {
             "value": "berth assigned",
             "probability": 0.0014108634318052283
            },
            {
             "value": "waitlist",
             "probability": 0.9975372870201489
            }
           ]
          }
         },
         "joint_probability": 0.9964651937668632,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0035410684221280093,
        "field_brier": 7.284383155232095e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 1,
           "probability": 0.9975442379036364,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007553590453817988
            },
            {
             "value": 1,
             "probability": 0.9975442379036364
            },
            {
             "value": 2,
             "probability": 0.001029574666150275
            },
            {
             "value": 3,
             "probability": 0.0006708283848315064
            }
           ]
          },
          "oversize": {
           "value": false,
           "probability": 0.9984867416490913,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984867416490913
            },
            {
             "value": true,
             "probability": 0.001513258350908624
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9975372870201489,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0010518495480458562
            },
            {
             "value": "berth assigned",
             "probability": 0.0014108634318052283
            },
            {
             "value": "waitlist",
             "probability": 0.9975372870201489
            }
           ]
          }
         },
         "joint_probability": 0.9964651937668632,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0035410684221280093,
        "field_brier": 7.284383155232095e-06
       }
      },
      "timing_ms": {
       "scoring": 34.910544054582715,
       "base_joint": 0.06554019637405872,
       "constrained_joint": 0.046669039875268936,
       "constrained_total": 34.957213094457984
      },
      "input_tokens": 582,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aea0120a31b76a4-0038",
      "group": "bendsynth-0aea0120a31b76a4",
      "provenance": "bendsynth-v1; family=bendsynth-0aea0120a31b76a4; domain=harbour berth assignment intake desk; parameters=fishing_len_limit=25,large_len_limit=139,draught_limit=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "waitlist",
       "oversize": false,
       "issues": 0
      },
      "logits": [
       [
        -5.816792964935303,
        1.071107268333435,
        -5.671090126037598
       ],
       [
        1.07183837890625,
        -5.452256202697754
       ],
       [
        1.0711842775344849,
        -6.008629322052002,
        -6.259860515594482,
        -6.304932594299316
       ]
      ],
      "joint_logits": [
       -7.793758392333984,
       -7.81896448135376,
       -8.006454467773438,
       -8.046728134155273,
       -7.8108906745910645,
       -7.755301475524902,
       -7.961472034454346,
       -8.025713920593262,
       1.0704071521759033,
       -7.855848789215088,
       -8.234546661376953,
       -8.365546226501465,
       -7.960019588470459,
       -8.103568077087402,
       -8.27401065826416,
       -8.292381286621094,
       -7.277325630187988,
       -7.706335067749023,
       -7.976788520812988,
       -7.987818241119385,
       -7.799665451049805,
       -7.6271491050720215,
       -7.9133124351501465,
       -7.936558723449707
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9978047252185483,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0010178143663328457
            },
            {
             "value": "waitlist",
             "probability": 0.9978047252185483
            },
            {
             "value": "berth assigned",
             "probability": 0.0011774604151188696
            }
           ]
          },
          "oversize": {
           "value": false,
           "probability": 0.9985345026612975,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985345026612975
            },
            {
             "value": true,
             "probability": 0.0014654973387025925
            }
           ]
          },
          "issues": {
           "value": 0,
           "probability": 0.9978816504496142,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978816504496142
            },
            {
             "value": 1,
             "probability": 0.0008401466964567666
            },
            {
             "value": 2,
             "probability": 0.0006535016826465241
            },
            {
             "value": 3,
             "probability": 0.0006247011712825668
            }
           ]
          }
         },
         "joint_probability": 0.9971063947026259,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002897799866755645,
        "field_brier": 5.849174223620711e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9978047252185483,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0010178143663328457
            },
            {
             "value": "waitlist",
             "probability": 0.9978047252185483
            },
            {
             "value": "berth assigned",
             "probability": 0.0011774604151188696
            }
           ]
          },
          "oversize": {
           "value": false,
           "probability": 0.9985345026612975,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985345026612975
            },
            {
             "value": true,
             "probability": 0.0014654973387025925
            }
           ]
          },
          "issues": {
           "value": 0,
           "probability": 0.9978816504496142,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978816504496142
            },
            {
             "value": 1,
             "probability": 0.0008401466964567666
            },
            {
             "value": 2,
             "probability": 0.0006535016826465241
            },
            {
             "value": 3,
             "probability": 0.0006247011712825668
            }
           ]
          }
         },
         "joint_probability": 0.9971063947026259,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002897799866755645,
        "field_brier": 5.849174223620711e-06
       }
      },
      "timing_ms": {
       "scoring": 32.82372606918216,
       "base_joint": 0.06341002881526947,
       "constrained_joint": 0.04783901385962963,
       "constrained_total": 32.87156508304179
      },
      "input_tokens": 588,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-004517920bd8227e-0008",
      "group": "bendsynth-004517920bd8227e",
      "provenance": "bendsynth-v1; family=bendsynth-004517920bd8227e; domain=open-pit blast clearance sign-off at a mining site; parameters=hole_quota=17; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "void permit",
       "within_quota": false,
       "followups": 3
      },
      "logits": [
       [
        -7.246180057525635,
        -7.380932807922363,
        -0.16851358115673065
       ],
       [
        -0.16806809604167938,
        -6.9491729736328125
       ],
       [
        -7.655100345611572,
        -7.451711654663086,
        -7.062263011932373,
        -0.16919434070587158
       ]
      ],
      "joint_logits": [
       -9.399084091186523,
       -9.17052173614502,
       -9.037690162658691,
       -9.255228996276855,
       -9.579883575439453,
       -9.400132179260254,
       -9.198519706726074,
       -9.74532699584961,
       -9.378191947937012,
       -9.25572681427002,
       -9.22225570678711,
       -9.505263328552246,
       -9.776017189025879,
       -9.670620918273926,
       -9.329537391662598,
       -9.705245018005371,
       -9.312836647033691,
       -8.984943389892578,
       -7.981294631958008,
       -0.16962137818336487,
       -9.314435958862305,
       -9.117610931396484,
       -9.138697624206543,
       -9.553373336791992
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "void permit",
           "probability": 0.9984213850678868,
           "distribution": [
            {
             "value": "hold pending sweep",
             "probability": 0.0008424077424059466
            },
            {
             "value": "clear to fire",
             "probability": 0.0007362071897073064
            },
            {
             "value": "void permit",
             "probability": 0.9984213850678868
            }
           ]
          },
          "within_quota": {
           "value": false,
           "probability": 0.9988662666917134,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988662666917134
            },
            {
             "value": true,
             "probability": 0.0011337333082865866
            }
           ]
          },
          "followups": {
           "value": 3,
           "probability": 0.9977419280504587,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005596681679936569
            },
            {
             "value": 1,
             "probability": 0.0006859005291213717
            },
            {
             "value": 2,
             "probability": 0.0010125032524263074
            },
            {
             "value": 3,
             "probability": 0.9977419280504587
            }
           ]
          }
         },
         "joint_probability": 0.9973158613695473,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002687747389599472,
        "field_brier": 4.407373040921119e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "void permit",
           "probability": 0.9984213850678868,
           "distribution": [
            {
             "value": "hold pending sweep",
             "probability": 0.0008424077424059466
            },
            {
             "value": "clear to fire",
             "probability": 0.0007362071897073064
            },
            {
             "value": "void permit",
             "probability": 0.9984213850678868
            }
           ]
          },
          "within_quota": {
           "value": false,
           "probability": 0.9988662666917134,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988662666917134
            },
            {
             "value": true,
             "probability": 0.0011337333082865866
            }
           ]
          },
          "followups": {
           "value": 3,
           "probability": 0.9977419280504587,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005596681679936569
            },
            {
             "value": 1,
             "probability": 0.0006859005291213717
            },
            {
             "value": 2,
             "probability": 0.0010125032524263074
            },
            {
             "value": 3,
             "probability": 0.9977419280504587
            }
           ]
          }
         },
         "joint_probability": 0.9973158613695473,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002687747389599472,
        "field_brier": 4.407373040921119e-06
       }
      },
      "timing_ms": {
       "scoring": 38.22944802232087,
       "base_joint": 0.06755907088518143,
       "constrained_joint": 0.048168934881687164,
       "constrained_total": 38.277616957202554
      },
      "input_tokens": 744,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-004517920bd8227e-0010",
      "group": "bendsynth-004517920bd8227e",
      "provenance": "bendsynth-v1; family=bendsynth-004517920bd8227e; domain=open-pit blast clearance sign-off at a mining site; parameters=hole_quota=25; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "void permit",
       "within_quota": false,
       "followups": 2
      },
      "logits": [
       [
        -5.455414295196533,
        1.398708701133728,
        -5.631826877593994
       ],
       [
        1.3991985321044922,
        -5.137742042541504
       ],
       [
        -5.646439552307129,
        1.3973866701126099,
        -4.924641132354736,
        -6.027898788452148
       ]
      ],
      "joint_logits": [
       -7.3442559242248535,
       -7.53580904006958,
       -7.733700752258301,
       -7.700050354003906,
       -7.400641918182373,
       -7.427389144897461,
       -7.368601322174072,
       -7.912307262420654,
       -7.373637676239014,
       1.3967902660369873,
       -5.289173126220703,
       -7.980988025665283,
       -7.484799861907959,
       -7.850099086761475,
       -7.915598392486572,
       -7.750576019287109,
       -7.621719837188721,
       -7.713974952697754,
       -7.906871795654297,
       -7.940645217895508,
       -7.428509712219238,
       -7.713531017303467,
       -7.780934810638428,
       -7.677420139312744
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "void permit",
           "probability": 0.9980641998838625,
           "distribution": [
            {
             "value": "hold pending sweep",
             "probability": 0.0010530540408780617
            },
            {
             "value": "void permit",
             "probability": 0.9980641998838625
            },
            {
             "value": "clear to fire",
             "probability": 0.0008827460752593632
            }
           ]
          },
          "within_quota": {
           "value": false,
           "probability": 0.998553182160137,
           "distribution": [
            {
             "value": false,
             "probability": 0.998553182160137
            },
            {
             "value": true,
             "probability": 0.0014468178398630066
            }
           ]
          },
          "followups": {
           "value": 1,
           "probability": 0.9967455572718998,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008699401527577447
            },
            {
             "value": 1,
             "probability": 0.9967455572718998
            },
            {
             "value": 2,
             "probability": 0.001790451392047087
            },
            {
             "value": 3,
             "probability": 0.0005940511832953091
            }
           ]
          }
         },
         "joint_probability": 0.9961513061813663,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 6.689819511356254,
        "field_brier": 0.6633116468717669
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "void permit",
           "probability": 0.9980641998838625,
           "distribution": [
            {
             "value": "hold pending sweep",
             "probability": 0.0010530540408780617
            },
            {
             "value": "void permit",
             "probability": 0.9980641998838625
            },
            {
             "value": "clear to fire",
             "probability": 0.0008827460752593632
            }
           ]
          },
          "within_quota": {
           "value": false,
           "probability": 0.998553182160137,
           "distribution": [
            {
             "value": false,
             "probability": 0.998553182160137
            },
            {
             "value": true,
             "probability": 0.0014468178398630066
            }
           ]
          },
          "followups": {
           "value": 1,
           "probability": 0.9967455572718998,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008699401527577447
            },
            {
             "value": 1,
             "probability": 0.9967455572718998
            },
            {
             "value": 2,
             "probability": 0.001790451392047087
            },
            {
             "value": 3,
             "probability": 0.0005940511832953091
            }
           ]
          }
         },
         "joint_probability": 0.9961513061813663,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 6.689819511356254,
        "field_brier": 0.6633116468717669
       }
      },
      "timing_ms": {
       "scoring": 37.4495058786124,
       "base_joint": 0.06586895324289799,
       "constrained_joint": 0.04792911931872368,
       "constrained_total": 37.49743499793112
      },
      "input_tokens": 756,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-08c9d1b1061197c6-0010",
      "group": "bendsynth-08c9d1b1061197c6",
      "provenance": "bendsynth-v1; family=bendsynth-08c9d1b1061197c6; domain=student visa document check filed on-site by an inspector; parameters=min_validity=180; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "referral",
       "problems": 2,
       "documents_ok": false
      },
      "logits": [
       [
        -0.2701510488986969,
        -7.292362213134766,
        -7.235892295837402
       ],
       [
        -7.7151970863342285,
        -7.623053073883057,
        -0.2701793909072876,
        -7.6261749267578125
       ],
       [
        -0.2696532607078552,
        -6.887059211730957
       ]
      ],
      "joint_logits": [
       -9.638985633850098,
       -9.481907844543457,
       -9.283567428588867,
       -9.417214393615723,
       -0.27091535925865173,
       -9.112503051757812,
       -9.485652923583984,
       -9.416658401489258,
       -9.512402534484863,
       -9.485268592834473,
       -9.476201057434082,
       -9.500648498535156,
       -8.953079223632812,
       -9.327281951904297,
       -9.50601577758789,
       -9.353479385375977,
       -9.44947624206543,
       -9.484548568725586,
       -9.371182441711426,
       -9.456308364868164,
       -8.980263710021973,
       -9.141334533691406,
       -9.37087345123291,
       -9.38474178314209
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "referral",
           "probability": 0.9981678486467038,
           "distribution": [
            {
             "value": "referral",
             "probability": 0.9981678486467038
            },
            {
             "value": "reject",
             "probability": 0.0008902170692602412
            },
            {
             "value": "clear",
             "probability": 0.0009419342840360212
            }
           ]
          },
          "problems": {
           "value": 2,
           "probability": 0.9981396223395036,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005832585582763052
            },
            {
             "value": 1,
             "probability": 0.0006395563109383301
            },
            {
             "value": 2,
             "probability": 0.9981396223395036
            },
            {
             "value": 3,
             "probability": 0.0006375627912817345
            }
           ]
          },
          "documents_ok": {
           "value": false,
           "probability": 0.9986648905814297,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986648905814297
            },
            {
             "value": true,
             "probability": 0.0013351094185702409
            }
           ]
          }
         },
         "joint_probability": 0.9974051908145937,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025981815377644464,
        "field_brier": 4.406084566501481e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "referral",
           "probability": 0.9981678486467038,
           "distribution": [
            {
             "value": "referral",
             "probability": 0.9981678486467038
            },
            {
             "value": "reject",
             "probability": 0.0008902170692602412
            },
            {
             "value": "clear",
             "probability": 0.0009419342840360212
            }
           ]
          },
          "problems": {
           "value": 2,
           "probability": 0.9981396223395036,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005832585582763052
            },
            {
             "value": 1,
             "probability": 0.0006395563109383301
            },
            {
             "value": 2,
             "probability": 0.9981396223395036
            },
            {
             "value": 3,
             "probability": 0.0006375627912817345
            }
           ]
          },
          "documents_ok": {
           "value": false,
           "probability": 0.9986648905814297,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986648905814297
            },
            {
             "value": true,
             "probability": 0.0013351094185702409
            }
           ]
          }
         },
         "joint_probability": 0.9974051908145937,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025981815377644464,
        "field_brier": 4.406084566501481e-06
       }
      },
      "timing_ms": {
       "scoring": 34.437397960573435,
       "base_joint": 0.06758887320756912,
       "constrained_joint": 0.06317999213933945,
       "constrained_total": 34.500577952712774
      },
      "input_tokens": 639,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-08c9d1b1061197c6-0021",
      "group": "bendsynth-08c9d1b1061197c6",
      "provenance": "bendsynth-v1; family=bendsynth-08c9d1b1061197c6; domain=student visa document check filed on-site by an inspector; parameters=min_validity=139; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "problems": 2,
       "decision": "reject",
       "documents_ok": false
      },
      "logits": [
       [
        -7.190288543701172,
        -7.179648399353027,
        0.3598010838031769,
        -7.102189064025879
       ],
       [
        -6.799321174621582,
        -6.8343377113342285,
        0.35990461707115173
       ],
       [
        0.3603813946247101,
        -6.4974517822265625
       ]
      ],
      "joint_logits": [
       -8.88391399383545,
       -9.02259349822998,
       -8.930896759033203,
       -9.003947257995605,
       -9.04266357421875,
       -9.018071174621582,
       -8.915704727172852,
       -9.052642822265625,
       -8.971840858459473,
       -9.054752349853516,
       -8.792691230773926,
       -9.070724487304688,
       -8.598307609558105,
       -8.893671035766602,
       -8.670403480529785,
       -8.964020729064941,
       0.3592751622200012,
       -8.930534362792969,
       -8.811232566833496,
       -8.92357349395752,
       -8.856050491333008,
       -8.90867805480957,
       -8.905744552612305,
       -8.965787887573242
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "problems": {
           "value": 2,
           "probability": 0.9983703950571198,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005252056902208723
            },
            {
             "value": 1,
             "probability": 0.0005308237919677118
            },
            {
             "value": 2,
             "probability": 0.9983703950571198
            },
            {
             "value": 3,
             "probability": 0.000573575460691636
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9984737796573567,
           "distribution": [
            {
             "value": "referral",
             "probability": 0.0007764696135837633
            },
            {
             "value": "clear",
             "probability": 0.0007497507290594807
            },
            {
             "value": "reject",
             "probability": 0.9984737796573567
            }
           ]
          },
          "documents_ok": {
           "value": false,
           "probability": 0.9989499148719574,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989499148719574
            },
            {
             "value": true,
             "probability": 0.0010500851280425677
            }
           ]
          }
         },
         "joint_probability": 0.9978454617940874,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0021568625625631955,
        "field_brier": 3.0806510990804564e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "problems": {
           "value": 2,
           "probability": 0.9983703950571198,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005252056902208723
            },
            {
             "value": 1,
             "probability": 0.0005308237919677118
            },
            {
             "value": 2,
             "probability": 0.9983703950571198
            },
            {
             "value": 3,
             "probability": 0.000573575460691636
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9984737796573567,
           "distribution": [
            {
             "value": "referral",
             "probability": 0.0007764696135837633
            },
            {
             "value": "clear",
             "probability": 0.0007497507290594807
            },
            {
             "value": "reject",
             "probability": 0.9984737796573567
            }
           ]
          },
          "documents_ok": {
           "value": false,
           "probability": 0.9989499148719574,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989499148719574
            },
            {
             "value": true,
             "probability": 0.0010500851280425677
            }
           ]
          }
         },
         "joint_probability": 0.9978454617940874,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0021568625625631955,
        "field_brier": 3.0806510990804564e-06
       }
      },
      "timing_ms": {
       "scoring": 33.94049499183893,
       "base_joint": 0.06496906280517578,
       "constrained_joint": 0.05694897845387459,
       "constrained_total": 33.99744397029281
      },
      "input_tokens": 624,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-08c9d1b1061197c6-0038",
      "group": "bendsynth-08c9d1b1061197c6",
      "provenance": "bendsynth-v1; family=bendsynth-08c9d1b1061197c6; domain=student visa document check filed on-site by an inspector; parameters=min_validity=99; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "documents_ok": true,
       "problems": 0,
       "decision": "clear"
      },
      "logits": [
       [
        -5.445003986358643,
        1.3013827800750732
       ],
       [
        1.300768494606018,
        -6.094367504119873,
        -6.104923725128174,
        -6.172325134277344
       ],
       [
        -5.703997611999512,
        -5.774234771728516,
        1.3008050918579102
       ]
      ],
      "joint_logits": [
       -7.740095615386963,
       -7.890219688415527,
       -7.860875606536865,
       -7.878123760223389,
       -7.925176620483398,
       -8.051520347595215,
       -7.879378318786621,
       -7.924500942230225,
       -8.03470230102539,
       -7.963253974914551,
       -7.96349573135376,
       -8.09990119934082,
       -7.53663969039917,
       -7.652461051940918,
       1.3001683950424194,
       -7.753729820251465,
       -7.820959091186523,
       -7.912675380706787,
       -7.762124538421631,
       -7.808699131011963,
       -7.998832702636719,
       -7.813403606414795,
       -7.878872871398926,
       -8.098753929138184
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "documents_ok": {
           "value": true,
           "probability": 0.9988262613650691,
           "distribution": [
            {
             "value": false,
             "probability": 0.001173738634930849
            },
            {
             "value": true,
             "probability": 0.9988262613650691
            }
           ]
          },
          "problems": {
           "value": 0,
           "probability": 0.9982130147533586,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9982130147533586
            },
            {
             "value": 1,
             "probability": 0.0006131354260951687
            },
            {
             "value": 2,
             "probability": 0.0006066971696660578
            },
            {
             "value": 3,
             "probability": 0.00056715265088026
            }
           ]
          },
          "decision": {
           "value": "clear",
           "probability": 0.9982495972183648,
           "distribution": [
            {
             "value": "referral",
             "probability": 0.0009059245639768961
            },
            {
             "value": "reject",
             "probability": 0.0008444782176582784
            },
            {
             "value": "clear",
             "probability": 0.9982495972183648
            }
           ]
          }
         },
         "joint_probability": 0.9976141567650728,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002388693893950991,
        "field_brier": 3.8706907824864475e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "documents_ok": {
           "value": true,
           "probability": 0.9988262613650691,
           "distribution": [
            {
             "value": false,
             "probability": 0.001173738634930849
            },
            {
             "value": true,
             "probability": 0.9988262613650691
            }
           ]
          },
          "problems": {
           "value": 0,
           "probability": 0.9982130147533586,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9982130147533586
            },
            {
             "value": 1,
             "probability": 0.0006131354260951687
            },
            {
             "value": 2,
             "probability": 0.0006066971696660578
            },
            {
             "value": 3,
             "probability": 0.00056715265088026
            }
           ]
          },
          "decision": {
           "value": "clear",
           "probability": 0.9982495972183648,
           "distribution": [
            {
             "value": "referral",
             "probability": 0.0009059245639768961
            },
            {
             "value": "reject",
             "probability": 0.0008444782176582784
            },
            {
             "value": "clear",
             "probability": 0.9982495972183648
            }
           ]
          }
         },
         "joint_probability": 0.9976141567650728,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002388693893950991,
        "field_brier": 3.8706907824864475e-06
       }
      },
      "timing_ms": {
       "scoring": 33.64298678934574,
       "base_joint": 0.06615906022489071,
       "constrained_joint": 0.04806998185813427,
       "constrained_total": 33.691056771203876
      },
      "input_tokens": 636,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a1d092d2c9efe4a-0010",
      "group": "bendsynth-0a1d092d2c9efe4a",
      "provenance": "bendsynth-v1; family=bendsynth-0a1d092d2c9efe4a; domain=dental practice recall appointment scheduling desk; parameters=recall_gap=18; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "long_gap": false,
       "decision": "book recall plus hygiene",
       "combined_reasons": 1
      },
      "logits": [
       [
        0.6699424386024475,
        -6.2952070236206055
       ],
       [
        -6.851747512817383,
        -7.023137092590332,
        0.6694223284721375,
        -6.9994215965271
       ],
       [
        -6.9589996337890625,
        0.669491708278656,
        -7.07383918762207,
        -6.981579780578613
       ]
      ],
      "joint_logits": [
       -9.02952766418457,
       -8.465929985046387,
       -9.108236312866211,
       -8.973518371582031,
       -9.061488151550293,
       -8.794604301452637,
       -9.204031944274902,
       -9.142836570739746,
       -8.899938583374023,
       0.6689682602882385,
       -9.042826652526855,
       -9.020946502685547,
       -8.983297348022461,
       -8.915327072143555,
       -9.133039474487305,
       -9.058362007141113,
       -9.029167175292969,
       -8.842174530029297,
       -9.157282829284668,
       -9.028220176696777,
       -9.171669006347656,
       -9.11507511138916,
       -9.241229057312012,
       -9.162364959716797,
       -8.996796607971191,
       -8.75951862335205,
       -9.132301330566406,
       -9.007542610168457,
       -9.164596557617188,
       -9.077659606933594,
       -9.222455024719238,
       -9.11057186126709
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "long_gap": {
           "value": false,
           "probability": 0.9990566688159735,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990566688159735
            },
            {
             "value": true,
             "probability": 0.0009433311840264087
            }
           ]
          },
          "decision": {
           "value": "book recall plus hygiene",
           "probability": 0.9985372776004553,
           "distribution": [
            {
             "value": "book recall only",
             "probability": 0.0005407065558739938
            },
            {
             "value": "send reminder letter",
             "probability": 0.0004555416414891264
            },
            {
             "value": "book recall plus hygiene",
             "probability": 0.9985372776004553
            },
            {
             "value": "urgent slot today",
             "probability": 0.0004664742021815872
            }
           ]
          },
          "combined_reasons": {
           "value": 1,
           "probability": 0.9986063915884438,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00048571627630656223
            },
            {
             "value": 1,
             "probability": 0.9986063915884438
            },
            {
             "value": 2,
             "probability": 0.0004330205919465528
            },
            {
             "value": 3,
             "probability": 0.0004748715433031782
            }
           ]
          }
         },
         "joint_probability": 0.9980838708398073,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001917967284102495,
        "field_brier": 2.409286244386053e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "long_gap": {
           "value": false,
           "probability": 0.9990566688159735,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990566688159735
            },
            {
             "value": true,
             "probability": 0.0009433311840264087
            }
           ]
          },
          "decision": {
           "value": "book recall plus hygiene",
           "probability": 0.9985372776004553,
           "distribution": [
            {
             "value": "book recall only",
             "probability": 0.0005407065558739938
            },
            {
             "value": "send reminder letter",
             "probability": 0.0004555416414891264
            },
            {
             "value": "book recall plus hygiene",
             "probability": 0.9985372776004553
            },
            {
             "value": "urgent slot today",
             "probability": 0.0004664742021815872
            }
           ]
          },
          "combined_reasons": {
           "value": 1,
           "probability": 0.9986063915884438,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00048571627630656223
            },
            {
             "value": 1,
             "probability": 0.9986063915884438
            },
            {
             "value": 2,
             "probability": 0.0004330205919465528
            },
            {
             "value": 3,
             "probability": 0.0004748715433031782
            }
           ]
          }
         },
         "joint_probability": 0.9980838708398073,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001917967284102495,
        "field_brier": 2.409286244386053e-06
       }
      },
      "timing_ms": {
       "scoring": 38.15027908422053,
       "base_joint": 0.07423991337418556,
       "constrained_joint": 0.05666911602020264,
       "constrained_total": 38.20694820024073
      },
      "input_tokens": 724,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a1d092d2c9efe4a-0023",
      "group": "bendsynth-0a1d092d2c9efe4a",
      "provenance": "bendsynth-v1; family=bendsynth-0a1d092d2c9efe4a; domain=dental practice recall appointment scheduling desk; parameters=recall_gap=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "long_gap": false,
       "decision": "book recall plus hygiene",
       "combined_reasons": 1
      },
      "logits": [
       [
        1.2503589391708374,
        -5.173414707183838
       ],
       [
        -6.049098014831543,
        -6.139169216156006,
        1.2499123811721802,
        -5.911858558654785
       ],
       [
        -6.053003311157227,
        1.2501050233840942,
        -6.193922996520996,
        -6.138281345367432
       ]
      ],
      "joint_logits": [
       -8.279631614685059,
       -7.770272254943848,
       -8.409564018249512,
       -8.379332542419434,
       -8.389742851257324,
       -7.858096122741699,
       -8.544515609741211,
       -8.489251136779785,
       -7.993635654449463,
       1.2490614652633667,
       -8.094170570373535,
       -8.115979194641113,
       -8.337788581848145,
       -7.133831024169922,
       -8.511141777038574,
       -8.490185737609863,
       -8.122364044189453,
       -7.838449478149414,
       -8.261039733886719,
       -8.171923637390137,
       -8.200353622436523,
       -7.924572944641113,
       -8.332197189331055,
       -8.233142852783203,
       -7.760908603668213,
       -7.0457868576049805,
       -7.943814754486084,
       -7.888362407684326,
       -8.12644100189209,
       -7.804514408111572,
       -8.23935604095459,
       -8.12535285949707
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "long_gap": {
           "value": false,
           "probability": 0.9983801066734778,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983801066734778
            },
            {
             "value": true,
             "probability": 0.0016198933265222622
            }
           ]
          },
          "decision": {
           "value": "send reminder letter",
           "probability": 0.9979344252653055,
           "distribution": [
            {
             "value": "book recall plus hygiene",
             "probability": 0.0006748107074349308
            },
            {
             "value": "urgent slot today",
             "probability": 0.0006166868649724105
            },
            {
             "value": "send reminder letter",
             "probability": 0.9979344252653055
            },
            {
             "value": "book recall only",
             "probability": 0.0007740771622871573
            }
           ]
          },
          "combined_reasons": {
           "value": 1,
           "probability": 0.9981267559068119,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000672180834899647
            },
            {
             "value": 1,
             "probability": 0.9981267559068119
            },
            {
             "value": 2,
             "probability": 0.0005838287107978315
            },
            {
             "value": 3,
             "probability": 0.0006172345474906589
            }
           ]
          }
         },
         "joint_probability": 0.997085555436067,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 9.022252420034537,
        "field_brier": 0.6648449537986986
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "long_gap": {
           "value": false,
           "probability": 0.9983801066734778,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983801066734778
            },
            {
             "value": true,
             "probability": 0.0016198933265222622
            }
           ]
          },
          "decision": {
           "value": "send reminder letter",
           "probability": 0.9979344252653055,
           "distribution": [
            {
             "value": "book recall plus hygiene",
             "probability": 0.0006748107074349308
            },
            {
             "value": "urgent slot today",
             "probability": 0.0006166868649724105
            },
            {
             "value": "send reminder letter",
             "probability": 0.9979344252653055
            },
            {
             "value": "book recall only",
             "probability": 0.0007740771622871573
            }
           ]
          },
          "combined_reasons": {
           "value": 1,
           "probability": 0.9981267559068119,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000672180834899647
            },
            {
             "value": 1,
             "probability": 0.9981267559068119
            },
            {
             "value": 2,
             "probability": 0.0005838287107978315
            },
            {
             "value": 3,
             "probability": 0.0006172345474906589
            }
           ]
          }
         },
         "joint_probability": 0.997085555436067,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 9.022252420034537,
        "field_brier": 0.6648449537986986
       }
      },
      "timing_ms": {
       "scoring": 36.897862097248435,
       "base_joint": 0.08452893234789371,
       "constrained_joint": 0.05685887299478054,
       "constrained_total": 36.954720970243216
      },
      "input_tokens": 709,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a1d092d2c9efe4a-0024",
      "group": "bendsynth-0a1d092d2c9efe4a",
      "provenance": "bendsynth-v1; family=bendsynth-0a1d092d2c9efe4a; domain=dental practice recall appointment scheduling desk; parameters=recall_gap=7; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "combined_reasons": 2,
       "decision": "book recall plus hygiene",
       "long_gap": false
      },
      "logits": [
       [
        -5.830419540405273,
        -5.340327739715576,
        -0.22882571816444397,
        -5.592646598815918
       ],
       [
        -5.915142059326172,
        -0.22628358006477356,
        -5.802836894989014,
        -5.5955586433410645
       ],
       [
        -0.2963685989379883,
        -2.758131265640259
       ]
      ],
      "joint_logits": [
       -8.940703392028809,
       -8.611824035644531,
       -7.135177135467529,
       -6.7476348876953125,
       -8.916282653808594,
       -8.616823196411133,
       -8.817780494689941,
       -8.566277503967285,
       -8.637914657592773,
       -8.312141418457031,
       -6.528039455413818,
       -6.2219133377075195,
       -8.543451309204102,
       -8.307527542114258,
       -8.448844909667969,
       -8.222304344177246,
       -7.250959873199463,
       -6.9886860847473145,
       -0.30694344639778137,
       -2.923607110977173,
       -7.037926197052002,
       -6.88141393661499,
       -6.597879409790039,
       -6.780447483062744,
       -8.822946548461914,
       -8.603035926818848,
       -6.782131671905518,
       -6.496489524841309,
       -8.741909980773926,
       -8.527735710144043,
       -8.670631408691406,
       -8.41053295135498
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "combined_reasons": {
           "value": 2,
           "probability": 0.9858024945391174,
           "distribution": [
            {
             "value": 0,
             "probability": 0.003639558411943733
            },
            {
             "value": 1,
             "probability": 0.005941454413637741
            },
            {
             "value": 2,
             "probability": 0.9858024945391174
            },
            {
             "value": 3,
             "probability": 0.004616492635301144
            }
           ]
          },
          "decision": {
           "value": "send reminder letter",
           "probability": 0.988311680116814,
           "distribution": [
            {
             "value": "book recall plus hygiene",
             "probability": 0.0033439064953580087
            },
            {
             "value": "send reminder letter",
             "probability": 0.988311680116814
            },
            {
             "value": "urgent slot today",
             "probability": 0.00374134418315069
            },
            {
             "value": "book recall only",
             "probability": 0.0046030692046773024
            }
           ]
          },
          "long_gap": {
           "value": false,
           "probability": 0.9214173823049338,
           "distribution": [
            {
             "value": false,
             "probability": 0.9214173823049338
            },
            {
             "value": true,
             "probability": 0.07858261769506629
            }
           ]
          }
         },
         "joint_probability": 0.9117249525486979,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 7.036433348313683,
        "field_brier": 0.660913471905606
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "combined_reasons": {
           "value": 2,
           "probability": 0.9858024945391174,
           "distribution": [
            {
             "value": 0,
             "probability": 0.003639558411943733
            },
            {
             "value": 1,
             "probability": 0.005941454413637741
            },
            {
             "value": 2,
             "probability": 0.9858024945391174
            },
            {
             "value": 3,
             "probability": 0.004616492635301144
            }
           ]
          },
          "decision": {
           "value": "send reminder letter",
           "probability": 0.988311680116814,
           "distribution": [
            {
             "value": "book recall plus hygiene",
             "probability": 0.0033439064953580087
            },
            {
             "value": "send reminder letter",
             "probability": 0.988311680116814
            },
            {
             "value": "urgent slot today",
             "probability": 0.00374134418315069
            },
            {
             "value": "book recall only",
             "probability": 0.0046030692046773024
            }
           ]
          },
          "long_gap": {
           "value": false,
           "probability": 0.9214173823049338,
           "distribution": [
            {
             "value": false,
             "probability": 0.9214173823049338
            },
            {
             "value": true,
             "probability": 0.07858261769506629
            }
           ]
          }
         },
         "joint_probability": 0.9117249525486979,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 7.036433348313683,
        "field_brier": 0.660913471905606
       }
      },
      "timing_ms": {
       "scoring": 37.86275093443692,
       "base_joint": 0.07488904520869255,
       "constrained_joint": 0.0581799540668726,
       "constrained_total": 37.92093088850379
      },
      "input_tokens": 724,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a1d092d2c9efe4a-0037",
      "group": "bendsynth-0a1d092d2c9efe4a",
      "provenance": "bendsynth-v1; family=bendsynth-0a1d092d2c9efe4a; domain=dental practice recall appointment scheduling desk; parameters=recall_gap=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "long_gap": false,
       "decision": "book recall only",
       "combined_reasons": 0
      },
      "logits": [
       [
        1.7276989221572876,
        -5.2361531257629395
       ],
       [
        -5.961121559143066,
        -5.802983283996582,
        -5.886028289794922,
        1.7271562814712524
       ],
       [
        1.7272279262542725,
        -5.9296369552612305,
        -5.8903117179870605,
        -5.973610877990723
       ]
      ],
      "joint_logits": [
       -7.794178485870361,
       -8.074926376342773,
       -8.03077507019043,
       -8.12090015411377,
       -7.636099815368652,
       -7.947439670562744,
       -7.858284950256348,
       -7.950164318084717,
       -7.584603786468506,
       -8.046645164489746,
       -8.00154972076416,
       -8.077763557434082,
       1.7267264127731323,
       -7.833357810974121,
       -7.925576686859131,
       -8.054868698120117,
       -8.042558670043945,
       -8.136373519897461,
       -8.062115669250488,
       -8.106247901916504,
       -7.894139289855957,
       -7.958065986633301,
       -7.9026641845703125,
       -7.9558258056640625,
       -8.002312660217285,
       -8.07174301147461,
       -7.987441062927246,
       -8.054523468017578,
       -7.853963851928711,
       -8.036609649658203,
       -8.0066499710083,
       -8.12066650390625
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "long_gap": {
           "value": false,
           "probability": 0.999055445329029,
           "distribution": [
            {
             "value": false,
             "probability": 0.999055445329029
            },
            {
             "value": true,
             "probability": 0.0009445546709709582
            }
           ]
          },
          "decision": {
           "value": "book recall only",
           "probability": 0.9985134865627069,
           "distribution": [
            {
             "value": "urgent slot today",
             "probability": 0.00045748545323308815
            },
            {
             "value": "book recall plus hygiene",
             "probability": 0.0005358656745612611
            },
            {
             "value": "send reminder letter",
             "probability": 0.0004931623094988098
            },
            {
             "value": "book recall only",
             "probability": 0.9985134865627069
            }
           ]
          },
          "combined_reasons": {
           "value": 0,
           "probability": 0.9985850197786568,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9985850197786568
            },
            {
             "value": 1,
             "probability": 0.0004721183172993808
            },
            {
             "value": 2,
             "probability": 0.0004910544308544614
            },
            {
             "value": 3,
             "probability": 0.0004518074731893672
            }
           ]
          }
         },
         "joint_probability": 0.9980843360056891,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019175012253001803,
        "field_brier": 2.4680241519919768e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "long_gap": {
           "value": false,
           "probability": 0.999055445329029,
           "distribution": [
            {
             "value": false,
             "probability": 0.999055445329029
            },
            {
             "value": true,
             "probability": 0.0009445546709709582
            }
           ]
          },
          "decision": {
           "value": "book recall only",
           "probability": 0.9985134865627069,
           "distribution": [
            {
             "value": "urgent slot today",
             "probability": 0.00045748545323308815
            },
            {
             "value": "book recall plus hygiene",
             "probability": 0.0005358656745612611
            },
            {
             "value": "send reminder letter",
             "probability": 0.0004931623094988098
            },
            {
             "value": "book recall only",
             "probability": 0.9985134865627069
            }
           ]
          },
          "combined_reasons": {
           "value": 0,
           "probability": 0.9985850197786568,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9985850197786568
            },
            {
             "value": 1,
             "probability": 0.0004721183172993808
            },
            {
             "value": 2,
             "probability": 0.0004910544308544614
            },
            {
             "value": 3,
             "probability": 0.0004518074731893672
            }
           ]
          }
         },
         "joint_probability": 0.9980843360056891,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019175012253001803,
        "field_brier": 2.4680241519919768e-06
       }
      },
      "timing_ms": {
       "scoring": 36.61491582170129,
       "base_joint": 0.07555913180112839,
       "constrained_joint": 0.05653989501297474,
       "constrained_total": 36.67145571671426
      },
      "input_tokens": 709,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b844142c7d6e1cf-0008",
      "group": "bendsynth-0b844142c7d6e1cf",
      "provenance": "bendsynth-v1; family=bendsynth-0b844142c7d6e1cf; domain=bakery wholesale delivery inspection at the loading dock; parameters=product_class=1; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "temp_ok": false,
       "issues": 1,
       "decision": "reject"
      },
      "logits": [
       [
        0.6581360697746277,
        -5.45159912109375
       ],
       [
        -6.200577259063721,
        0.656413197517395,
        -5.494330406188965,
        -6.518782138824463
       ],
       [
        0.6575625538825989,
        -5.762906074523926,
        -6.094789981842041
       ]
      ],
      "joint_logits": [
       -8.001532554626465,
       -8.087865829467773,
       -8.31363582611084,
       0.6553430557250977,
       -7.487083911895752,
       -7.9594316482543945,
       -6.09324836730957,
       -7.76861572265625,
       -8.234745979309082,
       -8.473175048828125,
       -8.32470703125,
       -8.55991268157959,
       -8.241303443908691,
       -7.598344802856445,
       -7.885671615600586,
       -7.4802937507629395,
       -7.952873706817627,
       -8.349417686462402,
       -7.696464538574219,
       -7.828231334686279,
       -8.075352668762207,
       -8.542463302612305,
       -7.935149192810059,
       -8.18276309967041
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "temp_ok": {
           "value": false,
           "probability": 0.9977837826294402,
           "distribution": [
            {
             "value": false,
             "probability": 0.9977837826294402
            },
            {
             "value": true,
             "probability": 0.002216217370559691
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.9960662274879357,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0010479369737542684
            },
            {
             "value": 1,
             "probability": 0.9960662274879357
            },
            {
             "value": 2,
             "probability": 0.0021235099774462706
            },
            {
             "value": 3,
             "probability": 0.0007623255608638225
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.997211774343996,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.997211774343996
            },
            {
             "value": "accept",
             "probability": 0.0016233543741304463
            },
            {
             "value": "accept with note",
             "probability": 0.0011648712818735133
            }
           ]
          }
         },
         "joint_probability": 0.9950009085644175,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0050116286938933375,
        "field_brier": 1.4417606221992142e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "temp_ok": {
           "value": false,
           "probability": 0.9977837826294402,
           "distribution": [
            {
             "value": false,
             "probability": 0.9977837826294402
            },
            {
             "value": true,
             "probability": 0.002216217370559691
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.9960662274879357,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0010479369737542684
            },
            {
             "value": 1,
             "probability": 0.9960662274879357
            },
            {
             "value": 2,
             "probability": 0.0021235099774462706
            },
            {
             "value": 3,
             "probability": 0.0007623255608638225
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.997211774343996,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.997211774343996
            },
            {
             "value": "accept",
             "probability": 0.0016233543741304463
            },
            {
             "value": "accept with note",
             "probability": 0.0011648712818735133
            }
           ]
          }
         },
         "joint_probability": 0.9950009085644175,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0050116286938933375,
        "field_brier": 1.4417606221992142e-05
       }
      },
      "timing_ms": {
       "scoring": 35.652835853397846,
       "base_joint": 0.06859912537038326,
       "constrained_joint": 0.048479996621608734,
       "constrained_total": 35.701315850019455
      },
      "input_tokens": 696,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-02d3b7ef3a349f3f-0015",
      "group": "bendsynth-02d3b7ef3a349f3f",
      "provenance": "bendsynth-v1; family=bendsynth-02d3b7ef3a349f3f; domain=apiary relocation permit desk at a regional agriculture office; parameters=wait_months=11,large_hive_cap=9; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "refer to senior officer",
       "approval_steps": 3,
       "eligibility_met": true
      },
      "logits": [
       [
        -6.951365947723389,
        -6.845432281494141,
        -7.0311198234558105,
        0.5735792517662048
       ],
       [
        -7.01188850402832,
        -7.150686740875244,
        -6.875494480133057,
        0.5736849308013916
       ],
       [
        -6.493438720703125,
        0.5743634700775146
       ]
      ],
      "joint_logits": [
       -9.118236541748047,
       -8.94067668914795,
       -9.409049987792969,
       -9.076642990112305,
       -9.028886795043945,
       -8.880631446838379,
       -9.336983680725098,
       -8.659991264343262,
       -9.156036376953125,
       -8.863104820251465,
       -9.250272750854492,
       -8.964812278747559,
       -9.042999267578125,
       -8.719462394714355,
       -9.171911239624023,
       -8.478996276855469,
       -9.241570472717285,
       -8.936883926391602,
       -9.471955299377441,
       -9.065752983093262,
       -9.2861909866333,
       -8.911670684814453,
       -9.320001602172852,
       -8.832496643066406,
       -9.44160270690918,
       -9.155085563659668,
       -9.55528450012207,
       -9.202731132507324,
       -9.396652221679688,
       -8.612930297851562,
       -9.205214500427246,
       0.5731724500656128
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "refer to senior officer",
           "probability": 0.9983653711878101,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0005385763625845361
            },
            {
             "value": "approve with inspection",
             "probability": 0.000598761377628256
            },
            {
             "value": "approve",
             "probability": 0.0004972910719771546
            },
            {
             "value": "refer to senior officer",
             "probability": 0.9983653711878101
            }
           ]
          },
          "approval_steps": {
           "value": 3,
           "probability": 0.9984707750520148,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005069471308054641
            },
            {
             "value": 1,
             "probability": 0.00044124869784954734
            },
            {
             "value": 2,
             "probability": 0.0005810291193301859
            },
            {
             "value": 3,
             "probability": 0.9984707750520148
            }
           ]
          },
          "eligibility_met": {
           "value": true,
           "probability": 0.9991486214912381,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008513785087618994
            },
            {
             "value": true,
             "probability": 0.9991486214912381
            }
           ]
          }
         },
         "joint_probability": 0.9979592533028098,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002042831858070992,
        "field_brier": 2.7151332552609472e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "refer to senior officer",
           "probability": 0.9983653711878101,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0005385763625845361
            },
            {
             "value": "approve with inspection",
             "probability": 0.000598761377628256
            },
            {
             "value": "approve",
             "probability": 0.0004972910719771546
            },
            {
             "value": "refer to senior officer",
             "probability": 0.9983653711878101
            }
           ]
          },
          "approval_steps": {
           "value": 3,
           "probability": 0.9984707750520148,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005069471308054641
            },
            {
             "value": 1,
             "probability": 0.00044124869784954734
            },
            {
             "value": 2,
             "probability": 0.0005810291193301859
            },
            {
             "value": 3,
             "probability": 0.9984707750520148
            }
           ]
          },
          "eligibility_met": {
           "value": true,
           "probability": 0.9991486214912381,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008513785087618994
            },
            {
             "value": true,
             "probability": 0.9991486214912381
            }
           ]
          }
         },
         "joint_probability": 0.9979592533028098,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002042831858070992,
        "field_brier": 2.7151332552609472e-06
       }
      },
      "timing_ms": {
       "scoring": 33.84188516065478,
       "base_joint": 0.07412000559270382,
       "constrained_joint": 0.05646888166666031,
       "constrained_total": 33.898354042321444
      },
      "input_tokens": 569,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-02d3b7ef3a349f3f-0020",
      "group": "bendsynth-02d3b7ef3a349f3f",
      "provenance": "bendsynth-v1; family=bendsynth-02d3b7ef3a349f3f; domain=apiary relocation permit desk at a regional agriculture office; parameters=wait_months=13,large_hive_cap=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "deny",
       "eligibility_met": false,
       "approval_steps": 2
      },
      "logits": [
       [
        -7.2213134765625,
        0.566209077835083,
        -7.002270698547363,
        -7.322177886962891
       ],
       [
        0.5666094422340393,
        -6.440303802490234
       ],
       [
        -7.179697036743164,
        -7.259117126464844,
        0.5662359595298767,
        -7.145900726318359
       ]
      ],
      "joint_logits": [
       -9.472180366516113,
       -9.5276460647583,
       -9.335192680358887,
       -9.385897636413574,
       -9.150247573852539,
       -9.252579689025879,
       -9.21796703338623,
       -9.137014389038086,
       -9.325868606567383,
       -9.128104209899902,
       0.565791666507721,
       -9.307530403137207,
       -9.128334045410156,
       -9.172682762145996,
       -9.075660705566406,
       -9.029749870300293,
       -9.076343536376953,
       -9.16388988494873,
       -8.946015357971191,
       -8.974373817443848,
       -9.14158821105957,
       -9.415552139282227,
       -8.820021629333496,
       -9.235223770141602,
       -9.363229751586914,
       -9.439253807067871,
       -9.282258987426758,
       -9.253999710083008,
       -9.513778686523438,
       -9.774154663085938,
       -9.118003845214844,
       -9.623334884643555
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "deny",
           "probability": 0.9986952736455308,
           "distribution": [
            {
             "value": "refer to senior officer",
             "probability": 0.0004143381568677432
            },
            {
             "value": "deny",
             "probability": 0.9986952736455308
            },
            {
             "value": "approve with inspection",
             "probability": 0.0005158033445278496
            },
            {
             "value": "approve",
             "probability": 0.00037458485307359307
            }
           ]
          },
          "eligibility_met": {
           "value": false,
           "probability": 0.9990952193885148,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990952193885148
            },
            {
             "value": true,
             "probability": 0.0009047806114852298
            }
           ]
          },
          "approval_steps": {
           "value": 2,
           "probability": 0.9987222947482309,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004319453528702308
            },
            {
             "value": 1,
             "probability": 0.00039896697895515365
            },
            {
             "value": 2,
             "probability": 0.9987222947482309
            },
            {
             "value": 3,
             "probability": 0.0004467929199436457
            }
           ]
          }
         },
         "joint_probability": 0.9982785823615539,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001722900980334128,
        "field_brier": 2.031838614124457e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "deny",
           "probability": 0.9986952736455308,
           "distribution": [
            {
             "value": "refer to senior officer",
             "probability": 0.0004143381568677432
            },
            {
             "value": "deny",
             "probability": 0.9986952736455308
            },
            {
             "value": "approve with inspection",
             "probability": 0.0005158033445278496
            },
            {
             "value": "approve",
             "probability": 0.00037458485307359307
            }
           ]
          },
          "eligibility_met": {
           "value": false,
           "probability": 0.9990952193885148,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990952193885148
            },
            {
             "value": true,
             "probability": 0.0009047806114852298
            }
           ]
          },
          "approval_steps": {
           "value": 2,
           "probability": 0.9987222947482309,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004319453528702308
            },
            {
             "value": 1,
             "probability": 0.00039896697895515365
            },
            {
             "value": 2,
             "probability": 0.9987222947482309
            },
            {
             "value": 3,
             "probability": 0.0004467929199436457
            }
           ]
          }
         },
         "joint_probability": 0.9982785823615539,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001722900980334128,
        "field_brier": 2.031838614124457e-06
       }
      },
      "timing_ms": {
       "scoring": 33.72307610698044,
       "base_joint": 0.07689907215535641,
       "constrained_joint": 0.06320886313915253,
       "constrained_total": 33.786284970119596
      },
      "input_tokens": 569,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-02d3b7ef3a349f3f-0023",
      "group": "bendsynth-02d3b7ef3a349f3f",
      "provenance": "bendsynth-v1; family=bendsynth-02d3b7ef3a349f3f; domain=apiary relocation permit desk at a regional agriculture office; parameters=wait_months=5,large_hive_cap=13; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "approval_steps": 1,
       "eligibility_met": true,
       "decision": "approve"
      },
      "logits": [
       [
        -5.910589218139648,
        1.6365537643432617,
        -5.836864471435547,
        -5.907116889953613
       ],
       [
        -5.296566009521484,
        1.637203574180603
       ],
       [
        -5.770687103271484,
        -5.774997711181641,
        1.636415958404541,
        -5.86603307723999
       ]
      ],
      "joint_logits": [
       -8.074625968933105,
       -8.067399978637695,
       -8.258580207824707,
       -8.149697303771973,
       -7.851561546325684,
       -7.872172832489014,
       -7.799737453460693,
       -7.936062335968018,
       -8.014615058898926,
       -8.017681121826172,
       -7.879554271697998,
       -8.120007514953613,
       -7.40013313293457,
       -7.4267096519470215,
       1.6359611749649048,
       -7.5993781089782715,
       -7.98947811126709,
       -7.989080429077148,
       -8.184043884277344,
       -8.044017791748047,
       -7.774072647094727,
       -7.775187969207764,
       -7.80936336517334,
       -7.840959548950195,
       -8.044464111328125,
       -8.007920265197754,
       -8.244807243347168,
       -8.094155311584473,
       -7.836666584014893,
       -7.842498779296875,
       -7.983692169189453,
       -7.902700424194336
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "approval_steps": {
           "value": 1,
           "probability": 0.9983775865469866,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000526759535871401
            },
            {
             "value": 1,
             "probability": 0.9983775865469866
            },
            {
             "value": 2,
             "probability": 0.0005670620777135915
            },
            {
             "value": 3,
             "probability": 0.0005285918394283788
            }
           ]
          },
          "eligibility_met": {
           "value": true,
           "probability": 0.9990266271994498,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009733728005501663
            },
            {
             "value": true,
             "probability": 0.9990266271994498
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9982401286350703,
           "distribution": [
            {
             "value": "approve with inspection",
             "probability": 0.0006058583801538747
            },
            {
             "value": "deny",
             "probability": 0.0006032524242192815
            },
            {
             "value": "approve",
             "probability": 0.9982401286350703
            },
            {
             "value": "refer to senior officer",
             "probability": 0.0005507605605565757
            }
           ]
          }
         },
         "joint_probability": 0.9977861927827576,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022162613110360023,
        "field_brier": 3.1790137506121915e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "approval_steps": {
           "value": 1,
           "probability": 0.9983775865469866,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000526759535871401
            },
            {
             "value": 1,
             "probability": 0.9983775865469866
            },
            {
             "value": 2,
             "probability": 0.0005670620777135915
            },
            {
             "value": 3,
             "probability": 0.0005285918394283788
            }
           ]
          },
          "eligibility_met": {
           "value": true,
           "probability": 0.9990266271994498,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009733728005501663
            },
            {
             "value": true,
             "probability": 0.9990266271994498
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9982401286350703,
           "distribution": [
            {
             "value": "approve with inspection",
             "probability": 0.0006058583801538747
            },
            {
             "value": "deny",
             "probability": 0.0006032524242192815
            },
            {
             "value": "approve",
             "probability": 0.9982401286350703
            },
            {
             "value": "refer to senior officer",
             "probability": 0.0005507605605565757
            }
           ]
          }
         },
         "joint_probability": 0.9977861927827576,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022162613110360023,
        "field_brier": 3.1790137506121915e-06
       }
      },
      "timing_ms": {
       "scoring": 34.85820512287319,
       "base_joint": 0.0742599368095398,
       "constrained_joint": 0.06780889816582203,
       "constrained_total": 34.92601402103901
      },
      "input_tokens": 587,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b1df5fa608628a9-0018",
      "group": "bendsynth-0b1df5fa608628a9",
      "provenance": "bendsynth-v1; family=bendsynth-0b1df5fa608628a9; domain=university chemistry stockroom request audit; parameters=auto_limit=1337; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "needs_review": false,
       "decision": "conditional",
       "risk_flags": 1
      },
      "logits": [
       [
        0.21686062216758728,
        -6.582397937774658
       ],
       [
        -7.008267402648926,
        -7.080617904663086,
        0.21656998991966248
       ],
       [
        -7.251150131225586,
        0.21635982394218445,
        -7.339454174041748,
        -7.34198522567749
       ]
      ],
      "joint_logits": [
       -8.934471130371094,
       -8.99666690826416,
       -9.058509826660156,
       -9.064788818359375,
       -9.047809600830078,
       -9.202380180358887,
       -9.13626480102539,
       -9.206448554992676,
       -9.201866149902344,
       0.2158934623003006,
       -9.247673034667969,
       -9.32133674621582,
       -9.228779792785645,
       -8.956908226013184,
       -9.320626258850098,
       -9.208883285522461,
       -9.153182983398438,
       -9.099998474121094,
       -9.247575759887695,
       -9.202038764953613,
       -8.77185344696045,
       -9.081697463989258,
       -8.852096557617188,
       -8.865105628967285
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_review": {
           "value": false,
           "probability": 0.998886639775309,
           "distribution": [
            {
             "value": false,
             "probability": 0.998886639775309
            },
            {
             "value": true,
             "probability": 0.0011133602246909484
            }
           ]
          },
          "decision": {
           "value": "conditional",
           "probability": 0.9985962612055271,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0007272485649194531
            },
            {
             "value": "escalate",
             "probability": 0.0006764902295534128
            },
            {
             "value": "conditional",
             "probability": 0.9985962612055271
            }
           ]
          },
          "risk_flags": {
           "value": 1,
           "probability": 0.9983864600951752,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005704274280040352
            },
            {
             "value": 1,
             "probability": 0.9983864600951752
            },
            {
             "value": 2,
             "probability": 0.0005222162054650962
            },
            {
             "value": 3,
             "probability": 0.0005208962713557032
            }
           ]
          }
         },
         "joint_probability": 0.9979209727343284,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002081191442966632,
        "field_brier": 2.9696984182195544e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_review": {
           "value": false,
           "probability": 0.998886639775309,
           "distribution": [
            {
             "value": false,
             "probability": 0.998886639775309
            },
            {
             "value": true,
             "probability": 0.0011133602246909484
            }
           ]
          },
          "decision": {
           "value": "conditional",
           "probability": 0.9985962612055271,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0007272485649194531
            },
            {
             "value": "escalate",
             "probability": 0.0006764902295534128
            },
            {
             "value": "conditional",
             "probability": 0.9985962612055271
            }
           ]
          },
          "risk_flags": {
           "value": 1,
           "probability": 0.9983864600951752,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005704274280040352
            },
            {
             "value": 1,
             "probability": 0.9983864600951752
            },
            {
             "value": 2,
             "probability": 0.0005222162054650962
            },
            {
             "value": 3,
             "probability": 0.0005208962713557032
            }
           ]
          }
         },
         "joint_probability": 0.9979209727343284,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002081191442966632,
        "field_brier": 2.9696984182195544e-06
       }
      },
      "timing_ms": {
       "scoring": 34.67036713846028,
       "base_joint": 0.06669992581009865,
       "constrained_joint": 0.04872889257967472,
       "constrained_total": 34.71909603103995
      },
      "input_tokens": 662,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b1df5fa608628a9-0029",
      "group": "bendsynth-0b1df5fa608628a9",
      "provenance": "bendsynth-v1; family=bendsynth-0b1df5fa608628a9; domain=university chemistry stockroom request audit; parameters=auto_limit=746; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_flags": 0,
       "decision": "approve",
       "needs_review": false
      },
      "logits": [
       [
        1.0243014097213745,
        -6.416436672210693,
        -6.383867263793945,
        -6.469287872314453
       ],
       [
        1.0243873596191406,
        -6.1044769287109375,
        -6.031803131103516
       ],
       [
        1.024766206741333,
        -5.632752418518066
       ]
      ],
      "joint_logits": [
       1.0236632823944092,
       -7.959585189819336,
       -7.921459197998047,
       -8.03320026397705,
       -7.8423333168029785,
       -7.965694904327393,
       -8.20541000366211,
       -8.1343994140625,
       -8.31293773651123,
       -8.223020553588867,
       -8.242401123046875,
       -8.142026901245117,
       -8.21806526184082,
       -8.105788230895996,
       -8.274949073791504,
       -8.157071113586426,
       -8.211846351623535,
       -8.098100662231445,
       -8.310644149780273,
       -8.170711517333984,
       -8.346796035766602,
       -8.281859397888184,
       -8.274709701538086,
       -8.193214416503906
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 0,
           "probability": 0.9982532826285411,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9982532826285411
            },
            {
             "value": 1,
             "probability": 0.0005858269506869838
            },
            {
             "value": 2,
             "probability": 0.0006052212900850282
            },
            {
             "value": 3,
             "probability": 0.0005556691306868581
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9983390777940815,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.9983390777940815
            },
            {
             "value": "escalate",
             "probability": 0.0008002979358904968
            },
            {
             "value": "conditional",
             "probability": 0.0008606242700279428
            }
           ]
          },
          "needs_review": {
           "value": false,
           "probability": 0.9987173177519202,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987173177519202
            },
            {
             "value": true,
             "probability": 0.0012826822480798928
            }
           ]
          }
         },
         "joint_probability": 0.997616513403507,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002386331622290525,
        "field_brier": 3.833212259407128e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 0,
           "probability": 0.9982532826285411,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9982532826285411
            },
            {
             "value": 1,
             "probability": 0.0005858269506869838
            },
            {
             "value": 2,
             "probability": 0.0006052212900850282
            },
            {
             "value": 3,
             "probability": 0.0005556691306868581
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9983390777940815,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.9983390777940815
            },
            {
             "value": "escalate",
             "probability": 0.0008002979358904968
            },
            {
             "value": "conditional",
             "probability": 0.0008606242700279428
            }
           ]
          },
          "needs_review": {
           "value": false,
           "probability": 0.9987173177519202,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987173177519202
            },
            {
             "value": true,
             "probability": 0.0012826822480798928
            }
           ]
          }
         },
         "joint_probability": 0.997616513403507,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002386331622290525,
        "field_brier": 3.833212259407128e-06
       }
      },
      "timing_ms": {
       "scoring": 35.75880499556661,
       "base_joint": 0.06468896754086018,
       "constrained_joint": 0.04547904245555401,
       "constrained_total": 35.80428403802216
      },
      "input_tokens": 656,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c9195364e722952-0003",
      "group": "bendsynth-0c9195364e722952",
      "provenance": "bendsynth-v1; family=bendsynth-0c9195364e722952; domain=dental practice recall scheduling audit; parameters=review_gap=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "beyond_gap": true,
       "decision": "defer file",
       "factors": 1
      },
      "logits": [
       [
        -5.588335037231445,
        1.3273847103118896
       ],
       [
        -6.144960403442383,
        -6.284611701965332,
        1.3268612623214722,
        -6.373631000518799
       ],
       [
        -6.26183557510376,
        1.3269294500350952,
        -6.308509349822998,
        -6.360030651092529
       ]
      ],
      "joint_logits": [
       -8.271632194519043,
       -8.182670593261719,
       -8.321023941040039,
       -8.356342315673828,
       -8.386631965637207,
       -8.339707374572754,
       -8.437002182006836,
       -8.456598281860352,
       -8.33222770690918,
       -8.036296844482422,
       -8.339356422424316,
       -8.382049560546875,
       -8.488986015319824,
       -8.470227241516113,
       -8.543874740600586,
       -8.569154739379883,
       -8.26020336151123,
       -7.855627059936523,
       -8.313446044921875,
       -8.336871147155762,
       -8.298291206359863,
       -8.195191383361816,
       -8.406745910644531,
       -8.419261932373047,
       -8.245615005493164,
       1.326390266418457,
       -8.230658531188965,
       -8.448777198791504,
       -8.478610038757324,
       -8.056075096130371,
       -8.557357788085938,
       -8.574585914611816
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "beyond_gap": {
           "value": true,
           "probability": 0.999008916122634,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009910838773659614
            },
            {
             "value": true,
             "probability": 0.999008916122634
            }
           ]
          },
          "decision": {
           "value": "defer file",
           "probability": 0.9984860577598433,
           "distribution": [
            {
             "value": "extend interval",
             "probability": 0.0005680296368744206
            },
            {
             "value": "schedule routine",
             "probability": 0.0004939934363566556
            },
            {
             "value": "defer file",
             "probability": 0.9984860577598433
            },
            {
             "value": "schedule now",
             "probability": 0.000451919166925607
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9985541903625705,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000505373907707862
            },
            {
             "value": 1,
             "probability": 0.9985541903625705
            },
            {
             "value": 2,
             "probability": 0.00048232838146285996
            },
            {
             "value": 3,
             "probability": 0.00045810734825869675
            }
           ]
          }
         },
         "joint_probability": 0.9980158647243653,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001986106279621267,
        "field_brier": 2.605235010605028e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "beyond_gap": {
           "value": true,
           "probability": 0.999008916122634,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009910838773659614
            },
            {
             "value": true,
             "probability": 0.999008916122634
            }
           ]
          },
          "decision": {
           "value": "defer file",
           "probability": 0.9984860577598433,
           "distribution": [
            {
             "value": "extend interval",
             "probability": 0.0005680296368744206
            },
            {
             "value": "schedule routine",
             "probability": 0.0004939934363566556
            },
            {
             "value": "defer file",
             "probability": 0.9984860577598433
            },
            {
             "value": "schedule now",
             "probability": 0.000451919166925607
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9985541903625705,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000505373907707862
            },
            {
             "value": 1,
             "probability": 0.9985541903625705
            },
            {
             "value": 2,
             "probability": 0.00048232838146285996
            },
            {
             "value": 3,
             "probability": 0.00045810734825869675
            }
           ]
          }
         },
         "joint_probability": 0.9980158647243653,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001986106279621267,
        "field_brier": 2.605235010605028e-06
       }
      },
      "timing_ms": {
       "scoring": 35.18875106237829,
       "base_joint": 0.07267901673913002,
       "constrained_joint": 0.05647004581987858,
       "constrained_total": 35.245221108198166
      },
      "input_tokens": 590,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c9195364e722952-0005",
      "group": "bendsynth-0c9195364e722952",
      "provenance": "bendsynth-v1; family=bendsynth-0c9195364e722952; domain=dental practice recall scheduling audit; parameters=review_gap=11; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "defer file",
       "factors": 3,
       "beyond_gap": true
      },
      "logits": [
       [
        0.10648881644010544,
        -7.635838985443115,
        -7.567162036895752,
        -7.671463966369629
       ],
       [
        -7.637389659881592,
        -7.544378280639648,
        -7.558596611022949,
        0.10642875730991364
       ],
       [
        -7.04680061340332,
        0.10702414810657501
       ]
      ],
      "joint_logits": [
       -9.742734909057617,
       -9.566349029541016,
       -9.699861526489258,
       -9.32610034942627,
       -9.659096717834473,
       -9.143658638000488,
       -9.641449928283691,
       0.10602591186761856,
       -9.78743839263916,
       -9.65764331817627,
       -9.737838745117188,
       -9.554551124572754,
       -9.900527000427246,
       -9.65556812286377,
       -10.013237953186035,
       -9.51359748840332,
       -9.887295722961426,
       -9.562093734741211,
       -9.808744430541992,
       -9.532955169677734,
       -9.831836700439453,
       -9.541152954101562,
       -9.850871086120605,
       -9.311785697937012,
       -9.877570152282715,
       -9.708630561828613,
       -9.793316841125488,
       -9.631752967834473,
       -9.966330528259277,
       -9.654276847839355,
       -10.009891510009766,
       -9.479894638061523
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "defer file",
           "probability": 0.9986838883566026,
           "distribution": [
            {
             "value": "defer file",
             "probability": 0.9986838883566026
            },
            {
             "value": "extend interval",
             "probability": 0.00043348869616421324
            },
            {
             "value": "schedule routine",
             "probability": 0.00046430539302373165
            },
            {
             "value": "schedule now",
             "probability": 0.0004183175542094854
            }
           ]
          },
          "factors": {
           "value": 3,
           "probability": 0.9986238777008657,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004328169739443347
            },
            {
             "value": 1,
             "probability": 0.00047500564261193634
            },
            {
             "value": 2,
             "probability": 0.00046829968257802593
            },
            {
             "value": 3,
             "probability": 0.9986238777008657
            }
           ]
          },
          "beyond_gap": {
           "value": true,
           "probability": 0.9992187432077859,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007812567922140342
            },
            {
             "value": true,
             "probability": 0.9992187432077859
            }
           ]
          }
         },
         "joint_probability": 0.9982216606909845,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0017799224315289157,
        "field_brier": 2.0191112669417205e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "defer file",
           "probability": 0.9986838883566026,
           "distribution": [
            {
             "value": "defer file",
             "probability": 0.9986838883566026
            },
            {
             "value": "extend interval",
             "probability": 0.00043348869616421324
            },
            {
             "value": "schedule routine",
             "probability": 0.00046430539302373165
            },
            {
             "value": "schedule now",
             "probability": 0.0004183175542094854
            }
           ]
          },
          "factors": {
           "value": 3,
           "probability": 0.9986238777008657,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004328169739443347
            },
            {
             "value": 1,
             "probability": 0.00047500564261193634
            },
            {
             "value": 2,
             "probability": 0.00046829968257802593
            },
            {
             "value": 3,
             "probability": 0.9986238777008657
            }
           ]
          },
          "beyond_gap": {
           "value": true,
           "probability": 0.9992187432077859,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007812567922140342
            },
            {
             "value": true,
             "probability": 0.9992187432077859
            }
           ]
          }
         },
         "joint_probability": 0.9982216606909845,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0017799224315289157,
        "field_brier": 2.0191112669417205e-06
       }
      },
      "timing_ms": {
       "scoring": 33.55451882816851,
       "base_joint": 0.07752887904644012,
       "constrained_joint": 0.05656899884343147,
       "constrained_total": 33.61108782701194
      },
      "input_tokens": 626,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c9195364e722952-0017",
      "group": "bendsynth-0c9195364e722952",
      "provenance": "bendsynth-v1; family=bendsynth-0c9195364e722952; domain=dental practice recall scheduling audit; parameters=review_gap=17; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "beyond_gap": true,
       "decision": "schedule now",
       "factors": 1
      },
      "logits": [
       [
        -5.5466461181640625,
        1.0768054723739624
       ],
       [
        -6.447196960449219,
        1.0760003328323364,
        -6.2221503257751465,
        -5.917156219482422
       ],
       [
        -6.353607654571533,
        1.0765384435653687,
        -6.543147087097168,
        -6.499797821044922
       ]
      ],
      "joint_logits": [
       -8.538713455200195,
       -8.2149076461792,
       -8.637874603271484,
       -8.630400657653809,
       -8.817689895629883,
       -7.991629123687744,
       -8.94704532623291,
       -8.930727005004883,
       -8.24416446685791,
       -8.105382919311523,
       -8.525115013122559,
       -8.425633430480957,
       -7.946174621582031,
       -7.7429518699646,
       -8.244780540466309,
       -8.122281074523926,
       -8.583664894104004,
       -8.32376766204834,
       -8.732403755187988,
       -8.669882774353027,
       -8.804466247558594,
       1.0756080150604248,
       -8.937366485595703,
       -9.02292537689209,
       -8.523994445800781,
       -7.695495128631592,
       -8.652491569519043,
       -8.65375804901123,
       -8.318424224853516,
       -7.282707214355469,
       -8.49657917022705,
       -8.46792984008789
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "beyond_gap": {
           "value": true,
           "probability": 0.9986729268008862,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013270731991139025
            },
            {
             "value": true,
             "probability": 0.9986729268008862
            }
           ]
          },
          "decision": {
           "value": "schedule now",
           "probability": 0.9978692151215985,
           "distribution": [
            {
             "value": "defer file",
             "probability": 0.000539250541354079
            },
            {
             "value": "schedule now",
             "probability": 0.9978692151215985
            },
            {
             "value": "extend interval",
             "probability": 0.0006753471615401536
            },
            {
             "value": "schedule routine",
             "probability": 0.0009161871755071509
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9984063118152032,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005921556330543314
            },
            {
             "value": 1,
             "probability": 0.9984063118152032
            },
            {
             "value": 2,
             "probability": 0.0004899139723137922
            },
            {
             "value": 3,
             "probability": 0.0005116185794286965
            }
           ]
          }
         },
         "joint_probability": 0.9974776729611615,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002525513464950882,
        "field_brier": 4.347011406785164e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "beyond_gap": {
           "value": true,
           "probability": 0.9986729268008862,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013270731991139025
            },
            {
             "value": true,
             "probability": 0.9986729268008862
            }
           ]
          },
          "decision": {
           "value": "schedule now",
           "probability": 0.9978692151215985,
           "distribution": [
            {
             "value": "defer file",
             "probability": 0.000539250541354079
            },
            {
             "value": "schedule now",
             "probability": 0.9978692151215985
            },
            {
             "value": "extend interval",
             "probability": 0.0006753471615401536
            },
            {
             "value": "schedule routine",
             "probability": 0.0009161871755071509
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9984063118152032,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005921556330543314
            },
            {
             "value": 1,
             "probability": 0.9984063118152032
            },
            {
             "value": 2,
             "probability": 0.0004899139723137922
            },
            {
             "value": 3,
             "probability": 0.0005116185794286965
            }
           ]
          }
         },
         "joint_probability": 0.9974776729611615,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002525513464950882,
        "field_brier": 4.347011406785164e-06
       }
      },
      "timing_ms": {
       "scoring": 35.002742893993855,
       "base_joint": 0.07619895040988922,
       "constrained_joint": 0.05714897997677326,
       "constrained_total": 35.05989187397063
      },
      "input_tokens": 632,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b813d53a09c7518-0004",
      "group": "bendsynth-0b813d53a09c7518",
      "provenance": "bendsynth-v1; family=bendsynth-0b813d53a09c7518; domain=overtime pay dispute desk at a warehouse staffing office; parameters=cap=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "send to auditor",
       "over_cap": true,
       "factors": 2
      },
      "logits": [
       [
        -0.42540064454078674,
        0.1052493005990982,
        -6.441967487335205,
        -5.6498284339904785
       ],
       [
        -5.445884704589844,
        0.568316638469696
       ],
       [
        -6.547743320465088,
        -5.860398769378662,
        0.5672537684440613,
        -6.262757778167725
       ]
      ],
      "joint_logits": [
       -8.110182762145996,
       -7.5574049949646,
       -7.394856929779053,
       -8.051107406616211,
       -8.1810884475708,
       -7.199893951416016,
       -0.43037357926368713,
       -7.685639381408691,
       -9.743812561035156,
       -9.213479042053223,
       -8.738174438476562,
       -9.712075233459473,
       -9.531315803527832,
       -8.526155471801758,
       0.10455217212438583,
       -8.99252986907959,
       -8.722945213317871,
       -8.233112335205078,
       -8.57628345489502,
       -8.504389762878418,
       -8.978455543518066,
       -8.460485458374023,
       -8.115425109863281,
       -8.89558219909668,
       -8.274848937988281,
       -7.686646938323975,
       -8.12405014038086,
       -7.959763050079346,
       -8.633537292480469,
       -7.966128349304199,
       -6.5859785079956055,
       -8.27751636505127
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "send to auditor",
           "probability": 0.6278160524603957,
           "distribution": [
            {
             "value": "deny claim",
             "probability": 0.369295515152476
            },
            {
             "value": "send to auditor",
             "probability": 0.6278160524603957
            },
            {
             "value": "regular rate",
             "probability": 0.0009003519216525894
            },
            {
             "value": "overtime rate",
             "probability": 0.001988080465475694
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.99756215767952,
           "distribution": [
            {
             "value": false,
             "probability": 0.0024378423204800908
            },
            {
             "value": true,
             "probability": 0.99756215767952
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9965023666961929,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008099801050992954
            },
            {
             "value": 1,
             "probability": 0.0016105878737230632
            },
            {
             "value": 2,
             "probability": 0.9965023666961929
            },
            {
             "value": 3,
             "probability": 0.0010770653249848276
            }
           ]
          }
         },
         "joint_probability": 0.6273784794828182,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.46620528492507957,
        "field_brier": 0.09164445371176215
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "send to auditor",
           "probability": 0.6278160524603957,
           "distribution": [
            {
             "value": "deny claim",
             "probability": 0.369295515152476
            },
            {
             "value": "send to auditor",
             "probability": 0.6278160524603957
            },
            {
             "value": "regular rate",
             "probability": 0.0009003519216525894
            },
            {
             "value": "overtime rate",
             "probability": 0.001988080465475694
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.99756215767952,
           "distribution": [
            {
             "value": false,
             "probability": 0.0024378423204800908
            },
            {
             "value": true,
             "probability": 0.99756215767952
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9965023666961929,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008099801050992954
            },
            {
             "value": 1,
             "probability": 0.0016105878737230632
            },
            {
             "value": 2,
             "probability": 0.9965023666961929
            },
            {
             "value": 3,
             "probability": 0.0010770653249848276
            }
           ]
          }
         },
         "joint_probability": 0.6273784794828182,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.46620528492507957,
        "field_brier": 0.09164445371176215
       }
      },
      "timing_ms": {
       "scoring": 34.52853811904788,
       "base_joint": 0.07595005445182323,
       "constrained_joint": 0.056829070672392845,
       "constrained_total": 34.58536718972027
      },
      "input_tokens": 613,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b813d53a09c7518-0008",
      "group": "bendsynth-0b813d53a09c7518",
      "provenance": "bendsynth-v1; family=bendsynth-0b813d53a09c7518; domain=overtime pay dispute desk at a warehouse staffing office; parameters=cap=13; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "factors": 3,
       "decision": "deny claim",
       "over_cap": true
      },
      "logits": [
       [
        -7.084371566772461,
        -6.921955108642578,
        -6.8501877784729,
        0.5218222141265869
       ],
       [
        -6.936399459838867,
        0.5213810205459595,
        -7.022219181060791,
        -6.34075403213501
       ],
       [
        -6.442217826843262,
        0.5225875973701477
       ]
      ],
      "joint_logits": [
       -9.214987754821777,
       -9.015364646911621,
       -9.249384880065918,
       -9.051667213439941,
       -9.312689781188965,
       -8.973505973815918,
       -9.258867263793945,
       -9.300121307373047,
       -9.079865455627441,
       -8.901985168457031,
       -9.078457832336426,
       -8.800122261047363,
       -9.239258766174316,
       -8.773819923400879,
       -9.11548900604248,
       -9.122306823730469,
       -9.03813362121582,
       -8.861727714538574,
       -9.106613159179688,
       -8.490274429321289,
       -9.14052963256836,
       -8.866608619689941,
       -9.131023406982422,
       -8.98031997680664,
       -9.424753189086914,
       -8.743853569030762,
       -9.1668062210083,
       0.5208470821380615,
       -9.496590614318848,
       -9.214240074157715,
       -9.5524263381958,
       -6.8570637702941895
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 3,
           "probability": 0.9982918866363165,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00049651196894599
            },
            {
             "value": 1,
             "probability": 0.0005840716205185054
            },
            {
             "value": 2,
             "probability": 0.0006275297742189702
            },
            {
             "value": 3,
             "probability": 0.9982918866363165
            }
           ]
          },
          "decision": {
           "value": "deny claim",
           "probability": 0.9978515255663909,
           "distribution": [
            {
             "value": "overtime rate",
             "probability": 0.0005756956874742273
            },
            {
             "value": "deny claim",
             "probability": 0.9978515255663909
            },
            {
             "value": "regular rate",
             "probability": 0.000528350458356327
            },
            {
             "value": "send to auditor",
             "probability": 0.0010444282877785204
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9990563448031704,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009436551968295521
            },
            {
             "value": true,
             "probability": 0.9990563448031704
            }
           ]
          }
         },
         "joint_probability": 0.997318927265422,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002684673247012419,
        "field_brier": 3.999143835513258e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 3,
           "probability": 0.9982918866363165,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00049651196894599
            },
            {
             "value": 1,
             "probability": 0.0005840716205185054
            },
            {
             "value": 2,
             "probability": 0.0006275297742189702
            },
            {
             "value": 3,
             "probability": 0.9982918866363165
            }
           ]
          },
          "decision": {
           "value": "deny claim",
           "probability": 0.9978515255663909,
           "distribution": [
            {
             "value": "overtime rate",
             "probability": 0.0005756956874742273
            },
            {
             "value": "deny claim",
             "probability": 0.9978515255663909
            },
            {
             "value": "regular rate",
             "probability": 0.000528350458356327
            },
            {
             "value": "send to auditor",
             "probability": 0.0010444282877785204
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9990563448031704,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009436551968295521
            },
            {
             "value": true,
             "probability": 0.9990563448031704
            }
           ]
          }
         },
         "joint_probability": 0.997318927265422,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002684673247012419,
        "field_brier": 3.999143835513258e-06
       }
      },
      "timing_ms": {
       "scoring": 34.821275155991316,
       "base_joint": 0.07348903454840183,
       "constrained_joint": 0.057958997786045074,
       "constrained_total": 34.87923415377736
      },
      "input_tokens": 628,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b813d53a09c7518-0011",
      "group": "bendsynth-0b813d53a09c7518",
      "provenance": "bendsynth-v1; family=bendsynth-0b813d53a09c7518; domain=overtime pay dispute desk at a warehouse staffing office; parameters=cap=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "send to auditor",
       "factors": 1,
       "over_cap": true
      },
      "logits": [
       [
        1.0717583894729614,
        -6.562862873077393,
        -5.56511116027832,
        -6.3943772315979
       ],
       [
        -6.307760715484619,
        1.0723516941070557,
        -6.395381927490234,
        -6.384496688842773
       ],
       [
        -5.646227836608887,
        1.0729154348373413
       ]
      ],
      "joint_logits": [
       -8.921931266784668,
       -8.737390518188477,
       -8.532501220703125,
       1.071410059928894,
       -9.098731994628906,
       -8.726046562194824,
       -9.048983573913574,
       -8.91165542602539,
       -8.692831039428711,
       -8.585538864135742,
       -8.579377174377441,
       -8.254112243652344,
       -8.873350143432617,
       -8.77918815612793,
       -8.793825149536133,
       -8.718725204467773,
       -7.868668556213379,
       -7.843478202819824,
       -7.799070835113525,
       -6.7947235107421875,
       -7.85414981842041,
       -7.893425941467285,
       -7.8379034996032715,
       -7.905519485473633,
       -8.589252471923828,
       -8.455656051635742,
       -8.413559913635254,
       -7.947274684906006,
       -8.687556266784668,
       -8.672890663146973,
       -8.66954517364502,
       -8.59460735321045
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "send to auditor",
           "probability": 0.997638906554063,
           "distribution": [
            {
             "value": "send to auditor",
             "probability": 0.997638906554063
            },
            {
             "value": "regular rate",
             "probability": 0.000482280226898622
            },
            {
             "value": "overtime rate",
             "probability": 0.0013080290941265703
            },
            {
             "value": "deny claim",
             "probability": 0.0005707841249119106
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9982309097474625,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006224277305564763
            },
            {
             "value": 1,
             "probability": 0.9982309097474625
            },
            {
             "value": 2,
             "probability": 0.0005702108156566663
            },
            {
             "value": 3,
             "probability": 0.0005764517063242991
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9987938841396017,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012061158603982806
            },
            {
             "value": true,
             "probability": 0.9987938841396017
            }
           ]
          }
         },
         "joint_probability": 0.9972914017605055,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027122731290723503,
        "field_brier": 4.976018529616912e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "send to auditor",
           "probability": 0.997638906554063,
           "distribution": [
            {
             "value": "send to auditor",
             "probability": 0.997638906554063
            },
            {
             "value": "regular rate",
             "probability": 0.000482280226898622
            },
            {
             "value": "overtime rate",
             "probability": 0.0013080290941265703
            },
            {
             "value": "deny claim",
             "probability": 0.0005707841249119106
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9982309097474625,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006224277305564763
            },
            {
             "value": 1,
             "probability": 0.9982309097474625
            },
            {
             "value": 2,
             "probability": 0.0005702108156566663
            },
            {
             "value": 3,
             "probability": 0.0005764517063242991
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9987938841396017,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012061158603982806
            },
            {
             "value": true,
             "probability": 0.9987938841396017
            }
           ]
          }
         },
         "joint_probability": 0.9972914017605055,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027122731290723503,
        "field_brier": 4.976018529616912e-06
       }
      },
      "timing_ms": {
       "scoring": 35.586107056587934,
       "base_joint": 0.08472893387079239,
       "constrained_joint": 0.05564000457525253,
       "constrained_total": 35.64174706116319
      },
      "input_tokens": 661,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b813d53a09c7518-0038",
      "group": "bendsynth-0b813d53a09c7518",
      "provenance": "bendsynth-v1; family=bendsynth-0b813d53a09c7518; domain=overtime pay dispute desk at a warehouse staffing office; parameters=cap=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "overtime rate",
       "over_cap": true,
       "factors": 1
      },
      "logits": [
       [
        -5.415310859680176,
        2.190825939178467,
        -5.044127464294434,
        -5.488188743591309
       ],
       [
        -4.78843879699707,
        2.191575288772583
       ],
       [
        -5.365618705749512,
        2.190969467163086,
        -5.3594160079956055,
        -5.433926105499268
       ]
      ],
      "joint_logits": [
       -7.670836448669434,
       -7.679853439331055,
       -7.557103633880615,
       -7.6399736404418945,
       -7.373261451721191,
       -7.2479681968688965,
       -7.405086517333984,
       -7.471858501434326,
       -7.693052768707275,
       -7.501932621002197,
       -7.775737285614014,
       -7.830973148345947,
       -7.497553825378418,
       2.1904525756835938,
       -7.559995651245117,
       -7.7360711097717285,
       -7.358034610748291,
       -7.34442663192749,
       -7.206069469451904,
       -7.259936809539795,
       -7.032086372375488,
       -6.759589672088623,
       -7.061237335205078,
       -7.108989715576172,
       -7.643651485443115,
       -7.77646017074585,
       -7.629754543304443,
       -7.693668842315674,
       -7.469022274017334,
       -7.304800987243652,
       -7.511289119720459,
       -7.58974552154541
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "overtime rate",
           "probability": 0.9983220590025478,
           "distribution": [
            {
             "value": "deny claim",
             "probability": 0.0004965551593510189
            },
            {
             "value": "overtime rate",
             "probability": 0.9983220590025478
            },
            {
             "value": "send to auditor",
             "probability": 0.0007197313315017068
            },
            {
             "value": "regular rate",
             "probability": 0.0004616545065995888
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9990705748053343,
           "distribution": [
            {
             "value": false,
             "probability": 0.000929425194665616
            },
            {
             "value": true,
             "probability": 0.9990705748053343
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9984656491248424,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005218533831740561
            },
            {
             "value": 1,
             "probability": 0.9984656491248424
            },
            {
             "value": 2,
             "probability": 0.0005251004327923483
            },
            {
             "value": 3,
             "probability": 0.000487397059191118
            }
           ]
          }
         },
         "joint_probability": 0.997949511847446,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0020525932815729504,
        "field_brier": 2.8869011315435226e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "overtime rate",
           "probability": 0.9983220590025478,
           "distribution": [
            {
             "value": "deny claim",
             "probability": 0.0004965551593510189
            },
            {
             "value": "overtime rate",
             "probability": 0.9983220590025478
            },
            {
             "value": "send to auditor",
             "probability": 0.0007197313315017068
            },
            {
             "value": "regular rate",
             "probability": 0.0004616545065995888
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9990705748053343,
           "distribution": [
            {
             "value": false,
             "probability": 0.000929425194665616
            },
            {
             "value": true,
             "probability": 0.9990705748053343
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9984656491248424,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005218533831740561
            },
            {
             "value": 1,
             "probability": 0.9984656491248424
            },
            {
             "value": 2,
             "probability": 0.0005251004327923483
            },
            {
             "value": 3,
             "probability": 0.000487397059191118
            }
           ]
          }
         },
         "joint_probability": 0.997949511847446,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0020525932815729504,
        "field_brier": 2.8869011315435226e-06
       }
      },
      "timing_ms": {
       "scoring": 34.95953301899135,
       "base_joint": 0.07620896212756634,
       "constrained_joint": 0.05710986442863941,
       "constrained_total": 35.01664288341999
      },
      "input_tokens": 643,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-077f5107d67be5e0-0004",
      "group": "bendsynth-077f5107d67be5e0",
      "provenance": "bendsynth-v1; family=bendsynth-077f5107d67be5e0; domain=national park campsite permit chatbot; parameters=capacity=11; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "waitlist",
       "extra_steps": 1,
       "quota_hold": true
      },
      "logits": [
       [
        -6.3179802894592285,
        0.7124549150466919,
        -6.433586597442627
       ],
       [
        -6.543763160705566,
        0.7122212052345276,
        -6.742702960968018,
        -6.670061111450195
       ],
       [
        -5.881069183349609,
        0.7127581238746643
       ]
      ],
      "joint_logits": [
       -8.282124519348145,
       -8.401159286499023,
       -8.353554725646973,
       -8.154228210449219,
       -8.534367561340332,
       -8.601221084594727,
       -8.400627136230469,
       -8.52876091003418,
       -8.205445289611816,
       -8.293120384216309,
       -8.002694129943848,
       0.7115912437438965,
       -8.404196739196777,
       -8.408740997314453,
       -8.352474212646484,
       -8.42380428314209,
       -8.396995544433594,
       -8.456482887268066,
       -8.489190101623535,
       -8.420575141906738,
       -8.616823196411133,
       -8.673210144042969,
       -8.506240844726562,
       -8.577461242675781
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9983302687236327,
           "distribution": [
            {
             "value": "auto approve",
             "probability": 0.0008830697516035603
            },
            {
             "value": "waitlist",
             "probability": 0.9983302687236327
            },
            {
             "value": "refer to ranger",
             "probability": 0.0007866615247637711
            }
           ]
          },
          "extra_steps": {
           "value": 1,
           "probability": 0.9980969266759304,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007045935941970928
            },
            {
             "value": 1,
             "probability": 0.9980969266759304
            },
            {
             "value": 2,
             "probability": 0.0005774845237782378
            },
            {
             "value": 3,
             "probability": 0.0006209952060942269
            }
           ]
          },
          "quota_hold": {
           "value": true,
           "probability": 0.9986330801045068,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013669198954932285
            },
            {
             "value": true,
             "probability": 0.9986330801045068
            }
           ]
          }
         },
         "joint_probability": 0.9974684493897227,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002534760402836062,
        "field_brier": 4.253618236124829e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9983302687236327,
           "distribution": [
            {
             "value": "auto approve",
             "probability": 0.0008830697516035603
            },
            {
             "value": "waitlist",
             "probability": 0.9983302687236327
            },
            {
             "value": "refer to ranger",
             "probability": 0.0007866615247637711
            }
           ]
          },
          "extra_steps": {
           "value": 1,
           "probability": 0.9980969266759304,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007045935941970928
            },
            {
             "value": 1,
             "probability": 0.9980969266759304
            },
            {
             "value": 2,
             "probability": 0.0005774845237782378
            },
            {
             "value": 3,
             "probability": 0.0006209952060942269
            }
           ]
          },
          "quota_hold": {
           "value": true,
           "probability": 0.9986330801045068,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013669198954932285
            },
            {
             "value": true,
             "probability": 0.9986330801045068
            }
           ]
          }
         },
         "joint_probability": 0.9974684493897227,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002534760402836062,
        "field_brier": 4.253618236124829e-06
       }
      },
      "timing_ms": {
       "scoring": 34.23337102867663,
       "base_joint": 0.0634088646620512,
       "constrained_joint": 0.057579949498176575,
       "constrained_total": 34.290950978174806
      },
      "input_tokens": 621,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-077f5107d67be5e0-0010",
      "group": "bendsynth-077f5107d67be5e0",
      "provenance": "bendsynth-v1; family=bendsynth-077f5107d67be5e0; domain=national park campsite permit chatbot; parameters=capacity=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_hold": false,
       "decision": "refer to ranger",
       "extra_steps": 2
      },
      "logits": [
       [
        -0.16566099226474762,
        -6.354087829589844
       ],
       [
        -7.110517978668213,
        -6.637650966644287,
        -0.1661175936460495
       ],
       [
        -7.388243198394775,
        -7.025637149810791,
        -0.1660730540752411,
        -7.449804306030273
       ]
      ],
      "joint_logits": [
       -9.484187126159668,
       -9.204329490661621,
       -9.059077262878418,
       -9.543817520141602,
       -9.099024772644043,
       -8.743138313293457,
       -8.411824226379395,
       -9.178954124450684,
       -9.533978462219238,
       -8.920811653137207,
       -0.16712047159671783,
       -9.587661743164062,
       -9.22305965423584,
       -8.864882469177246,
       -9.054434776306152,
       -9.261847496032715,
       -8.769086837768555,
       -8.434357643127441,
       -8.548829078674316,
       -8.82608699798584,
       -9.170071601867676,
       -8.898967742919922,
       -8.345568656921387,
       -9.247801780700684
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_hold": {
           "value": false,
           "probability": 0.9979511527789551,
           "distribution": [
            {
             "value": false,
             "probability": 0.9979511527789551
            },
            {
             "value": true,
             "probability": 0.0020488472210449765
            }
           ]
          },
          "decision": {
           "value": "refer to ranger",
           "probability": 0.9974954180884572,
           "distribution": [
            {
             "value": "auto approve",
             "probability": 0.0009616037347842176
            },
            {
             "value": "waitlist",
             "probability": 0.0015429781767586203
            },
            {
             "value": "refer to ranger",
             "probability": 0.9974954180884572
            }
           ]
          },
          "extra_steps": {
           "value": 2,
           "probability": 0.9975398609733614,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000728419551721919
            },
            {
             "value": 1,
             "probability": 0.0010467898147141408
            },
            {
             "value": 2,
             "probability": 0.9975398609733614
            },
            {
             "value": 3,
             "probability": 0.0006849296602025094
            }
           ]
          }
         },
         "joint_probability": 0.9964956114829838,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003510543269758905,
        "field_brier": 8.707240149344709e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_hold": {
           "value": false,
           "probability": 0.9979511527789551,
           "distribution": [
            {
             "value": false,
             "probability": 0.9979511527789551
            },
            {
             "value": true,
             "probability": 0.0020488472210449765
            }
           ]
          },
          "decision": {
           "value": "refer to ranger",
           "probability": 0.9974954180884572,
           "distribution": [
            {
             "value": "auto approve",
             "probability": 0.0009616037347842176
            },
            {
             "value": "waitlist",
             "probability": 0.0015429781767586203
            },
            {
             "value": "refer to ranger",
             "probability": 0.9974954180884572
            }
           ]
          },
          "extra_steps": {
           "value": 2,
           "probability": 0.9975398609733614,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000728419551721919
            },
            {
             "value": 1,
             "probability": 0.0010467898147141408
            },
            {
             "value": 2,
             "probability": 0.9975398609733614
            },
            {
             "value": 3,
             "probability": 0.0006849296602025094
            }
           ]
          }
         },
         "joint_probability": 0.9964956114829838,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003510543269758905,
        "field_brier": 8.707240149344709e-06
       }
      },
      "timing_ms": {
       "scoring": 34.56601803191006,
       "base_joint": 0.06691901944577694,
       "constrained_joint": 0.04846882075071335,
       "constrained_total": 34.614486852660775
      },
      "input_tokens": 624,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-077f5107d67be5e0-0012",
      "group": "bendsynth-077f5107d67be5e0",
      "provenance": "bendsynth-v1; family=bendsynth-077f5107d67be5e0; domain=national park campsite permit chatbot; parameters=capacity=4; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "refer to ranger",
       "quota_hold": false,
       "extra_steps": 2
      },
      "logits": [
       [
        -7.588390350341797,
        -7.424989223480225,
        -0.5683643817901611
       ],
       [
        -0.5678771138191223,
        -7.098450660705566
       ],
       [
        -7.905664443969727,
        -7.692527770996094,
        -0.568626880645752,
        -7.760507583618164
       ]
      ],
      "joint_logits": [
       -9.772191047668457,
       -9.58133602142334,
       -9.544346809387207,
       -9.59579849243164,
       -9.83781623840332,
       -9.719542503356934,
       -9.652239799499512,
       -9.674221992492676,
       -9.653777122497559,
       -9.435039520263672,
       -9.238343238830566,
       -9.460372924804688,
       -9.679010391235352,
       -9.555424690246582,
       -9.526884078979492,
       -9.556680679321289,
       -9.72039794921875,
       -9.239547729492188,
       -0.569310188293457,
       -9.57237720489502,
       -9.546817779541016,
       -9.442170143127441,
       -9.422466278076172,
       -9.470251083374023
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "refer to ranger",
           "probability": 0.9980575181627219,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0008920659444105728
            },
            {
             "value": "auto approve",
             "probability": 0.0010504158928676024
            },
            {
             "value": "refer to ranger",
             "probability": 0.9980575181627219
            }
           ]
          },
          "quota_hold": {
           "value": false,
           "probability": 0.9985439542594501,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985439542594501
            },
            {
             "value": true,
             "probability": 0.0014560457405499814
            }
           ]
          },
          "extra_steps": {
           "value": 2,
           "probability": 0.9977956038739286,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006495411555511865
            },
            {
             "value": 1,
             "probability": 0.0008038420634985997
            },
            {
             "value": 2,
             "probability": 0.9977956038739286
            },
            {
             "value": 3,
             "probability": 0.0007510129070216669
            }
           ]
          }
         },
         "joint_probability": 0.9971139458648703,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028902268196946148,
        "field_brier": 5.467992575141515e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "refer to ranger",
           "probability": 0.9980575181627219,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0008920659444105728
            },
            {
             "value": "auto approve",
             "probability": 0.0010504158928676024
            },
            {
             "value": "refer to ranger",
             "probability": 0.9980575181627219
            }
           ]
          },
          "quota_hold": {
           "value": false,
           "probability": 0.9985439542594501,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985439542594501
            },
            {
             "value": true,
             "probability": 0.0014560457405499814
            }
           ]
          },
          "extra_steps": {
           "value": 2,
           "probability": 0.9977956038739286,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006495411555511865
            },
            {
             "value": 1,
             "probability": 0.0008038420634985997
            },
            {
             "value": 2,
             "probability": 0.9977956038739286
            },
            {
             "value": 3,
             "probability": 0.0007510129070216669
            }
           ]
          }
         },
         "joint_probability": 0.9971139458648703,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028902268196946148,
        "field_brier": 5.467992575141515e-06
       }
      },
      "timing_ms": {
       "scoring": 34.15670315735042,
       "base_joint": 0.06480002775788307,
       "constrained_joint": 0.047958921641111374,
       "constrained_total": 34.20466207899153
      },
      "input_tokens": 621,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-077f5107d67be5e0-0039",
      "group": "bendsynth-077f5107d67be5e0",
      "provenance": "bendsynth-v1; family=bendsynth-077f5107d67be5e0; domain=national park campsite permit chatbot; parameters=capacity=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "waitlist",
       "quota_hold": true,
       "extra_steps": 1
      },
      "logits": [
       [
        -6.013279438018799,
        1.0662912130355835,
        -5.990229606628418
       ],
       [
        -5.587541103363037,
        1.0667058229446411
       ],
       [
        -6.205583095550537,
        1.066110372543335,
        -6.3989033699035645,
        -6.324047088623047
       ]
      ],
      "joint_logits": [
       -7.986134052276611,
       -8.094319343566895,
       -8.212130546569824,
       -8.089590072631836,
       -8.03520679473877,
       -7.975554943084717,
       -8.233050346374512,
       -8.148269653320312,
       -8.00108528137207,
       -7.92417049407959,
       -8.203697204589844,
       -8.120396614074707,
       -7.994141578674316,
       1.0655384063720703,
       -8.128469467163086,
       -8.162588119506836,
       -7.957217216491699,
       -8.149330139160156,
       -8.140670776367188,
       -8.040289878845215,
       -8.01197624206543,
       -7.927404403686523,
       -8.231212615966797,
       -8.138848304748535
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9982989923093074,
           "distribution": [
            {
             "value": "refer to ranger",
             "probability": 0.0008407024092463315
            },
            {
             "value": "waitlist",
             "probability": 0.9982989923093074
            },
            {
             "value": "auto approve",
             "probability": 0.0008603052814462186
            }
           ]
          },
          "quota_hold": {
           "value": true,
           "probability": 0.9987131199645676,
           "distribution": [
            {
             "value": false,
             "probability": 0.001286880035432519
            },
            {
             "value": true,
             "probability": 0.9987131199645676
            }
           ]
          },
          "extra_steps": {
           "value": 1,
           "probability": 0.9981185363011008,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006936266185003957
            },
            {
             "value": 1,
             "probability": 0.9981185363011008
            },
            {
             "value": 2,
             "probability": 0.0005716997108027607
            },
            {
             "value": 3,
             "probability": 0.000616137369595987
            }
           ]
          }
         },
         "joint_probability": 0.9975478017679277,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002455209794463069,
        "field_brier": 4.126647562297958e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9982989923093074,
           "distribution": [
            {
             "value": "refer to ranger",
             "probability": 0.0008407024092463315
            },
            {
             "value": "waitlist",
             "probability": 0.9982989923093074
            },
            {
             "value": "auto approve",
             "probability": 0.0008603052814462186
            }
           ]
          },
          "quota_hold": {
           "value": true,
           "probability": 0.9987131199645676,
           "distribution": [
            {
             "value": false,
             "probability": 0.001286880035432519
            },
            {
             "value": true,
             "probability": 0.9987131199645676
            }
           ]
          },
          "extra_steps": {
           "value": 1,
           "probability": 0.9981185363011008,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006936266185003957
            },
            {
             "value": 1,
             "probability": 0.9981185363011008
            },
            {
             "value": 2,
             "probability": 0.0005716997108027607
            },
            {
             "value": 3,
             "probability": 0.000616137369595987
            }
           ]
          }
         },
         "joint_probability": 0.9975478017679277,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002455209794463069,
        "field_brier": 4.126647562297958e-06
       }
      },
      "timing_ms": {
       "scoring": 34.544187830761075,
       "base_joint": 0.06605894304811954,
       "constrained_joint": 0.04801992326974869,
       "constrained_total": 34.592207754030824
      },
      "input_tokens": 639,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07c47162b60922ca-0000",
      "group": "bendsynth-07c47162b60922ca",
      "provenance": "bendsynth-v1; family=bendsynth-07c47162b60922ca; domain=language school placement inspection; parameters=min_tests=54; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "conditional approval",
       "deficiencies": 1,
       "both_flags": true
      },
      "logits": [
       [
        -6.3107194900512695,
        -6.3374857902526855,
        0.5365871787071228
       ],
       [
        -6.502951145172119,
        0.5362182855606079,
        -6.589474201202393,
        -6.613402366638184
       ],
       [
        0.5365937948226929,
        -5.634060859680176
       ]
      ],
      "joint_logits": [
       -8.286520957946777,
       -8.296953201293945,
       -8.548298835754395,
       -8.61098575592041,
       -8.478376388549805,
       -8.249655723571777,
       -8.428544998168945,
       -8.286920547485352,
       -8.391963005065918,
       -8.281322479248047,
       -8.465458869934082,
       -8.58897590637207,
       -8.560030937194824,
       -8.283543586730957,
       -8.518840789794922,
       -8.301397323608398,
       -8.274727821350098,
       -8.243041038513184,
       0.5351899266242981,
       -6.923359394073486,
       -8.340810775756836,
       -8.40983772277832,
       -8.492552757263184,
       -8.425982475280762
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "conditional approval",
           "probability": 0.9979078170320516,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.0010600905567137273
            },
            {
             "value": "standard approval",
             "probability": 0.0010320924112347087
            },
            {
             "value": "conditional approval",
             "probability": 0.9979078170320516
            }
           ]
          },
          "deficiencies": {
           "value": 1,
           "probability": 0.9975398746450644,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008746972826628399
            },
            {
             "value": 1,
             "probability": 0.9975398746450644
            },
            {
             "value": 2,
             "probability": 0.0008021977522668408
            },
            {
             "value": 3,
             "probability": 0.0007832303200058378
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.9979144915175192,
           "distribution": [
            {
             "value": false,
             "probability": 0.9979144915175192
            },
            {
             "value": true,
             "probability": 0.0020855084824808645
            }
           ]
          }
         },
         "joint_probability": 0.9965144882010688,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 7.4620409210448875,
        "field_brier": 0.663893768426903
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "conditional approval",
           "probability": 0.9979078170320516,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.0010600905567137273
            },
            {
             "value": "standard approval",
             "probability": 0.0010320924112347087
            },
            {
             "value": "conditional approval",
             "probability": 0.9979078170320516
            }
           ]
          },
          "deficiencies": {
           "value": 1,
           "probability": 0.9975398746450644,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008746972826628399
            },
            {
             "value": 1,
             "probability": 0.9975398746450644
            },
            {
             "value": 2,
             "probability": 0.0008021977522668408
            },
            {
             "value": 3,
             "probability": 0.0007832303200058378
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.9979144915175192,
           "distribution": [
            {
             "value": false,
             "probability": 0.9979144915175192
            },
            {
             "value": true,
             "probability": 0.0020855084824808645
            }
           ]
          }
         },
         "joint_probability": 0.9965144882010688,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 7.4620409210448875,
        "field_brier": 0.663893768426903
       }
      },
      "timing_ms": {
       "scoring": 34.6562871709466,
       "base_joint": 0.0679700169712305,
       "constrained_joint": 0.046588946133852005,
       "constrained_total": 34.70287611708045
      },
      "input_tokens": 579,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07c47162b60922ca-0003",
      "group": "bendsynth-07c47162b60922ca",
      "provenance": "bendsynth-v1; family=bendsynth-07c47162b60922ca; domain=language school placement inspection; parameters=min_tests=60; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "deficiencies": 0,
       "decision": "standard approval",
       "both_flags": true
      },
      "logits": [
       [
        0.35110750794410706,
        -6.931219577789307,
        -7.116487979888916,
        -7.133136749267578
       ],
       [
        -6.81898307800293,
        0.35107845067977905,
        -6.479174613952637
       ],
       [
        -6.265590667724609,
        0.3515903651714325
       ]
      ],
      "joint_logits": [
       -8.71085262298584,
       -8.899711608886719,
       -8.704875946044922,
       0.35047781467437744,
       -8.47577953338623,
       -8.426901817321777,
       -8.668396949768066,
       -8.89111042022705,
       -8.851075172424316,
       -8.924407958984375,
       -8.507265090942383,
       -8.574058532714844,
       -8.948543548583984,
       -9.086321830749512,
       -9.082221984863281,
       -9.223322868347168,
       -8.60669231414795,
       -8.660916328430176,
       -8.975296974182129,
       -9.093017578125,
       -9.133979797363281,
       -9.308472633361816,
       -8.603248596191406,
       -8.638399124145508
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "deficiencies": {
           "value": 0,
           "probability": 0.9981825575098507,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981825575098507
            },
            {
             "value": 1,
             "probability": 0.0006863340320205297
            },
            {
             "value": 2,
             "probability": 0.0005702620178577853
            },
            {
             "value": 3,
             "probability": 0.0005608464402709789
            }
           ]
          },
          "decision": {
           "value": "standard approval",
           "probability": 0.9981535555636991,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.0007678550030497681
            },
            {
             "value": "standard approval",
             "probability": 0.9981535555636991
            },
            {
             "value": "conditional approval",
             "probability": 0.0010785894332511247
            }
           ]
          },
          "both_flags": {
           "value": true,
           "probability": 0.9986645904658742,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013354095341258773
            },
            {
             "value": true,
             "probability": 0.9986645904658742
            }
           ]
          }
         },
         "joint_probability": 0.9975541637910058,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00244883215242978,
        "field_brier": 4.380949960776599e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "deficiencies": {
           "value": 0,
           "probability": 0.9981825575098507,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981825575098507
            },
            {
             "value": 1,
             "probability": 0.0006863340320205297
            },
            {
             "value": 2,
             "probability": 0.0005702620178577853
            },
            {
             "value": 3,
             "probability": 0.0005608464402709789
            }
           ]
          },
          "decision": {
           "value": "standard approval",
           "probability": 0.9981535555636991,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.0007678550030497681
            },
            {
             "value": "standard approval",
             "probability": 0.9981535555636991
            },
            {
             "value": "conditional approval",
             "probability": 0.0010785894332511247
            }
           ]
          },
          "both_flags": {
           "value": true,
           "probability": 0.9986645904658742,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013354095341258773
            },
            {
             "value": true,
             "probability": 0.9986645904658742
            }
           ]
          }
         },
         "joint_probability": 0.9975541637910058,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00244883215242978,
        "field_brier": 4.380949960776599e-06
       }
      },
      "timing_ms": {
       "scoring": 34.09651294350624,
       "base_joint": 0.06688898429274559,
       "constrained_joint": 0.04863901995122433,
       "constrained_total": 34.145151963457465
      },
      "input_tokens": 606,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07c47162b60922ca-0011",
      "group": "bendsynth-07c47162b60922ca",
      "provenance": "bendsynth-v1; family=bendsynth-07c47162b60922ca; domain=language school placement inspection; parameters=min_tests=21; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "both_flags": false,
       "decision": "standard approval",
       "deficiencies": 1
      },
      "logits": [
       [
        0.7381588816642761,
        -5.964240074157715
       ],
       [
        -6.525154113769531,
        -6.085792541503906,
        0.7375985980033875
       ],
       [
        -6.722959041595459,
        0.7376455068588257,
        -6.656196117401123,
        -6.764562129974365
       ]
      ],
      "joint_logits": [
       -8.617335319519043,
       -8.579666137695312,
       -8.58552074432373,
       -8.638152122497559,
       -8.25318431854248,
       -7.957161903381348,
       -8.259963035583496,
       -8.303190231323242,
       -8.817004203796387,
       0.7370536923408508,
       -8.63762092590332,
       -8.905801773071289,
       -8.638031959533691,
       -8.572614669799805,
       -8.559046745300293,
       -8.650699615478516,
       -8.16415786743164,
       -8.135285377502441,
       -8.099732398986816,
       -8.192643165588379,
       -8.794921875,
       -8.437103271484375,
       -8.692505836486816,
       -8.861602783203125
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "both_flags": {
           "value": false,
           "probability": 0.9987735432387705,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987735432387705
            },
            {
             "value": true,
             "probability": 0.0012264567612295366
            }
           ]
          },
          "decision": {
           "value": "standard approval",
           "probability": 0.9982139948277069,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.0006999231420997142
            },
            {
             "value": "conditional approval",
             "probability": 0.0010860820301934727
            },
            {
             "value": "standard approval",
             "probability": 0.9982139948277069
            }
           ]
          },
          "deficiencies": {
           "value": 1,
           "probability": 0.9982608279244053,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005743078405788895
            },
            {
             "value": 1,
             "probability": 0.9982608279244053
            },
            {
             "value": 2,
             "probability": 0.0006139591327369891
            },
            {
             "value": 3,
             "probability": 0.0005509051022788487
            }
           ]
          }
         },
         "joint_probability": 0.9976703029358011,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002332415030583836,
        "field_brier": 3.967554894537179e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "both_flags": {
           "value": false,
           "probability": 0.9987735432387705,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987735432387705
            },
            {
             "value": true,
             "probability": 0.0012264567612295366
            }
           ]
          },
          "decision": {
           "value": "standard approval",
           "probability": 0.9982139948277069,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.0006999231420997142
            },
            {
             "value": "conditional approval",
             "probability": 0.0010860820301934727
            },
            {
             "value": "standard approval",
             "probability": 0.9982139948277069
            }
           ]
          },
          "deficiencies": {
           "value": 1,
           "probability": 0.9982608279244053,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005743078405788895
            },
            {
             "value": 1,
             "probability": 0.9982608279244053
            },
            {
             "value": 2,
             "probability": 0.0006139591327369891
            },
            {
             "value": 3,
             "probability": 0.0005509051022788487
            }
           ]
          }
         },
         "joint_probability": 0.9976703029358011,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002332415030583836,
        "field_brier": 3.967554894537179e-06
       }
      },
      "timing_ms": {
       "scoring": 34.642416052520275,
       "base_joint": 0.07672887295484543,
       "constrained_joint": 0.04736008122563362,
       "constrained_total": 34.68977613374591
      },
      "input_tokens": 606,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07c47162b60922ca-0015",
      "group": "bendsynth-07c47162b60922ca",
      "provenance": "bendsynth-v1; family=bendsynth-07c47162b60922ca; domain=language school placement inspection; parameters=min_tests=70; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "standard approval",
       "deficiencies": 1,
       "both_flags": false
      },
      "logits": [
       [
        -6.076358795166016,
        -6.56920862197876,
        0.6713297367095947
       ],
       [
        -6.785110950469971,
        0.6713879704475403,
        -6.676990509033203,
        -6.729412078857422
       ],
       [
        0.6719911098480225,
        -6.030265808105469
       ]
      ],
      "joint_logits": [
       -8.253507614135742,
       -8.309052467346191,
       -7.811951637268066,
       -8.131343841552734,
       -8.184017181396484,
       -8.214922904968262,
       -8.192607879638672,
       -8.237641334533691,
       -8.633512496948242,
       -8.79904556274414,
       -8.551188468933105,
       -8.643415451049805,
       -8.576821327209473,
       -8.718571662902832,
       -8.56230640411377,
       -8.732420921325684,
       -8.792649269104004,
       -8.851961135864258,
       0.6707320809364319,
       -8.44654655456543,
       -8.574236869812012,
       -8.685347557067871,
       -8.798138618469238,
       -8.801060676574707
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "standard approval",
           "probability": 0.9981130524266917,
           "distribution": [
            {
             "value": "conditional approval",
             "probability": 0.001171374694278108
            },
            {
             "value": "escalate to regional review",
             "probability": 0.0007155728790302084
            },
            {
             "value": "standard approval",
             "probability": 0.9981130524266917
            }
           ]
          },
          "deficiencies": {
           "value": 1,
           "probability": 0.9981712764400951,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005766186471855657
            },
            {
             "value": 1,
             "probability": 0.9981712764400951
            },
            {
             "value": 2,
             "probability": 0.0006424580035085126
            },
            {
             "value": 3,
             "probability": 0.0006096469092108719
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.9987733693937542,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987733693937542
            },
            {
             "value": true,
             "probability": 0.001226630606245876
            }
           ]
          }
         },
         "joint_probability": 0.9975167215511532,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024863668988209836,
        "field_brier": 4.30504007179567e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "standard approval",
           "probability": 0.9981130524266917,
           "distribution": [
            {
             "value": "conditional approval",
             "probability": 0.001171374694278108
            },
            {
             "value": "escalate to regional review",
             "probability": 0.0007155728790302084
            },
            {
             "value": "standard approval",
             "probability": 0.9981130524266917
            }
           ]
          },
          "deficiencies": {
           "value": 1,
           "probability": 0.9981712764400951,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005766186471855657
            },
            {
             "value": 1,
             "probability": 0.9981712764400951
            },
            {
             "value": 2,
             "probability": 0.0006424580035085126
            },
            {
             "value": 3,
             "probability": 0.0006096469092108719
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.9987733693937542,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987733693937542
            },
            {
             "value": true,
             "probability": 0.001226630606245876
            }
           ]
          }
         },
         "joint_probability": 0.9975167215511532,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024863668988209836,
        "field_brier": 4.30504007179567e-06
       }
      },
      "timing_ms": {
       "scoring": 34.69605604186654,
       "base_joint": 0.06318907253444195,
       "constrained_joint": 0.04727998748421669,
       "constrained_total": 34.74333602935076
      },
      "input_tokens": 582,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07c47162b60922ca-0024",
      "group": "bendsynth-07c47162b60922ca",
      "provenance": "bendsynth-v1; family=bendsynth-07c47162b60922ca; domain=language school placement inspection; parameters=min_tests=40; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "standard approval",
       "deficiencies": 0,
       "both_flags": true
      },
      "logits": [
       [
        -6.459259033203125,
        0.6289302110671997,
        -6.17640495300293
       ],
       [
        0.6289001703262329,
        -6.594239234924316,
        -6.769659042358398,
        -6.738868236541748
       ],
       [
        -5.943027496337891,
        0.629473865032196
       ]
      ],
      "joint_logits": [
       -8.574131965637207,
       -8.641888618469238,
       -8.33607006072998,
       -8.490346908569336,
       -8.645896911621094,
       -8.545275688171387,
       -8.641668319702148,
       -8.475969314575195,
       -7.985544681549072,
       0.6282382011413574,
       -8.614078521728516,
       -8.614672660827637,
       -8.738961219787598,
       -8.91959285736084,
       -8.721037864685059,
       -8.893379211425781,
       -8.287782669067383,
       -8.167180061340332,
       -8.15455436706543,
       -8.207160949707031,
       -8.393604278564453,
       -8.264472961425781,
       -8.366754531860352,
       -8.23165512084961
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "standard approval",
           "probability": 0.9980610102121221,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.0008332888958615194
            },
            {
             "value": "standard approval",
             "probability": 0.9980610102121221
            },
            {
             "value": "conditional approval",
             "probability": 0.0011057008920163497
            }
           ]
          },
          "deficiencies": {
           "value": 0,
           "probability": 0.9980309665629833,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980309665629833
            },
            {
             "value": 1,
             "probability": 0.000728072396316133
            },
            {
             "value": 2,
             "probability": 0.000610928812390188
            },
            {
             "value": 3,
             "probability": 0.0006300322283104366
            }
           ]
          },
          "both_flags": {
           "value": true,
           "probability": 0.9986036572845011,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013963427154989642
            },
            {
             "value": true,
             "probability": 0.9986036572845011
            }
           ]
          }
         },
         "joint_probability": 0.9973704788972436,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026329843658524014,
        "field_brier": 4.917842971735386e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "standard approval",
           "probability": 0.9980610102121221,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.0008332888958615194
            },
            {
             "value": "standard approval",
             "probability": 0.9980610102121221
            },
            {
             "value": "conditional approval",
             "probability": 0.0011057008920163497
            }
           ]
          },
          "deficiencies": {
           "value": 0,
           "probability": 0.9980309665629833,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980309665629833
            },
            {
             "value": 1,
             "probability": 0.000728072396316133
            },
            {
             "value": 2,
             "probability": 0.000610928812390188
            },
            {
             "value": 3,
             "probability": 0.0006300322283104366
            }
           ]
          },
          "both_flags": {
           "value": true,
           "probability": 0.9986036572845011,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013963427154989642
            },
            {
             "value": true,
             "probability": 0.9986036572845011
            }
           ]
          }
         },
         "joint_probability": 0.9973704788972436,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026329843658524014,
        "field_brier": 4.917842971735386e-06
       }
      },
      "timing_ms": {
       "scoring": 34.043262945488095,
       "base_joint": 0.0655089970678091,
       "constrained_joint": 0.0467000063508749,
       "constrained_total": 34.08996295183897
      },
      "input_tokens": 579,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07c47162b60922ca-0030",
      "group": "bendsynth-07c47162b60922ca",
      "provenance": "bendsynth-v1; family=bendsynth-07c47162b60922ca; domain=language school placement inspection; parameters=min_tests=40; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "both_flags": true,
       "decision": "standard approval",
       "deficiencies": 0
      },
      "logits": [
       [
        -5.790317058563232,
        0.951449990272522
       ],
       [
        0.9508626461029053,
        -5.946052551269531,
        -6.234143257141113
       ],
       [
        0.9508824348449707,
        -6.399108409881592,
        -6.5719099044799805,
        -6.526474952697754
       ]
      ],
      "joint_logits": [
       -8.347825050354004,
       -8.398137092590332,
       -8.65206241607666,
       -8.619123458862305,
       -8.08669376373291,
       -8.016881942749023,
       -8.102148056030273,
       -8.056017875671387,
       -8.241217613220215,
       -8.204560279846191,
       -8.4319429397583,
       -8.3899507522583,
       0.9503124356269836,
       -8.33128833770752,
       -8.620331764221191,
       -8.597139358520508,
       -7.821180820465088,
       -8.03963851928711,
       -8.091412544250488,
       -8.019786834716797,
       -8.23432731628418,
       -8.212596893310547,
       -8.435090065002441,
       -8.396345138549805
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "both_flags": {
           "value": true,
           "probability": 0.9988208327496035,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011791672503964807
            },
            {
             "value": true,
             "probability": 0.9988208327496035
            }
           ]
          },
          "decision": {
           "value": "standard approval",
           "probability": 0.9982343593905454,
           "distribution": [
            {
             "value": "standard approval",
             "probability": 0.9982343593905454
            },
            {
             "value": "conditional approval",
             "probability": 0.0010091142699889303
            },
            {
             "value": "escalate to regional review",
             "probability": 0.0007565263394656762
            }
           ]
          },
          "deficiencies": {
           "value": 0,
           "probability": 0.9982540834862047,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9982540834862047
            },
            {
             "value": 1,
             "probability": 0.0006414765775465807
            },
            {
             "value": 2,
             "probability": 0.0005396770187655724
            },
            {
             "value": 3,
             "probability": 0.0005647629174831298
            }
           ]
          }
         },
         "joint_probability": 0.9976852880995246,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023173949872480263,
        "field_brier": 3.8529754643342605e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "both_flags": {
           "value": true,
           "probability": 0.9988208327496035,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011791672503964807
            },
            {
             "value": true,
             "probability": 0.9988208327496035
            }
           ]
          },
          "decision": {
           "value": "standard approval",
           "probability": 0.9982343593905454,
           "distribution": [
            {
             "value": "standard approval",
             "probability": 0.9982343593905454
            },
            {
             "value": "conditional approval",
             "probability": 0.0010091142699889303
            },
            {
             "value": "escalate to regional review",
             "probability": 0.0007565263394656762
            }
           ]
          },
          "deficiencies": {
           "value": 0,
           "probability": 0.9982540834862047,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9982540834862047
            },
            {
             "value": 1,
             "probability": 0.0006414765775465807
            },
            {
             "value": 2,
             "probability": 0.0005396770187655724
            },
            {
             "value": 3,
             "probability": 0.0005647629174831298
            }
           ]
          }
         },
         "joint_probability": 0.9976852880995246,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023173949872480263,
        "field_brier": 3.8529754643342605e-06
       }
      },
      "timing_ms": {
       "scoring": 33.341510919854045,
       "base_joint": 0.06614904850721359,
       "constrained_joint": 0.04697917029261589,
       "constrained_total": 33.38849009014666
      },
      "input_tokens": 618,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07c47162b60922ca-0039",
      "group": "bendsynth-07c47162b60922ca",
      "provenance": "bendsynth-v1; family=bendsynth-07c47162b60922ca; domain=language school placement inspection; parameters=min_tests=60; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "both_flags": false,
       "decision": "escalate to regional review",
       "deficiencies": 2
      },
      "logits": [
       [
        -0.2934340238571167,
        -6.837559700012207
       ],
       [
        -0.2937316596508026,
        -7.39743709564209,
        -7.2909345626831055
       ],
       [
        -7.725412845611572,
        -7.733672618865967,
        -0.29387661814689636,
        -7.5553741455078125
       ]
      ],
      "joint_logits": [
       -9.663115501403809,
       -9.376506805419922,
       -0.2945619523525238,
       -9.387940406799316,
       -9.619961738586426,
       -9.679689407348633,
       -9.399994850158691,
       -9.505324363708496,
       -9.517659187316895,
       -9.662505149841309,
       -9.200321197509766,
       -9.381722450256348,
       -9.396204948425293,
       -9.379122734069824,
       -9.002936363220215,
       -9.249250411987305,
       -9.515027046203613,
       -9.623867988586426,
       -9.249316215515137,
       -9.30831527709961,
       -9.41893196105957,
       -9.479838371276855,
       -9.136058807373047,
       -9.272137641906738
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "both_flags": {
           "value": false,
           "probability": 0.9985635246925592,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985635246925592
            },
            {
             "value": true,
             "probability": 0.001436475307440924
            }
           ]
          },
          "decision": {
           "value": "escalate to regional review",
           "probability": 0.9982665209163188,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.9982665209163188
            },
            {
             "value": "standard approval",
             "probability": 0.0008206280160616085
            },
            {
             "value": "conditional approval",
             "probability": 0.0009128510676196446
            }
           ]
          },
          "deficiencies": {
           "value": 2,
           "probability": 0.9981217963495574,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000591164567850121
            },
            {
             "value": 1,
             "probability": 0.0005863018678004393
            },
            {
             "value": 2,
             "probability": 0.9981217963495574
            },
            {
             "value": 3,
             "probability": 0.0007007372147919443
            }
           ]
          }
         },
         "joint_probability": 0.9974379618608353,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002565325775444864,
        "field_brier": 4.4501689289773914e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "both_flags": {
           "value": false,
           "probability": 0.9985635246925592,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985635246925592
            },
            {
             "value": true,
             "probability": 0.001436475307440924
            }
           ]
          },
          "decision": {
           "value": "escalate to regional review",
           "probability": 0.9982665209163188,
           "distribution": [
            {
             "value": "escalate to regional review",
             "probability": 0.9982665209163188
            },
            {
             "value": "standard approval",
             "probability": 0.0008206280160616085
            },
            {
             "value": "conditional approval",
             "probability": 0.0009128510676196446
            }
           ]
          },
          "deficiencies": {
           "value": 2,
           "probability": 0.9981217963495574,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000591164567850121
            },
            {
             "value": 1,
             "probability": 0.0005863018678004393
            },
            {
             "value": 2,
             "probability": 0.9981217963495574
            },
            {
             "value": 3,
             "probability": 0.0007007372147919443
            }
           ]
          }
         },
         "joint_probability": 0.9974379618608353,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002565325775444864,
        "field_brier": 4.4501689289773914e-06
       }
      },
      "timing_ms": {
       "scoring": 33.77855592407286,
       "base_joint": 0.07521919906139374,
       "constrained_joint": 0.04660920239984989,
       "constrained_total": 33.82516512647271
      },
      "input_tokens": 621,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0ca63e46388731ef-0001",
      "group": "bendsynth-0ca63e46388731ef",
      "provenance": "bendsynth-v1; family=bendsynth-0ca63e46388731ef; domain=dental practice recall scheduling intake; parameters=interval=18; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "overdue": false,
       "decision": "book urgent exam",
       "risk_flags": 2
      },
      "logits": [
       [
        2.3542535305023193,
        -4.738004684448242
       ],
       [
        -5.485186576843262,
        2.353787660598755,
        -5.453241348266602,
        -5.253015041351318
       ],
       [
        -5.716056823730469,
        -5.6750407218933105,
        2.353738307952881,
        -5.570074081420898,
        -5.613984107971191
       ]
      ],
      "joint_logits": [
       -7.889284610748291,
       -7.84734582901001,
       -7.5175700187683105,
       -7.730215549468994,
       -7.791802406311035,
       -7.809121131896973,
       -7.420592784881592,
       2.3533995151519775,
       -7.573326587677002,
       -7.823243618011475,
       -7.846846580505371,
       -7.796837329864502,
       -7.555339813232422,
       -7.681906700134277,
       -7.746118068695068,
       -7.662586688995361,
       -7.647196292877197,
       -7.307175159454346,
       -7.519413948059082,
       -7.5429229736328125,
       -7.875350475311279,
       -7.975376129150391,
       -7.796907424926758,
       -7.780688762664795,
       -7.742978096008301,
       -7.8431620597839355,
       -7.805684566497803,
       -7.65516471862793,
       -7.702489376068115,
       -7.729125022888184,
       -7.851711750030518,
       -7.940610885620117,
       -7.744410991668701,
       -7.748717308044434,
       -7.697327136993408,
       -7.622460842132568,
       -7.716745853424072,
       -7.57948112487793,
       -7.500872611999512,
       -7.518083095550537
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "overdue": {
           "value": false,
           "probability": 0.9991691735705979,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991691735705979
            },
            {
             "value": true,
             "probability": 0.0008308264294021818
            }
           ]
          },
          "decision": {
           "value": "book urgent exam",
           "probability": 0.9987036857115935,
           "distribution": [
            {
             "value": "close file",
             "probability": 0.00039356227145795743
            },
            {
             "value": "book urgent exam",
             "probability": 0.9987036857115935
            },
            {
             "value": "send reminder letter",
             "probability": 0.0004063377444030634
            },
            {
             "value": "book hygiene visit",
             "probability": 0.0004964142725454989
            }
           ]
          },
          "risk_flags": {
           "value": 2,
           "probability": 0.9986545328976157,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00031242638589542595
            },
            {
             "value": 1,
             "probability": 0.0003255074024448063
            },
            {
             "value": 2,
             "probability": 0.9986545328976157
            },
            {
             "value": 3,
             "probability": 0.00036153235603857484
            },
            {
             "value": 4,
             "probability": 0.0003460009580054246
            }
           ]
          }
         },
         "joint_probability": 0.9983161668132154,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0016852524272849216,
        "field_brier": 1.963891315531162e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "overdue": {
           "value": false,
           "probability": 0.9991691735705979,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991691735705979
            },
            {
             "value": true,
             "probability": 0.0008308264294021818
            }
           ]
          },
          "decision": {
           "value": "book urgent exam",
           "probability": 0.9987036857115935,
           "distribution": [
            {
             "value": "close file",
             "probability": 0.00039356227145795743
            },
            {
             "value": "book urgent exam",
             "probability": 0.9987036857115935
            },
            {
             "value": "send reminder letter",
             "probability": 0.0004063377444030634
            },
            {
             "value": "book hygiene visit",
             "probability": 0.0004964142725454989
            }
           ]
          },
          "risk_flags": {
           "value": 2,
           "probability": 0.9986545328976157,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00031242638589542595
            },
            {
             "value": 1,
             "probability": 0.0003255074024448063
            },
            {
             "value": 2,
             "probability": 0.9986545328976157
            },
            {
             "value": 3,
             "probability": 0.00036153235603857484
            },
            {
             "value": 4,
             "probability": 0.0003460009580054246
            }
           ]
          }
         },
         "joint_probability": 0.9983161668132154,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0016852524272849216,
        "field_brier": 1.963891315531162e-06
       }
      },
      "timing_ms": {
       "scoring": 37.784002954140306,
       "base_joint": 0.0855091493576765,
       "constrained_joint": 0.06777001544833183,
       "constrained_total": 37.85177296958864
      },
      "input_tokens": 721,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0ca63e46388731ef-0020",
      "group": "bendsynth-0ca63e46388731ef",
      "provenance": "bendsynth-v1; family=bendsynth-0ca63e46388731ef; domain=dental practice recall scheduling intake; parameters=interval=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "overdue": true,
       "risk_flags": 1,
       "decision": "close file"
      },
      "logits": [
       [
        -4.337026119232178,
        2.7436957359313965
       ],
       [
        -4.850759506225586,
        2.743036985397339,
        -5.3284711837768555,
        -5.251241683959961,
        -5.2220258712768555
       ],
       [
        2.7425758838653564,
        -5.122440338134766,
        -4.023959159851074,
        -5.0163373947143555
       ]
      ],
      "joint_logits": [
       -7.261350154876709,
       -7.251882076263428,
       -6.700555324554443,
       -7.168539047241211,
       -7.314881324768066,
       -7.7077765464782715,
       -6.978032112121582,
       -7.605260848999023,
       -7.669058322906494,
       -7.688511371612549,
       -7.176307678222656,
       -7.566526412963867,
       -7.59304141998291,
       -7.5834760665893555,
       -7.09834098815918,
       -7.462479114532471,
       -7.5617194175720215,
       -7.551293849945068,
       -7.045413017272949,
       -7.449891090393066,
       -6.791223049163818,
       -7.099379062652588,
       -6.467577934265137,
       -7.003643989562988,
       2.742205858230591,
       -7.216641902923584,
       -4.739584445953369,
       -7.03049373626709,
       -7.43710994720459,
       -7.518834590911865,
       -6.9776740074157715,
       -7.452184200286865,
       -7.442638397216797,
       -7.4259538650512695,
       -6.907678604125977,
       -7.344464302062988,
       -7.438920497894287,
       -7.397501468658447,
       -6.878682613372803,
       -7.3114118576049805
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "overdue": {
           "value": true,
           "probability": 0.9991595410351755,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008404589648245448
            },
            {
             "value": true,
             "probability": 0.9991595410351755
            }
           ]
          },
          "risk_flags": {
           "value": 1,
           "probability": 0.998501593729566,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005028110273290228
            },
            {
             "value": 1,
             "probability": 0.998501593729566
            },
            {
             "value": 2,
             "probability": 0.00031184393600058825
            },
            {
             "value": 3,
             "probability": 0.00033688193044739695
            },
            {
             "value": 4,
             "probability": 0.00034686937665686713
            }
           ]
          },
          "decision": {
           "value": "close file",
           "probability": 0.9980413023002312,
           "distribution": [
            {
             "value": "close file",
             "probability": 0.9980413023002312
            },
            {
             "value": "book urgent exam",
             "probability": 0.00038319109795184646
            },
            {
             "value": "send reminder letter",
             "probability": 0.0011494225034195736
            },
            {
             "value": "book hygiene visit",
             "probability": 0.00042608409839743877
            }
           ]
          }
         },
         "joint_probability": 0.9976720285992701,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023306853389497718,
        "field_brier": 3.2426297034784706e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "overdue": {
           "value": true,
           "probability": 0.9991595410351755,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008404589648245448
            },
            {
             "value": true,
             "probability": 0.9991595410351755
            }
           ]
          },
          "risk_flags": {
           "value": 1,
           "probability": 0.998501593729566,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005028110273290228
            },
            {
             "value": 1,
             "probability": 0.998501593729566
            },
            {
             "value": 2,
             "probability": 0.00031184393600058825
            },
            {
             "value": 3,
             "probability": 0.00033688193044739695
            },
            {
             "value": 4,
             "probability": 0.00034686937665686713
            }
           ]
          },
          "decision": {
           "value": "close file",
           "probability": 0.9980413023002312,
           "distribution": [
            {
             "value": "close file",
             "probability": 0.9980413023002312
            },
            {
             "value": "book urgent exam",
             "probability": 0.00038319109795184646
            },
            {
             "value": "send reminder letter",
             "probability": 0.0011494225034195736
            },
            {
             "value": "book hygiene visit",
             "probability": 0.00042608409839743877
            }
           ]
          }
         },
         "joint_probability": 0.9976720285992701,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023306853389497718,
        "field_brier": 3.2426297034784706e-06
       }
      },
      "timing_ms": {
       "scoring": 38.368266774341464,
       "base_joint": 0.08592917583882809,
       "constrained_joint": 0.07743015885353088,
       "constrained_total": 38.445696933194995
      },
      "input_tokens": 724,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0ca63e46388731ef-0021",
      "group": "bendsynth-0ca63e46388731ef",
      "provenance": "bendsynth-v1; family=bendsynth-0ca63e46388731ef; domain=dental practice recall scheduling intake; parameters=interval=15; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_flags": 2,
       "decision": "close file",
       "overdue": true
      },
      "logits": [
       [
        -5.841984272003174,
        -5.617768287658691,
        1.6209805011749268,
        -5.843400955200195,
        -5.676074028015137
       ],
       [
        -5.863519668579102,
        -5.772420406341553,
        -0.8510352373123169,
        1.534279704093933
       ],
       [
        -4.843817234039307,
        1.6219656467437744
       ]
      ],
      "joint_logits": [
       -8.226458549499512,
       -8.216026306152344,
       -8.162403106689453,
       -8.130274772644043,
       -7.473146438598633,
       -7.457425594329834,
       -8.031163215637207,
       -8.062315940856934,
       -8.121319770812988,
       -8.095478057861328,
       -8.005086898803711,
       -8.003890991210938,
       -7.331580638885498,
       -7.098088264465332,
       -7.854700088500977,
       -7.613740921020508,
       -8.529314041137695,
       -7.989223480224609,
       -8.450521469116211,
       -7.881138324737549,
       -7.149726390838623,
       -0.8649437427520752,
       -7.860100269317627,
       1.5335657596588135,
       -8.257552146911621,
       -8.251944541931152,
       -8.137333869934082,
       -8.164947509765625,
       -7.476318836212158,
       -7.439970970153809,
       -8.047593116760254,
       -8.020837783813477,
       -8.043478965759277,
       -8.036360740661621,
       -7.971092700958252,
       -7.955092430114746,
       -7.299346923828125,
       -7.329014301300049,
       -7.8368239402771,
       -7.932136058807373
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 2,
           "probability": 0.9974636169654061,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005724961090855346
            },
            {
             "value": 1,
             "probability": 0.0007163881528009386
            },
            {
             "value": 2,
             "probability": 0.9974636169654061
            },
            {
             "value": 3,
             "probability": 0.0005716858756414469
            },
            {
             "value": 4,
             "probability": 0.0006758128970660085
            }
           ]
          },
          "decision": {
           "value": "close file",
           "probability": 0.9146255556051486,
           "distribution": [
            {
             "value": "book urgent exam",
             "probability": 0.0005602989389154627
            },
            {
             "value": "book hygiene visit",
             "probability": 0.0006137390292375521
            },
            {
             "value": "send reminder letter",
             "probability": 0.08420040642669838
            },
            {
             "value": "close file",
             "probability": 0.9146255556051486
            }
           ]
          },
          "overdue": {
           "value": true,
           "probability": 0.9984466438125371,
           "distribution": [
            {
             "value": false,
             "probability": 0.001553356187462831
            },
            {
             "value": true,
             "probability": 0.9984466438125371
            }
           ]
          }
         },
         "joint_probability": 0.9139727925139057,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.08995447546130386,
        "field_brier": 0.004797359463342934
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 2,
           "probability": 0.9974636169654061,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005724961090855346
            },
            {
             "value": 1,
             "probability": 0.0007163881528009386
            },
            {
             "value": 2,
             "probability": 0.9974636169654061
            },
            {
             "value": 3,
             "probability": 0.0005716858756414469
            },
            {
             "value": 4,
             "probability": 0.0006758128970660085
            }
           ]
          },
          "decision": {
           "value": "close file",
           "probability": 0.9146255556051486,
           "distribution": [
            {
             "value": "book urgent exam",
             "probability": 0.0005602989389154627
            },
            {
             "value": "book hygiene visit",
             "probability": 0.0006137390292375521
            },
            {
             "value": "send reminder letter",
             "probability": 0.08420040642669838
            },
            {
             "value": "close file",
             "probability": 0.9146255556051486
            }
           ]
          },
          "overdue": {
           "value": true,
           "probability": 0.9984466438125371,
           "distribution": [
            {
             "value": false,
             "probability": 0.001553356187462831
            },
            {
             "value": true,
             "probability": 0.9984466438125371
            }
           ]
          }
         },
         "joint_probability": 0.9139727925139057,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.08995447546130386,
        "field_brier": 0.004797359463342934
       }
      },
      "timing_ms": {
       "scoring": 38.613043958321214,
       "base_joint": 0.08690916001796722,
       "constrained_joint": 0.0671090092509985,
       "constrained_total": 38.68015296757221
      },
      "input_tokens": 709,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0ca63e46388731ef-0032",
      "group": "bendsynth-0ca63e46388731ef",
      "provenance": "bendsynth-v1; family=bendsynth-0ca63e46388731ef; domain=dental practice recall scheduling intake; parameters=interval=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_flags": 1,
       "overdue": true,
       "decision": "send reminder letter"
      },
      "logits": [
       [
        -5.144662857055664,
        2.4563851356506348,
        -5.54960823059082,
        -5.448387622833252,
        -5.391345024108887
       ],
       [
        -4.584158420562744,
        2.457102060317993
       ],
       [
        -5.108938694000244,
        -5.085498809814453,
        2.4563074111938477,
        -4.925743579864502
       ]
      ],
      "joint_logits": [
       -7.2464070320129395,
       -7.30363655090332,
       -7.858358860015869,
       -7.167222023010254,
       -7.118926048278809,
       -7.096537113189697,
       -7.23055362701416,
       -6.9857916831970215,
       -7.669605731964111,
       -7.64961576461792,
       -7.764922618865967,
       -7.526297092437744,
       -7.298746109008789,
       -7.120560646057129,
       2.4559969902038574,
       -6.781216621398926,
       -7.706602096557617,
       -7.629340171813965,
       -8.2386474609375,
       -7.4814348220825195,
       -7.4778032302856445,
       -7.495382785797119,
       -7.908082485198975,
       -7.359431743621826,
       -7.578704833984375,
       -7.553496360778809,
       -8.129806518554688,
       -7.3935089111328125,
       -7.352591514587402,
       -7.371392726898193,
       -7.823030948638916,
       -7.282367706298828,
       -7.509517192840576,
       -7.518225193023682,
       -8.052960395812988,
       -7.34710168838501,
       -7.320855617523193,
       -7.319913387298584,
       -7.732348442077637,
       -7.2087836265563965
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 1,
           "probability": 0.9984095320710525,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004991322610779096
            },
            {
             "value": 1,
             "probability": 0.9984095320710525
            },
            {
             "value": 2,
             "probability": 0.0003329277054653273
            },
            {
             "value": 3,
             "probability": 0.00036839148931467404
            },
            {
             "value": 4,
             "probability": 0.00039001647308953985
            }
           ]
          },
          "overdue": {
           "value": true,
           "probability": 0.9991257422875136,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008742577124864174
            },
            {
             "value": true,
             "probability": 0.9991257422875136
            }
           ]
          },
          "decision": {
           "value": "send reminder letter",
           "probability": 0.9983318753618755,
           "distribution": [
            {
             "value": "book urgent exam",
             "probability": 0.0005172856571030891
            },
            {
             "value": "book hygiene visit",
             "probability": 0.0005295538219161308
            },
            {
             "value": "send reminder letter",
             "probability": 0.9983318753618755
            },
            {
             "value": "close file",
             "probability": 0.0006212851591052619
            }
           ]
          }
         },
         "joint_probability": 0.9980220948740355,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019798637634003014,
        "field_brier": 2.8075623656943842e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 1,
           "probability": 0.9984095320710525,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004991322610779096
            },
            {
             "value": 1,
             "probability": 0.9984095320710525
            },
            {
             "value": 2,
             "probability": 0.0003329277054653273
            },
            {
             "value": 3,
             "probability": 0.00036839148931467404
            },
            {
             "value": 4,
             "probability": 0.00039001647308953985
            }
           ]
          },
          "overdue": {
           "value": true,
           "probability": 0.9991257422875136,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008742577124864174
            },
            {
             "value": true,
             "probability": 0.9991257422875136
            }
           ]
          },
          "decision": {
           "value": "send reminder letter",
           "probability": 0.9983318753618755,
           "distribution": [
            {
             "value": "book urgent exam",
             "probability": 0.0005172856571030891
            },
            {
             "value": "book hygiene visit",
             "probability": 0.0005295538219161308
            },
            {
             "value": "send reminder letter",
             "probability": 0.9983318753618755
            },
            {
             "value": "close file",
             "probability": 0.0006212851591052619
            }
           ]
          }
         },
         "joint_probability": 0.9980220948740355,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019798637634003014,
        "field_brier": 2.8075623656943842e-06
       }
      },
      "timing_ms": {
       "scoring": 37.45721699669957,
       "base_joint": 0.08545001037418842,
       "constrained_joint": 0.06736884824931622,
       "constrained_total": 37.52458584494889
      },
      "input_tokens": 679,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0ca63e46388731ef-0033",
      "group": "bendsynth-0ca63e46388731ef",
      "provenance": "bendsynth-v1; family=bendsynth-0ca63e46388731ef; domain=dental practice recall scheduling intake; parameters=interval=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_flags": 3,
       "decision": "send reminder letter",
       "overdue": true
      },
      "logits": [
       [
        -6.727900505065918,
        -6.927633285522461,
        -6.538475036621094,
        1.0186444520950317,
        -6.694395065307617
       ],
       [
        -5.912964344024658,
        -6.475017070770264,
        -6.270371913909912,
        1.0181838274002075
       ],
       [
        -5.825674057006836,
        1.019334316253662
       ]
      ],
      "joint_logits": [
       -8.642184257507324,
       -8.762993812561035,
       -8.958972930908203,
       -8.820158958435059,
       -8.722635269165039,
       -8.809860229492188,
       -9.178094863891602,
       -8.665491104125977,
       -9.012211799621582,
       -8.780652046203613,
       -9.18349552154541,
       -8.890796661376953,
       -9.110745429992676,
       -8.810827255249023,
       -9.300292015075684,
       -9.084782600402832,
       -8.673521041870117,
       -8.462018013000488,
       -8.812188148498535,
       -8.67149543762207,
       -8.633696556091309,
       -8.658329963684082,
       -9.05042839050293,
       -8.198877334594727,
       -8.484512329101562,
       -6.753610610961914,
       -8.994817733764648,
       -8.120179176330566,
       -8.811620712280273,
       -7.600527763366699,
       -8.262777328491211,
       1.01766836643219,
       -8.631311416625977,
       -8.730894088745117,
       -8.833975791931152,
       -8.893598556518555,
       -8.625932693481445,
       -8.876871109008789,
       -9.244512557983398,
       -8.520670890808105
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 3,
           "probability": 0.9982475274114461,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00043147575829748157
            },
            {
             "value": 1,
             "probability": 0.0003533570012624874
            },
            {
             "value": 2,
             "probability": 0.0005214623122328915
            },
            {
             "value": 3,
             "probability": 0.9982475274114461
            },
            {
             "value": 4,
             "probability": 0.0004461775167611311
            }
           ]
          },
          "decision": {
           "value": "send reminder letter",
           "probability": 0.9977878537881715,
           "distribution": [
            {
             "value": "close file",
             "probability": 0.0009747177513451781
            },
            {
             "value": "book urgent exam",
             "probability": 0.000555625758011337
            },
            {
             "value": "book hygiene visit",
             "probability": 0.0006818027024720847
            },
            {
             "value": "send reminder letter",
             "probability": 0.9977878537881715
            }
           ]
          },
          "overdue": {
           "value": true,
           "probability": 0.9989363748318937,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010636251681064204
            },
            {
             "value": true,
             "probability": 0.9989363748318937
            }
           ]
          }
         },
         "joint_probability": 0.9972736058148879,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002730117566882541,
        "field_brier": 4.244342485110714e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 3,
           "probability": 0.9982475274114461,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00043147575829748157
            },
            {
             "value": 1,
             "probability": 0.0003533570012624874
            },
            {
             "value": 2,
             "probability": 0.0005214623122328915
            },
            {
             "value": 3,
             "probability": 0.9982475274114461
            },
            {
             "value": 4,
             "probability": 0.0004461775167611311
            }
           ]
          },
          "decision": {
           "value": "send reminder letter",
           "probability": 0.9977878537881715,
           "distribution": [
            {
             "value": "close file",
             "probability": 0.0009747177513451781
            },
            {
             "value": "book urgent exam",
             "probability": 0.000555625758011337
            },
            {
             "value": "book hygiene visit",
             "probability": 0.0006818027024720847
            },
            {
             "value": "send reminder letter",
             "probability": 0.9977878537881715
            }
           ]
          },
          "overdue": {
           "value": true,
           "probability": 0.9989363748318937,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010636251681064204
            },
            {
             "value": true,
             "probability": 0.9989363748318937
            }
           ]
          }
         },
         "joint_probability": 0.9972736058148879,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002730117566882541,
        "field_brier": 4.244342485110714e-06
       }
      },
      "timing_ms": {
       "scoring": 37.56351489573717,
       "base_joint": 0.08795899339020252,
       "constrained_joint": 0.06743893027305603,
       "constrained_total": 37.63095382601023
      },
      "input_tokens": 733,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a5decb0e18debdd-0026",
      "group": "bendsynth-0a5decb0e18debdd",
      "provenance": "bendsynth-v1; family=bendsynth-0a5decb0e18debdd; domain=season pass reinstatement audit at a ski resort field inspection office; parameters=wait_days=7; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "conditions": 0,
       "decision": "reinstate restricted pass",
       "wait_satisfied": true
      },
      "logits": [
       [
        1.3779337406158447,
        -5.716497421264648,
        -5.909502983093262,
        -5.990071773529053
       ],
       [
        1.3780367374420166,
        -5.472165584564209,
        -5.547470569610596
       ],
       [
        -5.2256035804748535,
        1.3787227869033813
       ]
      ],
      "joint_logits": [
       -7.6838603019714355,
       1.3772069215774536,
       -7.712874889373779,
       -7.031242847442627,
       -7.767761707305908,
       -7.311574935913086,
       -7.7775774002075195,
       -7.2049760818481445,
       -7.5449066162109375,
       -7.516589641571045,
       -7.51559591293335,
       -7.577175617218018,
       -7.922511577606201,
       -7.593725204467773,
       -7.635504245758057,
       -7.692979335784912,
       -7.654290199279785,
       -7.741487979888916,
       -7.958237171173096,
       -7.69998025894165,
       -7.707596778869629,
       -7.799203872680664,
       -7.73852014541626,
       -7.809891223907471
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "conditions": {
           "value": 0,
           "probability": 0.9978596730905818,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978596730905818
            },
            {
             "value": 1,
             "probability": 0.000827936642008682
            },
            {
             "value": 2,
             "probability": 0.0006826152578876758
            },
            {
             "value": 3,
             "probability": 0.000629775009521804
            }
           ]
          },
          "decision": {
           "value": "reinstate restricted pass",
           "probability": 0.9979625137297566,
           "distribution": [
            {
             "value": "reinstate restricted pass",
             "probability": 0.9979625137297566
            },
            {
             "value": "keep suspended",
             "probability": 0.0010570831433140626
            },
            {
             "value": "reinstate full pass",
             "probability": 0.000980403126929288
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9986473366909991,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013526633090008248
            },
            {
             "value": true,
             "probability": 0.9986473366909991
            }
           ]
          }
         },
         "joint_probability": 0.9971346825905522,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028694302897267683,
        "field_brier": 5.339473311638513e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "conditions": {
           "value": 0,
           "probability": 0.9978596730905818,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978596730905818
            },
            {
             "value": 1,
             "probability": 0.000827936642008682
            },
            {
             "value": 2,
             "probability": 0.0006826152578876758
            },
            {
             "value": 3,
             "probability": 0.000629775009521804
            }
           ]
          },
          "decision": {
           "value": "reinstate restricted pass",
           "probability": 0.9979625137297566,
           "distribution": [
            {
             "value": "reinstate restricted pass",
             "probability": 0.9979625137297566
            },
            {
             "value": "keep suspended",
             "probability": 0.0010570831433140626
            },
            {
             "value": "reinstate full pass",
             "probability": 0.000980403126929288
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9986473366909991,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013526633090008248
            },
            {
             "value": true,
             "probability": 0.9986473366909991
            }
           ]
          }
         },
         "joint_probability": 0.9971346825905522,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028694302897267683,
        "field_brier": 5.339473311638513e-06
       }
      },
      "timing_ms": {
       "scoring": 37.24184911698103,
       "base_joint": 0.0624100212007761,
       "constrained_joint": 0.04646903835237026,
       "constrained_total": 37.2883181553334
      },
      "input_tokens": 694,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0076a1062480b4bf-0001",
      "group": "bendsynth-0076a1062480b4bf",
      "provenance": "bendsynth-v1; family=bendsynth-0076a1062480b4bf; domain=bicycle courier dispatch intake desk; parameters=max_distance=22,max_weight=13; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 2,
       "decision": "hold for review",
       "long_route": false
      },
      "logits": [
       [
        -6.6902594566345215,
        -6.400975227355957,
        0.9966440796852112,
        -6.674041748046875
       ],
       [
        -6.301648139953613,
        -6.658534526824951,
        -6.1665873527526855,
        0.9962570071220398
       ],
       [
        0.997153639793396,
        -5.88339900970459
       ]
      ],
      "joint_logits": [
       -8.567428588867188,
       -8.587187767028809,
       -8.917098999023438,
       -8.930304527282715,
       -8.818614959716797,
       -8.722369194030762,
       -8.813881874084473,
       -8.871183395385742,
       -8.294404029846191,
       -8.340815544128418,
       -8.62277889251709,
       -8.670294761657715,
       -8.584277153015137,
       -8.551220893859863,
       -8.3167085647583,
       -8.541744232177734,
       -7.910165786743164,
       -8.46105670928955,
       -8.339344024658203,
       -8.810068130493164,
       -6.9678850173950195,
       -8.70037841796875,
       0.9958043694496155,
       -8.521245002746582,
       -8.530839920043945,
       -8.556564331054688,
       -8.891814231872559,
       -8.878158569335938,
       -8.860447883605957,
       -8.75836181640625,
       -8.814964294433594,
       -8.809710502624512
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 2,
           "probability": 0.9984645567768619,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004580922909815668
            },
            {
             "value": 1,
             "probability": 0.0006117689933310044
            },
            {
             "value": 2,
             "probability": 0.9984645567768619
            },
            {
             "value": 3,
             "probability": 0.0004655819388255996
            }
           ]
          },
          "decision": {
           "value": "hold for review",
           "probability": 0.998078129610816,
           "distribution": [
            {
             "value": "dispatch with note",
             "probability": 0.000675654133464329
            },
            {
             "value": "dispatch now",
             "probability": 0.00047285800747434676
            },
            {
             "value": "decline",
             "probability": 0.0007733582482453894
            },
            {
             "value": "hold for review",
             "probability": 0.998078129610816
            }
           ]
          },
          "long_route": {
           "value": false,
           "probability": 0.9989734786084576,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989734786084576
            },
            {
             "value": true,
             "probability": 0.001026521391542407
            }
           ]
          }
         },
         "joint_probability": 0.9976264569459464,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002376364372599785,
        "field_brier": 3.4125755307797215e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 2,
           "probability": 0.9984645567768619,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004580922909815668
            },
            {
             "value": 1,
             "probability": 0.0006117689933310044
            },
            {
             "value": 2,
             "probability": 0.9984645567768619
            },
            {
             "value": 3,
             "probability": 0.0004655819388255996
            }
           ]
          },
          "decision": {
           "value": "hold for review",
           "probability": 0.998078129610816,
           "distribution": [
            {
             "value": "dispatch with note",
             "probability": 0.000675654133464329
            },
            {
             "value": "dispatch now",
             "probability": 0.00047285800747434676
            },
            {
             "value": "decline",
             "probability": 0.0007733582482453894
            },
            {
             "value": "hold for review",
             "probability": 0.998078129610816
            }
           ]
          },
          "long_route": {
           "value": false,
           "probability": 0.9989734786084576,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989734786084576
            },
            {
             "value": true,
             "probability": 0.001026521391542407
            }
           ]
          }
         },
         "joint_probability": 0.9976264569459464,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002376364372599785,
        "field_brier": 3.4125755307797215e-06
       }
      },
      "timing_ms": {
       "scoring": 33.09664409607649,
       "base_joint": 0.07635913789272308,
       "constrained_joint": 0.058699166402220726,
       "constrained_total": 33.15534326247871
      },
      "input_tokens": 567,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0076a1062480b4bf-0020",
      "group": "bendsynth-0076a1062480b4bf",
      "provenance": "bendsynth-v1; family=bendsynth-0076a1062480b4bf; domain=bicycle courier dispatch intake desk; parameters=max_distance=24,max_weight=7; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "hold for review",
       "long_route": false,
       "risk_factors": 1
      },
      "logits": [
       [
        -6.22214412689209,
        -6.0708513259887695,
        -0.2494189292192459,
        0.391810804605484
       ],
       [
        0.8148479461669922,
        -5.432275772094727
       ],
       [
        -6.261964797973633,
        0.8140982389450073,
        -6.170226097106934,
        -6.1812968254089355
       ]
      ],
      "joint_logits": [
       -8.473089218139648,
       -8.266314506530762,
       -8.284029006958008,
       -8.301946640014648,
       -8.268296241760254,
       -8.28478717803955,
       -8.35560417175293,
       -8.200702667236328,
       -8.468596458435059,
       -7.340246677398682,
       -8.390289306640625,
       -8.403355598449707,
       -8.37878131866455,
       -8.354911804199219,
       -8.181640625,
       -8.276166915893555,
       -8.229236602783203,
       -0.25212109088897705,
       -8.215909957885742,
       -8.263461112976074,
       -8.123330116271973,
       -7.996316909790039,
       -7.986263751983643,
       -8.002251625061035,
       -8.514986991882324,
       0.3906896710395813,
       -8.470976829528809,
       -8.514431953430176,
       -8.340786933898926,
       -8.259270668029785,
       -8.193355560302734,
       -8.203845977783203
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "hold for review",
           "probability": 0.6537885255231803,
           "distribution": [
            {
             "value": "decline",
             "probability": 0.0008770678463006581
            },
            {
             "value": "dispatch now",
             "probability": 0.001020325778304286
            },
            {
             "value": "dispatch with note",
             "probability": 0.34431408085221477
            },
            {
             "value": "hold for review",
             "probability": 0.6537885255231803
            }
           ]
          },
          "long_route": {
           "value": false,
           "probability": 0.9980677262647945,
           "distribution": [
            {
             "value": false,
             "probability": 0.9980677262647945
            },
            {
             "value": true,
             "probability": 0.0019322737352055308
            }
           ]
          },
          "risk_factors": {
           "value": 1,
           "probability": 0.9973197295499809,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008428288685762471
            },
            {
             "value": 1,
             "probability": 0.9973197295499809
            },
            {
             "value": 2,
             "probability": 0.0009238060866180954
            },
            {
             "value": 3,
             "probability": 0.0009136354948246899
            }
           ]
          }
         },
         "joint_probability": 0.6530559490699122,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.4260924733302242,
        "field_brier": 0.07947781045549517
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "hold for review",
           "probability": 0.6537885255231803,
           "distribution": [
            {
             "value": "decline",
             "probability": 0.0008770678463006581
            },
            {
             "value": "dispatch now",
             "probability": 0.001020325778304286
            },
            {
             "value": "dispatch with note",
             "probability": 0.34431408085221477
            },
            {
             "value": "hold for review",
             "probability": 0.6537885255231803
            }
           ]
          },
          "long_route": {
           "value": false,
           "probability": 0.9980677262647945,
           "distribution": [
            {
             "value": false,
             "probability": 0.9980677262647945
            },
            {
             "value": true,
             "probability": 0.0019322737352055308
            }
           ]
          },
          "risk_factors": {
           "value": 1,
           "probability": 0.9973197295499809,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008428288685762471
            },
            {
             "value": 1,
             "probability": 0.9973197295499809
            },
            {
             "value": 2,
             "probability": 0.0009238060866180954
            },
            {
             "value": 3,
             "probability": 0.0009136354948246899
            }
           ]
          }
         },
         "joint_probability": 0.6530559490699122,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.4260924733302242,
        "field_brier": 0.07947781045549517
       }
      },
      "timing_ms": {
       "scoring": 33.915374195203185,
       "base_joint": 0.07555889897048473,
       "constrained_joint": 0.056658871471881866,
       "constrained_total": 33.97203306667507
      },
      "input_tokens": 573,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-02ac678b60db1e2b-0015",
      "group": "bendsynth-02ac678b60db1e2b",
      "provenance": "bendsynth-v1; family=bendsynth-02ac678b60db1e2b; domain=coworking space membership site inspection; parameters=capacity=149; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "conditional pass",
       "flags": 1,
       "exit_ok": true
      },
      "logits": [
       [
        -5.915597915649414,
        1.629642367362976,
        -5.834129333496094,
        -6.05472469329834
       ],
       [
        -6.000699043273926,
        1.6297799348831177,
        -6.054200172424316,
        -6.014348983764648
       ],
       [
        -5.342460632324219,
        1.630266547203064
       ]
      ],
      "joint_logits": [
       -8.163368225097656,
       -7.960285663604736,
       -7.979400157928467,
       -7.670846462249756,
       -8.194872856140137,
       -7.988677978515625,
       -8.078914642333984,
       -8.019238471984863,
       -8.160898208618164,
       -7.942556381225586,
       -7.809833526611328,
       1.6292051076889038,
       -8.250487327575684,
       -8.004985809326172,
       -8.187406539916992,
       -8.075186729431152,
       -8.025250434875488,
       -7.980053424835205,
       -7.909278869628906,
       -7.3572821617126465,
       -8.138853073120117,
       -8.042866706848145,
       -8.095197677612305,
       -8.007238388061523,
       -8.397510528564453,
       -8.087307929992676,
       -8.052605628967285,
       -7.83848237991333,
       -8.383821487426758,
       -8.125783920288086,
       -8.193503379821777,
       -8.10981559753418
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "conditional pass",
           "probability": 0.9984403649241332,
           "distribution": [
            {
             "value": "revisit",
             "probability": 0.0005277959646486491
            },
            {
             "value": "conditional pass",
             "probability": 0.9984403649241332
            },
            {
             "value": "pass",
             "probability": 0.0005725946782521507
            },
            {
             "value": "fail",
             "probability": 0.0004592444329659481
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9985776105339812,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00048473796076684955
            },
            {
             "value": 1,
             "probability": 0.9985776105339812
            },
            {
             "value": 2,
             "probability": 0.0004594853065252397
            },
            {
             "value": 3,
             "probability": 0.0004781661987267599
            }
           ]
          },
          "exit_ok": {
           "value": true,
           "probability": 0.9990637838463152,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009362161536847886
            },
            {
             "value": true,
             "probability": 0.9990637838463152
            }
           ]
          }
         },
         "joint_probability": 0.9980038004187384,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019981946431156545,
        "field_brier": 2.566911327317975e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "conditional pass",
           "probability": 0.9984403649241332,
           "distribution": [
            {
             "value": "revisit",
             "probability": 0.0005277959646486491
            },
            {
             "value": "conditional pass",
             "probability": 0.9984403649241332
            },
            {
             "value": "pass",
             "probability": 0.0005725946782521507
            },
            {
             "value": "fail",
             "probability": 0.0004592444329659481
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9985776105339812,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00048473796076684955
            },
            {
             "value": 1,
             "probability": 0.9985776105339812
            },
            {
             "value": 2,
             "probability": 0.0004594853065252397
            },
            {
             "value": 3,
             "probability": 0.0004781661987267599
            }
           ]
          },
          "exit_ok": {
           "value": true,
           "probability": 0.9990637838463152,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009362161536847886
            },
            {
             "value": true,
             "probability": 0.9990637838463152
            }
           ]
          }
         },
         "joint_probability": 0.9980038004187384,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019981946431156545,
        "field_brier": 2.566911327317975e-06
       }
      },
      "timing_ms": {
       "scoring": 37.27832902222872,
       "base_joint": 0.07811002433300018,
       "constrained_joint": 0.05860906094312668,
       "constrained_total": 37.336938083171844
      },
      "input_tokens": 688,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-02f8cc357855294f-0022",
      "group": "bendsynth-02f8cc357855294f",
      "provenance": "bendsynth-v1; family=bendsynth-02f8cc357855294f; domain=cold-storage seafood wholesaler intake desk; parameters=limit=17; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "downgrade",
       "risk_marks": 2,
       "hot_lot": false
      },
      "logits": [
       [
        -5.984259605407715,
        -5.7853546142578125,
        -5.922619819641113,
        1.7104543447494507
       ],
       [
        -5.877017021179199,
        -5.775285720825195,
        1.7104291915893555,
        -5.991222381591797
       ],
       [
        1.7109737396240234,
        -5.222485542297363
       ]
      ],
      "joint_logits": [
       -8.057051658630371,
       -8.109601974487305,
       -7.989370822906494,
       -8.05091667175293,
       -7.860772609710693,
       -8.097537994384766,
       -8.187467575073242,
       -8.199679374694824,
       -7.882927894592285,
       -7.930966377258301,
       -7.80687952041626,
       -7.887912273406982,
       -7.546937942504883,
       -7.931959629058838,
       -8.000765800476074,
       -8.011988639831543,
       -8.040645599365234,
       -8.097989082336426,
       -7.962407112121582,
       -8.0267915725708,
       -7.598858833312988,
       -8.050634384155273,
       -8.182022094726562,
       -8.189841270446777,
       -7.73070764541626,
       -7.865686893463135,
       -7.452941417694092,
       -7.802690029144287,
       1.709920048713684,
       -7.777595520019531,
       -7.843657493591309,
       -8.011127471923828
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "downgrade",
           "probability": 0.9985074263393192,
           "distribution": [
            {
             "value": "retest",
             "probability": 0.00045454773224180606
            },
            {
             "value": "reject",
             "probability": 0.000554578262656654
            },
            {
             "value": "release",
             "probability": 0.0004834476657823544
            },
            {
             "value": "downgrade",
             "probability": 0.9985074263393192
            }
           ]
          },
          "risk_marks": {
           "value": 2,
           "probability": 0.998482411080954,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005060044761677659
            },
            {
             "value": 1,
             "probability": 0.0005601905396184847
            },
            {
             "value": 2,
             "probability": 0.998482411080954
            },
            {
             "value": 3,
             "probability": 0.0004513939032597651
            }
           ]
          },
          "hot_lot": {
           "value": false,
           "probability": 0.9990263254333451,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990263254333451
            },
            {
             "value": true,
             "probability": 0.0009736745666549401
            }
           ]
          }
         },
         "joint_probability": 0.9979741546128009,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002027900187573078,
        "field_brier": 2.649479781866057e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "downgrade",
           "probability": 0.9985074263393192,
           "distribution": [
            {
             "value": "retest",
             "probability": 0.00045454773224180606
            },
            {
             "value": "reject",
             "probability": 0.000554578262656654
            },
            {
             "value": "release",
             "probability": 0.0004834476657823544
            },
            {
             "value": "downgrade",
             "probability": 0.9985074263393192
            }
           ]
          },
          "risk_marks": {
           "value": 2,
           "probability": 0.998482411080954,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005060044761677659
            },
            {
             "value": 1,
             "probability": 0.0005601905396184847
            },
            {
             "value": 2,
             "probability": 0.998482411080954
            },
            {
             "value": 3,
             "probability": 0.0004513939032597651
            }
           ]
          },
          "hot_lot": {
           "value": false,
           "probability": 0.9990263254333451,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990263254333451
            },
            {
             "value": true,
             "probability": 0.0009736745666549401
            }
           ]
          }
         },
         "joint_probability": 0.9979741546128009,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002027900187573078,
        "field_brier": 2.649479781866057e-06
       }
      },
      "timing_ms": {
       "scoring": 36.87230311334133,
       "base_joint": 0.08410890586674213,
       "constrained_joint": 0.05923910066485405,
       "constrained_total": 36.931542214006186
      },
      "input_tokens": 639,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-06acb453c9f440e3-0001",
      "group": "bendsynth-06acb453c9f440e3",
      "provenance": "bendsynth-v1; family=bendsynth-06acb453c9f440e3; domain=coworking space membership site inspection; parameters=min_days=17; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "wait_satisfied": true,
       "flags_count": 2,
       "decision": "reject"
      },
      "logits": [
       [
        -6.395282745361328,
        0.41908711194992065
       ],
       [
        -7.136486053466797,
        -7.079705238342285,
        0.4183274805545807,
        -6.736477851867676
       ],
       [
        0.418684720993042,
        -6.862354278564453,
        -6.696674346923828
       ]
      ],
      "joint_logits": [
       -8.86541748046875,
       -9.235252380371094,
       -9.060043334960938,
       -8.87757682800293,
       -9.250468254089355,
       -9.06668472290039,
       -8.465375900268555,
       -9.078577041625977,
       -8.938993453979492,
       -8.578692436218262,
       -8.820687294006348,
       -8.668829917907715,
       -8.734081268310547,
       -8.968890190124512,
       -8.789766311645508,
       -8.481267929077148,
       -8.958815574645996,
       -8.771374702453613,
       0.41781535744667053,
       -8.796585083007812,
       -8.676133155822754,
       -8.206117630004883,
       -8.600824356079102,
       -8.411439895629883
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_satisfied": {
           "value": true,
           "probability": 0.9989033193107525,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010966806892475185
            },
            {
             "value": true,
             "probability": 0.9989033193107525
            }
           ]
          },
          "flags_count": {
           "value": 2,
           "probability": 0.9981445899997288,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005226122962338439
            },
            {
             "value": 1,
             "probability": 0.0005531454258848083
            },
            {
             "value": 2,
             "probability": 0.9981445899997288
            },
            {
             "value": 3,
             "probability": 0.0007796522781526292
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9985012504775267,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.9985012504775267
            },
            {
             "value": "approve",
             "probability": 0.000687438208727298
            },
            {
             "value": "conditional approval",
             "probability": 0.0008113113137459682
            }
           ]
          }
         },
         "joint_probability": 0.9976336649887637,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002369139206579486,
        "field_brier": 3.470653985456114e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_satisfied": {
           "value": true,
           "probability": 0.9989033193107525,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010966806892475185
            },
            {
             "value": true,
             "probability": 0.9989033193107525
            }
           ]
          },
          "flags_count": {
           "value": 2,
           "probability": 0.9981445899997288,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005226122962338439
            },
            {
             "value": 1,
             "probability": 0.0005531454258848083
            },
            {
             "value": 2,
             "probability": 0.9981445899997288
            },
            {
             "value": 3,
             "probability": 0.0007796522781526292
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9985012504775267,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.9985012504775267
            },
            {
             "value": "approve",
             "probability": 0.000687438208727298
            },
            {
             "value": "conditional approval",
             "probability": 0.0008113113137459682
            }
           ]
          }
         },
         "joint_probability": 0.9976336649887637,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002369139206579486,
        "field_brier": 3.470653985456114e-06
       }
      },
      "timing_ms": {
       "scoring": 39.92093983106315,
       "base_joint": 0.06702914834022522,
       "constrained_joint": 0.0471300445497036,
       "constrained_total": 39.968069875612855
      },
      "input_tokens": 766,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-06acb453c9f440e3-0012",
      "group": "bendsynth-06acb453c9f440e3",
      "provenance": "bendsynth-v1; family=bendsynth-06acb453c9f440e3; domain=coworking space membership site inspection; parameters=min_days=30; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flags_count": 2,
       "wait_satisfied": true,
       "decision": "reject"
      },
      "logits": [
       [
        -8.038219451904297,
        -8.01512336730957,
        -0.7461413145065308,
        -7.803201198577881
       ],
       [
        -7.33894157409668,
        -0.7452729344367981
       ],
       [
        -8.035988807678223,
        -0.7455657124519348,
        -7.673282623291016
       ]
      ],
      "joint_logits": [
       -10.257052421569824,
       -9.69102668762207,
       -9.871960639953613,
       -10.17900276184082,
       -9.38953971862793,
       -9.84930419921875,
       -10.317648887634277,
       -9.763470649719238,
       -9.937703132629395,
       -10.211579322814941,
       -9.175031661987305,
       -9.879507064819336,
       -10.224323272705078,
       -9.298968315124512,
       -9.869709014892578,
       -9.847142219543457,
       -0.7467957735061646,
       -9.460923194885254,
       -10.066286087036133,
       -9.458208084106445,
       -9.684094429016113,
       -9.921942710876465,
       -9.140942573547363,
       -9.576300621032715
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags_count": {
           "value": 2,
           "probability": 0.9977659632295661,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006793901354359501
            },
            {
             "value": 1,
             "probability": 0.0006952640537057546
            },
            {
             "value": 2,
             "probability": 0.9977659632295661
            },
            {
             "value": 3,
             "probability": 0.0008593825812921775
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9986328633832239,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013671366167761056
            },
            {
             "value": true,
             "probability": 0.9986328633832239
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9983404832438153,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0006809072772558646
            },
            {
             "value": "reject",
             "probability": 0.9983404832438153
            },
            {
             "value": "conditional approval",
             "probability": 0.0009786094789288444
            }
           ]
          }
         },
         "joint_probability": 0.9971131628307974,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002891012120495637,
        "field_brier": 4.862617975856437e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags_count": {
           "value": 2,
           "probability": 0.9977659632295661,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006793901354359501
            },
            {
             "value": 1,
             "probability": 0.0006952640537057546
            },
            {
             "value": 2,
             "probability": 0.9977659632295661
            },
            {
             "value": 3,
             "probability": 0.0008593825812921775
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9986328633832239,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013671366167761056
            },
            {
             "value": true,
             "probability": 0.9986328633832239
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9983404832438153,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0006809072772558646
            },
            {
             "value": "reject",
             "probability": 0.9983404832438153
            },
            {
             "value": "conditional approval",
             "probability": 0.0009786094789288444
            }
           ]
          }
         },
         "joint_probability": 0.9971131628307974,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002891012120495637,
        "field_brier": 4.862617975856437e-06
       }
      },
      "timing_ms": {
       "scoring": 38.71367289684713,
       "base_joint": 0.06546895019710064,
       "constrained_joint": 0.04740012809634209,
       "constrained_total": 38.76107302494347
      },
      "input_tokens": 763,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-06acb453c9f440e3-0029",
      "group": "bendsynth-06acb453c9f440e3",
      "provenance": "bendsynth-v1; family=bendsynth-06acb453c9f440e3; domain=coworking space membership site inspection; parameters=min_days=20; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "approve",
       "flags_count": 0,
       "wait_satisfied": true
      },
      "logits": [
       [
        -6.187379837036133,
        -6.234344482421875,
        0.7918344736099243
       ],
       [
        0.7918004989624023,
        -6.564015865325928,
        -6.588438034057617,
        -6.641909122467041
       ],
       [
        -5.883328437805176,
        0.7923915982246399
       ]
      ],
      "joint_logits": [
       -8.316924095153809,
       -8.00595474243164,
       -8.24365234375,
       -8.317954063415527,
       -8.278995513916016,
       -8.323511123657227,
       -8.317944526672363,
       -8.37856388092041,
       -8.326556205749512,
       -8.247586250305176,
       -8.298407554626465,
       -8.314469337463379,
       -8.29085636138916,
       -8.325845718383789,
       -8.336769104003906,
       -8.374651908874512,
       -8.409913063049316,
       0.7912093997001648,
       -8.503189086914062,
       -8.485140800476074,
       -8.558248519897461,
       -8.543931007385254,
       -8.613405227661133,
       -8.628917694091797
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve",
           "probability": 0.9981839495477401,
           "distribution": [
            {
             "value": "conditional approval",
             "probability": 0.0009293438624057074
            },
            {
             "value": "reject",
             "probability": 0.0008867065898542174
            },
            {
             "value": "approve",
             "probability": 0.9981839495477401
            }
           ]
          },
          "flags_count": {
           "value": 0,
           "probability": 0.9981501191421367,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981501191421367
            },
            {
             "value": 1,
             "probability": 0.0006376840395278649
            },
            {
             "value": 2,
             "probability": 0.0006222990072614572
            },
            {
             "value": 3,
             "probability": 0.0005898978110738966
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9987404245450492,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012595754549507567
            },
            {
             "value": true,
             "probability": 0.9987404245450492
            }
           ]
          }
         },
         "joint_probability": 0.9975602999537181,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024426809637899303,
        "field_brier": 4.228321364723418e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve",
           "probability": 0.9981839495477401,
           "distribution": [
            {
             "value": "conditional approval",
             "probability": 0.0009293438624057074
            },
            {
             "value": "reject",
             "probability": 0.0008867065898542174
            },
            {
             "value": "approve",
             "probability": 0.9981839495477401
            }
           ]
          },
          "flags_count": {
           "value": 0,
           "probability": 0.9981501191421367,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981501191421367
            },
            {
             "value": 1,
             "probability": 0.0006376840395278649
            },
            {
             "value": 2,
             "probability": 0.0006222990072614572
            },
            {
             "value": 3,
             "probability": 0.0005898978110738966
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9987404245450492,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012595754549507567
            },
            {
             "value": true,
             "probability": 0.9987404245450492
            }
           ]
          }
         },
         "joint_probability": 0.9975602999537181,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024426809637899303,
        "field_brier": 4.228321364723418e-06
       }
      },
      "timing_ms": {
       "scoring": 38.44455606304109,
       "base_joint": 0.06584892980754375,
       "constrained_joint": 0.047108856961131096,
       "constrained_total": 38.49166492000222
      },
      "input_tokens": 760,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-06acb453c9f440e3-0034",
      "group": "bendsynth-06acb453c9f440e3",
      "provenance": "bendsynth-v1; family=bendsynth-06acb453c9f440e3; domain=coworking space membership site inspection; parameters=min_days=23; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flags_count": 1,
       "wait_satisfied": false,
       "decision": "conditional approval"
      },
      "logits": [
       [
        -6.456132888793945,
        0.7521854043006897,
        -6.664650917053223,
        -6.69648551940918
       ],
       [
        0.7527446150779724,
        -5.8449883460998535
       ],
       [
        -6.320563316345215,
        -6.142464637756348,
        0.7522479891777039
       ]
      ],
      "joint_logits": [
       -8.342339515686035,
       -8.206619262695312,
       -8.363166809082031,
       -8.211385726928711,
       -8.004652976989746,
       -8.41712474822998,
       -8.431221961975098,
       -8.216012001037598,
       0.7516179084777832,
       -8.34606647491455,
       -8.20478343963623,
       -8.493349075317383,
       -8.521584510803223,
       -8.38093376159668,
       -8.728848457336426,
       -8.404909133911133,
       -8.19632625579834,
       -8.591901779174805,
       -8.540434837341309,
       -8.41020679473877,
       -8.805429458618164,
       -8.442116737365723,
       -8.210467338562012,
       -8.622817993164062
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags_count": {
           "value": 1,
           "probability": 0.9980800295325021,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000738979892124985
            },
            {
             "value": 1,
             "probability": 0.9980800295325021
            },
            {
             "value": 2,
             "probability": 0.0005998935801079926
            },
            {
             "value": 3,
             "probability": 0.000581096995264933
            }
           ]
          },
          "wait_satisfied": {
           "value": false,
           "probability": 0.998638400887765,
           "distribution": [
            {
             "value": false,
             "probability": 0.998638400887765
            },
            {
             "value": true,
             "probability": 0.0013615991122350544
            }
           ]
          },
          "decision": {
           "value": "conditional approval",
           "probability": 0.9981424826975925,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0008462714339771961
            },
            {
             "value": "reject",
             "probability": 0.001011245868430327
            },
            {
             "value": "conditional approval",
             "probability": 0.9981424826975925
            }
           ]
          }
         },
         "joint_probability": 0.9975137521786527,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002489343667887643,
        "field_brier": 4.60899742079823e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags_count": {
           "value": 1,
           "probability": 0.9980800295325021,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000738979892124985
            },
            {
             "value": 1,
             "probability": 0.9980800295325021
            },
            {
             "value": 2,
             "probability": 0.0005998935801079926
            },
            {
             "value": 3,
             "probability": 0.000581096995264933
            }
           ]
          },
          "wait_satisfied": {
           "value": false,
           "probability": 0.998638400887765,
           "distribution": [
            {
             "value": false,
             "probability": 0.998638400887765
            },
            {
             "value": true,
             "probability": 0.0013615991122350544
            }
           ]
          },
          "decision": {
           "value": "conditional approval",
           "probability": 0.9981424826975925,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0008462714339771961
            },
            {
             "value": "reject",
             "probability": 0.001011245868430327
            },
            {
             "value": "conditional approval",
             "probability": 0.9981424826975925
            }
           ]
          }
         },
         "joint_probability": 0.9975137521786527,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002489343667887643,
        "field_brier": 4.60899742079823e-06
       }
      },
      "timing_ms": {
       "scoring": 38.43950596638024,
       "base_joint": 0.06627989932894707,
       "constrained_joint": 0.05494896322488785,
       "constrained_total": 38.494454929605126
      },
      "input_tokens": 760,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c127bf8b26278ab-0012",
      "group": "bendsynth-0c127bf8b26278ab",
      "provenance": "bendsynth-v1; family=bendsynth-0c127bf8b26278ab; domain=wind farm turbine maintenance escalation review; parameters=threshold=20; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "priority inspection",
       "over_threshold": true,
       "red_flags": 0
      },
      "logits": [
       [
        0.14299215376377106,
        -6.647555351257324,
        -6.456521987915039
       ],
       [
        -6.199739933013916,
        0.14371779561042786
       ],
       [
        0.14315462112426758,
        -6.584899425506592,
        -6.646220684051514
       ]
      ],
      "joint_logits": [
       -8.631339073181152,
       -8.57305908203125,
       -8.646931648254395,
       0.14223498106002808,
       -8.645596504211426,
       -8.778043746948242,
       -8.45842456817627,
       -8.334725379943848,
       -8.405339241027832,
       -8.652212142944336,
       -8.390894889831543,
       -8.423182487487793,
       -8.415840148925781,
       -8.111494064331055,
       -8.147284507751465,
       -8.202214241027832,
       -8.298041343688965,
       -8.3505220413208
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "priority inspection",
           "probability": 0.9975207798021251,
           "distribution": [
            {
             "value": "priority inspection",
             "probability": 0.9975207798021251
            },
            {
             "value": "immediate shutdown",
             "probability": 0.001121565404085072
            },
            {
             "value": "routine monitoring",
             "probability": 0.0013576547937899162
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9982448731700021,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017551268299978995
            },
            {
             "value": true,
             "probability": 0.9982448731700021
            }
           ]
          },
          "red_flags": {
           "value": 0,
           "probability": 0.997682849803363,
           "distribution": [
            {
             "value": 0,
             "probability": 0.997682849803363
            },
            {
             "value": 1,
             "probability": 0.001194086590764424
            },
            {
             "value": 2,
             "probability": 0.0011230636058725577
            }
           ]
          }
         },
         "joint_probability": 0.9967657982650837,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 18,
         "valid_assignments": 18,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0032394430694204796,
        "field_brier": 7.821636115301059e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "priority inspection",
           "probability": 0.9975207798021251,
           "distribution": [
            {
             "value": "priority inspection",
             "probability": 0.9975207798021251
            },
            {
             "value": "immediate shutdown",
             "probability": 0.001121565404085072
            },
            {
             "value": "routine monitoring",
             "probability": 0.0013576547937899162
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9982448731700021,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017551268299978995
            },
            {
             "value": true,
             "probability": 0.9982448731700021
            }
           ]
          },
          "red_flags": {
           "value": 0,
           "probability": 0.997682849803363,
           "distribution": [
            {
             "value": 0,
             "probability": 0.997682849803363
            },
            {
             "value": 1,
             "probability": 0.001194086590764424
            },
            {
             "value": 2,
             "probability": 0.0011230636058725577
            }
           ]
          }
         },
         "joint_probability": 0.9967657982650837,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 18,
         "valid_assignments": 18,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0032394430694204796,
        "field_brier": 7.821636115301059e-06
       }
      },
      "timing_ms": {
       "scoring": 37.37618797458708,
       "base_joint": 0.0575000885874033,
       "constrained_joint": 0.04067900590598583,
       "constrained_total": 37.41686698049307
      },
      "input_tokens": 711,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c127bf8b26278ab-0021",
      "group": "bendsynth-0c127bf8b26278ab",
      "provenance": "bendsynth-v1; family=bendsynth-0c127bf8b26278ab; domain=wind farm turbine maintenance escalation review; parameters=threshold=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "over_threshold": false,
       "red_flags": 1,
       "decision": "routine monitoring"
      },
      "logits": [
       [
        0.03894729167222977,
        -6.49807596206665
       ],
       [
        -6.792351245880127,
        0.03832213580608368,
        -6.87469482421875
       ],
       [
        -7.011898040771484,
        0.03854454308748245,
        -6.884202480316162
       ]
      ],
      "joint_logits": [
       -8.647561073303223,
       -8.57783317565918,
       -8.49223804473877,
       -9.228456497192383,
       0.03771911934018135,
       -8.970175743103027,
       -8.757861137390137,
       -8.869543075561523,
       -8.600118637084961,
       -8.698911666870117,
       -8.491034507751465,
       -8.614706039428711,
       -8.95095157623291,
       -8.9376802444458,
       -8.872529029846191,
       -8.657973289489746,
       -8.557896614074707,
       -8.590070724487305
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_threshold": {
           "value": false,
           "probability": 0.9985533011305048,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985533011305048
            },
            {
             "value": true,
             "probability": 0.0014466988694952657
            }
           ]
          },
          "red_flags": {
           "value": 1,
           "probability": 0.9979294137127993,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0010778939411096949
            },
            {
             "value": 1,
             "probability": 0.9979294137127993
            },
            {
             "value": 2,
             "probability": 0.000992692346091098
            }
           ]
          },
          "decision": {
           "value": "routine monitoring",
           "probability": 0.9981512788452445,
           "distribution": [
            {
             "value": "immediate shutdown",
             "probability": 0.0008654223511762967
            },
            {
             "value": "routine monitoring",
             "probability": 0.9981512788452445
            },
            {
             "value": "priority inspection",
             "probability": 0.0009832988035792418
            }
           ]
          }
         },
         "joint_probability": 0.9973277194406566,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 18,
         "valid_assignments": 18,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002675857474806257,
        "field_brier": 5.251366181362185e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_threshold": {
           "value": false,
           "probability": 0.9985533011305048,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985533011305048
            },
            {
             "value": true,
             "probability": 0.0014466988694952657
            }
           ]
          },
          "red_flags": {
           "value": 1,
           "probability": 0.9979294137127993,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0010778939411096949
            },
            {
             "value": 1,
             "probability": 0.9979294137127993
            },
            {
             "value": 2,
             "probability": 0.000992692346091098
            }
           ]
          },
          "decision": {
           "value": "routine monitoring",
           "probability": 0.9981512788452445,
           "distribution": [
            {
             "value": "immediate shutdown",
             "probability": 0.0008654223511762967
            },
            {
             "value": "routine monitoring",
             "probability": 0.9981512788452445
            },
            {
             "value": "priority inspection",
             "probability": 0.0009832988035792418
            }
           ]
          }
         },
         "joint_probability": 0.9973277194406566,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 18,
         "valid_assignments": 18,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002675857474806257,
        "field_brier": 5.251366181362185e-06
       }
      },
      "timing_ms": {
       "scoring": 35.56488687172532,
       "base_joint": 0.05924887955188751,
       "constrained_joint": 0.039190053939819336,
       "constrained_total": 35.60407692566514
      },
      "input_tokens": 729,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c127bf8b26278ab-0033",
      "group": "bendsynth-0c127bf8b26278ab",
      "provenance": "bendsynth-v1; family=bendsynth-0c127bf8b26278ab; domain=wind farm turbine maintenance escalation review; parameters=threshold=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "red_flags": 0,
       "over_threshold": true,
       "decision": "priority inspection"
      },
      "logits": [
       [
        0.00532673392444849,
        -6.676264762878418,
        -6.766827583312988
       ],
       [
        -6.303404808044434,
        0.005905162077397108
       ],
       [
        -6.63437557220459,
        -6.731783390045166,
        0.005232281982898712
       ]
      ],
      "joint_logits": [
       -8.567831039428711,
       -8.537129402160645,
       -8.74101448059082,
       -8.454453468322754,
       -8.811164855957031,
       0.004423059057444334,
       -8.256549835205078,
       -8.378068923950195,
       -8.633379936218262,
       -8.446690559387207,
       -8.466243743896484,
       -8.691868782043457,
       -8.33017635345459,
       -8.476954460144043,
       -8.696405410766602,
       -8.537229537963867,
       -8.523343086242676,
       -8.873808860778809
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "red_flags": {
           "value": 0,
           "probability": 0.997606735200549,
           "distribution": [
            {
             "value": 0,
             "probability": 0.997606735200549
            },
            {
             "value": 1,
             "probability": 0.0012507804254961355
            },
            {
             "value": 2,
             "probability": 0.0011424843739549766
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9981840152767406,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018159847232594548
            },
            {
             "value": true,
             "probability": 0.9981840152767406
            }
           ]
          },
          "decision": {
           "value": "priority inspection",
           "probability": 0.9975124813177866,
           "distribution": [
            {
             "value": "routine monitoring",
             "probability": 0.0013042872482801188
            },
            {
             "value": "immediate shutdown",
             "probability": 0.0011832314339331974
            },
            {
             "value": "priority inspection",
             "probability": 0.9975124813177866
            }
           ]
          }
         },
         "joint_probability": 0.9967056432718705,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 18,
         "valid_assignments": 18,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0032997950684289603,
        "field_brier": 8.160663564894867e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "red_flags": {
           "value": 0,
           "probability": 0.997606735200549,
           "distribution": [
            {
             "value": 0,
             "probability": 0.997606735200549
            },
            {
             "value": 1,
             "probability": 0.0012507804254961355
            },
            {
             "value": 2,
             "probability": 0.0011424843739549766
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9981840152767406,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018159847232594548
            },
            {
             "value": true,
             "probability": 0.9981840152767406
            }
           ]
          },
          "decision": {
           "value": "priority inspection",
           "probability": 0.9975124813177866,
           "distribution": [
            {
             "value": "routine monitoring",
             "probability": 0.0013042872482801188
            },
            {
             "value": "immediate shutdown",
             "probability": 0.0011832314339331974
            },
            {
             "value": "priority inspection",
             "probability": 0.9975124813177866
            }
           ]
          }
         },
         "joint_probability": 0.9967056432718705,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 18,
         "valid_assignments": 18,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0032997950684289603,
        "field_brier": 8.160663564894867e-06
       }
      },
      "timing_ms": {
       "scoring": 35.585376899689436,
       "base_joint": 0.06541889160871506,
       "constrained_joint": 0.03988994285464287,
       "constrained_total": 35.62526684254408
      },
      "input_tokens": 717,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c127bf8b26278ab-0038",
      "group": "bendsynth-0c127bf8b26278ab",
      "provenance": "bendsynth-v1; family=bendsynth-0c127bf8b26278ab; domain=wind farm turbine maintenance escalation review; parameters=threshold=25; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "priority inspection",
       "over_threshold": true,
       "red_flags": 0
      },
      "logits": [
       [
        0.39924126863479614,
        -6.161431312561035,
        -6.449762344360352
       ],
       [
        -5.957391262054443,
        0.399981290102005
       ],
       [
        0.3994382619857788,
        -6.381011009216309,
        -6.375312805175781
       ]
      ],
      "joint_logits": [
       -8.258368492126465,
       -8.383434295654297,
       -8.333517074584961,
       0.3984651565551758,
       -8.402515411376953,
       -8.49600601196289,
       -8.040545463562012,
       -7.910778999328613,
       -7.868714332580566,
       -7.908014297485352,
       -8.032358169555664,
       -7.9712724685668945,
       -8.185433387756348,
       -8.222739219665527,
       -8.334742546081543,
       -8.41028881072998,
       -8.179094314575195,
       -8.143277168273926
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "priority inspection",
           "probability": 0.9975306670093437,
           "distribution": [
            {
             "value": "priority inspection",
             "probability": 0.9975306670093437
            },
            {
             "value": "routine monitoring",
             "probability": 0.001411440102756645
            },
            {
             "value": "immediate shutdown",
             "probability": 0.0010578928878997014
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9982690847726027,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017309152273973342
            },
            {
             "value": true,
             "probability": 0.9982690847726027
            }
           ]
          },
          "red_flags": {
           "value": 0,
           "probability": 0.9977271579071519,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9977271579071519
            },
            {
             "value": 1,
             "probability": 0.0011331833451560499
            },
            {
             "value": 2,
             "probability": 0.0011396587476920135
            }
           ]
          }
         },
         "joint_probability": 0.9967566922237651,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 18,
         "valid_assignments": 18,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003248578698802362,
        "field_brier": 7.649926242501998e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "priority inspection",
           "probability": 0.9975306670093437,
           "distribution": [
            {
             "value": "priority inspection",
             "probability": 0.9975306670093437
            },
            {
             "value": "routine monitoring",
             "probability": 0.001411440102756645
            },
            {
             "value": "immediate shutdown",
             "probability": 0.0010578928878997014
            }
           ]
          },
          "over_threshold": {
           "value": true,
           "probability": 0.9982690847726027,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017309152273973342
            },
            {
             "value": true,
             "probability": 0.9982690847726027
            }
           ]
          },
          "red_flags": {
           "value": 0,
           "probability": 0.9977271579071519,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9977271579071519
            },
            {
             "value": 1,
             "probability": 0.0011331833451560499
            },
            {
             "value": 2,
             "probability": 0.0011396587476920135
            }
           ]
          }
         },
         "joint_probability": 0.9967566922237651,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 18,
         "valid_assignments": 18,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003248578698802362,
        "field_brier": 7.649926242501998e-06
       }
      },
      "timing_ms": {
       "scoring": 36.01767192594707,
       "base_joint": 0.06079999729990959,
       "constrained_joint": 0.040329061448574066,
       "constrained_total": 36.058000987395644
      },
      "input_tokens": 690,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-046da519f77bd7b0-0006",
      "group": "bendsynth-046da519f77bd7b0",
      "provenance": "bendsynth-v1; family=bendsynth-046da519f77bd7b0; domain=cemetery plot allocation site inspection; parameters=min_depth=90; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "factors": 2,
       "depth_ok": false,
       "decision": "flag for review"
      },
      "logits": [
       [
        -6.8333353996276855,
        -6.88749885559082,
        0.43559736013412476,
        -6.675167560577393
       ],
       [
        0.4362725019454956,
        -6.068144798278809
       ],
       [
        -6.473230361938477,
        -6.537143707275391,
        0.43583452701568604
       ]
      ],
      "joint_logits": [
       -8.63914680480957,
       -8.702735900878906,
       -8.858267784118652,
       -8.643107414245605,
       -8.524201393127441,
       -8.43585205078125,
       -8.783866882324219,
       -8.853882789611816,
       -8.52318000793457,
       -8.701361656188965,
       -8.63803482055664,
       -8.611215591430664,
       -8.133936882019043,
       -8.728894233703613,
       0.43497800827026367,
       -8.681010246276855,
       -8.67392635345459,
       -8.73310661315918,
       -8.530919075012207,
       -8.522171020507812,
       -8.682191848754883,
       -8.464385032653809,
       -8.368813514709473,
       -8.281465530395508
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 2,
           "probability": 0.9978314712825049,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006953442959679853
            },
            {
             "value": 1,
             "probability": 0.0006586839450349053
            },
            {
             "value": 2,
             "probability": 0.9978314712825049
            },
            {
             "value": 3,
             "probability": 0.0008145004764921421
            }
           ]
          },
          "depth_ok": {
           "value": false,
           "probability": 0.9985054244719537,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985054244719537
            },
            {
             "value": true,
             "probability": 0.0014945755280462755
            }
           ]
          },
          "decision": {
           "value": "flag for review",
           "probability": 0.9980681896215592,
           "distribution": [
            {
             "value": "standard allocation",
             "probability": 0.0009967617627512806
            },
            {
             "value": "reject",
             "probability": 0.000935048615689568
            },
            {
             "value": "flag for review",
             "probability": 0.9980681896215592
            }
           ]
          }
         },
         "joint_probability": 0.9972136358209627,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027902533177576357,
        "field_brier": 5.450183111965365e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 2,
           "probability": 0.9978314712825049,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006953442959679853
            },
            {
             "value": 1,
             "probability": 0.0006586839450349053
            },
            {
             "value": 2,
             "probability": 0.9978314712825049
            },
            {
             "value": 3,
             "probability": 0.0008145004764921421
            }
           ]
          },
          "depth_ok": {
           "value": false,
           "probability": 0.9985054244719537,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985054244719537
            },
            {
             "value": true,
             "probability": 0.0014945755280462755
            }
           ]
          },
          "decision": {
           "value": "flag for review",
           "probability": 0.9980681896215592,
           "distribution": [
            {
             "value": "standard allocation",
             "probability": 0.0009967617627512806
            },
            {
             "value": "reject",
             "probability": 0.000935048615689568
            },
            {
             "value": "flag for review",
             "probability": 0.9980681896215592
            }
           ]
          }
         },
         "joint_probability": 0.9972136358209627,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027902533177576357,
        "field_brier": 5.450183111965365e-06
       }
      },
      "timing_ms": {
       "scoring": 37.33470686711371,
       "base_joint": 0.06607896648347378,
       "constrained_joint": 0.047389883548021317,
       "constrained_total": 37.38209675066173
      },
      "input_tokens": 685,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-02692cc16a98b9bd-0014",
      "group": "bendsynth-02692cc16a98b9bd",
      "provenance": "bendsynth-v1; family=bendsynth-02692cc16a98b9bd; domain=esports tournament seeding appeal desk; parameters=point_threshold=206; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "concerns": 1,
       "ruling": "keep default seed",
       "flagged_for_review": false
      },
      "logits": [
       [
        -6.91672420501709,
        0.5539948344230652,
        -6.8195929527282715,
        -6.920032024383545
       ],
       [
        -6.5056257247924805,
        -6.520835876464844,
        0.5540545582771301
       ],
       [
        0.5545428395271301,
        -6.157696723937988
       ]
      ],
      "joint_logits": [
       -8.575329780578613,
       -8.672853469848633,
       -8.684350967407227,
       -8.686391830444336,
       -8.809459686279297,
       -8.847076416015625,
       -8.566515922546387,
       -8.551185607910156,
       -8.453295707702637,
       -8.604694366455078,
       0.5534176826477051,
       -8.403335571289062,
       -8.496408462524414,
       -8.572815895080566,
       -8.55358600616455,
       -8.5620756149292,
       -8.756418228149414,
       -8.757584571838379,
       -8.609061241149902,
       -8.64721393585205,
       -8.638569831848145,
       -8.641046524047852,
       -8.923116683959961,
       -8.855301856994629
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 1,
           "probability": 0.9982383402077575,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000568515239425819
            },
            {
             "value": 1,
             "probability": 0.9982383402077575
            },
            {
             "value": 2,
             "probability": 0.0006265067239713163
            },
            {
             "value": 3,
             "probability": 0.0005666378288453491
            }
           ]
          },
          "ruling": {
           "value": "keep default seed",
           "probability": 0.9982977644678342,
           "distribution": [
            {
             "value": "escalate to committee",
             "probability": 0.0008575904209997278
            },
            {
             "value": "downgrade one seed",
             "probability": 0.00084464511116603
            },
            {
             "value": "keep default seed",
             "probability": 0.9982977644678342
            }
           ]
          },
          "flagged_for_review": {
           "value": false,
           "probability": 0.9987855391354405,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987855391354405
            },
            {
             "value": true,
             "probability": 0.0012144608645594998
            }
           ]
          }
         },
         "joint_probability": 0.9976622654961732,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002340471271187569,
        "field_brier": 3.812188929797029e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 1,
           "probability": 0.9982383402077575,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000568515239425819
            },
            {
             "value": 1,
             "probability": 0.9982383402077575
            },
            {
             "value": 2,
             "probability": 0.0006265067239713163
            },
            {
             "value": 3,
             "probability": 0.0005666378288453491
            }
           ]
          },
          "ruling": {
           "value": "keep default seed",
           "probability": 0.9982977644678342,
           "distribution": [
            {
             "value": "escalate to committee",
             "probability": 0.0008575904209997278
            },
            {
             "value": "downgrade one seed",
             "probability": 0.00084464511116603
            },
            {
             "value": "keep default seed",
             "probability": 0.9982977644678342
            }
           ]
          },
          "flagged_for_review": {
           "value": false,
           "probability": 0.9987855391354405,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987855391354405
            },
            {
             "value": true,
             "probability": 0.0012144608645594998
            }
           ]
          }
         },
         "joint_probability": 0.9976622654961732,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002340471271187569,
        "field_brier": 3.812188929797029e-06
       }
      },
      "timing_ms": {
       "scoring": 34.39696878194809,
       "base_joint": 0.06608921103179455,
       "constrained_joint": 0.046508852392435074,
       "constrained_total": 34.443477634340525
      },
      "input_tokens": 634,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-02692cc16a98b9bd-0033",
      "group": "bendsynth-02692cc16a98b9bd",
      "provenance": "bendsynth-v1; family=bendsynth-02692cc16a98b9bd; domain=esports tournament seeding appeal desk; parameters=point_threshold=100; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flagged_for_review": true,
       "ruling": "escalate to committee",
       "concerns": 2
      },
      "logits": [
       [
        -6.538216590881348,
        0.11165425181388855
       ],
       [
        -7.0293288230896,
        0.11145451664924622,
        -7.150125980377197
       ],
       [
        -7.4808349609375,
        -7.23823881149292,
        0.11120469868183136,
        -7.3123779296875
       ]
      ],
      "joint_logits": [
       -9.125612258911133,
       -8.963943481445312,
       -9.042547225952148,
       -8.94857120513916,
       -9.100563049316406,
       -8.893023490905762,
       -8.740108489990234,
       -8.963669776916504,
       -9.241323471069336,
       -9.067879676818848,
       -9.19477653503418,
       -9.103238105773926,
       -9.409063339233398,
       -9.19656753540039,
       -9.013474464416504,
       -9.25713062286377,
       -9.320951461791992,
       -8.842577934265137,
       0.1106639951467514,
       -9.092177391052246,
       -9.499545097351074,
       -9.292157173156738,
       -9.1851806640625,
       -9.31510066986084
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flagged_for_review": {
           "value": true,
           "probability": 0.9987074834344365,
           "distribution": [
            {
             "value": false,
             "probability": 0.001292516565563475
            },
            {
             "value": true,
             "probability": 0.9987074834344365
            }
           ]
          },
          "ruling": {
           "value": "escalate to committee",
           "probability": 0.9985081001251439,
           "distribution": [
            {
             "value": "keep default seed",
             "probability": 0.000790949583513976
            },
            {
             "value": "escalate to committee",
             "probability": 0.9985081001251439
            },
            {
             "value": "downgrade one seed",
             "probability": 0.0007009502913421529
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.9982586300091716,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005035726863567127
            },
            {
             "value": 1,
             "probability": 0.000641830284655234
            },
            {
             "value": 2,
             "probability": 0.9982586300091716
            },
            {
             "value": 3,
             "probability": 0.0005959670198164129
            }
           ]
          }
         },
         "joint_probability": 0.9977189777627721,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022836277313320248,
        "field_brier": 3.5789912113710105e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flagged_for_review": {
           "value": true,
           "probability": 0.9987074834344365,
           "distribution": [
            {
             "value": false,
             "probability": 0.001292516565563475
            },
            {
             "value": true,
             "probability": 0.9987074834344365
            }
           ]
          },
          "ruling": {
           "value": "escalate to committee",
           "probability": 0.9985081001251439,
           "distribution": [
            {
             "value": "keep default seed",
             "probability": 0.000790949583513976
            },
            {
             "value": "escalate to committee",
             "probability": 0.9985081001251439
            },
            {
             "value": "downgrade one seed",
             "probability": 0.0007009502913421529
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.9982586300091716,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005035726863567127
            },
            {
             "value": 1,
             "probability": 0.000641830284655234
            },
            {
             "value": 2,
             "probability": 0.9982586300091716
            },
            {
             "value": 3,
             "probability": 0.0005959670198164129
            }
           ]
          }
         },
         "joint_probability": 0.9977189777627721,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022836277313320248,
        "field_brier": 3.5789912113710105e-06
       }
      },
      "timing_ms": {
       "scoring": 34.399430034682155,
       "base_joint": 0.06442004814743996,
       "constrained_joint": 0.048249028623104095,
       "constrained_total": 34.44767906330526
      },
      "input_tokens": 625,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-02692cc16a98b9bd-0037",
      "group": "bendsynth-02692cc16a98b9bd",
      "provenance": "bendsynth-v1; family=bendsynth-02692cc16a98b9bd; domain=esports tournament seeding appeal desk; parameters=point_threshold=149; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "concerns": 0,
       "ruling": "keep default seed",
       "flagged_for_review": false
      },
      "logits": [
       [
        1.308964729309082,
        -6.099424362182617,
        -6.075667858123779,
        -6.152979850769043
       ],
       [
        -5.807986736297607,
        1.3091380596160889,
        -5.801066875457764
       ],
       [
        1.3095192909240723,
        -5.378352642059326
       ]
      ],
      "joint_logits": [
       -7.854280471801758,
       -7.896306037902832,
       1.308467149734497,
       -8.019754409790039,
       -8.016033172607422,
       -7.769873142242432,
       -8.000174522399902,
       -7.796460151672363,
       -7.948537826538086,
       -7.857250690460205,
       -7.862768173217773,
       -7.894881248474121,
       -7.952054977416992,
       -7.785400867462158,
       -7.992683410644531,
       -7.813568115234375,
       -7.807379722595215,
       -7.871266841888428,
       -8.008882522583008,
       -7.832881927490234,
       -8.116724014282227,
       -7.892738342285156,
       -7.885145664215088,
       -7.957820415496826
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 0,
           "probability": 0.998201835760117,
           "distribution": [
            {
             "value": 0,
             "probability": 0.998201835760117
            },
            {
             "value": 1,
             "probability": 0.0006050566100561257
            },
            {
             "value": 2,
             "probability": 0.0006196026504883941
            },
            {
             "value": 3,
             "probability": 0.0005735049793384711
            }
           ]
          },
          "ruling": {
           "value": "keep default seed",
           "probability": 0.9983748220626639,
           "distribution": [
            {
             "value": "downgrade one seed",
             "probability": 0.0008097774299186285
            },
            {
             "value": "keep default seed",
             "probability": 0.9983748220626639
            },
            {
             "value": "escalate to committee",
             "probability": 0.0008154005074175008
            }
           ]
          },
          "flagged_for_review": {
           "value": false,
           "probability": 0.998755618807063,
           "distribution": [
            {
             "value": false,
             "probability": 0.998755618807063
            },
            {
             "value": true,
             "probability": 0.00124438119293708
            }
           ]
          }
         },
         "joint_probability": 0.9977053425137518,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002297294247152178,
        "field_brier": 3.7903644829873324e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 0,
           "probability": 0.998201835760117,
           "distribution": [
            {
             "value": 0,
             "probability": 0.998201835760117
            },
            {
             "value": 1,
             "probability": 0.0006050566100561257
            },
            {
             "value": 2,
             "probability": 0.0006196026504883941
            },
            {
             "value": 3,
             "probability": 0.0005735049793384711
            }
           ]
          },
          "ruling": {
           "value": "keep default seed",
           "probability": 0.9983748220626639,
           "distribution": [
            {
             "value": "downgrade one seed",
             "probability": 0.0008097774299186285
            },
            {
             "value": "keep default seed",
             "probability": 0.9983748220626639
            },
            {
             "value": "escalate to committee",
             "probability": 0.0008154005074175008
            }
           ]
          },
          "flagged_for_review": {
           "value": false,
           "probability": 0.998755618807063,
           "distribution": [
            {
             "value": false,
             "probability": 0.998755618807063
            },
            {
             "value": true,
             "probability": 0.00124438119293708
            }
           ]
          }
         },
         "joint_probability": 0.9977053425137518,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002297294247152178,
        "field_brier": 3.7903644829873324e-06
       }
      },
      "timing_ms": {
       "scoring": 35.12397105805576,
       "base_joint": 0.06359908729791641,
       "constrained_joint": 0.04624016582965851,
       "constrained_total": 35.17021122388542
      },
      "input_tokens": 613,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0356a9a675d005a7-0007",
      "group": "bendsynth-0356a9a675d005a7",
      "provenance": "bendsynth-v1; family=bendsynth-0356a9a675d005a7; domain=placement appeal review board at a language school; parameters=gap=55; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "concerns": 2,
       "waiver_met": true,
       "decision": "uphold placement"
      },
      "logits": [
       [
        -7.452763080596924,
        -7.432711124420166,
        0.11358768492937088,
        -7.325981140136719
       ],
       [
        -6.582703113555908,
        0.11398527026176453
       ],
       [
        -6.376833915710449,
        -7.6715192794799805,
        -7.411764621734619,
        0.11274736374616623
       ]
      ],
      "joint_logits": [
       -9.474841117858887,
       -9.631103515625,
       -9.403175354003906,
       -9.108009338378906,
       -9.928936004638672,
       -9.900667190551758,
       -9.704699516296387,
       -9.381057739257812,
       -9.542469024658203,
       -9.684385299682617,
       -9.449078559875488,
       -9.139290809631348,
       -9.7344970703125,
       -9.9719877243042,
       -9.748111724853516,
       -9.141106605529785,
       -9.464664459228516,
       -9.693887710571289,
       -9.480121612548828,
       -8.865979194641113,
       -6.7038798332214355,
       -9.865971565246582,
       -9.384893417358398,
       0.11205225437879562,
       -9.406892776489258,
       -9.515089988708496,
       -9.252213478088379,
       -9.028961181640625,
       -9.751530647277832,
       -9.828850746154785,
       -9.607215881347656,
       -9.140636444091797
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 2,
           "probability": 0.9983694852794152,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005167339158522858
            },
            {
             "value": 1,
             "probability": 0.0005272001113014084
            },
            {
             "value": 2,
             "probability": 0.9983694852794152
            },
            {
             "value": 3,
             "probability": 0.0005865806934312035
            }
           ]
          },
          "waiver_met": {
           "value": true,
           "probability": 0.9987665283787825,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012334716212174086
            },
            {
             "value": true,
             "probability": 0.9987665283787825
            }
           ]
          },
          "decision": {
           "value": "uphold placement",
           "probability": 0.9975310017371686,
           "distribution": [
            {
             "value": "one level up",
             "probability": 0.0015154340137908278
            },
            {
             "value": "full retest",
             "probability": 0.0004152047936401414
            },
            {
             "value": "direct to advanced",
             "probability": 0.0005383594554004802
            },
            {
             "value": "uphold placement",
             "probability": 0.9975310017371686
            }
           ]
          }
         },
         "joint_probability": 0.9968378243512634,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0031671858911189064,
        "field_brier": 5.14841071256516e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 2,
           "probability": 0.9983694852794152,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005167339158522858
            },
            {
             "value": 1,
             "probability": 0.0005272001113014084
            },
            {
             "value": 2,
             "probability": 0.9983694852794152
            },
            {
             "value": 3,
             "probability": 0.0005865806934312035
            }
           ]
          },
          "waiver_met": {
           "value": true,
           "probability": 0.9987665283787825,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012334716212174086
            },
            {
             "value": true,
             "probability": 0.9987665283787825
            }
           ]
          },
          "decision": {
           "value": "uphold placement",
           "probability": 0.9975310017371686,
           "distribution": [
            {
             "value": "one level up",
             "probability": 0.0015154340137908278
            },
            {
             "value": "full retest",
             "probability": 0.0004152047936401414
            },
            {
             "value": "direct to advanced",
             "probability": 0.0005383594554004802
            },
            {
             "value": "uphold placement",
             "probability": 0.9975310017371686
            }
           ]
          }
         },
         "joint_probability": 0.9968378243512634,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0031671858911189064,
        "field_brier": 5.14841071256516e-06
       }
      },
      "timing_ms": {
       "scoring": 37.058710819110274,
       "base_joint": 0.07678894326090813,
       "constrained_joint": 0.0592090655118227,
       "constrained_total": 37.1179198846221
      },
      "input_tokens": 722,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0356a9a675d005a7-0039",
      "group": "bendsynth-0356a9a675d005a7",
      "provenance": "bendsynth-v1; family=bendsynth-0356a9a675d005a7; domain=placement appeal review board at a language school; parameters=gap=79; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "waiver_met": true,
       "decision": "one level up",
       "concerns": 1
      },
      "logits": [
       [
        -5.035046577453613,
        1.6821951866149902
       ],
       [
        -5.908690929412842,
        -5.705683708190918,
        -5.861476421356201,
        1.6817517280578613
       ],
       [
        -5.9376749992370605,
        1.6819126605987549,
        -5.863659858703613,
        -5.973302364349365
       ]
      ],
      "joint_logits": [
       -7.9570488929748535,
       -7.8080949783325195,
       -7.841567516326904,
       -7.955504417419434,
       -7.773905277252197,
       -7.552547931671143,
       -7.700727462768555,
       -7.806670188903809,
       -7.885227680206299,
       -7.720695495605469,
       -7.899280071258545,
       -7.966213226318359,
       -8.039420127868652,
       -7.35910177230835,
       -7.906836032867432,
       -8.000328063964844,
       -8.175673484802246,
       -7.899903774261475,
       -8.11274528503418,
       -8.241634368896484,
       -8.032426834106445,
       -7.521693229675293,
       -7.981479167938232,
       -8.064699172973633,
       -8.135292053222656,
       -7.701948165893555,
       -8.134922981262207,
       -8.21286678314209,
       -8.219099044799805,
       1.6812845468521118,
       -8.042513847351074,
       -8.267605781555176
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "waiver_met": {
           "value": true,
           "probability": 0.9987915915102791,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012084084897209292
            },
            {
             "value": true,
             "probability": 0.9987915915102791
            }
           ]
          },
          "decision": {
           "value": "one level up",
           "probability": 0.9983488073270864,
           "distribution": [
            {
             "value": "full retest",
             "probability": 0.0005044230855683221
            },
            {
             "value": "direct to advanced",
             "probability": 0.0006179592367933973
            },
            {
             "value": "uphold placement",
             "probability": 0.0005288103505518074
            },
            {
             "value": "one level up",
             "probability": 0.9983488073270864
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9985094678275663,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004900127565352391
            },
            {
             "value": 1,
             "probability": 0.9985094678275663
            },
            {
             "value": 2,
             "probability": 0.0005276571553154777
            },
            {
             "value": 3,
             "probability": 0.00047286226058294617
            }
           ]
          }
         },
         "joint_probability": 0.9978825068825149,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002119738175860046,
        "field_brier": 3.1755718344174397e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "waiver_met": {
           "value": true,
           "probability": 0.9987915915102791,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012084084897209292
            },
            {
             "value": true,
             "probability": 0.9987915915102791
            }
           ]
          },
          "decision": {
           "value": "one level up",
           "probability": 0.9983488073270864,
           "distribution": [
            {
             "value": "full retest",
             "probability": 0.0005044230855683221
            },
            {
             "value": "direct to advanced",
             "probability": 0.0006179592367933973
            },
            {
             "value": "uphold placement",
             "probability": 0.0005288103505518074
            },
            {
             "value": "one level up",
             "probability": 0.9983488073270864
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9985094678275663,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004900127565352391
            },
            {
             "value": 1,
             "probability": 0.9985094678275663
            },
            {
             "value": 2,
             "probability": 0.0005276571553154777
            },
            {
             "value": 3,
             "probability": 0.00047286226058294617
            }
           ]
          }
         },
         "joint_probability": 0.9978825068825149,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002119738175860046,
        "field_brier": 3.1755718344174397e-06
       }
      },
      "timing_ms": {
       "scoring": 37.66459506005049,
       "base_joint": 0.07442920468747616,
       "constrained_joint": 0.06673019379377365,
       "constrained_total": 37.73132525384426
      },
      "input_tokens": 728,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05a2b74462b2b3b5-0018",
      "group": "bendsynth-05a2b74462b2b3b5",
      "provenance": "bendsynth-v1; family=bendsynth-05a2b74462b2b3b5; domain=regional archive document request intake, filed by a visiting field inspector; parameters=admin_limit=21,restricted_limit=100; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "barriers": 2,
       "decision": "supervised viewing",
       "past_limit": true
      },
      "logits": [
       [
        -6.676019668579102,
        -6.511602401733398,
        0.7075912952423096,
        -6.608720302581787
       ],
       [
        -6.2997918128967285,
        0.7078277468681335,
        -6.331757068634033
       ],
       [
        -5.963883399963379,
        0.7083429098129272
       ]
      ],
      "joint_logits": [
       -8.56627082824707,
       -8.363295555114746,
       -8.55189037322998,
       -8.555228233337402,
       -8.429180145263672,
       -8.364580154418945,
       -8.350419998168945,
       -8.272953033447266,
       -8.512543678283691,
       -7.96759033203125,
       -8.418274879455566,
       -8.393962860107422,
       -8.534005165100098,
       -8.288296699523926,
       -8.11219310760498,
       0.7070165276527405,
       -8.691301345825195,
       -8.293540000915527,
       -8.398233413696289,
       -8.302119255065918,
       -8.575411796569824,
       -8.402637481689453,
       -8.371513366699219,
       -8.373213768005371
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "barriers": {
           "value": 2,
           "probability": 0.9979857108090046,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006201015488151944
            },
            {
             "value": 1,
             "probability": 0.0007309174721652796
            },
            {
             "value": 2,
             "probability": 0.9979857108090046
            },
            {
             "value": 3,
             "probability": 0.0006632701700149638
            }
           ]
          },
          "decision": {
           "value": "supervised viewing",
           "probability": 0.9982217175928023,
           "distribution": [
            {
             "value": "release copy",
             "probability": 0.0009033507529927166
            },
            {
             "value": "supervised viewing",
             "probability": 0.9982217175928023
            },
            {
             "value": "deny",
             "probability": 0.0008749316542049563
            }
           ]
          },
          "past_limit": {
           "value": true,
           "probability": 0.9987360218039196,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012639781960803451
            },
            {
             "value": true,
             "probability": 0.9987360218039196
            }
           ]
          }
         },
         "joint_probability": 0.9974121612424494,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025911930003441877,
        "field_brier": 4.4517242026321614e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "barriers": {
           "value": 2,
           "probability": 0.9979857108090046,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006201015488151944
            },
            {
             "value": 1,
             "probability": 0.0007309174721652796
            },
            {
             "value": 2,
             "probability": 0.9979857108090046
            },
            {
             "value": 3,
             "probability": 0.0006632701700149638
            }
           ]
          },
          "decision": {
           "value": "supervised viewing",
           "probability": 0.9982217175928023,
           "distribution": [
            {
             "value": "release copy",
             "probability": 0.0009033507529927166
            },
            {
             "value": "supervised viewing",
             "probability": 0.9982217175928023
            },
            {
             "value": "deny",
             "probability": 0.0008749316542049563
            }
           ]
          },
          "past_limit": {
           "value": true,
           "probability": 0.9987360218039196,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012639781960803451
            },
            {
             "value": true,
             "probability": 0.9987360218039196
            }
           ]
          }
         },
         "joint_probability": 0.9974121612424494,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025911930003441877,
        "field_brier": 4.4517242026321614e-06
       }
      },
      "timing_ms": {
       "scoring": 35.1504220161587,
       "base_joint": 0.07430906407535076,
       "constrained_joint": 0.04713982343673706,
       "constrained_total": 35.19756183959544
      },
      "input_tokens": 625,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-012ff3a79b0376ec-0003",
      "group": "bendsynth-012ff3a79b0376ec",
      "provenance": "bendsynth-v1; family=bendsynth-012ff3a79b0376ec; domain=vineyard harvest labour request desk; parameters=wait_days=50; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "wait_ok": true,
       "decision": "priority crew",
       "score": 3
      },
      "logits": [
       [
        -6.703795909881592,
        -0.1352398693561554
       ],
       [
        -7.3653740882873535,
        -0.1354069709777832,
        -7.209668159484863
       ],
       [
        -7.576740741729736,
        -7.496906280517578,
        -7.516301155090332,
        -0.13568072021007538
       ]
      ],
      "joint_logits": [
       -9.357723236083984,
       -9.28027629852295,
       -9.389479637145996,
       -9.301709175109863,
       -9.20361328125,
       -9.114672660827637,
       -9.072271347045898,
       -8.868892669677734,
       -9.178579330444336,
       -9.165611267089844,
       -9.260194778442383,
       -9.183859825134277,
       -9.615734100341797,
       -9.520295143127441,
       -9.61850357055664,
       -9.540604591369629,
       -9.533170700073242,
       -9.367010116577148,
       -9.130858421325684,
       -0.1362459659576416,
       -9.397289276123047,
       -9.337126731872559,
       -9.486876487731934,
       -9.350760459899902
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_ok": {
           "value": true,
           "probability": 0.9985981445496122,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014018554503878804
            },
            {
             "value": true,
             "probability": 0.9985981445496122
            }
           ]
          },
          "decision": {
           "value": "priority crew",
           "probability": 0.9984313021338039,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0007234080956346348
            },
            {
             "value": "priority crew",
             "probability": 0.9984313021338039
            },
            {
             "value": "standard crew",
             "probability": 0.0008452897705614041
            }
           ]
          },
          "score": {
           "value": 3,
           "probability": 0.9981581026565539,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005855825431369069
            },
            {
             "value": 1,
             "probability": 0.0006342486914150447
            },
            {
             "value": 2,
             "probability": 0.0006220661088940864
            },
            {
             "value": 3,
             "probability": 0.9981581026565539
            }
           ]
          }
         },
         "joint_probability": 0.9975940187902904,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024088802334315696,
        "field_brier": 4.051258285678218e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_ok": {
           "value": true,
           "probability": 0.9985981445496122,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014018554503878804
            },
            {
             "value": true,
             "probability": 0.9985981445496122
            }
           ]
          },
          "decision": {
           "value": "priority crew",
           "probability": 0.9984313021338039,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0007234080956346348
            },
            {
             "value": "priority crew",
             "probability": 0.9984313021338039
            },
            {
             "value": "standard crew",
             "probability": 0.0008452897705614041
            }
           ]
          },
          "score": {
           "value": 3,
           "probability": 0.9981581026565539,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005855825431369069
            },
            {
             "value": 1,
             "probability": 0.0006342486914150447
            },
            {
             "value": 2,
             "probability": 0.0006220661088940864
            },
            {
             "value": 3,
             "probability": 0.9981581026565539
            }
           ]
          }
         },
         "joint_probability": 0.9975940187902904,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024088802334315696,
        "field_brier": 4.051258285678218e-06
       }
      },
      "timing_ms": {
       "scoring": 34.034903859719634,
       "base_joint": 0.06741005927324295,
       "constrained_joint": 0.047558918595314026,
       "constrained_total": 34.08246277831495
      },
      "input_tokens": 595,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-012ff3a79b0376ec-0032",
      "group": "bendsynth-012ff3a79b0376ec",
      "provenance": "bendsynth-v1; family=bendsynth-012ff3a79b0376ec; domain=vineyard harvest labour request desk; parameters=wait_days=30; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "score": 2,
       "wait_ok": true,
       "decision": "standard crew"
      },
      "logits": [
       [
        -7.428349018096924,
        -7.338014602661133,
        -0.7882791757583618,
        -7.536520481109619
       ],
       [
        -1.0674813985824585,
        -2.1845288276672363
       ],
       [
        -7.074990749359131,
        -6.978716850280762,
        -0.7882785797119141
       ]
      ],
      "joint_logits": [
       -9.186753273010254,
       -9.09007453918457,
       -9.526504516601562,
       -9.117379188537598,
       -9.018952369689941,
       -9.49439525604248,
       -9.209399223327637,
       -9.110843658447266,
       -9.196296691894531,
       -9.123250007629395,
       -9.020434379577637,
       -9.13007640838623,
       -9.051165580749512,
       -8.966154098510742,
       -1.0706892013549805,
       -8.999573707580566,
       -8.900198936462402,
       -2.1950809955596924,
       -9.345555305480957,
       -9.249197006225586,
       -9.527880668640137,
       -9.244844436645508,
       -9.151680946350098,
       -9.507875442504883
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "score": {
           "value": 2,
           "probability": 0.9961048608391228,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001301845405853336
            },
            {
             "value": 1,
             "probability": 0.001424921800516186
            },
            {
             "value": 2,
             "probability": 0.9961048608391228
            },
            {
             "value": 3,
             "probability": 0.0011683719545076882
            }
           ]
          },
          "wait_ok": {
           "value": false,
           "probability": 0.7534405896041239,
           "distribution": [
            {
             "value": false,
             "probability": 0.7534405896041239
            },
            {
             "value": true,
             "probability": 0.24655941039587617
            }
           ]
          },
          "decision": {
           "value": "standard crew",
           "probability": 0.9961054299900562,
           "distribution": [
            {
             "value": "priority crew",
             "probability": 0.0018536206520159935
            },
            {
             "value": "deny",
             "probability": 0.002040949357927719
            },
            {
             "value": "standard crew",
             "probability": 0.9961054299900562
            }
           ]
          }
         },
         "joint_probability": 0.7510276448244109,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 1.4107046114152428,
        "field_brier": 0.37846282519703006
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "score": {
           "value": 2,
           "probability": 0.9961048608391228,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001301845405853336
            },
            {
             "value": 1,
             "probability": 0.001424921800516186
            },
            {
             "value": 2,
             "probability": 0.9961048608391228
            },
            {
             "value": 3,
             "probability": 0.0011683719545076882
            }
           ]
          },
          "wait_ok": {
           "value": false,
           "probability": 0.7534405896041239,
           "distribution": [
            {
             "value": false,
             "probability": 0.7534405896041239
            },
            {
             "value": true,
             "probability": 0.24655941039587617
            }
           ]
          },
          "decision": {
           "value": "standard crew",
           "probability": 0.9961054299900562,
           "distribution": [
            {
             "value": "priority crew",
             "probability": 0.0018536206520159935
            },
            {
             "value": "deny",
             "probability": 0.002040949357927719
            },
            {
             "value": "standard crew",
             "probability": 0.9961054299900562
            }
           ]
          }
         },
         "joint_probability": 0.7510276448244109,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 1.4107046114152428,
        "field_brier": 0.37846282519703006
       }
      },
      "timing_ms": {
       "scoring": 33.953874139115214,
       "base_joint": 0.07379893213510513,
       "constrained_joint": 0.04856893792748451,
       "constrained_total": 34.0024430770427
      },
      "input_tokens": 580,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-064907c5b8b14819-0000",
      "group": "bendsynth-064907c5b8b14819",
      "provenance": "bendsynth-v1; family=bendsynth-064907c5b8b14819; domain=railway signal maintenance window intake desk; parameters=overdue_target=4; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "approve",
       "quota_ok": true,
       "factors": 2
      },
      "logits": [
       [
        -6.349743366241455,
        0.6465330719947815,
        -6.396644115447998
       ],
       [
        -5.911302089691162,
        0.6469026207923889
       ],
       [
        -6.731471538543701,
        -6.565583229064941,
        0.6463092565536499,
        -6.692320823669434
       ]
      ],
      "joint_logits": [
       -8.358220100402832,
       -8.34211540222168,
       -8.497651100158691,
       -8.294846534729004,
       -8.527070045471191,
       -8.565093994140625,
       -8.339212417602539,
       -8.55241584777832,
       -8.468009948730469,
       -8.416850090026855,
       -8.175958633422852,
       -8.452021598815918,
       -8.658491134643555,
       -8.023002624511719,
       0.6456878781318665,
       -8.595144271850586,
       -8.470632553100586,
       -8.325907707214355,
       -8.639554023742676,
       -8.388251304626465,
       -8.697399139404297,
       -8.57915210723877,
       -8.16502857208252,
       -8.670772552490234
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve",
           "probability": 0.9982145620589888,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0009136496136023562
            },
            {
             "value": "approve",
             "probability": 0.9982145620589888
            },
            {
             "value": "escalate",
             "probability": 0.0008717883274088491
            }
           ]
          },
          "quota_ok": {
           "value": true,
           "probability": 0.9985835787184787,
           "distribution": [
            {
             "value": false,
             "probability": 0.001416421281521353
            },
            {
             "value": true,
             "probability": 0.9985835787184787
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9979913565277015,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006237308173806529
            },
            {
             "value": 1,
             "probability": 0.0007362779286831982
            },
            {
             "value": 2,
             "probability": 0.9979913565277015
            },
            {
             "value": 3,
             "probability": 0.0006486347262346118
            }
           ]
          }
         },
         "joint_probability": 0.9973713583321541,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026321026127412938,
        "field_brier": 4.727192855546092e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve",
           "probability": 0.9982145620589888,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0009136496136023562
            },
            {
             "value": "approve",
             "probability": 0.9982145620589888
            },
            {
             "value": "escalate",
             "probability": 0.0008717883274088491
            }
           ]
          },
          "quota_ok": {
           "value": true,
           "probability": 0.9985835787184787,
           "distribution": [
            {
             "value": false,
             "probability": 0.001416421281521353
            },
            {
             "value": true,
             "probability": 0.9985835787184787
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9979913565277015,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006237308173806529
            },
            {
             "value": 1,
             "probability": 0.0007362779286831982
            },
            {
             "value": 2,
             "probability": 0.9979913565277015
            },
            {
             "value": 3,
             "probability": 0.0006486347262346118
            }
           ]
          }
         },
         "joint_probability": 0.9973713583321541,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026321026127412938,
        "field_brier": 4.727192855546092e-06
       }
      },
      "timing_ms": {
       "scoring": 38.54935406707227,
       "base_joint": 0.07178913801908493,
       "constrained_joint": 0.0505899079144001,
       "constrained_total": 38.59994397498667
      },
      "input_tokens": 744,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-064907c5b8b14819-0011",
      "group": "bendsynth-064907c5b8b14819",
      "provenance": "bendsynth-v1; family=bendsynth-064907c5b8b14819; domain=railway signal maintenance window intake desk; parameters=overdue_target=3; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_ok": true,
       "decision": "escalate",
       "factors": 2
      },
      "logits": [
       [
        -6.6120476722717285,
        -0.32462379336357117
       ],
       [
        -6.774805068969727,
        -7.32289457321167,
        -0.32525795698165894
       ],
       [
        -7.368777275085449,
        -7.396457672119141,
        -0.3255188763141632,
        -7.199799537658691
       ]
      ],
      "joint_logits": [
       -8.680357933044434,
       -8.735920906066895,
       -8.888501167297363,
       -9.141818046569824,
       -9.313957214355469,
       -9.320759773254395,
       -9.435104370117188,
       -9.495872497558594,
       -9.550168991088867,
       -9.579675674438477,
       -8.50784969329834,
       -9.299176216125488,
       -8.945836067199707,
       -9.028508186340332,
       -8.540886878967285,
       -9.01972770690918,
       -9.230498313903809,
       -9.365468978881836,
       -9.364508628845215,
       -9.787203788757324,
       -9.55115032196045,
       -9.332771301269531,
       -0.32648971676826477,
       -8.125768661499023
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_ok": {
           "value": true,
           "probability": 0.9981439067961432,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018560932038568154
            },
            {
             "value": true,
             "probability": 0.9981439067961432
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9975109332193509,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0015773022514868605
            },
            {
             "value": "approve",
             "probability": 0.0009117645291622232
            },
            {
             "value": "escalate",
             "probability": 0.9975109332193509
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9972508258257883,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008708757134314917
            },
            {
             "value": 1,
             "probability": 0.0008470999790431695
            },
            {
             "value": 2,
             "probability": 0.9972508258257883
            },
            {
             "value": 3,
             "probability": 0.0010311984817370856
            }
           ]
          }
         },
         "joint_probability": 0.9962831390091963,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003723785682686266,
        "field_brier": 8.834048727383665e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_ok": {
           "value": true,
           "probability": 0.9981439067961432,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018560932038568154
            },
            {
             "value": true,
             "probability": 0.9981439067961432
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9975109332193509,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0015773022514868605
            },
            {
             "value": "approve",
             "probability": 0.0009117645291622232
            },
            {
             "value": "escalate",
             "probability": 0.9975109332193509
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9972508258257883,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008708757134314917
            },
            {
             "value": 1,
             "probability": 0.0008470999790431695
            },
            {
             "value": 2,
             "probability": 0.9972508258257883
            },
            {
             "value": 3,
             "probability": 0.0010311984817370856
            }
           ]
          }
         },
         "joint_probability": 0.9962831390091963,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003723785682686266,
        "field_brier": 8.834048727383665e-06
       }
      },
      "timing_ms": {
       "scoring": 36.79810417816043,
       "base_joint": 0.06435904651880264,
       "constrained_joint": 0.04789000377058983,
       "constrained_total": 36.84599418193102
      },
      "input_tokens": 735,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-064907c5b8b14819-0022",
      "group": "bendsynth-064907c5b8b14819",
      "provenance": "bendsynth-v1; family=bendsynth-064907c5b8b14819; domain=railway signal maintenance window intake desk; parameters=overdue_target=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_ok": true,
       "factors": 1,
       "decision": "approve"
      },
      "logits": [
       [
        -7.070461273193359,
        -0.6423724293708801
       ],
       [
        -8.022714614868164,
        -0.6426684856414795,
        -7.972716331481934,
        -8.00739860534668
       ],
       [
        -7.559422969818115,
        -7.653369903564453,
        -0.6426494121551514
       ]
      ],
      "joint_logits": [
       -9.611593246459961,
       -9.727938652038574,
       -9.673748016357422,
       -9.32474136352539,
       -9.380388259887695,
       -9.253079414367676,
       -9.55982780456543,
       -9.691978454589844,
       -9.657946586608887,
       -9.563567161560059,
       -9.690126419067383,
       -9.677395820617676,
       -9.94083309173584,
       -9.98293399810791,
       -10.028518676757812,
       -9.450122833251953,
       -9.609169006347656,
       -0.6434583067893982,
       -9.910977363586426,
       -9.956168174743652,
       -9.874267578125,
       -9.948128700256348,
       -9.984908103942871,
       -10.025325775146484
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_ok": {
           "value": true,
           "probability": 0.9983870708612517,
           "distribution": [
            {
             "value": false,
             "probability": 0.0016129291387484057
            },
            {
             "value": true,
             "probability": 0.9983870708612517
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9980913391664862,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006223821480689911
            },
            {
             "value": 1,
             "probability": 0.9980913391664862
            },
            {
             "value": 2,
             "probability": 0.0006542909779995578
            },
            {
             "value": 3,
             "probability": 0.0006319877074452933
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.998110397018299,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0009891495392325114
            },
            {
             "value": "escalate",
             "probability": 0.0009004534424685334
            },
            {
             "value": "approve",
             "probability": 0.998110397018299
            }
           ]
          }
         },
         "joint_probability": 0.9973034008364514,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00270024153655997,
        "field_brier": 5.1402547721987076e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_ok": {
           "value": true,
           "probability": 0.9983870708612517,
           "distribution": [
            {
             "value": false,
             "probability": 0.0016129291387484057
            },
            {
             "value": true,
             "probability": 0.9983870708612517
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9980913391664862,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006223821480689911
            },
            {
             "value": 1,
             "probability": 0.9980913391664862
            },
            {
             "value": 2,
             "probability": 0.0006542909779995578
            },
            {
             "value": 3,
             "probability": 0.0006319877074452933
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.998110397018299,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0009891495392325114
            },
            {
             "value": "escalate",
             "probability": 0.0009004534424685334
            },
            {
             "value": "approve",
             "probability": 0.998110397018299
            }
           ]
          }
         },
         "joint_probability": 0.9973034008364514,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00270024153655997,
        "field_brier": 5.1402547721987076e-06
       }
      },
      "timing_ms": {
       "scoring": 37.237938959151506,
       "base_joint": 0.06462913006544113,
       "constrained_joint": 0.0466399360448122,
       "constrained_total": 37.28457889519632
      },
      "input_tokens": 750,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-064907c5b8b14819-0023",
      "group": "bendsynth-064907c5b8b14819",
      "provenance": "bendsynth-v1; family=bendsynth-064907c5b8b14819; domain=railway signal maintenance window intake desk; parameters=overdue_target=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "escalate",
       "quota_ok": true,
       "factors": 1
      },
      "logits": [
       [
        -0.25421667098999023,
        -6.6631975173950195,
        -7.214247703552246
       ],
       [
        -6.5461015701293945,
        -0.25347235798835754
       ],
       [
        -7.48547887802124,
        -0.25395071506500244,
        -7.285399913787842,
        -7.4872870445251465
       ]
      ],
      "joint_logits": [
       -9.533327102661133,
       -8.602496147155762,
       -9.500638961791992,
       -9.696226119995117,
       -9.647651672363281,
       -0.255016952753067,
       -9.255889892578125,
       -9.596927642822266,
       -8.828514099121094,
       -8.656183242797852,
       -8.612436294555664,
       -8.779068946838379,
       -9.046310424804688,
       -8.293161392211914,
       -8.894649505615234,
       -9.0654296875,
       -9.4473295211792,
       -9.09874153137207,
       -9.125003814697266,
       -9.344691276550293,
       -9.421329498291016,
       -9.142922401428223,
       -9.346579551696777,
       -9.514627456665039
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "escalate",
           "probability": 0.9974109522167096,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.9974109522167096
            },
            {
             "value": "waitlist",
             "probability": 0.0016424381095385176
            },
            {
             "value": "approve",
             "probability": 0.0009466096737518674
            }
           ]
          },
          "quota_ok": {
           "value": true,
           "probability": 0.998153525151235,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018464748487649705
            },
            {
             "value": true,
             "probability": 0.998153525151235
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9976762402056923,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007217334575554119
            },
            {
             "value": 1,
             "probability": 0.9976762402056923
            },
            {
             "value": 2,
             "probability": 0.0008815967799066388
            },
            {
             "value": 3,
             "probability": 0.0007204295568455544
            }
           ]
          }
         },
         "joint_probability": 0.996613014756262,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003392734062705767,
        "field_brier": 8.110923456874449e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "escalate",
           "probability": 0.9974109522167096,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.9974109522167096
            },
            {
             "value": "waitlist",
             "probability": 0.0016424381095385176
            },
            {
             "value": "approve",
             "probability": 0.0009466096737518674
            }
           ]
          },
          "quota_ok": {
           "value": true,
           "probability": 0.998153525151235,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018464748487649705
            },
            {
             "value": true,
             "probability": 0.998153525151235
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9976762402056923,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007217334575554119
            },
            {
             "value": 1,
             "probability": 0.9976762402056923
            },
            {
             "value": 2,
             "probability": 0.0008815967799066388
            },
            {
             "value": 3,
             "probability": 0.0007204295568455544
            }
           ]
          }
         },
         "joint_probability": 0.996613014756262,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003392734062705767,
        "field_brier": 8.110923456874449e-06
       }
      },
      "timing_ms": {
       "scoring": 37.961470894515514,
       "base_joint": 0.06542890332639217,
       "constrained_joint": 0.047449953854084015,
       "constrained_total": 38.0089208483696
      },
      "input_tokens": 741,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-064907c5b8b14819-0027",
      "group": "bendsynth-064907c5b8b14819",
      "provenance": "bendsynth-v1; family=bendsynth-064907c5b8b14819; domain=railway signal maintenance window intake desk; parameters=overdue_target=9; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_ok": true,
       "factors": 1,
       "decision": "approve"
      },
      "logits": [
       [
        -6.862968444824219,
        -0.5412028431892395
       ],
       [
        -7.76301383972168,
        -0.5416326522827148,
        -7.702788352966309,
        -7.777299404144287
       ],
       [
        -0.5416773557662964,
        -7.169687271118164,
        -7.501424789428711
       ]
      ],
      "joint_logits": [
       -9.572168350219727,
       -9.249968528747559,
       -9.600903511047363,
       -8.934800148010254,
       -9.043713569641113,
       -9.358933448791504,
       -9.62924575805664,
       -9.221221923828125,
       -9.469841957092285,
       -9.606260299682617,
       -9.245991706848145,
       -9.557158470153809,
       -9.66818904876709,
       -9.5162353515625,
       -9.811088562011719,
       -0.5426275730133057,
       -8.831135749816895,
       -9.246996879577637,
       -9.270176887512207,
       -9.605494499206543,
       -9.940591812133789,
       -9.709915161132812,
       -9.52798843383789,
       -9.880146026611328
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_ok": {
           "value": true,
           "probability": 0.9982064538529174,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017935461470826226
            },
            {
             "value": true,
             "probability": 0.9982064538529174
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9977775738699696,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007291683110956204
            },
            {
             "value": 1,
             "probability": 0.9977775738699696
            },
            {
             "value": 2,
             "probability": 0.0007744319693460744
            },
            {
             "value": 3,
             "probability": 0.000718825849588731
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9977330213590906,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.9977330213590906
            },
            {
             "value": "waitlist",
             "probability": 0.0013197944052771632
            },
            {
             "value": "escalate",
             "probability": 0.000947184235632301
            }
           ]
          }
         },
         "joint_probability": 0.996785348015428,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0032198300784206475,
        "field_brier": 6.9330475922435925e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_ok": {
           "value": true,
           "probability": 0.9982064538529174,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017935461470826226
            },
            {
             "value": true,
             "probability": 0.9982064538529174
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9977775738699696,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007291683110956204
            },
            {
             "value": 1,
             "probability": 0.9977775738699696
            },
            {
             "value": 2,
             "probability": 0.0007744319693460744
            },
            {
             "value": 3,
             "probability": 0.000718825849588731
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9977330213590906,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.9977330213590906
            },
            {
             "value": "waitlist",
             "probability": 0.0013197944052771632
            },
            {
             "value": "escalate",
             "probability": 0.000947184235632301
            }
           ]
          }
         },
         "joint_probability": 0.996785348015428,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0032198300784206475,
        "field_brier": 6.9330475922435925e-06
       }
      },
      "timing_ms": {
       "scoring": 35.57762689888477,
       "base_joint": 0.07255910895764828,
       "constrained_joint": 0.048770103603601456,
       "constrained_total": 35.626397002488375
      },
      "input_tokens": 747,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03ede7689b43c354-0030",
      "group": "bendsynth-03ede7689b43c354",
      "provenance": "bendsynth-v1; family=bendsynth-03ede7689b43c354; domain=audit of elevator inspector dispatch decisions; parameters=grace_days=45; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "minor lapse",
       "factors": 1,
       "timely": false
      },
      "logits": [
       [
        -1.0230318307876587,
        -6.560114860534668,
        -1.4032655954360962
       ],
       [
        -6.924108505249023,
        -0.5040156841278076,
        -7.103330135345459,
        -7.123424530029297
       ],
       [
        -0.5050297379493713,
        -5.737002372741699
       ]
      ],
      "joint_logits": [
       -9.214438438415527,
       -8.584848403930664,
       -1.0268274545669556,
       -7.401586532592773,
       -9.494744300842285,
       -8.752470970153809,
       -9.50232982635498,
       -8.75793170928955,
       -8.920366287231445,
       -8.22170639038086,
       -9.014554977416992,
       -8.459586143493652,
       -9.101519584655762,
       -8.344167709350586,
       -9.127625465393066,
       -8.413089752197266,
       -9.19072437286377,
       -8.553359031677246,
       -1.4091004133224487,
       -7.3358354568481445,
       -9.456192016601562,
       -8.730596542358398,
       -9.45663833618164,
       -8.707266807556152
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "compliant",
           "probability": 0.592543641020889,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.592543641020889
            },
            {
             "value": "violation",
             "probability": 0.002333435493496941
            },
            {
             "value": "minor lapse",
             "probability": 0.40512292348561413
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.995694590672376,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0016214939672393005
            },
            {
             "value": 1,
             "probability": 0.995694590672376
            },
            {
             "value": 2,
             "probability": 0.0013554402022238484
            },
            {
             "value": 3,
             "probability": 0.001328475158160921
            }
           ]
          },
          "timely": {
           "value": false,
           "probability": 0.9946854212767914,
           "distribution": [
            {
             "value": false,
             "probability": 0.9946854212767914
            },
            {
             "value": true,
             "probability": 0.005314578723208641
            }
           ]
          }
         },
         "joint_probability": 0.5902987624520916,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 0.9093994519660088,
        "field_brier": 0.23502446831611967
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "compliant",
           "probability": 0.592543641020889,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.592543641020889
            },
            {
             "value": "violation",
             "probability": 0.002333435493496941
            },
            {
             "value": "minor lapse",
             "probability": 0.40512292348561413
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.995694590672376,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0016214939672393005
            },
            {
             "value": 1,
             "probability": 0.995694590672376
            },
            {
             "value": 2,
             "probability": 0.0013554402022238484
            },
            {
             "value": 3,
             "probability": 0.001328475158160921
            }
           ]
          },
          "timely": {
           "value": false,
           "probability": 0.9946854212767914,
           "distribution": [
            {
             "value": false,
             "probability": 0.9946854212767914
            },
            {
             "value": true,
             "probability": 0.005314578723208641
            }
           ]
          }
         },
         "joint_probability": 0.5902987624520916,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 0.9093994519660088,
        "field_brier": 0.23502446831611967
       }
      },
      "timing_ms": {
       "scoring": 37.709693890064955,
       "base_joint": 0.07339916191995144,
       "constrained_joint": 0.047590117901563644,
       "constrained_total": 37.75728400796652
      },
      "input_tokens": 747,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03ede7689b43c354-0035",
      "group": "bendsynth-03ede7689b43c354",
      "provenance": "bendsynth-v1; family=bendsynth-03ede7689b43c354; domain=audit of elevator inspector dispatch decisions; parameters=grace_days=21; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "factors": 1,
       "decision": "minor lapse",
       "timely": true
      },
      "logits": [
       [
        -5.912777423858643,
        1.378132939338684,
        -6.027351379394531,
        -6.005924701690674
       ],
       [
        -5.646862030029297,
        1.3776365518569946,
        -5.112506866455078
       ],
       [
        -5.193271636962891,
        1.3786436319351196
       ]
      ],
      "joint_logits": [
       -7.675512313842773,
       -7.717137813568115,
       -7.642923355102539,
       -8.005060195922852,
       -7.596322059631348,
       -7.641746997833252,
       -7.571656703948975,
       -7.834326267242432,
       -7.660769939422607,
       1.3769537210464478,
       -7.699517250061035,
       -5.863583564758301,
       -7.656810760498047,
       -7.8311028480529785,
       -7.797238826751709,
       -8.153339385986328,
       -7.71754264831543,
       -7.8283538818359375,
       -7.74953556060791,
       -7.804990291595459,
       -7.724640369415283,
       -8.17252254486084,
       -7.667422294616699,
       -7.743528842926025
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 1,
           "probability": 0.9980929511860055,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006804071338594455
            },
            {
             "value": 1,
             "probability": 0.9980929511860055
            },
            {
             "value": 2,
             "probability": 0.0006067504042735581
            },
            {
             "value": 3,
             "probability": 0.0006198912758615505
            }
           ]
          },
          "decision": {
           "value": "minor lapse",
           "probability": 0.997597640812671,
           "distribution": [
            {
             "value": "violation",
             "probability": 0.0008876757973005578
            },
            {
             "value": "minor lapse",
             "probability": 0.997597640812671
            },
            {
             "value": "compliant",
             "probability": 0.0015146833900285248
            }
           ]
          },
          "timely": {
           "value": true,
           "probability": 0.9986028398401284,
           "distribution": [
            {
             "value": false,
             "probability": 0.001397160159871668
            },
            {
             "value": true,
             "probability": 0.9986028398401284
            }
           ]
          }
         },
         "joint_probability": 0.9969166737392148,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0030880895048106508,
        "field_brier": 5.8699590254968674e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 1,
           "probability": 0.9980929511860055,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006804071338594455
            },
            {
             "value": 1,
             "probability": 0.9980929511860055
            },
            {
             "value": 2,
             "probability": 0.0006067504042735581
            },
            {
             "value": 3,
             "probability": 0.0006198912758615505
            }
           ]
          },
          "decision": {
           "value": "minor lapse",
           "probability": 0.997597640812671,
           "distribution": [
            {
             "value": "violation",
             "probability": 0.0008876757973005578
            },
            {
             "value": "minor lapse",
             "probability": 0.997597640812671
            },
            {
             "value": "compliant",
             "probability": 0.0015146833900285248
            }
           ]
          },
          "timely": {
           "value": true,
           "probability": 0.9986028398401284,
           "distribution": [
            {
             "value": false,
             "probability": 0.001397160159871668
            },
            {
             "value": true,
             "probability": 0.9986028398401284
            }
           ]
          }
         },
         "joint_probability": 0.9969166737392148,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0030880895048106508,
        "field_brier": 5.8699590254968674e-06
       }
      },
      "timing_ms": {
       "scoring": 39.90709106437862,
       "base_joint": 0.06552995182573795,
       "constrained_joint": 0.04803878255188465,
       "constrained_total": 39.955129846930504
      },
      "input_tokens": 759,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03ede7689b43c354-0039",
      "group": "bendsynth-03ede7689b43c354",
      "provenance": "bendsynth-v1; family=bendsynth-03ede7689b43c354; domain=audit of elevator inspector dispatch decisions; parameters=grace_days=30; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "compliant",
       "timely": true,
       "factors": 0
      },
      "logits": [
       [
        1.7903814315795898,
        -5.212146759033203,
        -5.30194616317749
       ],
       [
        -4.849752902984619,
        1.7908153533935547
       ],
       [
        1.7902525663375854,
        -5.568173408508301,
        -5.596154689788818,
        -5.606639862060547
       ]
      ],
      "joint_logits": [
       -7.419127941131592,
       -7.363687992095947,
       -7.372337818145752,
       -7.377151012420654,
       1.7897090911865234,
       -7.535908222198486,
       -7.592415809631348,
       -7.597739219665527,
       -7.24251651763916,
       -7.2609968185424805,
       -7.282357215881348,
       -7.291977405548096,
       -7.312100410461426,
       -7.302462577819824,
       -7.3173441886901855,
       -7.325868606567383,
       -7.272998332977295,
       -7.350629806518555,
       -7.392752170562744,
       -7.410892009735107,
       -7.45419979095459,
       -7.367345333099365,
       -7.398343086242676,
       -7.414374351501465
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "compliant",
           "probability": 0.9982619865832058,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.9982619865832058
            },
            {
             "value": "minor lapse",
             "probability": 0.0009079986415348044
            },
            {
             "value": "violation",
             "probability": 0.0008300147752593782
            }
           ]
          },
          "timely": {
           "value": true,
           "probability": 0.9986954197176615,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013045802823385612
            },
            {
             "value": true,
             "probability": 0.9986954197176615
            }
           ]
          },
          "factors": {
           "value": 0,
           "probability": 0.9981335162767413,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981335162767413
            },
            {
             "value": 1,
             "probability": 0.0006360114180139875
            },
            {
             "value": 2,
             "probability": 0.0006184614871385428
            },
            {
             "value": 3,
             "probability": 0.0006120108181060993
            }
           ]
          }
         },
         "joint_probability": 0.9975911078081658,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024117982404734356,
        "field_brier": 4.194419996284674e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "compliant",
           "probability": 0.9982619865832058,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.9982619865832058
            },
            {
             "value": "minor lapse",
             "probability": 0.0009079986415348044
            },
            {
             "value": "violation",
             "probability": 0.0008300147752593782
            }
           ]
          },
          "timely": {
           "value": true,
           "probability": 0.9986954197176615,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013045802823385612
            },
            {
             "value": true,
             "probability": 0.9986954197176615
            }
           ]
          },
          "factors": {
           "value": 0,
           "probability": 0.9981335162767413,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981335162767413
            },
            {
             "value": 1,
             "probability": 0.0006360114180139875
            },
            {
             "value": 2,
             "probability": 0.0006184614871385428
            },
            {
             "value": 3,
             "probability": 0.0006120108181060993
            }
           ]
          }
         },
         "joint_probability": 0.9975911078081658,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024117982404734356,
        "field_brier": 4.194419996284674e-06
       }
      },
      "timing_ms": {
       "scoring": 38.34694717079401,
       "base_joint": 0.06507919169962406,
       "constrained_joint": 0.04800991155207157,
       "constrained_total": 38.39495708234608
      },
      "input_tokens": 753,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03aee97ccad64677-0005",
      "group": "bendsynth-03aee97ccad64677",
      "provenance": "bendsynth-v1; family=bendsynth-03aee97ccad64677; domain=community theatre audition scoring \u2014 director appeal panel; parameters=cast_threshold=70; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "over_threshold": false,
       "risk_factors": 3,
       "decision": "not cast"
      },
      "logits": [
       [
        0.6702194213867188,
        -6.355108261108398
       ],
       [
        -7.020606517791748,
        -6.95225191116333,
        -6.936949253082275,
        0.6696650981903076
       ],
       [
        -7.153622627258301,
        -6.812682628631592,
        -6.857348442077637,
        0.6696076989173889
       ]
      ],
      "joint_logits": [
       -9.249894142150879,
       -8.953258514404297,
       -8.981096267700195,
       -9.247254371643066,
       -9.092066764831543,
       -8.851766586303711,
       -8.854397773742676,
       -9.147104263305664,
       -9.168900489807129,
       -8.899625778198242,
       -8.90340805053711,
       -8.611319541931152,
       -9.171455383300781,
       -8.650797843933105,
       -8.759495735168457,
       0.6692228317260742,
       -9.244049072265625,
       -8.91769027709961,
       -8.959238052368164,
       -9.355607986450195,
       -9.281757354736328,
       -8.87282657623291,
       -8.962861061096191,
       -9.313680648803711,
       -9.379356384277344,
       -9.042083740234375,
       -9.067004203796387,
       -9.267828941345215,
       -9.305634498596191,
       -9.00007438659668,
       -9.042647361755371,
       -9.256813049316406
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_threshold": {
           "value": false,
           "probability": 0.9991117138344313,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991117138344313
            },
            {
             "value": true,
             "probability": 0.0008882861655687089
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.998558073671998,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000456594666048361
            },
            {
             "value": 1,
             "probability": 0.0004888963459603424
            },
            {
             "value": 2,
             "probability": 0.0004964353159932857
            },
            {
             "value": 3,
             "probability": 0.998558073671998
            }
           ]
          },
          "decision": {
           "value": "not cast",
           "probability": 0.9985005840684629,
           "distribution": [
            {
             "value": "escalate to director",
             "probability": 0.00039972617960706326
            },
            {
             "value": "cast in lead track",
             "probability": 0.0005621225481493179
            },
            {
             "value": "cast in ensemble",
             "probability": 0.0005375672037807159
            },
            {
             "value": "not cast",
             "probability": 0.9985005840684629
            }
           ]
          }
         },
         "joint_probability": 0.9981164759403666,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018853001215957022,
        "field_brier": 2.454730640172819e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_threshold": {
           "value": false,
           "probability": 0.9991117138344313,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991117138344313
            },
            {
             "value": true,
             "probability": 0.0008882861655687089
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.998558073671998,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000456594666048361
            },
            {
             "value": 1,
             "probability": 0.0004888963459603424
            },
            {
             "value": 2,
             "probability": 0.0004964353159932857
            },
            {
             "value": 3,
             "probability": 0.998558073671998
            }
           ]
          },
          "decision": {
           "value": "not cast",
           "probability": 0.9985005840684629,
           "distribution": [
            {
             "value": "escalate to director",
             "probability": 0.00039972617960706326
            },
            {
             "value": "cast in lead track",
             "probability": 0.0005621225481493179
            },
            {
             "value": "cast in ensemble",
             "probability": 0.0005375672037807159
            },
            {
             "value": "not cast",
             "probability": 0.9985005840684629
            }
           ]
          }
         },
         "joint_probability": 0.9981164759403666,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018853001215957022,
        "field_brier": 2.454730640172819e-06
       }
      },
      "timing_ms": {
       "scoring": 37.610195111483335,
       "base_joint": 0.09542913176119328,
       "constrained_joint": 0.059848884120583534,
       "constrained_total": 37.67004399560392
      },
      "input_tokens": 683,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03aee97ccad64677-0010",
      "group": "bendsynth-03aee97ccad64677",
      "provenance": "bendsynth-v1; family=bendsynth-03aee97ccad64677; domain=community theatre audition scoring \u2014 director appeal panel; parameters=cast_threshold=69; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "not cast",
       "risk_factors": 3,
       "over_threshold": false
      },
      "logits": [
       [
        -7.178177833557129,
        -6.837839126586914,
        -6.9272260665893555,
        0.641901969909668
       ],
       [
        -7.017480850219727,
        -6.985080718994141,
        -6.988184452056885,
        0.6419398188591003
       ],
       [
        0.6423932313919067,
        -6.275047302246094
       ]
      ],
      "joint_logits": [
       -9.36315631866455,
       -9.126297950744629,
       -9.238428115844727,
       -9.223127365112305,
       -9.364398956298828,
       -9.308815002441406,
       -9.305728912353516,
       -9.159124374389648,
       -9.078239440917969,
       -8.761066436767578,
       -8.9747314453125,
       -8.855048179626465,
       -9.112051963806152,
       -8.992324829101562,
       -8.774325370788574,
       -8.852603912353516,
       -9.145835876464844,
       -8.88652229309082,
       -8.9879150390625,
       -8.994949340820312,
       -9.15722942352295,
       -9.07490062713623,
       -8.94100570678711,
       -8.90284252166748,
       -9.330867767333984,
       -9.239806175231934,
       -9.199398040771484,
       -9.111800193786621,
       -8.538127899169922,
       -9.252493858337402,
       0.6414886116981506,
       -9.246201515197754
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "not cast",
           "probability": 0.9985200582224656,
           "distribution": [
            {
             "value": "escalate to director",
             "probability": 0.0004009953052715134
            },
            {
             "value": "cast in lead track",
             "probability": 0.0005635681685727658
            },
            {
             "value": "cast in ensemble",
             "probability": 0.0005153783036901841
            },
            {
             "value": "not cast",
             "probability": 0.9985200582224656
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.998557791800674,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00047090056742615593
            },
            {
             "value": 1,
             "probability": 0.00048640744126042637
            },
            {
             "value": 2,
             "probability": 0.00048490019063936814
            },
            {
             "value": 3,
             "probability": 0.998557791800674
            }
           ]
          },
          "over_threshold": {
           "value": false,
           "probability": 0.9990106180591677,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990106180591677
            },
            {
             "value": true,
             "probability": 0.0009893819408322573
            }
           ]
          }
         },
         "joint_probability": 0.9981073193015262,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018944740818122347,
        "field_brier": 2.555144751458509e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "not cast",
           "probability": 0.9985200582224656,
           "distribution": [
            {
             "value": "escalate to director",
             "probability": 0.0004009953052715134
            },
            {
             "value": "cast in lead track",
             "probability": 0.0005635681685727658
            },
            {
             "value": "cast in ensemble",
             "probability": 0.0005153783036901841
            },
            {
             "value": "not cast",
             "probability": 0.9985200582224656
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.998557791800674,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00047090056742615593
            },
            {
             "value": 1,
             "probability": 0.00048640744126042637
            },
            {
             "value": 2,
             "probability": 0.00048490019063936814
            },
            {
             "value": 3,
             "probability": 0.998557791800674
            }
           ]
          },
          "over_threshold": {
           "value": false,
           "probability": 0.9990106180591677,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990106180591677
            },
            {
             "value": true,
             "probability": 0.0009893819408322573
            }
           ]
          }
         },
         "joint_probability": 0.9981073193015262,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018944740818122347,
        "field_brier": 2.555144751458509e-06
       }
      },
      "timing_ms": {
       "scoring": 37.39520790986717,
       "base_joint": 0.07438985630869865,
       "constrained_joint": 0.057928962633013725,
       "constrained_total": 37.45313687250018
      },
      "input_tokens": 692,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03aee97ccad64677-0037",
      "group": "bendsynth-03aee97ccad64677",
      "provenance": "bendsynth-v1; family=bendsynth-03aee97ccad64677; domain=community theatre audition scoring \u2014 director appeal panel; parameters=cast_threshold=69; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "over_threshold": true,
       "risk_factors": 0,
       "decision": "cast in lead track"
      },
      "logits": [
       [
        -5.203734397888184,
        1.6074527502059937
       ],
       [
        1.6069217920303345,
        -5.922636985778809,
        -5.8679304122924805,
        -5.937952995300293
       ],
       [
        -5.909140586853027,
        -5.799283027648926,
        1.6067519187927246,
        -5.729862213134766
       ]
      ],
      "joint_logits": [
       -8.011139869689941,
       -7.85986852645874,
       -7.824095726013184,
       -7.853238582611084,
       -8.034542083740234,
       -7.991917133331299,
       -8.223779678344727,
       -7.887935161590576,
       -7.989500999450684,
       -7.941912651062012,
       -8.129643440246582,
       -7.847384929656982,
       -8.055095672607422,
       -7.993630886077881,
       -8.188796997070312,
       -7.898362159729004,
       -7.778711795806885,
       -7.5571770668029785,
       1.6063121557235718,
       -7.430874824523926,
       -8.036005973815918,
       -7.9573445320129395,
       -8.002801895141602,
       -7.917741775512695,
       -7.986525535583496,
       -7.8614091873168945,
       -8.019659996032715,
       -7.83796501159668,
       -8.047022819519043,
       -7.944169998168945,
       -8.151998519897461,
       -7.9011921882629395
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_threshold": {
           "value": true,
           "probability": 0.9988998265912415,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011001734087585265
            },
            {
             "value": true,
             "probability": 0.9988998265912415
            }
           ]
          },
          "risk_factors": {
           "value": 0,
           "probability": 0.9983697038478669,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9983697038478669
            },
            {
             "value": 1,
             "probability": 0.0005360998153203226
            },
            {
             "value": 2,
             "probability": 0.0005662449076586435
            },
            {
             "value": 3,
             "probability": 0.0005279514291541083
            }
           ]
          },
          "decision": {
           "value": "cast in lead track",
           "probability": 0.9982000548636285,
           "distribution": [
            {
             "value": "escalate to director",
             "probability": 0.000543384209987994
            },
            {
             "value": "cast in ensemble",
             "probability": 0.0006064812793025029
            },
            {
             "value": "cast in lead track",
             "probability": 0.9982000548636285
            },
            {
             "value": "not cast",
             "probability": 0.0006500796470810135
            }
           ]
          }
         },
         "joint_probability": 0.9977612238288505,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022412859771507522,
        "field_brier": 3.4302965348750523e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_threshold": {
           "value": true,
           "probability": 0.9988998265912415,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011001734087585265
            },
            {
             "value": true,
             "probability": 0.9988998265912415
            }
           ]
          },
          "risk_factors": {
           "value": 0,
           "probability": 0.9983697038478669,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9983697038478669
            },
            {
             "value": 1,
             "probability": 0.0005360998153203226
            },
            {
             "value": 2,
             "probability": 0.0005662449076586435
            },
            {
             "value": 3,
             "probability": 0.0005279514291541083
            }
           ]
          },
          "decision": {
           "value": "cast in lead track",
           "probability": 0.9982000548636285,
           "distribution": [
            {
             "value": "escalate to director",
             "probability": 0.000543384209987994
            },
            {
             "value": "cast in ensemble",
             "probability": 0.0006064812793025029
            },
            {
             "value": "cast in lead track",
             "probability": 0.9982000548636285
            },
            {
             "value": "not cast",
             "probability": 0.0006500796470810135
            }
           ]
          }
         },
         "joint_probability": 0.9977612238288505,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022412859771507522,
        "field_brier": 3.4302965348750523e-06
       }
      },
      "timing_ms": {
       "scoring": 37.607024889439344,
       "base_joint": 0.07671886123716831,
       "constrained_joint": 0.058238860219717026,
       "constrained_total": 37.66526374965906
      },
      "input_tokens": 683,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03fb81348fdd9fa0-0003",
      "group": "bendsynth-03fb81348fdd9fa0",
      "provenance": "bendsynth-v1; family=bendsynth-03fb81348fdd9fa0; domain=regional archive document request intake desk; parameters=tolerance_pct=20; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_marks": 2,
       "decision": "block",
       "within_tolerance": true
      },
      "logits": [
       [
        -7.301688194274902,
        -6.336915493011475,
        0.08754462003707886,
        -7.17008113861084
       ],
       [
        0.08851973712444305,
        -6.741783142089844,
        -6.940263748168945
       ],
       [
        -6.2692060470581055,
        0.0887528657913208
       ]
      ],
      "joint_logits": [
       -8.921529769897461,
       -9.061960220336914,
       -9.01956844329834,
       -9.138792037963867,
       -9.18188762664795,
       -9.276495933532715,
       -8.280208587646484,
       -7.1867475509643555,
       -8.367674827575684,
       -8.601075172424316,
       -8.505757331848145,
       -8.864912033081055,
       -8.420909881591797,
       0.08691277354955673,
       -9.075088500976562,
       -8.775083541870117,
       -9.210339546203613,
       -9.208549499511719,
       -8.822802543640137,
       -8.969655990600586,
       -8.846671104431152,
       -9.002811431884766,
       -9.008118629455566,
       -9.157954216003418
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_marks": {
           "value": 2,
           "probability": 0.9970645844032919,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006160560851421125
            },
            {
             "value": 1,
             "probability": 0.0016166490661729076
            },
            {
             "value": 2,
             "probability": 0.9970645844032919
            },
            {
             "value": 3,
             "probability": 0.0007027104453931333
            }
           ]
          },
          "decision": {
           "value": "block",
           "probability": 0.9980373203925267,
           "distribution": [
            {
             "value": "block",
             "probability": 0.9980373203925267
            },
            {
             "value": "manual review",
             "probability": 0.0010784098245365838
            },
            {
             "value": "auto-approve",
             "probability": 0.000884269782936789
            }
           ]
          },
          "within_tolerance": {
           "value": true,
           "probability": 0.998270097882017,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017299021179829487
            },
            {
             "value": true,
             "probability": 0.998270097882017
            }
           ]
          }
         },
         "joint_probability": 0.996434814346134,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0035715560738996317,
        "field_brier": 7.961893571509169e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_marks": {
           "value": 2,
           "probability": 0.9970645844032919,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006160560851421125
            },
            {
             "value": 1,
             "probability": 0.0016166490661729076
            },
            {
             "value": 2,
             "probability": 0.9970645844032919
            },
            {
             "value": 3,
             "probability": 0.0007027104453931333
            }
           ]
          },
          "decision": {
           "value": "block",
           "probability": 0.9980373203925267,
           "distribution": [
            {
             "value": "block",
             "probability": 0.9980373203925267
            },
            {
             "value": "manual review",
             "probability": 0.0010784098245365838
            },
            {
             "value": "auto-approve",
             "probability": 0.000884269782936789
            }
           ]
          },
          "within_tolerance": {
           "value": true,
           "probability": 0.998270097882017,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017299021179829487
            },
            {
             "value": true,
             "probability": 0.998270097882017
            }
           ]
          }
         },
         "joint_probability": 0.996434814346134,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0035715560738996317,
        "field_brier": 7.961893571509169e-06
       }
      },
      "timing_ms": {
       "scoring": 37.29821881279349,
       "base_joint": 0.06456999108195305,
       "constrained_joint": 0.04836893640458584,
       "constrained_total": 37.34658774919808
      },
      "input_tokens": 680,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03fb81348fdd9fa0-0024",
      "group": "bendsynth-03fb81348fdd9fa0",
      "provenance": "bendsynth-v1; family=bendsynth-03fb81348fdd9fa0; domain=regional archive document request intake desk; parameters=tolerance_pct=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "within_tolerance": false,
       "decision": "block",
       "risk_marks": 2
      },
      "logits": [
       [
        0.7647285461425781,
        -6.0207085609436035
       ],
       [
        0.7643079161643982,
        -6.413900375366211,
        -6.38107967376709
       ],
       [
        -6.711641311645508,
        -6.617702484130859,
        0.7639256119728088,
        -6.438118934631348
       ]
      ],
      "joint_logits": [
       -8.401586532592773,
       -8.134848594665527,
       0.7634871602058411,
       -8.047402381896973,
       -8.45996379852295,
       -8.460877418518066,
       -8.696475982666016,
       -8.24995231628418,
       -8.463163375854492,
       -8.331491470336914,
       -8.317899703979492,
       -8.180049896240234,
       -8.419490814208984,
       -8.343841552734375,
       -8.594595909118652,
       -8.178686141967773,
       -8.567891120910645,
       -8.630812644958496,
       -8.655083656311035,
       -8.318596839904785,
       -8.74892520904541,
       -8.653131484985352,
       -8.6776123046875,
       -8.451807975769043
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_tolerance": {
           "value": false,
           "probability": 0.9988711619236997,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988711619236997
            },
            {
             "value": true,
             "probability": 0.00112883807630028
            }
           ]
          },
          "decision": {
           "value": "block",
           "probability": 0.998450877014401,
           "distribution": [
            {
             "value": "block",
             "probability": 0.998450877014401
            },
            {
             "value": "auto-approve",
             "probability": 0.0007618516091928351
            },
            {
             "value": "manual review",
             "probability": 0.0007872713764061569
            }
           ]
          },
          "risk_marks": {
           "value": 2,
           "probability": 0.9980693226307409,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005656701509947644
            },
            {
             "value": 1,
             "probability": 0.0006213844154920358
            },
            {
             "value": 2,
             "probability": 0.9980693226307409
            },
            {
             "value": 3,
             "probability": 0.0007436228027723352
            }
           ]
          }
         },
         "joint_probability": 0.9976318276035978,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002370980951623161,
        "field_brier": 3.7117127375249532e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_tolerance": {
           "value": false,
           "probability": 0.9988711619236997,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988711619236997
            },
            {
             "value": true,
             "probability": 0.00112883807630028
            }
           ]
          },
          "decision": {
           "value": "block",
           "probability": 0.998450877014401,
           "distribution": [
            {
             "value": "block",
             "probability": 0.998450877014401
            },
            {
             "value": "auto-approve",
             "probability": 0.0007618516091928351
            },
            {
             "value": "manual review",
             "probability": 0.0007872713764061569
            }
           ]
          },
          "risk_marks": {
           "value": 2,
           "probability": 0.9980693226307409,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005656701509947644
            },
            {
             "value": 1,
             "probability": 0.0006213844154920358
            },
            {
             "value": 2,
             "probability": 0.9980693226307409
            },
            {
             "value": 3,
             "probability": 0.0007436228027723352
            }
           ]
          }
         },
         "joint_probability": 0.9976318276035978,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002370980951623161,
        "field_brier": 3.7117127375249532e-06
       }
      },
      "timing_ms": {
       "scoring": 36.96499206125736,
       "base_joint": 0.06746011786162853,
       "constrained_joint": 0.04740920849144459,
       "constrained_total": 37.01240126974881
      },
      "input_tokens": 680,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03fb81348fdd9fa0-0035",
      "group": "bendsynth-03fb81348fdd9fa0",
      "provenance": "bendsynth-v1; family=bendsynth-03fb81348fdd9fa0; domain=regional archive document request intake desk; parameters=tolerance_pct=15; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_marks": 0,
       "within_tolerance": true,
       "decision": "auto-approve"
      },
      "logits": [
       [
        1.1617133617401123,
        -5.917242527008057,
        -6.253803253173828,
        -6.214409828186035
       ],
       [
        -5.313588619232178,
        1.1622426509857178
       ],
       [
        1.1617839336395264,
        -5.869006156921387,
        -5.6365203857421875
       ]
      ],
      "joint_logits": [
       -7.74977970123291,
       -7.917577743530273,
       -7.700703144073486,
       1.1610575914382935,
       -8.107758522033691,
       -7.513244152069092,
       -7.8258843421936035,
       -7.627866268157959,
       -7.476390838623047,
       -7.783036708831787,
       -7.90187931060791,
       -7.698379993438721,
       -8.160785675048828,
       -7.918953895568848,
       -7.746289253234863,
       -8.309800148010254,
       -8.170965194702148,
       -8.070830345153809,
       -8.102869033813477,
       -7.903304100036621,
       -7.661972999572754,
       -8.34077262878418,
       -8.154603958129883,
       -8.010885238647461
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_marks": {
           "value": 0,
           "probability": 0.9979337610248584,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9979337610248584
            },
            {
             "value": 1,
             "probability": 0.0008409114065143536
            },
            {
             "value": 2,
             "probability": 0.000600597948812311
            },
            {
             "value": 3,
             "probability": 0.0006247296198149473
            }
           ]
          },
          "within_tolerance": {
           "value": true,
           "probability": 0.9984621504664555,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015378495335445649
            },
            {
             "value": true,
             "probability": 0.9984621504664555
            }
           ]
          },
          "decision": {
           "value": "auto-approve",
           "probability": 0.9980040933014206,
           "distribution": [
            {
             "value": "auto-approve",
             "probability": 0.9980040933014206
            },
            {
             "value": "block",
             "probability": 0.0008824680587083103
            },
            {
             "value": "manual review",
             "probability": 0.0011134386398711138
            }
           ]
          }
         },
         "joint_probability": 0.9972796017467896,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027241052610898872,
        "field_brier": 5.4865272981568955e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_marks": {
           "value": 0,
           "probability": 0.9979337610248584,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9979337610248584
            },
            {
             "value": 1,
             "probability": 0.0008409114065143536
            },
            {
             "value": 2,
             "probability": 0.000600597948812311
            },
            {
             "value": 3,
             "probability": 0.0006247296198149473
            }
           ]
          },
          "within_tolerance": {
           "value": true,
           "probability": 0.9984621504664555,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015378495335445649
            },
            {
             "value": true,
             "probability": 0.9984621504664555
            }
           ]
          },
          "decision": {
           "value": "auto-approve",
           "probability": 0.9980040933014206,
           "distribution": [
            {
             "value": "auto-approve",
             "probability": 0.9980040933014206
            },
            {
             "value": "block",
             "probability": 0.0008824680587083103
            },
            {
             "value": "manual review",
             "probability": 0.0011134386398711138
            }
           ]
          }
         },
         "joint_probability": 0.9972796017467896,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027241052610898872,
        "field_brier": 5.4865272981568955e-06
       }
      },
      "timing_ms": {
       "scoring": 35.72952514514327,
       "base_joint": 0.0649790745228529,
       "constrained_joint": 0.047859037294983864,
       "constrained_total": 35.777384182438254
      },
      "input_tokens": 662,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c715d0d44701723-0016",
      "group": "bendsynth-0c715d0d44701723",
      "provenance": "bendsynth-v1; family=bendsynth-0c715d0d44701723; domain=airline duty-hour extension review by a crew supervisor; parameters=limit=80; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "exception": true,
       "breaches": 2,
       "decision": "deny"
      },
      "logits": [
       [
        -6.04909086227417,
        0.8264291286468506
       ],
       [
        -7.006809234619141,
        -6.470269203186035,
        0.8259750604629517,
        -6.966466426849365
       ],
       [
        0.8259038329124451,
        -6.786828517913818,
        -6.813375949859619,
        -6.620667457580566
       ]
      ],
      "joint_logits": [
       -9.100008010864258,
       -9.050951957702637,
       -9.156370162963867,
       -8.898386001586914,
       -8.571471214294434,
       -8.632975578308105,
       -8.781163215637207,
       -8.425392150878906,
       -8.401834487915039,
       -8.906489372253418,
       -8.989078521728516,
       -8.659601211547852,
       -8.981453895568848,
       -8.953007698059082,
       -9.004610061645508,
       -9.093249320983887,
       -9.091754913330078,
       -9.191628456115723,
       -9.292387008666992,
       -8.963532447814941,
       -8.155574798583984,
       -8.652186393737793,
       -8.79786205291748,
       -8.5400390625,
       0.8253961205482483,
       -8.654736518859863,
       -8.291752815246582,
       -8.340021133422852,
       -9.131067276000977,
       -9.050413131713867,
       -9.225744247436523,
       -8.958931922912598
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "exception": {
           "value": true,
           "probability": 0.9989683047278145,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010316952721853981
            },
            {
             "value": true,
             "probability": 0.9989683047278145
            }
           ]
          },
          "breaches": {
           "value": 2,
           "probability": 0.9985147644306315,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003959311486924543
            },
            {
             "value": 1,
             "probability": 0.0006770737779827476
            },
            {
             "value": 2,
             "probability": 0.9985147644306315
            },
            {
             "value": 3,
             "probability": 0.0004122306426934
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.9984436923937123,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.9984436923937123
            },
            {
             "value": "reassign",
             "probability": 0.0004933509908360018
            },
            {
             "value": "approve with rest",
             "probability": 0.0004804260942467296
            },
            {
             "value": "approve",
             "probability": 0.0005825305212048379
            }
           ]
          }
         },
         "joint_probability": 0.9979369198596693,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002065211221729067,
        "field_brier": 2.7851596832150943e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "exception": {
           "value": true,
           "probability": 0.9989683047278145,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010316952721853981
            },
            {
             "value": true,
             "probability": 0.9989683047278145
            }
           ]
          },
          "breaches": {
           "value": 2,
           "probability": 0.9985147644306315,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003959311486924543
            },
            {
             "value": 1,
             "probability": 0.0006770737779827476
            },
            {
             "value": 2,
             "probability": 0.9985147644306315
            },
            {
             "value": 3,
             "probability": 0.0004122306426934
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.9984436923937123,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.9984436923937123
            },
            {
             "value": "reassign",
             "probability": 0.0004933509908360018
            },
            {
             "value": "approve with rest",
             "probability": 0.0004804260942467296
            },
            {
             "value": "approve",
             "probability": 0.0005825305212048379
            }
           ]
          }
         },
         "joint_probability": 0.9979369198596693,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002065211221729067,
        "field_brier": 2.7851596832150943e-06
       }
      },
      "timing_ms": {
       "scoring": 38.08088996447623,
       "base_joint": 0.07648905739188194,
       "constrained_joint": 0.05832896567881107,
       "constrained_total": 38.13921893015504
      },
      "input_tokens": 725,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c715d0d44701723-0026",
      "group": "bendsynth-0c715d0d44701723",
      "provenance": "bendsynth-v1; family=bendsynth-0c715d0d44701723; domain=airline duty-hour extension review by a crew supervisor; parameters=limit=81; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "reassign",
       "exception": false,
       "breaches": 1
      },
      "logits": [
       [
        -5.628964424133301,
        1.6021480560302734,
        -5.959843158721924,
        -6.100071907043457
       ],
       [
        1.6028944253921509,
        -5.358151435852051
       ],
       [
        -5.694591045379639,
        1.6021769046783447,
        -6.033819198608398,
        -5.986690044403076
       ]
      ],
      "joint_logits": [
       -7.479767322540283,
       -7.156056880950928,
       -7.791145324707031,
       -7.716160297393799,
       -7.746248722076416,
       -8.048277854919434,
       -8.121956825256348,
       -7.986939907073975,
       -7.364834308624268,
       1.6015923023223877,
       -7.75242280960083,
       -7.911299228668213,
       -7.934703826904297,
       -7.768474578857422,
       -8.260623931884766,
       -8.149093627929688,
       -7.838379383087158,
       -7.450267314910889,
       -8.187251091003418,
       -8.122584342956543,
       -8.054987907409668,
       -8.302497863769531,
       -8.368285179138184,
       -8.367653846740723,
       -7.893827438354492,
       -8.032357215881348,
       -8.186456680297852,
       -8.098649024963379,
       -8.141247749328613,
       -8.403090476989746,
       -8.472644805908203,
       -8.343327522277832
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "reassign",
           "probability": 0.9983074921676981,
           "distribution": [
            {
             "value": "approve with rest",
             "probability": 0.0007224904123866249
            },
            {
             "value": "reassign",
             "probability": 0.9983074921676981
            },
            {
             "value": "approve",
             "probability": 0.0005189591683770822
            },
            {
             "value": "deny",
             "probability": 0.0004510582515381857
            }
           ]
          },
          "exception": {
           "value": false,
           "probability": 0.9990527933321056,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990527933321056
            },
            {
             "value": true,
             "probability": 0.0009472066678943462
            }
           ]
          },
          "breaches": {
           "value": 1,
           "probability": 0.9983362352472664,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006765981695685389
            },
            {
             "value": 1,
             "probability": 0.9983362352472664
            },
            {
             "value": 2,
             "probability": 0.00048195433973883324
            },
            {
             "value": 3,
             "probability": 0.000505212243426241
            }
           ]
          }
         },
         "joint_probability": 0.9977528251386628,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022497035477431556,
        "field_brier": 3.1223886328424024e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "reassign",
           "probability": 0.9983074921676981,
           "distribution": [
            {
             "value": "approve with rest",
             "probability": 0.0007224904123866249
            },
            {
             "value": "reassign",
             "probability": 0.9983074921676981
            },
            {
             "value": "approve",
             "probability": 0.0005189591683770822
            },
            {
             "value": "deny",
             "probability": 0.0004510582515381857
            }
           ]
          },
          "exception": {
           "value": false,
           "probability": 0.9990527933321056,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990527933321056
            },
            {
             "value": true,
             "probability": 0.0009472066678943462
            }
           ]
          },
          "breaches": {
           "value": 1,
           "probability": 0.9983362352472664,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006765981695685389
            },
            {
             "value": 1,
             "probability": 0.9983362352472664
            },
            {
             "value": 2,
             "probability": 0.00048195433973883324
            },
            {
             "value": 3,
             "probability": 0.000505212243426241
            }
           ]
          }
         },
         "joint_probability": 0.9977528251386628,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022497035477431556,
        "field_brier": 3.1223886328424024e-06
       }
      },
      "timing_ms": {
       "scoring": 36.96510195732117,
       "base_joint": 0.08507911115884781,
       "constrained_joint": 0.0573799479752779,
       "constrained_total": 37.022481905296445
      },
      "input_tokens": 731,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0078df3f28bfceb9-0036",
      "group": "bendsynth-0078df3f28bfceb9",
      "provenance": "bendsynth-v1; family=bendsynth-0078df3f28bfceb9; domain=coworking space membership chatbot; parameters=threshold_day=900,threshold_member=2400; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "exceeds_limit": true,
       "risk_level": 1,
       "decision": "overage charge"
      },
      "logits": [
       [
        -3.464988946914673,
        1.9387469291687012
       ],
       [
        -3.451082706451416,
        1.9369330406188965,
        -4.986564636230469,
        -5.234725475311279
       ],
       [
        -3.6497931480407715,
        -2.9886527061462402,
        1.9322397708892822
       ]
      ],
      "joint_logits": [
       -5.7457170486450195,
       -5.629383087158203,
       -6.002603530883789,
       -5.756094932556152,
       -5.273157119750977,
       -4.865985870361328,
       -6.676424980163574,
       -6.582201957702637,
       -7.348077297210693,
       -6.881206512451172,
       -6.746253490447998,
       -7.547039031982422,
       -5.559142589569092,
       -4.909523010253906,
       -4.489731311798096,
       -4.451778888702393,
       -3.5235750675201416,
       1.928661584854126,
       -6.896847724914551,
       -6.671291828155518,
       -6.674697399139404,
       -7.095783710479736,
       -6.969191551208496,
       -7.094766139984131
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "exceeds_limit": {
           "value": true,
           "probability": 0.9955204176694677,
           "distribution": [
            {
             "value": false,
             "probability": 0.004479582330532248
            },
            {
             "value": true,
             "probability": 0.9955204176694677
            }
           ]
          },
          "risk_level": {
           "value": 1,
           "probability": 0.9937162726124725,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0045423105406401645
            },
            {
             "value": 1,
             "probability": 0.9937162726124725
            },
            {
             "value": 2,
             "probability": 0.000978195142496479
            },
            {
             "value": 3,
             "probability": 0.0007632217043908178
            }
           ]
          },
          "decision": {
           "value": "overage charge",
           "probability": 0.9890634073507126,
           "distribution": [
            {
             "value": "no overage charge",
             "probability": 0.003723729004191814
            },
            {
             "value": "account flagged",
             "probability": 0.00721286364509555
            },
            {
             "value": "overage charge",
             "probability": 0.9890634073507126
            }
           ]
          }
         },
         "joint_probability": 0.9855306060236267,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.014575096531035343,
        "field_brier": 9.57637073943703e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "exceeds_limit": {
           "value": true,
           "probability": 0.9955204176694677,
           "distribution": [
            {
             "value": false,
             "probability": 0.004479582330532248
            },
            {
             "value": true,
             "probability": 0.9955204176694677
            }
           ]
          },
          "risk_level": {
           "value": 1,
           "probability": 0.9937162726124725,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0045423105406401645
            },
            {
             "value": 1,
             "probability": 0.9937162726124725
            },
            {
             "value": 2,
             "probability": 0.000978195142496479
            },
            {
             "value": 3,
             "probability": 0.0007632217043908178
            }
           ]
          },
          "decision": {
           "value": "overage charge",
           "probability": 0.9890634073507126,
           "distribution": [
            {
             "value": "no overage charge",
             "probability": 0.003723729004191814
            },
            {
             "value": "account flagged",
             "probability": 0.00721286364509555
            },
            {
             "value": "overage charge",
             "probability": 0.9890634073507126
            }
           ]
          }
         },
         "joint_probability": 0.9855306060236267,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.014575096531035343,
        "field_brier": 9.57637073943703e-05
       }
      },
      "timing_ms": {
       "scoring": 37.97065117396414,
       "base_joint": 0.06493902765214443,
       "constrained_joint": 0.048110028728842735,
       "constrained_total": 38.018761202692986
      },
      "input_tokens": 724,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0078df3f28bfceb9-0038",
      "group": "bendsynth-0078df3f28bfceb9",
      "provenance": "bendsynth-v1; family=bendsynth-0078df3f28bfceb9; domain=coworking space membership chatbot; parameters=threshold_day=599,threshold_member=1256; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "exceeds_limit": true,
       "risk_level": 1,
       "decision": "overage charge"
      },
      "logits": [
       [
        -4.217568397521973,
        1.5073294639587402
       ],
       [
        -4.949658393859863,
        1.506386399269104,
        -5.072790145874023,
        -5.177961349487305
       ],
       [
        -2.601747751235962,
        1.4914854764938354,
        -4.460453987121582
       ]
      ],
      "joint_logits": [
       -6.6150946617126465,
       -7.302900791168213,
       -6.54771614074707,
       -6.065423488616943,
       -6.547183990478516,
       -6.588359355926514,
       -6.697057247161865,
       -7.41281270980835,
       -6.65379524230957,
       -6.735311985015869,
       -7.432437419891357,
       -6.68607234954834,
       -6.4850077629089355,
       -6.97836446762085,
       -6.74147891998291,
       -2.7458457946777344,
       1.4902199506759644,
       -5.807486534118652,
       -6.645304203033447,
       -7.146305561065674,
       -6.85905647277832,
       -6.8693695068359375,
       -7.422131061553955,
       -6.933725357055664
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "exceeds_limit": {
           "value": true,
           "probability": 0.9967469308057453,
           "distribution": [
            {
             "value": false,
             "probability": 0.003253069194254694
            },
            {
             "value": true,
             "probability": 0.9967469308057453
            }
           ]
          },
          "risk_level": {
           "value": 1,
           "probability": 0.9958073322508252,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001564410466908474
            },
            {
             "value": 1,
             "probability": 0.9958073322508252
            },
            {
             "value": 2,
             "probability": 0.0013831691677775447
            },
            {
             "value": 3,
             "probability": 0.0012450881144888334
            }
           ]
          },
          "decision": {
           "value": "account flagged",
           "probability": 0.9810789211045289,
           "distribution": [
            {
             "value": "overage charge",
             "probability": 0.016369496655267626
            },
            {
             "value": "account flagged",
             "probability": 0.9810789211045289
            },
            {
             "value": "no overage charge",
             "probability": 0.0025515822402034523
            }
           ]
          }
         },
         "joint_probability": 0.9798380454243303,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 4.2564337260992104,
        "field_brier": 0.643365327093485
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "exceeds_limit": {
           "value": true,
           "probability": 0.9967469308057453,
           "distribution": [
            {
             "value": false,
             "probability": 0.003253069194254694
            },
            {
             "value": true,
             "probability": 0.9967469308057453
            }
           ]
          },
          "risk_level": {
           "value": 1,
           "probability": 0.9958073322508252,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001564410466908474
            },
            {
             "value": 1,
             "probability": 0.9958073322508252
            },
            {
             "value": 2,
             "probability": 0.0013831691677775447
            },
            {
             "value": 3,
             "probability": 0.0012450881144888334
            }
           ]
          },
          "decision": {
           "value": "account flagged",
           "probability": 0.9810789211045289,
           "distribution": [
            {
             "value": "overage charge",
             "probability": 0.016369496655267626
            },
            {
             "value": "account flagged",
             "probability": 0.9810789211045289
            },
            {
             "value": "no overage charge",
             "probability": 0.0025515822402034523
            }
           ]
          }
         },
         "joint_probability": 0.9798380454243303,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 4.2564337260992104,
        "field_brier": 0.643365327093485
       }
      },
      "timing_ms": {
       "scoring": 37.73560398258269,
       "base_joint": 0.06928900256752968,
       "constrained_joint": 0.04765018820762634,
       "constrained_total": 37.783254170790315
      },
      "input_tokens": 694,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0382acfb5ccec16b-0019",
      "group": "bendsynth-0382acfb5ccec16b",
      "provenance": "bendsynth-v1; family=bendsynth-0382acfb5ccec16b; domain=museum photography permit audit; parameters=max_group=3; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "major violation",
       "breaches": 3,
       "oversized_group": true
      },
      "logits": [
       [
        -7.276982307434082,
        -0.12495329231023788,
        -7.265743255615234
       ],
       [
        -7.4762091636657715,
        -7.512493133544922,
        -7.435639381408691,
        -0.12530697882175446
       ],
       [
        -6.920989513397217,
        -0.12449614703655243
       ]
      ],
      "joint_logits": [
       -9.475273132324219,
       -9.247485160827637,
       -9.526662826538086,
       -9.326149940490723,
       -9.451313018798828,
       -9.307511329650879,
       -9.573183059692383,
       -9.050383567810059,
       -9.226884841918945,
       -9.12641429901123,
       -9.34708023071289,
       -9.069697380065918,
       -9.301846504211426,
       -8.716347694396973,
       -9.179776191711426,
       -0.1258234977722168,
       -9.348823547363281,
       -9.218985557556152,
       -9.440458297729492,
       -9.185593605041504,
       -9.504772186279297,
       -9.305865287780762,
       -9.593894958496094,
       -9.23672103881836
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "major violation",
           "probability": 0.9984270788351554,
           "distribution": [
            {
             "value": "minor violation",
             "probability": 0.000782041017158435
            },
            {
             "value": "major violation",
             "probability": 0.9984270788351554
            },
            {
             "value": "compliant",
             "probability": 0.0007908801476862458
            }
           ]
          },
          "breaches": {
           "value": 3,
           "probability": 0.9980739735809055,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006407762940677172
            },
            {
             "value": 1,
             "probability": 0.0006179431258132354
            },
            {
             "value": 2,
             "probability": 0.0006673069992135268
            },
            {
             "value": 3,
             "probability": 0.9980739735809055
            }
           ]
          },
          "oversized_group": {
           "value": true,
           "probability": 0.9988835602756135,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011164397243864923
            },
            {
             "value": true,
             "probability": 0.9988835602756135
            }
           ]
          }
         },
         "joint_probability": 0.9975585935167868,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024443915755603704,
        "field_brier": 3.7171200772477006e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "major violation",
           "probability": 0.9984270788351554,
           "distribution": [
            {
             "value": "minor violation",
             "probability": 0.000782041017158435
            },
            {
             "value": "major violation",
             "probability": 0.9984270788351554
            },
            {
             "value": "compliant",
             "probability": 0.0007908801476862458
            }
           ]
          },
          "breaches": {
           "value": 3,
           "probability": 0.9980739735809055,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006407762940677172
            },
            {
             "value": 1,
             "probability": 0.0006179431258132354
            },
            {
             "value": 2,
             "probability": 0.0006673069992135268
            },
            {
             "value": 3,
             "probability": 0.9980739735809055
            }
           ]
          },
          "oversized_group": {
           "value": true,
           "probability": 0.9988835602756135,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011164397243864923
            },
            {
             "value": true,
             "probability": 0.9988835602756135
            }
           ]
          }
         },
         "joint_probability": 0.9975585935167868,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024443915755603704,
        "field_brier": 3.7171200772477006e-06
       }
      },
      "timing_ms": {
       "scoring": 38.1584691349417,
       "base_joint": 0.06335880607366562,
       "constrained_joint": 0.04880991764366627,
       "constrained_total": 38.20727905258536
      },
      "input_tokens": 715,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0382acfb5ccec16b-0027",
      "group": "bendsynth-0382acfb5ccec16b",
      "provenance": "bendsynth-v1; family=bendsynth-0382acfb5ccec16b; domain=museum photography permit audit; parameters=max_group=4; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "oversized_group": false,
       "decision": "compliant",
       "breaches": 1
      },
      "logits": [
       [
        0.265516996383667,
        -6.287970542907715
       ],
       [
        0.2646791934967041,
        -6.69089937210083,
        -6.372082233428955
       ],
       [
        -7.0511579513549805,
        0.2649061381816864,
        -6.9369659423828125,
        -7.1093645095825195
       ]
      ],
      "joint_logits": [
       -9.207054138183594,
       0.2640981078147888,
       -8.979840278625488,
       -9.335455894470215,
       -8.87818431854248,
       -8.397359848022461,
       -8.803366661071777,
       -8.938332557678223,
       -8.559113502502441,
       -8.091534614562988,
       -8.4775390625,
       -8.60978889465332,
       -9.180930137634277,
       -8.926109313964844,
       -9.13249683380127,
       -9.225794792175293,
       -8.873542785644531,
       -8.711959838867188,
       -8.75094223022461,
       -8.926237106323242,
       -8.558747291564941,
       -8.405726432800293,
       -8.419319152832031,
       -8.599058151245117
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "oversized_group": {
           "value": false,
           "probability": 0.9985768919683651,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985768919683651
            },
            {
             "value": true,
             "probability": 0.001423108031634893
            }
           ]
          },
          "decision": {
           "value": "compliant",
           "probability": 0.9977405471141063,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.9977405471141063
            },
            {
             "value": "minor violation",
             "probability": 0.0009511482443744225
            },
            {
             "value": "major violation",
             "probability": 0.0013083046415192595
            }
           ]
          },
          "breaches": {
           "value": 1,
           "probability": 0.9979669944319552,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006634221380938761
            },
            {
             "value": 1,
             "probability": 0.9979669944319552
            },
            {
             "value": 2,
             "probability": 0.0007436744656920749
            },
            {
             "value": 3,
             "probability": 0.0006259089642587054
            }
           ]
          }
         },
         "joint_probability": 0.9971609658125873,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028430718888971655,
        "field_brier": 5.763332872084695e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "oversized_group": {
           "value": false,
           "probability": 0.9985768919683651,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985768919683651
            },
            {
             "value": true,
             "probability": 0.001423108031634893
            }
           ]
          },
          "decision": {
           "value": "compliant",
           "probability": 0.9977405471141063,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.9977405471141063
            },
            {
             "value": "minor violation",
             "probability": 0.0009511482443744225
            },
            {
             "value": "major violation",
             "probability": 0.0013083046415192595
            }
           ]
          },
          "breaches": {
           "value": 1,
           "probability": 0.9979669944319552,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006634221380938761
            },
            {
             "value": 1,
             "probability": 0.9979669944319552
            },
            {
             "value": 2,
             "probability": 0.0007436744656920749
            },
            {
             "value": 3,
             "probability": 0.0006259089642587054
            }
           ]
          }
         },
         "joint_probability": 0.9971609658125873,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028430718888971655,
        "field_brier": 5.763332872084695e-06
       }
      },
      "timing_ms": {
       "scoring": 37.08494105376303,
       "base_joint": 0.06884895265102386,
       "constrained_joint": 0.04791910760104656,
       "constrained_total": 37.13286016136408
      },
      "input_tokens": 736,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01b0dcf40678d20f-0010",
      "group": "bendsynth-01b0dcf40678d20f",
      "provenance": "bendsynth-v1; family=bendsynth-01b0dcf40678d20f; domain=art gallery incoming loan request intake; parameters=min_value=355,max_months=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "return to curator",
       "paperwork_complete": false,
       "issues": 2
      },
      "logits": [
       [
        -5.953488349914551,
        -5.723203182220459,
        -5.935573101043701,
        1.7661563158035278
       ],
       [
        1.766715168952942,
        -5.251204967498779
       ],
       [
        -6.370390892028809,
        -5.975052356719971,
        1.766148567199707,
        -6.0834808349609375,
        -6.203697204589844
       ]
      ],
      "joint_logits": [
       -8.417190551757812,
       -8.127198219299316,
       -7.971905708312988,
       -8.194169044494629,
       -8.274166107177734,
       -8.50961685180664,
       -8.296441078186035,
       -8.224712371826172,
       -8.295354843139648,
       -8.35384464263916,
       -8.335624694824219,
       -8.031392097473145,
       -7.304419994354248,
       -8.120604515075684,
       -8.165729522705078,
       -8.372511863708496,
       -8.012364387512207,
       -8.144827842712402,
       -8.064831733703613,
       -8.169854164123535,
       -8.517958641052246,
       -8.125670433044434,
       -7.646982669830322,
       -8.203611373901367,
       -8.330089569091797,
       -8.588751792907715,
       -8.274877548217773,
       -8.275060653686523,
       -8.333601951599121,
       -8.423736572265625,
       -8.41447639465332,
       -7.6505231857299805,
       1.7656898498535156,
       -7.9763336181640625,
       -8.273609161376953,
       -8.466927528381348,
       -8.067215919494629,
       -7.891262054443359,
       -8.162758827209473,
       -8.301125526428223
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "return to curator",
           "probability": 0.998547049816487,
           "distribution": [
            {
             "value": "decline",
             "probability": 0.0004433733872633659
            },
            {
             "value": "counter-offer shorter loan",
             "probability": 0.0005581887882116118
            },
            {
             "value": "accept",
             "probability": 0.0004513880080379627
            },
            {
             "value": "return to curator",
             "probability": 0.998547049816487
            }
           ]
          },
          "paperwork_complete": {
           "value": false,
           "probability": 0.9991051148594174,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991051148594174
            },
            {
             "value": true,
             "probability": 0.0008948851405824885
            }
           ]
          },
          "issues": {
           "value": 2,
           "probability": 0.9985393101989519,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00029222073509448126
            },
            {
             "value": 1,
             "probability": 0.0004339147031277085
            },
            {
             "value": 2,
             "probability": 0.9985393101989519
            },
            {
             "value": 3,
             "probability": 0.00038932701279930737
            },
            {
             "value": 4,
             "probability": 0.00034522735002648363
            }
           ]
          }
         },
         "joint_probability": 0.9980813781685709,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019204647439071743,
        "field_brier": 2.3675519838857436e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "return to curator",
           "probability": 0.998547049816487,
           "distribution": [
            {
             "value": "decline",
             "probability": 0.0004433733872633659
            },
            {
             "value": "counter-offer shorter loan",
             "probability": 0.0005581887882116118
            },
            {
             "value": "accept",
             "probability": 0.0004513880080379627
            },
            {
             "value": "return to curator",
             "probability": 0.998547049816487
            }
           ]
          },
          "paperwork_complete": {
           "value": false,
           "probability": 0.9991051148594174,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991051148594174
            },
            {
             "value": true,
             "probability": 0.0008948851405824885
            }
           ]
          },
          "issues": {
           "value": 2,
           "probability": 0.9985393101989519,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00029222073509448126
            },
            {
             "value": 1,
             "probability": 0.0004339147031277085
            },
            {
             "value": 2,
             "probability": 0.9985393101989519
            },
            {
             "value": 3,
             "probability": 0.00038932701279930737
            },
            {
             "value": 4,
             "probability": 0.00034522735002648363
            }
           ]
          }
         },
         "joint_probability": 0.9980813781685709,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019204647439071743,
        "field_brier": 2.3675519838857436e-06
       }
      },
      "timing_ms": {
       "scoring": 33.950604032725096,
       "base_joint": 0.08309888653457165,
       "constrained_joint": 0.06572878919541836,
       "constrained_total": 34.016332821920514
      },
      "input_tokens": 560,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01b0dcf40678d20f-0026",
      "group": "bendsynth-01b0dcf40678d20f",
      "provenance": "bendsynth-v1; family=bendsynth-01b0dcf40678d20f; domain=art gallery incoming loan request intake; parameters=min_value=499,max_months=17; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "issues": 0,
       "decision": "accept",
       "paperwork_complete": true
      },
      "logits": [
       [
        1.6347984075546265,
        -6.350552558898926,
        -6.436910629272461,
        -6.496703624725342,
        -6.503067970275879
       ],
       [
        1.6347209215164185,
        -6.164634704589844,
        -6.062158584594727,
        -6.066579818725586
       ],
       [
        -5.5089311599731445,
        1.6352473497390747
       ]
      ],
      "joint_logits": [
       -7.8179707527160645,
       1.6342933177947998,
       -8.172171592712402,
       -7.763477325439453,
       -8.137713432312012,
       -7.775181770324707,
       -8.08166217803955,
       -7.689568996429443,
       -8.458479881286621,
       -8.138633728027344,
       -8.685205459594727,
       -8.443575859069824,
       -8.589536666870117,
       -8.303628921508789,
       -8.598711967468262,
       -8.337783813476562,
       -8.558183670043945,
       -8.264162063598633,
       -8.771655082702637,
       -8.535196304321289,
       -8.651294708251953,
       -8.358193397521973,
       -8.665350914001465,
       -8.428866386413574,
       -8.616877555847168,
       -8.34450626373291,
       -8.824912071228027,
       -8.621542930603027,
       -8.69202995300293,
       -8.392777442932129,
       -8.713579177856445,
       -8.498770713806152,
       -8.627036094665527,
       -8.34211540222168,
       -8.806065559387207,
       -8.6102294921875,
       -8.721381187438965,
       -8.437793731689453,
       -8.704700469970703,
       -8.49776554107666
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 0,
           "probability": 0.9987624847470088,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9987624847470088
            },
            {
             "value": 1,
             "probability": 0.00033999171890542975
            },
            {
             "value": 2,
             "probability": 0.00031186286612719594
            },
            {
             "value": 3,
             "probability": 0.00029376220606240204
            },
            {
             "value": 4,
             "probability": 0.0002918984618961544
            }
           ]
          },
          "decision": {
           "value": "accept",
           "probability": 0.9986852506977183,
           "distribution": [
            {
             "value": "accept",
             "probability": 0.9986852506977183
            },
            {
             "value": "return to curator",
             "probability": 0.00040946005885708576
            },
            {
             "value": "decline",
             "probability": 0.00045364532966231125
            },
            {
             "value": "counter-offer shorter loan",
             "probability": 0.0004516439137622523
            }
           ]
          },
          "paperwork_complete": {
           "value": true,
           "probability": 0.9992111761929724,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007888238070276424
            },
            {
             "value": true,
             "probability": 0.9992111761929724
            }
           ]
          }
         },
         "joint_probability": 0.9982582981980404,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0017432203280123414,
        "field_brier": 1.8220944460089037e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 0,
           "probability": 0.9987624847470088,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9987624847470088
            },
            {
             "value": 1,
             "probability": 0.00033999171890542975
            },
            {
             "value": 2,
             "probability": 0.00031186286612719594
            },
            {
             "value": 3,
             "probability": 0.00029376220606240204
            },
            {
             "value": 4,
             "probability": 0.0002918984618961544
            }
           ]
          },
          "decision": {
           "value": "accept",
           "probability": 0.9986852506977183,
           "distribution": [
            {
             "value": "accept",
             "probability": 0.9986852506977183
            },
            {
             "value": "return to curator",
             "probability": 0.00040946005885708576
            },
            {
             "value": "decline",
             "probability": 0.00045364532966231125
            },
            {
             "value": "counter-offer shorter loan",
             "probability": 0.0004516439137622523
            }
           ]
          },
          "paperwork_complete": {
           "value": true,
           "probability": 0.9992111761929724,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007888238070276424
            },
            {
             "value": true,
             "probability": 0.9992111761929724
            }
           ]
          }
         },
         "joint_probability": 0.9982582981980404,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0017432203280123414,
        "field_brier": 1.8220944460089037e-06
       }
      },
      "timing_ms": {
       "scoring": 35.331758903339505,
       "base_joint": 0.08693896234035492,
       "constrained_joint": 0.06674905307590961,
       "constrained_total": 35.398507956415415
      },
      "input_tokens": 560,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01b0dcf40678d20f-0029",
      "group": "bendsynth-01b0dcf40678d20f",
      "provenance": "bendsynth-v1; family=bendsynth-01b0dcf40678d20f; domain=art gallery incoming loan request intake; parameters=min_value=251,max_months=24; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "return to curator",
       "paperwork_complete": false,
       "issues": 1
      },
      "logits": [
       [
        -5.867450714111328,
        -5.967461585998535,
        1.892827033996582,
        -5.803501605987549
       ],
       [
        1.8932303190231323,
        -5.162034034729004
       ],
       [
        -6.1397528648376465,
        1.8927098512649536,
        -6.076198577880859,
        -6.036400318145752,
        -6.0562849044799805
       ]
      ],
      "joint_logits": [
       -8.222738265991211,
       -8.057326316833496,
       -8.191532135009766,
       -8.144768714904785,
       -8.169063568115234,
       -8.229469299316406,
       -8.308976173400879,
       -8.165750503540039,
       -8.103408813476562,
       -8.129886627197266,
       -8.37194538116455,
       -8.133604049682617,
       -8.307538032531738,
       -8.245734214782715,
       -8.281515121459961,
       -8.323272705078125,
       -8.280843734741211,
       -8.307723045349121,
       -8.2245512008667,
       -8.243132591247559,
       -8.050179481506348,
       1.8923780918121338,
       -7.884891033172607,
       -7.972269535064697,
       -8.012969017028809,
       -8.151613235473633,
       -7.896291732788086,
       -8.109407424926758,
       -8.03458023071289,
       -8.038630485534668,
       -8.254000663757324,
       -7.7357497215271,
       -8.186480522155762,
       -8.136249542236328,
       -8.1296968460083,
       -8.185359954833984,
       -8.17141056060791,
       -8.15621280670166,
       -8.094101905822754,
       -8.110248565673828
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "return to curator",
           "probability": 0.9987350089827224,
           "distribution": [
            {
             "value": "decline",
             "probability": 0.0004257988042611703
            },
            {
             "value": "counter-offer shorter loan",
             "probability": 0.00038527443788787235
            },
            {
             "value": "return to curator",
             "probability": 0.9987350089827224
            },
            {
             "value": "accept",
             "probability": 0.0004539177751286735
            }
           ]
          },
          "paperwork_complete": {
           "value": false,
           "probability": 0.9991378892567858,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991378892567858
            },
            {
             "value": true,
             "probability": 0.0008621107432142206
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.9986179850360161,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003242986226748365
            },
            {
             "value": 1,
             "probability": 0.9986179850360161
            },
            {
             "value": 2,
             "probability": 0.000345578273993575
            },
            {
             "value": 3,
             "probability": 0.0003596090937906726
            },
            {
             "value": 4,
             "probability": 0.00035252897352492605
            }
           ]
          }
         },
         "joint_probability": 0.9982867490596947,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0017147202331155194,
        "field_brier": 2.003536393337917e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "return to curator",
           "probability": 0.9987350089827224,
           "distribution": [
            {
             "value": "decline",
             "probability": 0.0004257988042611703
            },
            {
             "value": "counter-offer shorter loan",
             "probability": 0.00038527443788787235
            },
            {
             "value": "return to curator",
             "probability": 0.9987350089827224
            },
            {
             "value": "accept",
             "probability": 0.0004539177751286735
            }
           ]
          },
          "paperwork_complete": {
           "value": false,
           "probability": 0.9991378892567858,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991378892567858
            },
            {
             "value": true,
             "probability": 0.0008621107432142206
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.9986179850360161,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003242986226748365
            },
            {
             "value": 1,
             "probability": 0.9986179850360161
            },
            {
             "value": 2,
             "probability": 0.000345578273993575
            },
            {
             "value": 3,
             "probability": 0.0003596090937906726
            },
            {
             "value": 4,
             "probability": 0.00035252897352492605
            }
           ]
          }
         },
         "joint_probability": 0.9982867490596947,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0017147202331155194,
        "field_brier": 2.003536393337917e-06
       }
      },
      "timing_ms": {
       "scoring": 33.25543203391135,
       "base_joint": 0.08318014442920685,
       "constrained_joint": 0.06658886559307575,
       "constrained_total": 33.32202089950442
      },
      "input_tokens": 572,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01b0dcf40678d20f-0031",
      "group": "bendsynth-01b0dcf40678d20f",
      "provenance": "bendsynth-v1; family=bendsynth-01b0dcf40678d20f; domain=art gallery incoming loan request intake; parameters=min_value=500,max_months=24; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "issues": 0,
       "paperwork_complete": true,
       "decision": "accept"
      },
      "logits": [
       [
        1.782127022743225,
        -6.1820783615112305,
        -6.244311332702637,
        -6.28413724899292,
        -6.309320449829102
       ],
       [
        -5.324864387512207,
        1.7826019525527954
       ],
       [
        -5.818072319030762,
        1.7820708751678467,
        -5.911123275756836,
        -6.055844306945801
       ]
      ],
      "joint_logits": [
       -7.908807277679443,
       -7.658085346221924,
       -7.993460178375244,
       -8.12159538269043,
       -7.464816570281982,
       1.7816274166107178,
       -7.648983478546143,
       -7.718527317047119,
       -8.332436561584473,
       -8.282329559326172,
       -8.437387466430664,
       -8.563270568847656,
       -8.088223457336426,
       -7.994747638702393,
       -8.174657821655273,
       -8.339376449584961,
       -8.393196105957031,
       -8.329706192016602,
       -8.473419189453125,
       -8.6181058883667,
       -8.163919448852539,
       -8.08186149597168,
       -8.224382400512695,
       -8.411965370178223,
       -8.42725944519043,
       -8.395658493041992,
       -8.483765602111816,
       -8.634598731994629,
       -8.20378589630127,
       -8.150442123413086,
       -8.242209434509277,
       -8.464451789855957,
       -8.444174766540527,
       -8.411445617675781,
       -8.528343200683594,
       -8.67092514038086,
       -8.230155944824219,
       -8.163702964782715,
       -8.287027359008789,
       -8.47095012664795
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 0,
           "probability": 0.9987071756305348,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9987071756305348
            },
            {
             "value": 1,
             "probability": 0.00034723842673281335
            },
            {
             "value": 2,
             "probability": 0.0003262873369944046
            },
            {
             "value": 3,
             "probability": 0.00031354809556426194
            },
            {
             "value": 4,
             "probability": 0.0003057505101737323
            }
           ]
          },
          "paperwork_complete": {
           "value": true,
           "probability": 0.9991817030787808,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008182969212191917
            },
            {
             "value": true,
             "probability": 0.9991817030787808
            }
           ]
          },
          "decision": {
           "value": "accept",
           "probability": 0.9986510324396595,
           "distribution": [
            {
             "value": "counter-offer shorter loan",
             "probability": 0.0004997046600552813
            },
            {
             "value": "accept",
             "probability": 0.9986510324396595
            },
            {
             "value": "decline",
             "probability": 0.0004553045049955872
            },
            {
             "value": "return to curator",
             "probability": 0.0003939583952897623
            }
           ]
          }
         },
         "joint_probability": 0.9982083072303376,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001793299770941466,
        "field_brier": 1.953790640814507e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 0,
           "probability": 0.9987071756305348,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9987071756305348
            },
            {
             "value": 1,
             "probability": 0.00034723842673281335
            },
            {
             "value": 2,
             "probability": 0.0003262873369944046
            },
            {
             "value": 3,
             "probability": 0.00031354809556426194
            },
            {
             "value": 4,
             "probability": 0.0003057505101737323
            }
           ]
          },
          "paperwork_complete": {
           "value": true,
           "probability": 0.9991817030787808,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008182969212191917
            },
            {
             "value": true,
             "probability": 0.9991817030787808
            }
           ]
          },
          "decision": {
           "value": "accept",
           "probability": 0.9986510324396595,
           "distribution": [
            {
             "value": "counter-offer shorter loan",
             "probability": 0.0004997046600552813
            },
            {
             "value": "accept",
             "probability": 0.9986510324396595
            },
            {
             "value": "decline",
             "probability": 0.0004553045049955872
            },
            {
             "value": "return to curator",
             "probability": 0.0003939583952897623
            }
           ]
          }
         },
         "joint_probability": 0.9982083072303376,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001793299770941466,
        "field_brier": 1.953790640814507e-06
       }
      },
      "timing_ms": {
       "scoring": 35.511367954313755,
       "base_joint": 0.09208917617797852,
       "constrained_joint": 0.07559917867183685,
       "constrained_total": 35.58696713298559
      },
      "input_tokens": 566,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05dd068df1ba8f0b-0007",
      "group": "bendsynth-05dd068df1ba8f0b",
      "provenance": "bendsynth-v1; family=bendsynth-05dd068df1ba8f0b; domain=textile mill fabric quality inspection; parameters=limit=4,shrink_limit=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "reject",
       "over_shrink": false,
       "fail_count": 1
      },
      "logits": [
       [
        -6.605064392089844,
        -6.71941614151001,
        -6.722756385803223,
        0.8329230546951294
       ],
       [
        0.8336923122406006,
        -6.217587471008301
       ],
       [
        -7.300047874450684,
        0.8331705927848816,
        -6.89157772064209,
        -7.16473388671875,
        -7.2232465744018555
       ]
      ],
      "joint_logits": [
       -9.125947952270508,
       -8.247891426086426,
       -8.793935775756836,
       -8.99495792388916,
       -9.049924850463867,
       -9.297723770141602,
       -8.816619873046875,
       -8.880757331848145,
       -9.128100395202637,
       -9.18754768371582,
       -9.23819637298584,
       -8.406577110290527,
       -8.846691131591797,
       -9.137245178222656,
       -9.163777351379395,
       -9.401721954345703,
       -8.93258285522461,
       -8.964296340942383,
       -9.247367858886719,
       -9.298829078674316,
       -9.24632453918457,
       -8.383466720581055,
       -8.861178398132324,
       -9.112471580505371,
       -9.170143127441406,
       -9.41702651977539,
       -8.91254711151123,
       -8.996383666992188,
       -9.26684856414795,
       -9.342890739440918,
       -9.638726234436035,
       0.8326334357261658,
       -9.096983909606934,
       -9.532917022705078,
       -9.648221015930176,
       -9.858804702758789,
       -9.14663028717041,
       -9.481887817382812,
       -9.727593421936035,
       -9.768962860107422
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "reject",
           "probability": 0.9983661937048758,
           "distribution": [
            {
             "value": "rework",
             "probability": 0.0005875069237438302
            },
            {
             "value": "downgrade",
             "probability": 0.0005240233714096212
            },
            {
             "value": "pass",
             "probability": 0.0005222759999708773
            },
            {
             "value": "reject",
             "probability": 0.9983661937048758
            }
           ]
          },
          "over_shrink": {
           "value": false,
           "probability": 0.9991344502254975,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991344502254975
            },
            {
             "value": true,
             "probability": 0.0008655497745025914
            }
           ]
          },
          "fail_count": {
           "value": 1,
           "probability": 0.9986133186277943,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0002932145814486737
            },
            {
             "value": 1,
             "probability": 0.9986133186277943
            },
            {
             "value": 2,
             "probability": 0.00044114544629280164
            },
            {
             "value": 3,
             "probability": 0.0003357002211995426
            },
            {
             "value": 4,
             "probability": 0.00031662112326473387
            }
           ]
          }
         },
         "joint_probability": 0.9980770944419899,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019247567143491284,
        "field_brier": 2.492208610479445e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "reject",
           "probability": 0.9983661937048758,
           "distribution": [
            {
             "value": "rework",
             "probability": 0.0005875069237438302
            },
            {
             "value": "downgrade",
             "probability": 0.0005240233714096212
            },
            {
             "value": "pass",
             "probability": 0.0005222759999708773
            },
            {
             "value": "reject",
             "probability": 0.9983661937048758
            }
           ]
          },
          "over_shrink": {
           "value": false,
           "probability": 0.9991344502254975,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991344502254975
            },
            {
             "value": true,
             "probability": 0.0008655497745025914
            }
           ]
          },
          "fail_count": {
           "value": 1,
           "probability": 0.9986133186277943,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0002932145814486737
            },
            {
             "value": 1,
             "probability": 0.9986133186277943
            },
            {
             "value": 2,
             "probability": 0.00044114544629280164
            },
            {
             "value": 3,
             "probability": 0.0003357002211995426
            },
            {
             "value": 4,
             "probability": 0.00031662112326473387
            }
           ]
          }
         },
         "joint_probability": 0.9980770944419899,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019247567143491284,
        "field_brier": 2.492208610479445e-06
       }
      },
      "timing_ms": {
       "scoring": 34.85761513002217,
       "base_joint": 0.08508889004588127,
       "constrained_joint": 0.06727897562086582,
       "constrained_total": 34.924894105643034
      },
      "input_tokens": 634,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07d3c9374685dd2a-0002",
      "group": "bendsynth-07d3c9374685dd2a",
      "provenance": "bendsynth-v1; family=bendsynth-07d3c9374685dd2a; domain=hospital bed allocation office - auditor reviewing one patient file; parameters=wait_threshold=23; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "issues": 0,
       "decision": "allocate bed",
       "wait_ok": true
      },
      "logits": [
       [
        1.3421880006790161,
        -5.762661933898926,
        -5.917681694030762,
        -5.979402542114258
       ],
       [
        -5.214652061462402,
        1.3421719074249268,
        -5.812392234802246
       ],
       [
        -5.266153335571289,
        1.3430240154266357
       ]
      ],
      "joint_logits": [
       -7.487098693847656,
       -6.803405284881592,
       -7.9486260414123535,
       1.3414764404296875,
       -8.020275115966797,
       -7.9014482498168945,
       -7.285891532897949,
       -7.2240376472473145,
       -7.9030022621154785,
       -7.565765380859375,
       -7.805874824523926,
       -7.742705345153809,
       -7.472247123718262,
       -7.348300933837891,
       -8.020078659057617,
       -7.80816125869751,
       -7.933525562286377,
       -7.861281871795654,
       -7.518955707550049,
       -7.423515796661377,
       -8.042977333068848,
       -7.920994758605957,
       -7.988998889923096,
       -7.9100728034973145
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 0,
           "probability": 0.9978193428831967,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978193428831967
            },
            {
             "value": 1,
             "probability": 0.0008193223931553255
            },
            {
             "value": 2,
             "probability": 0.0007016662860227413
            },
            {
             "value": 3,
             "probability": 0.0006596684376251757
            }
           ]
          },
          "decision": {
           "value": "allocate bed",
           "probability": 0.9978031563111656,
           "distribution": [
            {
             "value": "hold for review",
             "probability": 0.0014172697625267577
            },
            {
             "value": "allocate bed",
             "probability": 0.9978031563111656
            },
            {
             "value": "return to referring team",
             "probability": 0.0007795739263077731
            }
           ]
          },
          "wait_ok": {
           "value": true,
           "probability": 0.9986538740968007,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013461259031992297
            },
            {
             "value": true,
             "probability": 0.9986538740968007
            }
           ]
          }
         },
         "joint_probability": 0.9971096193091707,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028945659076235974,
        "field_brier": 5.806891281193847e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 0,
           "probability": 0.9978193428831967,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978193428831967
            },
            {
             "value": 1,
             "probability": 0.0008193223931553255
            },
            {
             "value": 2,
             "probability": 0.0007016662860227413
            },
            {
             "value": 3,
             "probability": 0.0006596684376251757
            }
           ]
          },
          "decision": {
           "value": "allocate bed",
           "probability": 0.9978031563111656,
           "distribution": [
            {
             "value": "hold for review",
             "probability": 0.0014172697625267577
            },
            {
             "value": "allocate bed",
             "probability": 0.9978031563111656
            },
            {
             "value": "return to referring team",
             "probability": 0.0007795739263077731
            }
           ]
          },
          "wait_ok": {
           "value": true,
           "probability": 0.9986538740968007,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013461259031992297
            },
            {
             "value": true,
             "probability": 0.9986538740968007
            }
           ]
          }
         },
         "joint_probability": 0.9971096193091707,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028945659076235974,
        "field_brier": 5.806891281193847e-06
       }
      },
      "timing_ms": {
       "scoring": 38.87684200890362,
       "base_joint": 0.06547011435031891,
       "constrained_joint": 0.04725903272628784,
       "constrained_total": 38.92410104162991
      },
      "input_tokens": 738,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07d3c9374685dd2a-0012",
      "group": "bendsynth-07d3c9374685dd2a",
      "provenance": "bendsynth-v1; family=bendsynth-07d3c9374685dd2a; domain=hospital bed allocation office - auditor reviewing one patient file; parameters=wait_threshold=11; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "return to referring team",
       "issues": 1,
       "wait_ok": true
      },
      "logits": [
       [
        1.2299541234970093,
        -5.870513916015625,
        -5.791461944580078
       ],
       [
        -6.094922065734863,
        1.2297693490982056,
        -6.149263381958008,
        -6.157716274261475
       ],
       [
        -5.484775543212891,
        1.2304582595825195
       ]
      ],
      "joint_logits": [
       -7.9750752449035645,
       -7.929170608520508,
       -7.865464210510254,
       1.2292358875274658,
       -8.058233261108398,
       -7.846627712249756,
       -8.023194313049316,
       -8.00822639465332,
       -7.9410600662231445,
       -7.8778276443481445,
       -8.137545585632324,
       -7.753843307495117,
       -8.009896278381348,
       -7.968021392822266,
       -7.99342155456543,
       -7.9607391357421875,
       -7.828061580657959,
       -7.7824482917785645,
       -8.058470726013184,
       -7.817282199859619,
       -7.912869453430176,
       -7.867542743682861,
       -7.878816604614258,
       -7.845922946929932
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "return to referring team",
           "probability": 0.9982856642943571,
           "distribution": [
            {
             "value": "return to referring team",
             "probability": 0.9982856642943571
            },
            {
             "value": "allocate bed",
             "probability": 0.000823305229147387
            },
            {
             "value": "hold for review",
             "probability": 0.0008910304764955737
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.9981013940187897,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006578116294163645
            },
            {
             "value": 1,
             "probability": 0.9981013940187897
            },
            {
             "value": 2,
             "probability": 0.0006230192226022804
            },
            {
             "value": 3,
             "probability": 0.000617775129191635
            }
           ]
          },
          "wait_ok": {
           "value": true,
           "probability": 0.9987891651785936,
           "distribution": [
            {
             "value": false,
             "probability": 0.00121083482140634
            },
            {
             "value": true,
             "probability": 0.9987891651785936
            }
           ]
          }
         },
         "joint_probability": 0.9975690425695359,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024339170048509186,
        "field_brier": 4.050058508383514e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "return to referring team",
           "probability": 0.9982856642943571,
           "distribution": [
            {
             "value": "return to referring team",
             "probability": 0.9982856642943571
            },
            {
             "value": "allocate bed",
             "probability": 0.000823305229147387
            },
            {
             "value": "hold for review",
             "probability": 0.0008910304764955737
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.9981013940187897,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006578116294163645
            },
            {
             "value": 1,
             "probability": 0.9981013940187897
            },
            {
             "value": 2,
             "probability": 0.0006230192226022804
            },
            {
             "value": 3,
             "probability": 0.000617775129191635
            }
           ]
          },
          "wait_ok": {
           "value": true,
           "probability": 0.9987891651785936,
           "distribution": [
            {
             "value": false,
             "probability": 0.00121083482140634
            },
            {
             "value": true,
             "probability": 0.9987891651785936
            }
           ]
          }
         },
         "joint_probability": 0.9975690425695359,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024339170048509186,
        "field_brier": 4.050058508383514e-06
       }
      },
      "timing_ms": {
       "scoring": 39.37261621467769,
       "base_joint": 0.06621913053095341,
       "constrained_joint": 0.04670885391533375,
       "constrained_total": 39.419325068593025
      },
      "input_tokens": 690,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07d3c9374685dd2a-0033",
      "group": "bendsynth-07d3c9374685dd2a",
      "provenance": "bendsynth-v1; family=bendsynth-07d3c9374685dd2a; domain=hospital bed allocation office - auditor reviewing one patient file; parameters=wait_threshold=15; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "issues": 1,
       "wait_ok": false,
       "decision": "hold for review"
      },
      "logits": [
       [
        -6.371094703674316,
        0.18892404437065125,
        -6.950220108032227,
        -7.004977226257324
       ],
       [
        0.18999019265174866,
        -6.0802459716796875
       ],
       [
        0.1889461874961853,
        -6.623695373535156,
        -6.109968185424805
       ]
      ],
      "joint_logits": [
       -7.51925802230835,
       -8.727758407592773,
       -8.01162052154541,
       -8.725654602050781,
       -8.340285301208496,
       -8.205788612365723,
       0.18784363567829132,
       -8.80679988861084,
       -7.406875133514404,
       -8.739481925964355,
       -8.65167236328125,
       -8.4561185836792,
       -9.04201602935791,
       -8.968008041381836,
       -8.686686515808105,
       -9.004257202148438,
       -8.639927864074707,
       -8.318068504333496,
       -9.212305068969727,
       -8.962091445922852,
       -8.75725269317627,
       -8.985480308532715,
       -8.675381660461426,
       -8.394515037536621
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 1,
           "probability": 0.9970482951498711,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0014116801747382
            },
            {
             "value": 1,
             "probability": 0.9970482951498711
            },
            {
             "value": 2,
             "probability": 0.0007910890106694782
            },
            {
             "value": 3,
             "probability": 0.0007489356647211866
            }
           ]
          },
          "wait_ok": {
           "value": false,
           "probability": 0.9981117897652013,
           "distribution": [
            {
             "value": false,
             "probability": 0.9981117897652013
            },
            {
             "value": true,
             "probability": 0.0018882102347986972
            }
           ]
          },
          "decision": {
           "value": "hold for review",
           "probability": 0.9970705244103786,
           "distribution": [
            {
             "value": "hold for review",
             "probability": 0.9970705244103786
            },
            {
             "value": "return to referring team",
             "probability": 0.0010965618643186952
            },
            {
             "value": "allocate bed",
             "probability": 0.0018329137253027606
            }
           ]
          }
         },
         "joint_probability": 0.995971682819437,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004036452705884326,
        "field_brier": 1.0722217516394207e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 1,
           "probability": 0.9970482951498711,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0014116801747382
            },
            {
             "value": 1,
             "probability": 0.9970482951498711
            },
            {
             "value": 2,
             "probability": 0.0007910890106694782
            },
            {
             "value": 3,
             "probability": 0.0007489356647211866
            }
           ]
          },
          "wait_ok": {
           "value": false,
           "probability": 0.9981117897652013,
           "distribution": [
            {
             "value": false,
             "probability": 0.9981117897652013
            },
            {
             "value": true,
             "probability": 0.0018882102347986972
            }
           ]
          },
          "decision": {
           "value": "hold for review",
           "probability": 0.9970705244103786,
           "distribution": [
            {
             "value": "hold for review",
             "probability": 0.9970705244103786
            },
            {
             "value": "return to referring team",
             "probability": 0.0010965618643186952
            },
            {
             "value": "allocate bed",
             "probability": 0.0018329137253027606
            }
           ]
          }
         },
         "joint_probability": 0.995971682819437,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004036452705884326,
        "field_brier": 1.0722217516394207e-05
       }
      },
      "timing_ms": {
       "scoring": 38.45486603677273,
       "base_joint": 0.07485016249120235,
       "constrained_joint": 0.047588953748345375,
       "constrained_total": 38.50245499052107
      },
      "input_tokens": 738,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07ba328a90ed835e-0001",
      "group": "bendsynth-07ba328a90ed835e",
      "provenance": "bendsynth-v1; family=bendsynth-07ba328a90ed835e; domain=water utility meter dispute appeals desk; parameters=quota=3; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "close upheld",
       "settlement_path": false,
       "markers": 1
      },
      "logits": [
       [
        0.2595618665218353,
        -6.648127555847168,
        -6.635231018066406
       ],
       [
        0.25991109013557434,
        -6.13908576965332
       ],
       [
        0.25923195481300354,
        -6.712616443634033,
        -6.970715522766113,
        -7.032261848449707
       ]
      ],
      "joint_logits": [
       0.2584995627403259,
       -8.347970008850098,
       -8.665244102478027,
       -8.819747924804688,
       -8.368618965148926,
       -8.353156089782715,
       -8.619965553283691,
       -8.672978401184082,
       -8.674132347106934,
       -8.608331680297852,
       -8.816554069519043,
       -8.872026443481445,
       -8.5802640914917,
       -8.614191055297852,
       -8.829756736755371,
       -8.882473945617676,
       -8.68950366973877,
       -8.60938549041748,
       -8.856598854064941,
       -8.89614486694336,
       -8.57025146484375,
       -8.534972190856934,
       -8.811387062072754,
       -8.818133354187012
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "close upheld",
           "probability": 0.9979909321316521,
           "distribution": [
            {
             "value": "close upheld",
             "probability": 0.9979909321316521
            },
            {
             "value": "escalate to board",
             "probability": 0.0009980563988991687
            },
            {
             "value": "settle with credit",
             "probability": 0.001011011469448755
            }
           ]
          },
          "settlement_path": {
           "value": false,
           "probability": 0.9983395366384172,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983395366384172
            },
            {
             "value": true,
             "probability": 0.0016604633615827228
            }
           ]
          },
          "markers": {
           "value": 0,
           "probability": 0.9976616936330713,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9976616936330713
            },
            {
             "value": 1,
             "probability": 0.0009357246739288263
            },
            {
             "value": 2,
             "probability": 0.0007228649973525148
            },
            {
             "value": 3,
             "probability": 0.0006797166956472793
            }
           ]
          }
         },
         "joint_probability": 0.9969312921548976,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.60954299757431,
        "field_brier": 0.6644902782052745
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "close upheld",
           "probability": 0.9979909321316521,
           "distribution": [
            {
             "value": "close upheld",
             "probability": 0.9979909321316521
            },
            {
             "value": "escalate to board",
             "probability": 0.0009980563988991687
            },
            {
             "value": "settle with credit",
             "probability": 0.001011011469448755
            }
           ]
          },
          "settlement_path": {
           "value": false,
           "probability": 0.9983395366384172,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983395366384172
            },
            {
             "value": true,
             "probability": 0.0016604633615827228
            }
           ]
          },
          "markers": {
           "value": 0,
           "probability": 0.9976616936330713,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9976616936330713
            },
            {
             "value": 1,
             "probability": 0.0009357246739288263
            },
            {
             "value": 2,
             "probability": 0.0007228649973525148
            },
            {
             "value": 3,
             "probability": 0.0006797166956472793
            }
           ]
          }
         },
         "joint_probability": 0.9969312921548976,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.60954299757431,
        "field_brier": 0.6644902782052745
       }
      },
      "timing_ms": {
       "scoring": 35.28028097935021,
       "base_joint": 0.0676298514008522,
       "constrained_joint": 0.047378940507769585,
       "constrained_total": 35.32765991985798
      },
      "input_tokens": 638,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07ba328a90ed835e-0027",
      "group": "bendsynth-07ba328a90ed835e",
      "provenance": "bendsynth-v1; family=bendsynth-07ba328a90ed835e; domain=water utility meter dispute appeals desk; parameters=quota=2; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "markers": 1,
       "settlement_path": false,
       "decision": "close upheld"
      },
      "logits": [
       [
        -6.8180108070373535,
        -6.0245890617370605,
        -0.02532271482050419,
        -6.80487585067749
       ],
       [
        -0.02313648723065853,
        -5.996668815612793
       ],
       [
        -6.67379093170166,
        -0.027615822851657867,
        -5.186711311340332
       ]
      ],
      "joint_logits": [
       -9.071011543273926,
       -8.546050071716309,
       -8.490059852600098,
       -8.704659461975098,
       -8.91437816619873,
       -8.184731483459473,
       -8.534463882446289,
       -7.28316593170166,
       -7.276541233062744,
       -8.200539588928223,
       -8.47016716003418,
       -7.93148136138916,
       -8.734126091003418,
       -0.029428306967020035,
       -5.701972484588623,
       -9.515185356140137,
       -8.486105918884277,
       -8.447898864746094,
       -9.10468864440918,
       -8.517670631408691,
       -8.389692306518555,
       -8.704169273376465,
       -8.948079109191895,
       -8.201531410217285
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "markers": {
           "value": 2,
           "probability": 0.9952830520957642,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0011166567966798121
            },
            {
             "value": 1,
             "probability": 0.0024688704883229174
            },
            {
             "value": 2,
             "probability": 0.9952830520957642
            },
            {
             "value": 3,
             "probability": 0.0011314206192330981
            }
           ]
          },
          "settlement_path": {
           "value": false,
           "probability": 0.9974612261929567,
           "distribution": [
            {
             "value": false,
             "probability": 0.9974612261929567
            },
            {
             "value": true,
             "probability": 0.0025387738070432736
            }
           ]
          },
          "decision": {
           "value": "escalate to board",
           "probability": 0.9930034223851615,
           "distribution": [
            {
             "value": "settle with credit",
             "probability": 0.0012898924051878333
            },
            {
             "value": "escalate to board",
             "probability": 0.9930034223851615
            },
            {
             "value": "close upheld",
             "probability": 0.005706685209650704
            }
           ]
          }
         },
         "joint_probability": 0.9912051706587578,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 7.255946658212009,
        "field_brier": 1.320116260813883
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "markers": {
           "value": 2,
           "probability": 0.9952830520957642,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0011166567966798121
            },
            {
             "value": 1,
             "probability": 0.0024688704883229174
            },
            {
             "value": 2,
             "probability": 0.9952830520957642
            },
            {
             "value": 3,
             "probability": 0.0011314206192330981
            }
           ]
          },
          "settlement_path": {
           "value": false,
           "probability": 0.9974612261929567,
           "distribution": [
            {
             "value": false,
             "probability": 0.9974612261929567
            },
            {
             "value": true,
             "probability": 0.0025387738070432736
            }
           ]
          },
          "decision": {
           "value": "escalate to board",
           "probability": 0.9930034223851615,
           "distribution": [
            {
             "value": "settle with credit",
             "probability": 0.0012898924051878333
            },
            {
             "value": "escalate to board",
             "probability": 0.9930034223851615
            },
            {
             "value": "close upheld",
             "probability": 0.005706685209650704
            }
           ]
          }
         },
         "joint_probability": 0.9912051706587578,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 7.255946658212009,
        "field_brier": 1.320116260813883
       }
      },
      "timing_ms": {
       "scoring": 33.422200940549374,
       "base_joint": 0.06577000021934509,
       "constrained_joint": 0.04881899803876877,
       "constrained_total": 33.47101993858814
      },
      "input_tokens": 632,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07ba328a90ed835e-0031",
      "group": "bendsynth-07ba328a90ed835e",
      "provenance": "bendsynth-v1; family=bendsynth-07ba328a90ed835e; domain=water utility meter dispute appeals desk; parameters=quota=3; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "settlement_path": false,
       "decision": "close upheld",
       "markers": 1
      },
      "logits": [
       [
        -0.18055135011672974,
        -6.302892208099365
       ],
       [
        -0.18301628530025482,
        -6.9952216148376465,
        -5.819165229797363
       ],
       [
        -7.1537580490112305,
        -6.574270248413086,
        -0.18197056651115417,
        -7.085421562194824
       ]
      ],
      "joint_logits": [
       -9.112310409545898,
       -8.077055931091309,
       -0.18417885899543762,
       -8.991182327270508,
       -9.35721206665039,
       -8.967534065246582,
       -8.922920227050781,
       -9.34847640991211,
       -8.796549797058105,
       -7.9361701011657715,
       -6.600801944732666,
       -8.649832725524902,
       -9.333245277404785,
       -9.038678169250488,
       -8.968475341796875,
       -9.342456817626953,
       -9.060647964477539,
       -8.623477935791016,
       -9.670926094055176,
       -9.003668785095215,
       -8.379854202270508,
       -8.106330871582031,
       -8.766470909118652,
       -8.335359573364258
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "settlement_path": {
           "value": false,
           "probability": 0.9978114841901585,
           "distribution": [
            {
             "value": false,
             "probability": 0.9978114841901585
            },
            {
             "value": true,
             "probability": 0.0021885158098415222
            }
           ]
          },
          "decision": {
           "value": "escalate to board",
           "probability": 0.995354836921306,
           "distribution": [
            {
             "value": "escalate to board",
             "probability": 0.995354836921306
            },
            {
             "value": "settle with credit",
             "probability": 0.0010951529731427951
            },
            {
             "value": "close upheld",
             "probability": 0.003550010105551166
            }
           ]
          },
          "markers": {
           "value": 2,
           "probability": 0.9963963438131878,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0009345944807832474
            },
            {
             "value": 1,
             "probability": 0.0016683675375506245
            },
            {
             "value": 2,
             "probability": 0.9963963438131878
            },
            {
             "value": 3,
             "probability": 0.0010006941684783765
            }
           ]
          }
         },
         "joint_probability": 0.9941983849194429,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 7.75780975199589,
        "field_brier": 1.3243760698337703
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "settlement_path": {
           "value": false,
           "probability": 0.9978114841901585,
           "distribution": [
            {
             "value": false,
             "probability": 0.9978114841901585
            },
            {
             "value": true,
             "probability": 0.0021885158098415222
            }
           ]
          },
          "decision": {
           "value": "escalate to board",
           "probability": 0.995354836921306,
           "distribution": [
            {
             "value": "escalate to board",
             "probability": 0.995354836921306
            },
            {
             "value": "settle with credit",
             "probability": 0.0010951529731427951
            },
            {
             "value": "close upheld",
             "probability": 0.003550010105551166
            }
           ]
          },
          "markers": {
           "value": 2,
           "probability": 0.9963963438131878,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0009345944807832474
            },
            {
             "value": 1,
             "probability": 0.0016683675375506245
            },
            {
             "value": 2,
             "probability": 0.9963963438131878
            },
            {
             "value": 3,
             "probability": 0.0010006941684783765
            }
           ]
          }
         },
         "joint_probability": 0.9941983849194429,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 7.75780975199589,
        "field_brier": 1.3243760698337703
       }
      },
      "timing_ms": {
       "scoring": 34.87928514368832,
       "base_joint": 0.06530899554491043,
       "constrained_joint": 0.046770088374614716,
       "constrained_total": 34.926055232062936
      },
      "input_tokens": 638,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0afb665d93886563-0027",
      "group": "bendsynth-0afb665d93886563",
      "provenance": "bendsynth-v1; family=bendsynth-0afb665d93886563; domain=courier parcel claims desk at a neighborhood shipping store; parameters=threshold=497; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "escalate to carrier",
       "escalated": true,
       "support_factors": 2
      },
      "logits": [
       [
        -5.738577365875244,
        1.3567711114883423,
        -5.7776780128479
       ],
       [
        -5.236142158508301,
        1.3570271730422974
       ],
       [
        -6.1134538650512695,
        -6.158226013183594,
        1.3566501140594482,
        -6.009023666381836
       ]
      ],
      "joint_logits": [
       -7.676113128662109,
       -7.792415142059326,
       -7.703688144683838,
       -7.564938545227051,
       -7.997001647949219,
       -8.02194881439209,
       -8.015654563903809,
       -7.879704475402832,
       -7.8035888671875,
       -7.83751106262207,
       -7.532507419586182,
       -7.713442802429199,
       -8.296183586120605,
       -8.07300090789795,
       1.3560959100723267,
       -8.19823169708252,
       -7.759616851806641,
       -7.913533687591553,
       -7.768986225128174,
       -7.653179168701172,
       -8.021592140197754,
       -8.102724075317383,
       -7.7945990562438965,
       -7.920068264007568
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "escalate to carrier",
           "probability": 0.9983765233399021,
           "distribution": [
            {
             "value": "full payout",
             "probability": 0.0008276059503258522
            },
            {
             "value": "escalate to carrier",
             "probability": 0.9983765233399021
            },
            {
             "value": "standard payout",
             "probability": 0.0007958707097720044
            }
           ]
          },
          "escalated": {
           "value": true,
           "probability": 0.9986321813263014,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013678186736986735
            },
            {
             "value": true,
             "probability": 0.9986321813263014
            }
           ]
          },
          "support_factors": {
           "value": 2,
           "probability": 0.9982556623448127,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005688751529415115
            },
            {
             "value": 1,
             "probability": 0.0005439670157967721
            },
            {
             "value": 2,
             "probability": 0.9982556623448127
            },
            {
             "value": 3,
             "probability": 0.000631495486449083
            }
           ]
          }
         },
         "joint_probability": 0.9977025954778542,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023000476048444,
        "field_brier": 3.918964522808945e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "escalate to carrier",
           "probability": 0.9983765233399021,
           "distribution": [
            {
             "value": "full payout",
             "probability": 0.0008276059503258522
            },
            {
             "value": "escalate to carrier",
             "probability": 0.9983765233399021
            },
            {
             "value": "standard payout",
             "probability": 0.0007958707097720044
            }
           ]
          },
          "escalated": {
           "value": true,
           "probability": 0.9986321813263014,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013678186736986735
            },
            {
             "value": true,
             "probability": 0.9986321813263014
            }
           ]
          },
          "support_factors": {
           "value": 2,
           "probability": 0.9982556623448127,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005688751529415115
            },
            {
             "value": 1,
             "probability": 0.0005439670157967721
            },
            {
             "value": 2,
             "probability": 0.9982556623448127
            },
            {
             "value": 3,
             "probability": 0.000631495486449083
            }
           ]
          }
         },
         "joint_probability": 0.9977025954778542,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023000476048444,
        "field_brier": 3.918964522808945e-06
       }
      },
      "timing_ms": {
       "scoring": 35.65399604849517,
       "base_joint": 0.06724894046783447,
       "constrained_joint": 0.049350084736943245,
       "constrained_total": 35.70334613323212
      },
      "input_tokens": 667,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0afb665d93886563-0035",
      "group": "bendsynth-0afb665d93886563",
      "provenance": "bendsynth-v1; family=bendsynth-0afb665d93886563; domain=courier parcel claims desk at a neighborhood shipping store; parameters=threshold=155; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "support_factors": 1,
       "decision": "full payout",
       "escalated": false
      },
      "logits": [
       [
        -6.275423526763916,
        1.0508102178573608,
        -6.331092834472656,
        -6.413017749786377
       ],
       [
        -6.210038661956787,
        -5.740886688232422,
        1.0508383512496948
       ],
       [
        1.0513564348220825,
        -5.588954448699951
       ]
      ],
      "joint_logits": [
       -8.265928268432617,
       -8.239516258239746,
       -7.907985687255859,
       -7.87078857421875,
       -8.089302062988281,
       -8.096772193908691,
       -8.221805572509766,
       -8.084957122802734,
       -7.40095329284668,
       -7.816247463226318,
       1.0501443147659302,
       -8.069841384887695,
       -8.197200775146484,
       -8.544589042663574,
       -7.9685258865356445,
       -7.8173017501831055,
       -8.30416202545166,
       -8.067110061645508,
       -8.273445129394531,
       -8.59016227722168,
       -8.052690505981445,
       -7.871532917022705,
       -8.454012870788574,
       -8.158820152282715
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "support_factors": {
           "value": 1,
           "probability": 0.9981495108120146,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006568298237071091
            },
            {
             "value": 1,
             "probability": 0.9981495108120146
            },
            {
             "value": 2,
             "probability": 0.0006212636405224088
            },
            {
             "value": 3,
             "probability": 0.000572395723755923
            }
           ]
          },
          "decision": {
           "value": "full payout",
           "probability": 0.9981778048307279,
           "distribution": [
            {
             "value": "escalate to carrier",
             "probability": 0.0007012117300666026
            },
            {
             "value": "standard payout",
             "probability": 0.001120983439205501
            },
            {
             "value": "full payout",
             "probability": 0.9981778048307279
            }
           ]
          },
          "escalated": {
           "value": false,
           "probability": 0.9986950840766109,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986950840766109
            },
            {
             "value": true,
             "probability": 0.0013049159233891152
            }
           ]
          }
         },
         "joint_probability": 0.9974851739259237,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025179935607341397,
        "field_brier": 4.347883052758137e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "support_factors": {
           "value": 1,
           "probability": 0.9981495108120146,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006568298237071091
            },
            {
             "value": 1,
             "probability": 0.9981495108120146
            },
            {
             "value": 2,
             "probability": 0.0006212636405224088
            },
            {
             "value": 3,
             "probability": 0.000572395723755923
            }
           ]
          },
          "decision": {
           "value": "full payout",
           "probability": 0.9981778048307279,
           "distribution": [
            {
             "value": "escalate to carrier",
             "probability": 0.0007012117300666026
            },
            {
             "value": "standard payout",
             "probability": 0.001120983439205501
            },
            {
             "value": "full payout",
             "probability": 0.9981778048307279
            }
           ]
          },
          "escalated": {
           "value": false,
           "probability": 0.9986950840766109,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986950840766109
            },
            {
             "value": true,
             "probability": 0.0013049159233891152
            }
           ]
          }
         },
         "joint_probability": 0.9974851739259237,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025179935607341397,
        "field_brier": 4.347883052758137e-06
       }
      },
      "timing_ms": {
       "scoring": 36.31151909939945,
       "base_joint": 0.06461911834776402,
       "constrained_joint": 0.04801899194717407,
       "constrained_total": 36.35953809134662
      },
      "input_tokens": 658,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c514a056ad55817-0024",
      "group": "bendsynth-0c514a056ad55817",
      "provenance": "bendsynth-v1; family=bendsynth-0c514a056ad55817; domain=boarding admission desk at a veterinary kennel; parameters=grace_days=112; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "admit",
       "risk_factors": 3,
       "exception_applies": true
      },
      "logits": [
       [
        -5.812421798706055,
        2.13201904296875,
        -5.606464385986328,
        -5.195384502410889
       ],
       [
        -6.025710105895996,
        -5.799821376800537,
        -5.3699235916137695,
        2.1318981647491455,
        -5.767542362213135
       ],
       [
        -4.97230339050293,
        2.1326451301574707
       ]
      ],
      "joint_logits": [
       -8.523847579956055,
       -8.228013038635254,
       -8.271540641784668,
       -8.01987075805664,
       -7.991641998291016,
       -7.796636581420898,
       -8.508349418640137,
       -7.754146099090576,
       -8.30434513092041,
       -8.071492195129395,
       -8.104228973388672,
       -7.789314270019531,
       -7.866574764251709,
       -7.446878433227539,
       -7.572515487670898,
       -6.718210697174072,
       -7.384979724884033,
       2.131422758102417,
       -7.897391319274902,
       -7.3723320960998535,
       -8.241721153259277,
       -8.026921272277832,
       -8.093889236450195,
       -7.8554768562316895,
       -7.862061500549316,
       -7.62650203704834,
       -8.280462265014648,
       -7.4875335693359375,
       -8.074259757995605,
       -7.839939594268799,
       -8.14352035522461,
       -7.948742866516113,
       -7.979493141174316,
       -7.725329875946045,
       -7.546005725860596,
       -7.154500961303711,
       -7.767248153686523,
       -6.489634990692139,
       -7.850138187408447,
       -7.657006740570068
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "admit",
           "probability": 0.9985544545357724,
           "distribution": [
            {
             "value": "refuse",
             "probability": 0.00035411555641702856
            },
            {
             "value": "admit",
             "probability": 0.9985544545357724
            },
            {
             "value": "admit with vet check",
             "probability": 0.00043510199064860614
            },
            {
             "value": "waitlist",
             "probability": 0.000656327917161962
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.9984337132944796,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00028609811516894414
            },
            {
             "value": 1,
             "probability": 0.0003586056660826843
            },
            {
             "value": 2,
             "probability": 0.0005512129313234638
            },
            {
             "value": 3,
             "probability": 0.9984337132944796
            },
            {
             "value": 4,
             "probability": 0.0003703699929453551
            }
           ]
          },
          "exception_applies": {
           "value": true,
           "probability": 0.9991796415306191,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008203584693808318
            },
            {
             "value": true,
             "probability": 0.9991796415306191
            }
           ]
          }
         },
         "joint_probability": 0.9979591604087884,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0020429249420576092,
        "field_brier": 2.4285898188667214e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "admit",
           "probability": 0.9985544545357724,
           "distribution": [
            {
             "value": "refuse",
             "probability": 0.00035411555641702856
            },
            {
             "value": "admit",
             "probability": 0.9985544545357724
            },
            {
             "value": "admit with vet check",
             "probability": 0.00043510199064860614
            },
            {
             "value": "waitlist",
             "probability": 0.000656327917161962
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.9984337132944796,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00028609811516894414
            },
            {
             "value": 1,
             "probability": 0.0003586056660826843
            },
            {
             "value": 2,
             "probability": 0.0005512129313234638
            },
            {
             "value": 3,
             "probability": 0.9984337132944796
            },
            {
             "value": 4,
             "probability": 0.0003703699929453551
            }
           ]
          },
          "exception_applies": {
           "value": true,
           "probability": 0.9991796415306191,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008203584693808318
            },
            {
             "value": true,
             "probability": 0.9991796415306191
            }
           ]
          }
         },
         "joint_probability": 0.9979591604087884,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0020429249420576092,
        "field_brier": 2.4285898188667214e-06
       }
      },
      "timing_ms": {
       "scoring": 38.40634599328041,
       "base_joint": 0.09003886952996254,
       "constrained_joint": 0.0672689639031887,
       "constrained_total": 38.4736149571836
      },
      "input_tokens": 744,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c514a056ad55817-0025",
      "group": "bendsynth-0c514a056ad55817",
      "provenance": "bendsynth-v1; family=bendsynth-0c514a056ad55817; domain=boarding admission desk at a veterinary kennel; parameters=grace_days=51; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "admit",
       "exception_applies": true,
       "risk_factors": 2
      },
      "logits": [
       [
        -5.925453186035156,
        -6.204202651977539,
        1.5516326427459717,
        -6.36526346206665
       ],
       [
        -5.556978225708008,
        1.552173137664795
       ],
       [
        -6.50119161605835,
        -6.421565055847168,
        -6.045408248901367,
        1.5514267683029175,
        -6.273011207580566
       ]
      ],
      "joint_logits": [
       -8.534106254577637,
       -8.52344799041748,
       -8.257240295410156,
       -8.532166481018066,
       -8.36976432800293,
       -8.299428939819336,
       -8.301922798156738,
       -8.029237747192383,
       -7.697925090789795,
       -8.076476097106934,
       -8.92836856842041,
       -8.832420349121094,
       -8.5711030960083,
       -8.869333267211914,
       -8.71709156036377,
       -8.66086196899414,
       -8.618045806884766,
       -8.302181243896484,
       -7.738492488861084,
       -8.464759826660156,
       -8.52523136138916,
       -8.445449829101562,
       -8.212498664855957,
       -7.951427459716797,
       -8.342758178710938,
       -8.237683296203613,
       -8.066287994384766,
       -7.297344207763672,
       1.5510154962539673,
       -7.872284412384033,
       -8.968378067016602,
       -8.88670825958252,
       -8.584972381591797,
       -9.003357887268066,
       -8.777355194091797,
       -8.738679885864258,
       -8.592223167419434,
       -8.46557903289795,
       -8.342357635498047,
       -8.537330627441406
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "admit",
           "probability": 0.9986431710098517,
           "distribution": [
            {
             "value": "admit with vet check",
             "probability": 0.0005651362715783153
            },
            {
             "value": "waitlist",
             "probability": 0.00042765523509764457
            },
            {
             "value": "admit",
             "probability": 0.9986431710098517
            },
            {
             "value": "refuse",
             "probability": 0.0003640374834723712
            }
           ]
          },
          "exception_applies": {
           "value": true,
           "probability": 0.9991830790756513,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008169209243487934
            },
            {
             "value": true,
             "probability": 0.9991830790756513
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.9984376499405756,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00031777020287966304
            },
            {
             "value": 1,
             "probability": 0.000344107910808287
            },
            {
             "value": 2,
             "probability": 0.0005012534735816493
            },
            {
             "value": 3,
             "probability": 0.9984376499405756
            },
            {
             "value": 4,
             "probability": 0.00039921847215480463
            }
           ]
          }
         },
         "joint_probability": 0.9980270072940757,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.850334645637556,
        "field_brier": 0.6652932247953608
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "admit",
           "probability": 0.9986431710098517,
           "distribution": [
            {
             "value": "admit with vet check",
             "probability": 0.0005651362715783153
            },
            {
             "value": "waitlist",
             "probability": 0.00042765523509764457
            },
            {
             "value": "admit",
             "probability": 0.9986431710098517
            },
            {
             "value": "refuse",
             "probability": 0.0003640374834723712
            }
           ]
          },
          "exception_applies": {
           "value": true,
           "probability": 0.9991830790756513,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008169209243487934
            },
            {
             "value": true,
             "probability": 0.9991830790756513
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.9984376499405756,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00031777020287966304
            },
            {
             "value": 1,
             "probability": 0.000344107910808287
            },
            {
             "value": 2,
             "probability": 0.0005012534735816493
            },
            {
             "value": 3,
             "probability": 0.9984376499405756
            },
            {
             "value": 4,
             "probability": 0.00039921847215480463
            }
           ]
          }
         },
         "joint_probability": 0.9980270072940757,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.850334645637556,
        "field_brier": 0.6652932247953608
       }
      },
      "timing_ms": {
       "scoring": 39.056360023096204,
       "base_joint": 0.08666887879371643,
       "constrained_joint": 0.06699003279209137,
       "constrained_total": 39.123350055888295
      },
      "input_tokens": 759,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c514a056ad55817-0026",
      "group": "bendsynth-0c514a056ad55817",
      "provenance": "bendsynth-v1; family=bendsynth-0c514a056ad55817; domain=boarding admission desk at a veterinary kennel; parameters=grace_days=30; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 2,
       "decision": "waitlist",
       "exception_applies": false
      },
      "logits": [
       [
        -6.6128387451171875,
        -6.327410697937012,
        1.4387376308441162,
        -6.485740661621094,
        -6.600396156311035
       ],
       [
        1.4387283325195312,
        -6.305215835571289,
        -6.456154823303223,
        -5.930896759033203
       ],
       [
        1.4393398761749268,
        -5.662084102630615
       ]
      ],
      "joint_logits": [
       -8.695982933044434,
       -8.84158706665039,
       -8.70726490020752,
       -8.827588081359863,
       -8.7777738571167,
       -8.921799659729004,
       -8.36274242401123,
       -8.526328086853027,
       -8.080952644348145,
       -8.499150276184082,
       -8.443528175354004,
       -8.63752555847168,
       -8.584258079528809,
       -8.764826774597168,
       -8.121475219726562,
       -8.33164119720459,
       1.4383195638656616,
       -8.410599708557129,
       -8.189166069030762,
       -8.731603622436523,
       -8.617331504821777,
       -8.868825912475586,
       -7.4715189933776855,
       -8.469853401184082,
       -8.494239807128906,
       -8.712174415588379,
       -8.521743774414062,
       -8.732366561889648,
       -8.682255744934082,
       -8.821675300598145,
       -8.308368682861328,
       -8.3690767288208,
       -8.73779010772705,
       -8.83312702178955,
       -8.683485984802246,
       -8.782815933227539,
       -8.72243595123291,
       -8.884706497192383,
       -8.36784553527832,
       -8.530794143676758
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 2,
           "probability": 0.9985752241019566,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003181453569911884
            },
            {
             "value": 1,
             "probability": 0.0004232388930992163
            },
            {
             "value": 2,
             "probability": 0.9985752241019566
            },
            {
             "value": 3,
             "probability": 0.00036126303220590224
            },
            {
             "value": 4,
             "probability": 0.0003221286157470026
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9985659504812698,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.9985659504812698
            },
            {
             "value": "admit",
             "probability": 0.00043273756635848836
            },
            {
             "value": "refuse",
             "probability": 0.0003721111866025139
            },
            {
             "value": "admit with vet check",
             "probability": 0.0006292007657691752
            }
           ]
          },
          "exception_applies": {
           "value": false,
           "probability": 0.9991767474827523,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991767474827523
            },
            {
             "value": true,
             "probability": 0.0008232525172477696
            }
           ]
          }
         },
         "joint_probability": 0.9981578364707664,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018438623991859416,
        "field_brier": 2.226073796210196e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 2,
           "probability": 0.9985752241019566,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003181453569911884
            },
            {
             "value": 1,
             "probability": 0.0004232388930992163
            },
            {
             "value": 2,
             "probability": 0.9985752241019566
            },
            {
             "value": 3,
             "probability": 0.00036126303220590224
            },
            {
             "value": 4,
             "probability": 0.0003221286157470026
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9985659504812698,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.9985659504812698
            },
            {
             "value": "admit",
             "probability": 0.00043273756635848836
            },
            {
             "value": "refuse",
             "probability": 0.0003721111866025139
            },
            {
             "value": "admit with vet check",
             "probability": 0.0006292007657691752
            }
           ]
          },
          "exception_applies": {
           "value": false,
           "probability": 0.9991767474827523,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991767474827523
            },
            {
             "value": true,
             "probability": 0.0008232525172477696
            }
           ]
          }
         },
         "joint_probability": 0.9981578364707664,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018438623991859416,
        "field_brier": 2.226073796210196e-06
       }
      },
      "timing_ms": {
       "scoring": 36.984301172196865,
       "base_joint": 0.08807913400232792,
       "constrained_joint": 0.0763691496104002,
       "constrained_total": 37.060670321807265
      },
      "input_tokens": 783,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a82b03c8c1710e9-0008",
      "group": "bendsynth-0a82b03c8c1710e9",
      "provenance": "bendsynth-v1; family=bendsynth-0a82b03c8c1710e9; domain=loan intake desk at a contemporary art gallery; parameters=quota=11,value_cap=249; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "curator review",
       "exempt_from_quota": true,
       "risk_factors": 2
      },
      "logits": [
       [
        -6.0053558349609375,
        1.4980382919311523,
        -5.98235559463501,
        -6.22299861907959
       ],
       [
        -5.351382255554199,
        1.4985368251800537
       ],
       [
        -6.05881404876709,
        -6.1171417236328125,
        1.4980639219284058,
        -6.068084716796875
       ]
      ],
      "joint_logits": [
       -8.048192024230957,
       -8.173503875732422,
       -8.16838550567627,
       -8.05700397491455,
       -8.077418327331543,
       -8.190099716186523,
       -7.886590003967285,
       -8.112236022949219,
       -8.14621639251709,
       -8.195877075195312,
       -7.640269756317139,
       -8.129806518554688,
       -8.146814346313477,
       -8.001429557800293,
       1.4975347518920898,
       -8.158468246459961,
       -8.068324089050293,
       -8.135796546936035,
       -8.059030532836914,
       -8.053054809570312,
       -8.08521842956543,
       -8.141924858093262,
       -7.882763862609863,
       -8.09187126159668,
       -8.29133129119873,
       -8.418806076049805,
       -8.338235855102539,
       -8.304075241088867,
       -8.272342681884766,
       -8.377677917480469,
       -8.133647918701172,
       -8.308938026428223
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "curator review",
           "probability": 0.9984437795574876,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0005503524365781516
            },
            {
             "value": "curator review",
             "probability": 0.9984437795574876
            },
            {
             "value": "decline",
             "probability": 0.0005631574983734797
            },
            {
             "value": "approve with conditions",
             "probability": 0.0004427105075607578
            }
           ]
          },
          "exempt_from_quota": {
           "value": true,
           "probability": 0.9989415796831838,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010584203168163538
            },
            {
             "value": true,
             "probability": 0.9989415796831838
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9984692605838641,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005217042308112405
            },
            {
             "value": 1,
             "probability": 0.0004921449654754376
            },
            {
             "value": 2,
             "probability": 0.9984692605838641
            },
            {
             "value": 3,
             "probability": 0.0005168902198493098
            }
           ]
          }
         },
         "joint_probability": 0.9979411134354284,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002061008985230458,
        "field_brier": 2.867692198898669e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "curator review",
           "probability": 0.9984437795574876,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0005503524365781516
            },
            {
             "value": "curator review",
             "probability": 0.9984437795574876
            },
            {
             "value": "decline",
             "probability": 0.0005631574983734797
            },
            {
             "value": "approve with conditions",
             "probability": 0.0004427105075607578
            }
           ]
          },
          "exempt_from_quota": {
           "value": true,
           "probability": 0.9989415796831838,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010584203168163538
            },
            {
             "value": true,
             "probability": 0.9989415796831838
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9984692605838641,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005217042308112405
            },
            {
             "value": 1,
             "probability": 0.0004921449654754376
            },
            {
             "value": 2,
             "probability": 0.9984692605838641
            },
            {
             "value": 3,
             "probability": 0.0005168902198493098
            }
           ]
          }
         },
         "joint_probability": 0.9979411134354284,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002061008985230458,
        "field_brier": 2.867692198898669e-06
       }
      },
      "timing_ms": {
       "scoring": 37.4513769056648,
       "base_joint": 0.07606903091073036,
       "constrained_joint": 0.057499855756759644,
       "constrained_total": 37.50887676142156
      },
      "input_tokens": 693,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-080d96b292f4b27d-0014",
      "group": "bendsynth-080d96b292f4b27d",
      "provenance": "bendsynth-v1; family=bendsynth-080d96b292f4b27d; domain=kindergarten enrolment site inspection; parameters=micro_ratio=5,centre_ratio=10,cap=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "non-compliant",
       "ratio_ok": true,
       "deficiencies": 2
      },
      "logits": [
       [
        -6.737274169921875,
        -6.733675956726074,
        -0.6909891366958618
       ],
       [
        -4.571732521057129,
        -0.7070105671882629
       ],
       [
        -7.056962966918945,
        -6.853981971740723,
        -0.827837347984314,
        -2.7609753608703613,
        -6.633029937744141
       ]
      ],
      "joint_logits": [
       -9.170629501342773,
       -8.957984924316406,
       -9.005549430847168,
       -9.082743644714355,
       -8.643692016601562,
       -9.528440475463867,
       -9.342052459716797,
       -8.912455558776855,
       -8.9758882522583,
       -9.036274909973145,
       -9.515610694885254,
       -9.348755836486816,
       -8.684797286987305,
       -8.761065483093262,
       -9.033062934875488,
       -9.752008438110352,
       -9.578224182128906,
       -8.606948852539062,
       -8.607675552368164,
       -9.30239200592041,
       -8.480624198913574,
       -8.196285247802734,
       -5.196410179138184,
       -5.904344081878662,
       -7.98191499710083,
       -7.971192359924316,
       -7.806915283203125,
       -0.8420010209083557,
       -2.8146941661834717,
       -7.6654744148254395
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "non-compliant",
           "probability": 0.9952805715378981,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.0023554693532688207
            },
            {
             "value": "provisional",
             "probability": 0.0023639591088330477
            },
            {
             "value": "non-compliant",
             "probability": 0.9952805715378981
            }
           ]
          },
          "ratio_ok": {
           "value": true,
           "probability": 0.9794619098694032,
           "distribution": [
            {
             "value": false,
             "probability": 0.020538090130596744
            },
            {
             "value": true,
             "probability": 0.9794619098694032
            }
           ]
          },
          "deficiencies": {
           "value": 2,
           "probability": 0.8679867966290323,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0017109538969743083
            },
            {
             "value": 1,
             "probability": 0.0020960020025695466
            },
            {
             "value": 2,
             "probability": 0.8679867966290323
            },
            {
             "value": 3,
             "probability": 0.12559197946270378
            },
            {
             "value": 4,
             "probability": 0.002614268008720045
            }
           ]
          }
         },
         "joint_probability": 0.8557795783114794,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.1557424379722912,
        "field_brier": 0.011364007330151601
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "non-compliant",
           "probability": 0.9952805715378981,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.0023554693532688207
            },
            {
             "value": "provisional",
             "probability": 0.0023639591088330477
            },
            {
             "value": "non-compliant",
             "probability": 0.9952805715378981
            }
           ]
          },
          "ratio_ok": {
           "value": true,
           "probability": 0.9794619098694032,
           "distribution": [
            {
             "value": false,
             "probability": 0.020538090130596744
            },
            {
             "value": true,
             "probability": 0.9794619098694032
            }
           ]
          },
          "deficiencies": {
           "value": 2,
           "probability": 0.8679867966290323,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0017109538969743083
            },
            {
             "value": 1,
             "probability": 0.0020960020025695466
            },
            {
             "value": 2,
             "probability": 0.8679867966290323
            },
            {
             "value": 3,
             "probability": 0.12559197946270378
            },
            {
             "value": 4,
             "probability": 0.002614268008720045
            }
           ]
          }
         },
         "joint_probability": 0.8557795783114794,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.1557424379722912,
        "field_brier": 0.011364007330151601
       }
      },
      "timing_ms": {
       "scoring": 39.92141084745526,
       "base_joint": 0.0867689959704876,
       "constrained_joint": 0.05679996684193611,
       "constrained_total": 39.9782108142972
      },
      "input_tokens": 827,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-080d96b292f4b27d-0026",
      "group": "bendsynth-080d96b292f4b27d",
      "provenance": "bendsynth-v1; family=bendsynth-080d96b292f4b27d; domain=kindergarten enrolment site inspection; parameters=micro_ratio=3,centre_ratio=9,cap=22; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "provisional",
       "deficiencies": 1,
       "ratio_ok": true
      },
      "logits": [
       [
        -5.375110626220703,
        0.9296981692314148,
        -4.181896686553955
       ],
       [
        -5.905454635620117,
        0.9335370063781738,
        -5.606419563293457,
        -6.245364189147949,
        -6.3108649253845215
       ],
       [
        0.9341520071029663,
        -4.75714111328125
       ]
      ],
      "joint_logits": [
       -8.11911678314209,
       -7.98108434677124,
       -6.5590949058532715,
       -7.216914653778076,
       -7.779331207275391,
       -7.942647933959961,
       -8.506783485412598,
       -7.994135856628418,
       -8.577420234680176,
       -7.99210786819458,
       -7.347022533416748,
       -7.853977203369141,
       0.9274449348449707,
       -6.240905284881592,
       -6.98793888092041,
       -7.541991710662842,
       -7.835209846496582,
       -8.220760345458984,
       -7.950258731842041,
       -8.287964820861816,
       -7.302346706390381,
       -7.880048751831055,
       -4.636308193206787,
       -6.403763771057129,
       -6.929007053375244,
       -7.666216850280762,
       -7.734692096710205,
       -8.118623733520508,
       -7.850429058074951,
       -8.123971939086914
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "compliant",
           "probability": 0.9922072432944855,
           "distribution": [
            {
             "value": "non-compliant",
             "probability": 0.0018132545675356158
            },
            {
             "value": "compliant",
             "probability": 0.9922072432944855
            },
            {
             "value": "provisional",
             "probability": 0.005979502137978835
            }
           ]
          },
          "deficiencies": {
           "value": 1,
           "probability": 0.9960234747329479,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0010669236413348385
            },
            {
             "value": 1,
             "probability": 0.9960234747329479
            },
            {
             "value": 2,
             "probability": 0.0014388074410092596
            },
            {
             "value": 3,
             "probability": 0.0007594728940000483
            },
            {
             "value": 4,
             "probability": 0.0007113212907078048
            }
           ]
          },
          "ratio_ok": {
           "value": false,
           "probability": 0.9966361291093758,
           "distribution": [
            {
             "value": false,
             "probability": 0.9966361291093758
            },
            {
             "value": true,
             "probability": 0.0033638708906242433
            }
           ]
          }
         },
         "joint_probability": 0.9899740590283027,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 7.34128524510062,
        "field_brier": 1.3197141678078113
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "compliant",
           "probability": 0.9922072432944855,
           "distribution": [
            {
             "value": "non-compliant",
             "probability": 0.0018132545675356158
            },
            {
             "value": "compliant",
             "probability": 0.9922072432944855
            },
            {
             "value": "provisional",
             "probability": 0.005979502137978835
            }
           ]
          },
          "deficiencies": {
           "value": 1,
           "probability": 0.9960234747329479,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0010669236413348385
            },
            {
             "value": 1,
             "probability": 0.9960234747329479
            },
            {
             "value": 2,
             "probability": 0.0014388074410092596
            },
            {
             "value": 3,
             "probability": 0.0007594728940000483
            },
            {
             "value": 4,
             "probability": 0.0007113212907078048
            }
           ]
          },
          "ratio_ok": {
           "value": false,
           "probability": 0.9966361291093758,
           "distribution": [
            {
             "value": false,
             "probability": 0.9966361291093758
            },
            {
             "value": true,
             "probability": 0.0033638708906242433
            }
           ]
          }
         },
         "joint_probability": 0.9899740590283027,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 7.34128524510062,
        "field_brier": 1.3197141678078113
       }
      },
      "timing_ms": {
       "scoring": 39.56959396600723,
       "base_joint": 0.07224897854030132,
       "constrained_joint": 0.05500996485352516,
       "constrained_total": 39.62460393086076
      },
      "input_tokens": 827,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0458e03809c48680-0000",
      "group": "bendsynth-0458e03809c48680",
      "provenance": "bendsynth-v1; family=bendsynth-0458e03809c48680; domain=hospital bed allocation chatbot for transfer requests; parameters=tolerance=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "pressure_flags": 2,
       "decision": "waitlist",
       "within_tolerance": false
      },
      "logits": [
       [
        -6.347203254699707,
        -6.305582046508789,
        1.1282044649124146,
        -6.234036922454834
       ],
       [
        -6.159003257751465,
        -6.002482891082764,
        1.1285125017166138
       ],
       [
        1.1287297010421753,
        -5.542710781097412
       ]
      ],
      "joint_logits": [
       -8.366000175476074,
       -8.150162696838379,
       -8.206607818603516,
       -8.026741981506348,
       -8.196870803833008,
       -7.942442417144775,
       -8.39504337310791,
       -8.180624961853027,
       -8.231048583984375,
       -8.074616432189941,
       -7.874986171722412,
       -7.922934532165527,
       -8.215049743652344,
       -8.275554656982422,
       -8.060173034667969,
       -8.099130630493164,
       1.1277186870574951,
       -7.941351413726807,
       -8.279787063598633,
       -8.084738731384277,
       -8.118303298950195,
       -7.881036281585693,
       -8.008768081665039,
       -7.846011161804199
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "pressure_flags": {
           "value": 2,
           "probability": 0.9982106330257365,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005658405098156777
            },
            {
             "value": 1,
             "probability": 0.000589888221432186
            },
            {
             "value": 2,
             "probability": 0.9982106330257365
            },
            {
             "value": 3,
             "probability": 0.0006336382430156311
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9985182507203011,
           "distribution": [
            {
             "value": "offer alternative unit",
             "probability": 0.0006830118201336149
            },
            {
             "value": "admit as requested",
             "probability": 0.0007987374595653406
            },
            {
             "value": "waitlist",
             "probability": 0.9985182507203011
            }
           ]
          },
          "within_tolerance": {
           "value": false,
           "probability": 0.998735029233479,
           "distribution": [
            {
             "value": false,
             "probability": 0.998735029233479
            },
            {
             "value": true,
             "probability": 0.0012649707665209873
            }
           ]
          }
         },
         "joint_probability": 0.9977258811045299,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00227670863083454,
        "field_brier": 3.5906149572927866e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "pressure_flags": {
           "value": 2,
           "probability": 0.9982106330257365,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005658405098156777
            },
            {
             "value": 1,
             "probability": 0.000589888221432186
            },
            {
             "value": 2,
             "probability": 0.9982106330257365
            },
            {
             "value": 3,
             "probability": 0.0006336382430156311
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9985182507203011,
           "distribution": [
            {
             "value": "offer alternative unit",
             "probability": 0.0006830118201336149
            },
            {
             "value": "admit as requested",
             "probability": 0.0007987374595653406
            },
            {
             "value": "waitlist",
             "probability": 0.9985182507203011
            }
           ]
          },
          "within_tolerance": {
           "value": false,
           "probability": 0.998735029233479,
           "distribution": [
            {
             "value": false,
             "probability": 0.998735029233479
            },
            {
             "value": true,
             "probability": 0.0012649707665209873
            }
           ]
          }
         },
         "joint_probability": 0.9977258811045299,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00227670863083454,
        "field_brier": 3.5906149572927866e-06
       }
      },
      "timing_ms": {
       "scoring": 35.415058955550194,
       "base_joint": 0.06683915853500366,
       "constrained_joint": 0.04709884524345398,
       "constrained_total": 35.46215780079365
      },
      "input_tokens": 621,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0458e03809c48680-0015",
      "group": "bendsynth-0458e03809c48680",
      "provenance": "bendsynth-v1; family=bendsynth-0458e03809c48680; domain=hospital bed allocation chatbot for transfer requests; parameters=tolerance=21; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "within_tolerance": false,
       "pressure_flags": 0,
       "decision": "offer alternative unit"
      },
      "logits": [
       [
        1.4800927639007568,
        -4.952496528625488
       ],
       [
        1.479820966720581,
        -5.843179225921631,
        -5.863243103027344,
        -5.984287738800049
       ],
       [
        1.4795920848846436,
        -5.673596382141113,
        -5.144955635070801
       ]
      ],
      "joint_logits": [
       1.4788633584976196,
       -7.481255054473877,
       -6.804721355438232,
       -7.901977062225342,
       -7.932631969451904,
       -7.440977573394775,
       -7.987363815307617,
       -7.94450044631958,
       -7.444216728210449,
       -8.134675025939941,
       -8.052191734313965,
       -7.563321113586426,
       -7.109578609466553,
       -7.336596965789795,
       -6.846587657928467,
       -7.639925479888916,
       -7.800211429595947,
       -7.27371072769165,
       -7.6573028564453125,
       -7.781527042388916,
       -7.303902626037598,
       -7.762371063232422,
       -7.924417018890381,
       -7.4185590744018555
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_tolerance": {
           "value": false,
           "probability": 0.9983943008567011,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983943008567011
            },
            {
             "value": true,
             "probability": 0.001605699143298865
            }
           ]
          },
          "pressure_flags": {
           "value": 0,
           "probability": 0.9981229904762112,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981229904762112
            },
            {
             "value": 1,
             "probability": 0.0006589395150309431
            },
            {
             "value": 2,
             "probability": 0.0006458502971906264
            },
            {
             "value": 3,
             "probability": 0.0005722197115672308
            }
           ]
          },
          "decision": {
           "value": "offer alternative unit",
           "probability": 0.9978946954848312,
           "distribution": [
            {
             "value": "offer alternative unit",
             "probability": 0.9978946954848312
            },
            {
             "value": "waitlist",
             "probability": 0.0007807182537416073
            },
            {
             "value": "admit as requested",
             "probability": 0.0013245862614271677
            }
           ]
          }
         },
         "joint_probability": 0.9971677045143168,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002836314024128076,
        "field_brier": 5.5516067921362465e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_tolerance": {
           "value": false,
           "probability": 0.9983943008567011,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983943008567011
            },
            {
             "value": true,
             "probability": 0.001605699143298865
            }
           ]
          },
          "pressure_flags": {
           "value": 0,
           "probability": 0.9981229904762112,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981229904762112
            },
            {
             "value": 1,
             "probability": 0.0006589395150309431
            },
            {
             "value": 2,
             "probability": 0.0006458502971906264
            },
            {
             "value": 3,
             "probability": 0.0005722197115672308
            }
           ]
          },
          "decision": {
           "value": "offer alternative unit",
           "probability": 0.9978946954848312,
           "distribution": [
            {
             "value": "offer alternative unit",
             "probability": 0.9978946954848312
            },
            {
             "value": "waitlist",
             "probability": 0.0007807182537416073
            },
            {
             "value": "admit as requested",
             "probability": 0.0013245862614271677
            }
           ]
          }
         },
         "joint_probability": 0.9971677045143168,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002836314024128076,
        "field_brier": 5.5516067921362465e-06
       }
      },
      "timing_ms": {
       "scoring": 32.60084893554449,
       "base_joint": 0.06291992031037807,
       "constrained_joint": 0.04567904397845268,
       "constrained_total": 32.64652797952294
      },
      "input_tokens": 564,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0458e03809c48680-0022",
      "group": "bendsynth-0458e03809c48680",
      "provenance": "bendsynth-v1; family=bendsynth-0458e03809c48680; domain=hospital bed allocation chatbot for transfer requests; parameters=tolerance=15; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "pressure_flags": 1,
       "within_tolerance": true,
       "decision": "admit as requested"
      },
      "logits": [
       [
        -6.048271179199219,
        1.3176676034927368,
        -5.915514945983887,
        -6.083003044128418
       ],
       [
        -5.137065887451172,
        1.318060040473938
       ],
       [
        -5.814535140991211,
        1.3175121545791626,
        -5.310912132263184
       ]
      ],
      "joint_logits": [
       -7.901546478271484,
       -7.9481201171875,
       -7.451416969299316,
       -8.082340240478516,
       -8.127578735351562,
       -7.698299884796143,
       -7.726573467254639,
       -7.396590709686279,
       -7.218492031097412,
       -7.786048412322998,
       1.316814661026001,
       -6.926342010498047,
       -7.765498638153076,
       -7.789941787719727,
       -7.312577724456787,
       -7.942078113555908,
       -8.049843788146973,
       -7.566382884979248,
       -7.884958267211914,
       -7.992038726806641,
       -7.48047399520874,
       -8.137531280517578,
       -8.215635299682617,
       -7.728579998016357
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "pressure_flags": {
           "value": 1,
           "probability": 0.9980383626000469,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006311907890886851
            },
            {
             "value": 1,
             "probability": 0.9980383626000469
            },
            {
             "value": 2,
             "probability": 0.000720801876315088
            },
            {
             "value": 3,
             "probability": 0.0006096447345493032
            }
           ]
          },
          "within_tolerance": {
           "value": true,
           "probability": 0.9984300276982334,
           "distribution": [
            {
             "value": false,
             "probability": 0.00156997230176657
            },
            {
             "value": true,
             "probability": 0.9984300276982334
            }
           ]
          },
          "decision": {
           "value": "admit as requested",
           "probability": 0.9978831632829903,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0007973902963539533
            },
            {
             "value": "admit as requested",
             "probability": 0.9978831632829903
            },
            {
             "value": "offer alternative unit",
             "probability": 0.0013194464206558003
            }
           ]
          }
         },
         "joint_probability": 0.9971874163194966,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002816546426096589,
        "field_brier": 5.641679677732683e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "pressure_flags": {
           "value": 1,
           "probability": 0.9980383626000469,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006311907890886851
            },
            {
             "value": 1,
             "probability": 0.9980383626000469
            },
            {
             "value": 2,
             "probability": 0.000720801876315088
            },
            {
             "value": 3,
             "probability": 0.0006096447345493032
            }
           ]
          },
          "within_tolerance": {
           "value": true,
           "probability": 0.9984300276982334,
           "distribution": [
            {
             "value": false,
             "probability": 0.00156997230176657
            },
            {
             "value": true,
             "probability": 0.9984300276982334
            }
           ]
          },
          "decision": {
           "value": "admit as requested",
           "probability": 0.9978831632829903,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0007973902963539533
            },
            {
             "value": "admit as requested",
             "probability": 0.9978831632829903
            },
            {
             "value": "offer alternative unit",
             "probability": 0.0013194464206558003
            }
           ]
          }
         },
         "joint_probability": 0.9971874163194966,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002816546426096589,
        "field_brier": 5.641679677732683e-06
       }
      },
      "timing_ms": {
       "scoring": 35.69764504209161,
       "base_joint": 0.0737491063773632,
       "constrained_joint": 0.04821992479264736,
       "constrained_total": 35.745864966884255
      },
      "input_tokens": 624,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0458e03809c48680-0037",
      "group": "bendsynth-0458e03809c48680",
      "provenance": "bendsynth-v1; family=bendsynth-0458e03809c48680; domain=hospital bed allocation chatbot for transfer requests; parameters=tolerance=25; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "within_tolerance": true,
       "decision": "admit as requested",
       "pressure_flags": 1
      },
      "logits": [
       [
        3.1507790088653564,
        -0.1563025414943695
       ],
       [
        -3.8136439323425293,
        3.1715526580810547,
        -1.0700130462646484
       ],
       [
        -3.5601611137390137,
        -2.517512559890747,
        3.1811745166778564,
        -3.6713342666625977
       ]
      ],
      "joint_logits": [
       -7.930408954620361,
       -7.287847518920898,
       -6.267228603363037,
       -8.07532787322998,
       -6.943087577819824,
       -5.650915622711182,
       3.147077798843384,
       -7.048393726348877,
       -6.004261493682861,
       -4.902261257171631,
       -2.7258248329162598,
       -6.0803303718566895,
       -5.9731764793396,
       -5.399836540222168,
       -4.638205051422119,
       -6.176794528961182,
       -5.06502628326416,
       -3.868560314178467,
       -0.6201658844947815,
       -5.099302768707275,
       -4.148934364318848,
       -3.1320929527282715,
       -1.6496260166168213,
       -4.283298492431641
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_tolerance": {
           "value": false,
           "probability": 0.9646709499484896,
           "distribution": [
            {
             "value": false,
             "probability": 0.9646709499484896
            },
            {
             "value": true,
             "probability": 0.03532905005151046
            }
           ]
          },
          "decision": {
           "value": "offer alternative unit",
           "probability": 0.9849203466279232,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0009115255161473644
            },
            {
             "value": "offer alternative unit",
             "probability": 0.9849203466279232
            },
            {
             "value": "admit as requested",
             "probability": 0.01416812785592948
            }
           ]
          },
          "pressure_flags": {
           "value": 2,
           "probability": 0.9944428237450861,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001174505345805819
            },
            {
             "value": 1,
             "probability": 0.0033317427790100287
            },
            {
             "value": 2,
             "probability": 0.9944428237450861
            },
            {
             "value": 3,
             "probability": 0.0010509281300980537
            }
           ]
          }
         },
         "joint_probability": 0.9611071504940774,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 0,
        "joint_nll": 6.318840128845792,
        "field_brier": 1.9284600373757759
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_tolerance": {
           "value": false,
           "probability": 0.9646709499484896,
           "distribution": [
            {
             "value": false,
             "probability": 0.9646709499484896
            },
            {
             "value": true,
             "probability": 0.03532905005151046
            }
           ]
          },
          "decision": {
           "value": "offer alternative unit",
           "probability": 0.9849203466279232,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.0009115255161473644
            },
            {
             "value": "offer alternative unit",
             "probability": 0.9849203466279232
            },
            {
             "value": "admit as requested",
             "probability": 0.01416812785592948
            }
           ]
          },
          "pressure_flags": {
           "value": 2,
           "probability": 0.9944428237450861,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001174505345805819
            },
            {
             "value": 1,
             "probability": 0.0033317427790100287
            },
            {
             "value": 2,
             "probability": 0.9944428237450861
            },
            {
             "value": 3,
             "probability": 0.0010509281300980537
            }
           ]
          }
         },
         "joint_probability": 0.9611071504940774,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 0,
        "joint_nll": 6.318840128845792,
        "field_brier": 1.9284600373757759
       }
      },
      "timing_ms": {
       "scoring": 31.46727103739977,
       "base_joint": 0.06646895781159401,
       "constrained_joint": 0.04813005216419697,
       "constrained_total": 31.515401089563966
      },
      "input_tokens": 561,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0399bbadf5ddc82a-0034",
      "group": "bendsynth-0399bbadf5ddc82a",
      "provenance": "bendsynth-v1; family=bendsynth-0399bbadf5ddc82a; domain=city solar panel installation permit chatbot; parameters=queue_period=14,cap_limit=19; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "standard permit",
       "risk_flags": 0,
       "queue_ok": true
      },
      "logits": [
       [
        -6.533846855163574,
        -6.514084815979004,
        0.4830355942249298
       ],
       [
        0.4828037917613983,
        -6.672232151031494,
        -6.838151931762695,
        -6.934399604797363
       ],
       [
        -6.048834800720215,
        0.48339027166366577
       ]
      ],
      "joint_logits": [
       -8.65860366821289,
       -8.722612380981445,
       -8.412419319152832,
       -8.597822189331055,
       -8.517985343933105,
       -8.675134658813477,
       -8.61007022857666,
       -8.756498336791992,
       -8.62464427947998,
       -8.664031028747559,
       -8.385546684265137,
       -8.579426765441895,
       -8.50158405303955,
       -8.6853666305542,
       -8.593277931213379,
       -8.762504577636719,
       -8.409217834472656,
       0.482241153717041,
       -8.432233810424805,
       -8.398683547973633,
       -8.620549201965332,
       -8.812154769897461,
       -8.705740928649902,
       -8.974956512451172
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "standard permit",
           "probability": 0.9981921458402813,
           "distribution": [
            {
             "value": "expedited permit",
             "probability": 0.0008949957520830611
            },
            {
             "value": "hold pending review",
             "probability": 0.000912858407635625
            },
            {
             "value": "standard permit",
             "probability": 0.9981921458402813
            }
           ]
          },
          "risk_flags": {
           "value": 0,
           "probability": 0.9979608884170953,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9979608884170953
            },
            {
             "value": 1,
             "probability": 0.0007793292141120642
            },
            {
             "value": 2,
             "probability": 0.0006601805409046024
            },
            {
             "value": 3,
             "probability": 0.0005996018278880339
            }
           ]
          },
          "queue_ok": {
           "value": true,
           "probability": 0.9985463524300262,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014536475699738454
            },
            {
             "value": true,
             "probability": 0.9985463524300262
            }
           ]
          }
         },
         "joint_probability": 0.9973995301908861,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002603856904027566,
        "field_brier": 4.89651260564404e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "standard permit",
           "probability": 0.9981921458402813,
           "distribution": [
            {
             "value": "expedited permit",
             "probability": 0.0008949957520830611
            },
            {
             "value": "hold pending review",
             "probability": 0.000912858407635625
            },
            {
             "value": "standard permit",
             "probability": 0.9981921458402813
            }
           ]
          },
          "risk_flags": {
           "value": 0,
           "probability": 0.9979608884170953,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9979608884170953
            },
            {
             "value": 1,
             "probability": 0.0007793292141120642
            },
            {
             "value": 2,
             "probability": 0.0006601805409046024
            },
            {
             "value": 3,
             "probability": 0.0005996018278880339
            }
           ]
          },
          "queue_ok": {
           "value": true,
           "probability": 0.9985463524300262,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014536475699738454
            },
            {
             "value": true,
             "probability": 0.9985463524300262
            }
           ]
          }
         },
         "joint_probability": 0.9973995301908861,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002603856904027566,
        "field_brier": 4.89651260564404e-06
       }
      },
      "timing_ms": {
       "scoring": 40.35719600506127,
       "base_joint": 0.06501888856291771,
       "constrained_joint": 0.047339824959635735,
       "constrained_total": 40.404535830020905
      },
      "input_tokens": 865,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0399bbadf5ddc82a-0036",
      "group": "bendsynth-0399bbadf5ddc82a",
      "provenance": "bendsynth-v1; family=bendsynth-0399bbadf5ddc82a; domain=city solar panel installation permit chatbot; parameters=queue_period=11,cap_limit=29; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "queue_ok": true,
       "decision": "standard permit",
       "risk_flags": 0
      },
      "logits": [
       [
        -6.194116592407227,
        0.11187466233968735
       ],
       [
        -6.636723518371582,
        -6.723864555358887,
        0.11145266890525818
       ],
       [
        0.11068140715360641,
        -6.280542373657227,
        -7.129465103149414,
        -7.262124061584473
       ]
      ],
      "joint_logits": [
       -8.695734977722168,
       -8.232911109924316,
       -8.737092971801758,
       -8.888554573059082,
       -8.861035346984863,
       -8.360681533813477,
       -8.845023155212402,
       -8.971652030944824,
       -8.529250144958496,
       -8.434350967407227,
       -8.933862686157227,
       -9.058523178100586,
       -8.810307502746582,
       -8.683430671691895,
       -8.911235809326172,
       -8.976396560668945,
       -8.745360374450684,
       -8.671710014343262,
       -9.055420875549316,
       -9.135086059570312,
       0.10995139926671982,
       -7.112654209136963,
       -9.087576866149902,
       -9.359642028808594
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "queue_ok": {
           "value": true,
           "probability": 0.9981779894665286,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018220105334713745
            },
            {
             "value": true,
             "probability": 0.9981779894665286
            }
           ]
          },
          "decision": {
           "value": "standard permit",
           "probability": 0.9977568992330342,
           "distribution": [
            {
             "value": "hold pending review",
             "probability": 0.0011703860037126305
            },
            {
             "value": "expedited permit",
             "probability": 0.0010727147632531592
            },
            {
             "value": "standard permit",
             "probability": 0.9977568992330342
            }
           ]
          },
          "risk_flags": {
           "value": 0,
           "probability": 0.9969875878767868,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9969875878767868
            },
            {
             "value": 1,
             "probability": 0.001671154489301523
            },
            {
             "value": 2,
             "probability": 0.0007150462398662986
            },
            {
             "value": 3,
             "probability": 0.0006262113940454708
            }
           ]
          }
         },
         "joint_probability": 0.9962600937204239,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0037469172146850044,
        "field_brier": 8.98742738076186e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "queue_ok": {
           "value": true,
           "probability": 0.9981779894665286,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018220105334713745
            },
            {
             "value": true,
             "probability": 0.9981779894665286
            }
           ]
          },
          "decision": {
           "value": "standard permit",
           "probability": 0.9977568992330342,
           "distribution": [
            {
             "value": "hold pending review",
             "probability": 0.0011703860037126305
            },
            {
             "value": "expedited permit",
             "probability": 0.0010727147632531592
            },
            {
             "value": "standard permit",
             "probability": 0.9977568992330342
            }
           ]
          },
          "risk_flags": {
           "value": 0,
           "probability": 0.9969875878767868,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9969875878767868
            },
            {
             "value": 1,
             "probability": 0.001671154489301523
            },
            {
             "value": 2,
             "probability": 0.0007150462398662986
            },
            {
             "value": 3,
             "probability": 0.0006262113940454708
            }
           ]
          }
         },
         "joint_probability": 0.9962600937204239,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0037469172146850044,
        "field_brier": 8.98742738076186e-06
       }
      },
      "timing_ms": {
       "scoring": 41.18548706173897,
       "base_joint": 0.06775883957743645,
       "constrained_joint": 0.04698894917964935,
       "constrained_total": 41.23247601091862
      },
      "input_tokens": 883,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-063f21ebbfe2cde9-0020",
      "group": "bendsynth-063f21ebbfe2cde9",
      "provenance": "bendsynth-v1; family=bendsynth-063f21ebbfe2cde9; domain=municipal aviation office post-flight log audit; parameters=limit=119; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "infractions": 1,
       "restricted_zone": false,
       "decision": "minor note",
       "over_limit": false
      },
      "logits": [
       [
        -5.923506259918213,
        1.8929150104522705,
        -6.1219482421875,
        -5.860023021697998
       ],
       [
        1.8932833671569824,
        -5.244675636291504
       ],
       [
        -4.155411720275879,
        -5.63411808013916,
        1.8911762237548828
       ],
       [
        1.8932942152023315,
        -5.25834321975708
       ]
      ],
      "joint_logits": [
       -8.333809852600098,
       -8.329251289367676,
       -8.182937622070312,
       -8.531942367553711,
       -8.276369094848633,
       -8.221588134765625,
       -8.359048843383789,
       -8.755162239074707,
       -8.510774612426758,
       -8.855884552001953,
       -8.296008110046387,
       -8.472013473510742,
       -4.383020401000977,
       -8.337868690490723,
       -8.022431373596191,
       -8.049809455871582,
       1.8905558586120605,
       -7.789814472198486,
       -8.371830940246582,
       -8.27301025390625,
       -8.098736763000488,
       -8.345235824584961,
       -7.434629917144775,
       -8.349324226379395,
       -8.432252883911133,
       -8.54348373413086,
       -8.419127464294434,
       -8.743671417236328,
       -8.467129707336426,
       -8.415424346923828,
       -8.57412052154541,
       -8.947871208190918,
       -8.709183692932129,
       -9.10441780090332,
       -8.4830961227417,
       -8.693036079406738,
       -8.513883590698242,
       -8.290570259094238,
       -8.124505996704102,
       -8.424993515014648,
       -7.861711502075195,
       -8.323217391967773,
       -8.34488296508789,
       -8.66970157623291,
       -8.410795211791992,
       -8.675931930541992,
       -8.477095603942871,
       -8.312172889709473
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "infractions": {
           "value": 1,
           "probability": 0.9988382970615431,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00040259331841327364
            },
            {
             "value": 1,
             "probability": 0.9988382970615431
            },
            {
             "value": 2,
             "probability": 0.00033012962679467814
            },
            {
             "value": 3,
             "probability": 0.0004289799932489955
            }
           ]
          },
          "restricted_zone": {
           "value": false,
           "probability": 0.999206258726114,
           "distribution": [
            {
             "value": false,
             "probability": 0.999206258726114
            },
            {
             "value": true,
             "probability": 0.0007937412738859114
            }
           ]
          },
          "decision": {
           "value": "minor note",
           "probability": 0.9971032252521629,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.0023590669799387213
            },
            {
             "value": "violation",
             "probability": 0.0005377077678983982
            },
            {
             "value": "minor note",
             "probability": 0.9971032252521629
            }
           ]
          },
          "over_limit": {
           "value": false,
           "probability": 0.9992170334929733,
           "distribution": [
            {
             "value": false,
             "probability": 0.9992170334929733
            },
            {
             "value": true,
             "probability": 0.0007829665070267241
            }
           ]
          }
         },
         "joint_probability": 0.9964846909448116,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0035215022723731953,
        "field_brier": 4.634099655884864e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "infractions": {
           "value": 1,
           "probability": 0.9988382970615431,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00040259331841327364
            },
            {
             "value": 1,
             "probability": 0.9988382970615431
            },
            {
             "value": 2,
             "probability": 0.00033012962679467814
            },
            {
             "value": 3,
             "probability": 0.0004289799932489955
            }
           ]
          },
          "restricted_zone": {
           "value": false,
           "probability": 0.999206258726114,
           "distribution": [
            {
             "value": false,
             "probability": 0.999206258726114
            },
            {
             "value": true,
             "probability": 0.0007937412738859114
            }
           ]
          },
          "decision": {
           "value": "minor note",
           "probability": 0.9971032252521629,
           "distribution": [
            {
             "value": "compliant",
             "probability": 0.0023590669799387213
            },
            {
             "value": "violation",
             "probability": 0.0005377077678983982
            },
            {
             "value": "minor note",
             "probability": 0.9971032252521629
            }
           ]
          },
          "over_limit": {
           "value": false,
           "probability": 0.9992170334929733,
           "distribution": [
            {
             "value": false,
             "probability": 0.9992170334929733
            },
            {
             "value": true,
             "probability": 0.0007829665070267241
            }
           ]
          }
         },
         "joint_probability": 0.9964846909448116,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0035215022723731953,
        "field_brier": 4.634099655884864e-06
       }
      },
      "timing_ms": {
       "scoring": 39.44468591362238,
       "base_joint": 0.1001390628516674,
       "constrained_joint": 0.08400902152061462,
       "constrained_total": 39.528694935142994
      },
      "input_tokens": 811,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-063f21ebbfe2cde9-0023",
      "group": "bendsynth-063f21ebbfe2cde9",
      "provenance": "bendsynth-v1; family=bendsynth-063f21ebbfe2cde9; domain=municipal aviation office post-flight log audit; parameters=limit=63; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "restricted_zone": true,
       "infractions": 2,
       "over_limit": true,
       "decision": "violation"
      },
      "logits": [
       [
        -6.504572868347168,
        0.8810098767280579
       ],
       [
        -7.000014305114746,
        -6.9208269119262695,
        0.8804848790168762,
        -7.053236484527588
       ],
       [
        -6.467552185058594,
        0.8809866905212402
       ],
       [
        -6.859960079193115,
        -6.818218231201172,
        0.8807422518730164
       ]
      ],
      "joint_logits": [
       -10.211492538452148,
       -10.003829002380371,
       -9.630565643310547,
       -9.730927467346191,
       -9.704712867736816,
       -9.250809669494629,
       -10.204388618469238,
       -9.988282203674316,
       -9.598448753356934,
       -9.676602363586426,
       -9.701332092285156,
       -9.204773902893066,
       -9.96463394165039,
       -10.001994132995605,
       -9.527591705322266,
       -9.632787704467773,
       -9.604474067687988,
       -9.104514122009277,
       -10.243985176086426,
       -10.012765884399414,
       -9.66513442993164,
       -9.785125732421875,
       -9.744773864746094,
       -9.324922561645508,
       -9.664507865905762,
       -9.638858795166016,
       -9.19746208190918,
       -9.302239418029785,
       -9.275611877441406,
       -8.925773620605469,
       -9.59698486328125,
       -9.627129554748535,
       -9.134060859680176,
       -9.252370834350586,
       -9.218194007873535,
       -8.651495933532715,
       -9.569161415100098,
       -9.54934024810791,
       -9.050987243652344,
       -9.092828750610352,
       -9.033051490783691,
       0.880111813545227,
       -9.714515686035156,
       -9.695106506347656,
       -9.277566909790039,
       -9.345415115356445,
       -9.338659286499023,
       -8.980086326599121
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "restricted_zone": {
           "value": true,
           "probability": 0.999380254983885,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006197450161149045
            },
            {
             "value": true,
             "probability": 0.999380254983885
            }
           ]
          },
          "infractions": {
           "value": 2,
           "probability": 0.9988556187460711,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003776117508534953
            },
            {
             "value": 1,
             "probability": 0.00040872957256333527
            },
            {
             "value": 2,
             "probability": 0.9988556187460711
            },
            {
             "value": 3,
             "probability": 0.00035803993051205683
            }
           ]
          },
          "over_limit": {
           "value": true,
           "probability": 0.9993568817915612,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006431182084388503
            },
            {
             "value": true,
             "probability": 0.9993568817915612
            }
           ]
          },
          "decision": {
           "value": "violation",
           "probability": 0.9991127234035466,
           "distribution": [
            {
             "value": "minor note",
             "probability": 0.00043438038965966895
            },
            {
             "value": "compliant",
             "probability": 0.0004528962067936949
            },
            {
             "value": "violation",
             "probability": 0.9991127234035466
            }
           ]
          }
         },
         "joint_probability": 0.9984831157264488,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0015180359072448795,
        "field_brier": 1.130970607441995e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "restricted_zone": {
           "value": true,
           "probability": 0.999380254983885,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006197450161149045
            },
            {
             "value": true,
             "probability": 0.999380254983885
            }
           ]
          },
          "infractions": {
           "value": 2,
           "probability": 0.9988556187460711,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003776117508534953
            },
            {
             "value": 1,
             "probability": 0.00040872957256333527
            },
            {
             "value": 2,
             "probability": 0.9988556187460711
            },
            {
             "value": 3,
             "probability": 0.00035803993051205683
            }
           ]
          },
          "over_limit": {
           "value": true,
           "probability": 0.9993568817915612,
           "distribution": [
            {
             "value": false,
             "probability": 0.0006431182084388503
            },
            {
             "value": true,
             "probability": 0.9993568817915612
            }
           ]
          },
          "decision": {
           "value": "violation",
           "probability": 0.9991127234035466,
           "distribution": [
            {
             "value": "minor note",
             "probability": 0.00043438038965966895
            },
            {
             "value": "compliant",
             "probability": 0.0004528962067936949
            },
            {
             "value": "violation",
             "probability": 0.9991127234035466
            }
           ]
          }
         },
         "joint_probability": 0.9984831157264488,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0015180359072448795,
        "field_brier": 1.130970607441995e-06
       }
      },
      "timing_ms": {
       "scoring": 40.735322050750256,
       "base_joint": 0.10234909132122993,
       "constrained_joint": 0.08368911221623421,
       "constrained_total": 40.81901116296649
      },
      "input_tokens": 815,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-063f21ebbfe2cde9-0032",
      "group": "bendsynth-063f21ebbfe2cde9",
      "provenance": "bendsynth-v1; family=bendsynth-063f21ebbfe2cde9; domain=municipal aviation office post-flight log audit; parameters=limit=113; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "infractions": 1,
       "decision": "minor note",
       "over_limit": false,
       "restricted_zone": false
      },
      "logits": [
       [
        -6.15369176864624,
        1.5865731239318848,
        -6.401267051696777,
        -6.155674934387207
       ],
       [
        -6.0518035888671875,
        -4.280524253845215,
        1.584466814994812
       ],
       [
        1.5870001316070557,
        -5.567757606506348
       ],
       [
        1.5869779586791992,
        -5.539768218994141
       ]
      ],
      "joint_logits": [
       -8.52766227722168,
       -8.842268943786621,
       -8.892374038696289,
       -9.205269813537598,
       -8.536646842956543,
       -8.5612211227417,
       -8.58543872833252,
       -8.928378105163574,
       -8.473307609558105,
       -8.414645195007324,
       -8.366100311279297,
       -8.649786949157715,
       -8.565542221069336,
       -8.55926513671875,
       -8.541236877441406,
       -8.866693496704102,
       -4.475578308105469,
       -8.73425579071045,
       -8.715336799621582,
       -8.621427536010742,
       1.5838196277618408,
       -7.845609664916992,
       -8.255353927612305,
       -8.629783630371094,
       -8.796114921569824,
       -9.083909034729004,
       -9.125872611999512,
       -9.483183860778809,
       -8.746064186096191,
       -8.821840286254883,
       -8.834024429321289,
       -9.162745475769043,
       -8.700763702392578,
       -8.666696548461914,
       -8.62751579284668,
       -8.911731719970703,
       -8.507462501525879,
       -8.815518379211426,
       -8.850417137145996,
       -9.103622436523438,
       -8.71694278717041,
       -8.549973487854004,
       -8.555335998535156,
       -8.927779197692871,
       -8.368342399597168,
       -8.536581993103027,
       -8.42594051361084,
       -8.586627006530762
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "infractions": {
           "value": 1,
           "probability": 0.9987928411156876,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004344313396764843
            },
            {
             "value": 1,
             "probability": 0.9987928411156876
            },
            {
             "value": 2,
             "probability": 0.00033915677343286806
            },
            {
             "value": 3,
             "probability": 0.00043357077120312115
            }
           ]
          },
          "decision": {
           "value": "minor note",
           "probability": 0.9966913093556863,
           "distribution": [
            {
             "value": "violation",
             "probability": 0.0004810283323811138
            },
            {
             "value": "compliant",
             "probability": 0.0028276623119326888
            },
            {
             "value": "minor note",
             "probability": 0.9966913093556863
            }
           ]
          },
          "over_limit": {
           "value": false,
           "probability": 0.9992194709057907,
           "distribution": [
            {
             "value": false,
             "probability": 0.9992194709057907
            },
            {
             "value": true,
             "probability": 0.0007805290942092972
            }
           ]
          },
          "restricted_zone": {
           "value": false,
           "probability": 0.99919731598534,
           "distribution": [
            {
             "value": false,
             "probability": 0.99919731598534
            },
            {
             "value": true,
             "probability": 0.0008026840146599637
            }
           ]
          }
         },
         "joint_probability": 0.9960465079390518,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.003961327769779707,
        "field_brier": 5.907631216679344e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "infractions": {
           "value": 1,
           "probability": 0.9987928411156876,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004344313396764843
            },
            {
             "value": 1,
             "probability": 0.9987928411156876
            },
            {
             "value": 2,
             "probability": 0.00033915677343286806
            },
            {
             "value": 3,
             "probability": 0.00043357077120312115
            }
           ]
          },
          "decision": {
           "value": "minor note",
           "probability": 0.9966913093556863,
           "distribution": [
            {
             "value": "violation",
             "probability": 0.0004810283323811138
            },
            {
             "value": "compliant",
             "probability": 0.0028276623119326888
            },
            {
             "value": "minor note",
             "probability": 0.9966913093556863
            }
           ]
          },
          "over_limit": {
           "value": false,
           "probability": 0.9992194709057907,
           "distribution": [
            {
             "value": false,
             "probability": 0.9992194709057907
            },
            {
             "value": true,
             "probability": 0.0007805290942092972
            }
           ]
          },
          "restricted_zone": {
           "value": false,
           "probability": 0.99919731598534,
           "distribution": [
            {
             "value": false,
             "probability": 0.99919731598534
            },
            {
             "value": true,
             "probability": 0.0008026840146599637
            }
           ]
          }
         },
         "joint_probability": 0.9960465079390518,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.003961327769779707,
        "field_brier": 5.907631216679344e-06
       }
      },
      "timing_ms": {
       "scoring": 41.59815306775272,
       "base_joint": 0.10128901340067387,
       "constrained_joint": 0.0912791583687067,
       "constrained_total": 41.689432226121426
      },
      "input_tokens": 831,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0902aaaa9aee85c1-0027",
      "group": "bendsynth-0902aaaa9aee85c1",
      "provenance": "bendsynth-v1; family=bendsynth-0902aaaa9aee85c1; domain=employee emergency assistance fund chatbot at a large employer; parameters=wait_months=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "blockers": 0,
       "waiting_ok": true,
       "decision": "emergency grant"
      },
      "logits": [
       [
        1.8513723611831665,
        -5.732617378234863,
        -5.676290035247803,
        -5.682197093963623
       ],
       [
        -5.041084289550781,
        1.8519381284713745
       ],
       [
        -5.595467567443848,
        1.8512948751449585,
        -5.687766075134277,
        -5.665757179260254
       ]
      ],
      "joint_logits": [
       -7.790706634521484,
       -7.959102153778076,
       -7.897919178009033,
       -7.8853254318237305,
       -7.586079120635986,
       1.8509125709533691,
       -7.666658401489258,
       -7.7263360023498535,
       -7.737185001373291,
       -8.059617042541504,
       -7.794794082641602,
       -7.750648498535156,
       -7.697536945343018,
       -7.912612438201904,
       -7.808783054351807,
       -7.7807111740112305,
       -7.668063640594482,
       -7.9890666007995605,
       -7.739842891693115,
       -7.717538356781006,
       -7.634211540222168,
       -7.909213542938232,
       -7.735418319702148,
       -7.703054904937744,
       -7.671980381011963,
       -7.991888523101807,
       -7.774197578430176,
       -7.7058539390563965,
       -7.628159046173096,
       -7.9357147216796875,
       -7.736089706420898,
       -7.705195903778076
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "blockers": {
           "value": 0,
           "probability": 0.9984211480872222,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9984211480872222
            },
            {
             "value": 1,
             "probability": 0.0005077255515444544
            },
            {
             "value": 2,
             "probability": 0.000537144950382543
            },
            {
             "value": 3,
             "probability": 0.0005339814108508249
            }
           ]
          },
          "waiting_ok": {
           "value": true,
           "probability": 0.9989861864981661,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010138135018339179
            },
            {
             "value": true,
             "probability": 0.9989861864981661
            }
           ]
          },
          "decision": {
           "value": "emergency grant",
           "probability": 0.998343790552889,
           "distribution": [
            {
             "value": "referred to payroll advance",
             "probability": 0.0005823609580841913
            },
            {
             "value": "emergency grant",
             "probability": 0.998343790552889
            },
            {
             "value": "partial grant",
             "probability": 0.000531015970129172
            },
            {
             "value": "deny",
             "probability": 0.0005428325188975625
            }
           ]
          }
         },
         "joint_probability": 0.9979621978754015,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0020398812684173143,
        "field_brier": 3.0128913994453054e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "blockers": {
           "value": 0,
           "probability": 0.9984211480872222,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9984211480872222
            },
            {
             "value": 1,
             "probability": 0.0005077255515444544
            },
            {
             "value": 2,
             "probability": 0.000537144950382543
            },
            {
             "value": 3,
             "probability": 0.0005339814108508249
            }
           ]
          },
          "waiting_ok": {
           "value": true,
           "probability": 0.9989861864981661,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010138135018339179
            },
            {
             "value": true,
             "probability": 0.9989861864981661
            }
           ]
          },
          "decision": {
           "value": "emergency grant",
           "probability": 0.998343790552889,
           "distribution": [
            {
             "value": "referred to payroll advance",
             "probability": 0.0005823609580841913
            },
            {
             "value": "emergency grant",
             "probability": 0.998343790552889
            },
            {
             "value": "partial grant",
             "probability": 0.000531015970129172
            },
            {
             "value": "deny",
             "probability": 0.0005428325188975625
            }
           ]
          }
         },
         "joint_probability": 0.9979621978754015,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0020398812684173143,
        "field_brier": 3.0128913994453054e-06
       }
      },
      "timing_ms": {
       "scoring": 37.24852902814746,
       "base_joint": 0.07994985207915306,
       "constrained_joint": 0.06646895781159401,
       "constrained_total": 37.31499798595905
      },
      "input_tokens": 670,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0902aaaa9aee85c1-0031",
      "group": "bendsynth-0902aaaa9aee85c1",
      "provenance": "bendsynth-v1; family=bendsynth-0902aaaa9aee85c1; domain=employee emergency assistance fund chatbot at a large employer; parameters=wait_months=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "partial grant",
       "blockers": 0,
       "waiting_ok": true
      },
      "logits": [
       [
        -5.355027198791504,
        2.114631414413452,
        -5.499667644500732,
        -5.41776180267334
       ],
       [
        2.114703416824341,
        -5.399570941925049,
        -5.517221450805664,
        -5.4923553466796875
       ],
       [
        -4.848346710205078,
        2.115283966064453
       ]
      ],
      "joint_logits": [
       -7.570882320404053,
       -7.153692245483398,
       -7.460135459899902,
       -7.353118896484375,
       -7.566659450531006,
       -7.462189674377441,
       -7.548207759857178,
       -7.431845664978027,
       -7.645778179168701,
       2.1142208576202393,
       -7.7237653732299805,
       -7.369813919067383,
       -7.853217124938965,
       -7.557054042816162,
       -7.806769847869873,
       -7.5363359451293945,
       -7.6859354972839355,
       -7.412536144256592,
       -7.571989059448242,
       -7.490400314331055,
       -7.684328079223633,
       -7.588936805725098,
       -7.655048847198486,
       -7.576171875,
       -7.609394550323486,
       -7.318073749542236,
       -7.492686748504639,
       -7.418679237365723,
       -7.587660789489746,
       -7.520975112915039,
       -7.5555338859558105,
       -7.506877422332764
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "partial grant",
           "probability": 0.9984036279989436,
           "distribution": [
            {
             "value": "emergency grant",
             "probability": 0.0005692128014056265
            },
            {
             "value": "partial grant",
             "probability": 0.9984036279989436
            },
            {
             "value": "deny",
             "probability": 0.0004925588803508141
            },
            {
             "value": "referred to payroll advance",
             "probability": 0.0005346003192999889
            }
           ]
          },
          "blockers": {
           "value": 0,
           "probability": 0.9984754238896795,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9984754238896795
            },
            {
             "value": 1,
             "probability": 0.0005444143342986037
            },
            {
             "value": 2,
             "probability": 0.0004839880668860021
            },
            {
             "value": 3,
             "probability": 0.0004961737091358186
            }
           ]
          },
          "waiting_ok": {
           "value": true,
           "probability": 0.9990552362587984,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009447637412015601
            },
            {
             "value": true,
             "probability": 0.9990552362587984
            }
           ]
          }
         },
         "joint_probability": 0.9979938018792239,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002008213231818324,
        "field_brier": 2.7623758888343167e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "partial grant",
           "probability": 0.9984036279989436,
           "distribution": [
            {
             "value": "emergency grant",
             "probability": 0.0005692128014056265
            },
            {
             "value": "partial grant",
             "probability": 0.9984036279989436
            },
            {
             "value": "deny",
             "probability": 0.0004925588803508141
            },
            {
             "value": "referred to payroll advance",
             "probability": 0.0005346003192999889
            }
           ]
          },
          "blockers": {
           "value": 0,
           "probability": 0.9984754238896795,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9984754238896795
            },
            {
             "value": 1,
             "probability": 0.0005444143342986037
            },
            {
             "value": 2,
             "probability": 0.0004839880668860021
            },
            {
             "value": 3,
             "probability": 0.0004961737091358186
            }
           ]
          },
          "waiting_ok": {
           "value": true,
           "probability": 0.9990552362587984,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009447637412015601
            },
            {
             "value": true,
             "probability": 0.9990552362587984
            }
           ]
          }
         },
         "joint_probability": 0.9979938018792239,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002008213231818324,
        "field_brier": 2.7623758888343167e-06
       }
      },
      "timing_ms": {
       "scoring": 36.017653066664934,
       "base_joint": 0.09098881855607033,
       "constrained_joint": 0.058870064094662666,
       "constrained_total": 36.0765231307596
      },
      "input_tokens": 679,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b39a16ba7577422-0018",
      "group": "bendsynth-0b39a16ba7577422",
      "provenance": "bendsynth-v1; family=bendsynth-0b39a16ba7577422; domain=bicycle courier delivery file audit; parameters=grace=31; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "late": false,
       "concerns": 1,
       "decision": "escalated audit"
      },
      "logits": [
       [
        -0.8183963894844055,
        -6.834588050842285
       ],
       [
        -7.517952919006348,
        -1.6400200128555298,
        -1.3976044654846191,
        -7.587850570678711
       ],
       [
        -7.24729061126709,
        -0.8193596005439758,
        -7.145350933074951
       ]
      ],
      "joint_logits": [
       -9.282451629638672,
       -9.43492603302002,
       -9.20669937133789,
       -9.120701789855957,
       -1.6430810689926147,
       -8.977701187133789,
       -9.878357887268066,
       -1.398850917816162,
       -9.70533275604248,
       -9.419157028198242,
       -9.404251098632812,
       -9.314586639404297,
       -9.304208755493164,
       -9.510431289672852,
       -9.16311264038086,
       -8.974566459655762,
       -9.245851516723633,
       -8.916227340698242,
       -9.541407585144043,
       -9.92542839050293,
       -9.498104095458984,
       -9.340347290039062,
       -9.592347145080566,
       -9.2420072555542
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "late": {
           "value": false,
           "probability": 0.9975669939221897,
           "distribution": [
            {
             "value": false,
             "probability": 0.9975669939221897
            },
            {
             "value": true,
             "probability": 0.0024330060778103655
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.5589787070425537,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001228462066401636
            },
            {
             "value": 1,
             "probability": 0.43864730335485624
            },
            {
             "value": 2,
             "probability": 0.5589787070425537
            },
            {
             "value": 3,
             "probability": 0.0011455275361883256
            }
           ]
          },
          "decision": {
           "value": "escalated audit",
           "probability": 0.9966065718960823,
           "distribution": [
            {
             "value": "flagged audit",
             "probability": 0.001610307354928644
            },
            {
             "value": "escalated audit",
             "probability": 0.9966065718960823
            },
            {
             "value": "clear",
             "probability": 0.001783120748989161
            }
           ]
          }
         },
         "joint_probability": 0.5582823153809577,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 0.8271206541796128,
        "field_brier": 0.2092019977703177
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "late": {
           "value": false,
           "probability": 0.9975669939221897,
           "distribution": [
            {
             "value": false,
             "probability": 0.9975669939221897
            },
            {
             "value": true,
             "probability": 0.0024330060778103655
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.5589787070425537,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001228462066401636
            },
            {
             "value": 1,
             "probability": 0.43864730335485624
            },
            {
             "value": 2,
             "probability": 0.5589787070425537
            },
            {
             "value": 3,
             "probability": 0.0011455275361883256
            }
           ]
          },
          "decision": {
           "value": "escalated audit",
           "probability": 0.9966065718960823,
           "distribution": [
            {
             "value": "flagged audit",
             "probability": 0.001610307354928644
            },
            {
             "value": "escalated audit",
             "probability": 0.9966065718960823
            },
            {
             "value": "clear",
             "probability": 0.001783120748989161
            }
           ]
          }
         },
         "joint_probability": 0.5582823153809577,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 0.8271206541796128,
        "field_brier": 0.2092019977703177
       }
      },
      "timing_ms": {
       "scoring": 33.83929608389735,
       "base_joint": 0.06817001849412918,
       "constrained_joint": 0.04683900624513626,
       "constrained_total": 33.88613509014249
      },
      "input_tokens": 631,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b39a16ba7577422-0019",
      "group": "bendsynth-0b39a16ba7577422",
      "provenance": "bendsynth-v1; family=bendsynth-0b39a16ba7577422; domain=bicycle courier delivery file audit; parameters=grace=37; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "late": true,
       "concerns": 2,
       "decision": "escalated audit"
      },
      "logits": [
       [
        -7.0480146408081055,
        -0.41471678018569946
       ],
       [
        -7.703522205352783,
        -7.712310314178467,
        -0.4153941571712494,
        -7.780393600463867
       ],
       [
        -7.389348983764648,
        -7.356640815734863,
        -0.4153027832508087
       ]
      ],
      "joint_logits": [
       -9.429956436157227,
       -9.392585754394531,
       -9.698142051696777,
       -9.466662406921387,
       -9.438752174377441,
       -9.753016471862793,
       -9.42452621459961,
       -9.373552322387695,
       -9.361295700073242,
       -9.611780166625977,
       -9.58041000366211,
       -10.092928886413574,
       -9.442831993103027,
       -9.432459831237793,
       -9.612645149230957,
       -9.51678466796875,
       -9.43411922454834,
       -9.450581550598145,
       -9.13058090209961,
       -9.114618301391602,
       -0.4161064028739929,
       -9.877693176269531,
       -9.877166748046875,
       -8.889472007751465
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "late": {
           "value": true,
           "probability": 0.9986859127323813,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013140872676186775
            },
            {
             "value": true,
             "probability": 0.9986859127323813
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.9980097114620824,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006822460846611377
            },
            {
             "value": 1,
             "probability": 0.0006762766328257619
            },
            {
             "value": 2,
             "probability": 0.9980097114620824
            },
            {
             "value": 3,
             "probability": 0.0006317658204307597
            }
           ]
          },
          "decision": {
           "value": "escalated audit",
           "probability": 0.9981007803026424,
           "distribution": [
            {
             "value": "clear",
             "probability": 0.0009340813774008621
            },
            {
             "value": "flagged audit",
             "probability": 0.0009651383199567633
            },
            {
             "value": "escalated audit",
             "probability": 0.9981007803026424
            }
           ]
          }
         },
         "joint_probability": 0.9972990975303627,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027045564876263125,
        "field_brier": 4.715957489698254e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "late": {
           "value": true,
           "probability": 0.9986859127323813,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013140872676186775
            },
            {
             "value": true,
             "probability": 0.9986859127323813
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.9980097114620824,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006822460846611377
            },
            {
             "value": 1,
             "probability": 0.0006762766328257619
            },
            {
             "value": 2,
             "probability": 0.9980097114620824
            },
            {
             "value": 3,
             "probability": 0.0006317658204307597
            }
           ]
          },
          "decision": {
           "value": "escalated audit",
           "probability": 0.9981007803026424,
           "distribution": [
            {
             "value": "clear",
             "probability": 0.0009340813774008621
            },
            {
             "value": "flagged audit",
             "probability": 0.0009651383199567633
            },
            {
             "value": "escalated audit",
             "probability": 0.9981007803026424
            }
           ]
          }
         },
         "joint_probability": 0.9972990975303627,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027045564876263125,
        "field_brier": 4.715957489698254e-06
       }
      },
      "timing_ms": {
       "scoring": 34.941993886604905,
       "base_joint": 0.06622006185352802,
       "constrained_joint": 0.04717893898487091,
       "constrained_total": 34.989172825589776
      },
      "input_tokens": 637,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b39a16ba7577422-0025",
      "group": "bendsynth-0b39a16ba7577422",
      "provenance": "bendsynth-v1; family=bendsynth-0b39a16ba7577422; domain=bicycle courier delivery file audit; parameters=grace=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "escalated audit",
       "late": true,
       "concerns": 3
      },
      "logits": [
       [
        -0.8519060015678406,
        -7.9171462059021,
        -7.879867076873779
       ],
       [
        -7.537497043609619,
        -0.8514137268066406
       ],
       [
        -8.142475128173828,
        -8.278467178344727,
        -8.060193061828613,
        -0.8521825075149536
       ]
      ],
      "joint_logits": [
       -9.946525573730469,
       -10.18686294555664,
       -9.989808082580566,
       -10.01161003112793,
       -10.042723655700684,
       -10.069007873535156,
       -9.484976768493652,
       -0.8526989221572876,
       -9.948371887207031,
       -10.12624740600586,
       -9.955848693847656,
       -10.18069076538086,
       -9.92107105255127,
       -9.995121955871582,
       -9.960522651672363,
       -9.917640686035156,
       -9.874903678894043,
       -10.07090950012207,
       -9.903874397277832,
       -10.138065338134766,
       -9.880989074707031,
       -9.987691879272461,
       -9.91640853881836,
       -9.932759284973145
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "escalated audit",
           "probability": 0.9982619981662313,
           "distribution": [
            {
             "value": "escalated audit",
             "probability": 0.9982619981662313
            },
            {
             "value": "flagged audit",
             "probability": 0.0008528049128519304
            },
            {
             "value": "clear",
             "probability": 0.0008851969209168147
            }
           ]
          },
          "late": {
           "value": true,
           "probability": 0.9987533938578955,
           "distribution": [
            {
             "value": false,
             "probability": 0.001246606142104479
            },
            {
             "value": true,
             "probability": 0.9987533938578955
            }
           ]
          },
          "concerns": {
           "value": 3,
           "probability": 0.997985911354062,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006807546412708769
            },
            {
             "value": 1,
             "probability": 0.000594196203324325
            },
            {
             "value": 2,
             "probability": 0.0007391378013428453
            },
            {
             "value": 3,
             "probability": 0.997985911354062
            }
           ]
          }
         },
         "joint_probability": 0.99747070300731,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025325010681745714,
        "field_brier": 4.352975900720261e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "escalated audit",
           "probability": 0.9982619981662313,
           "distribution": [
            {
             "value": "escalated audit",
             "probability": 0.9982619981662313
            },
            {
             "value": "flagged audit",
             "probability": 0.0008528049128519304
            },
            {
             "value": "clear",
             "probability": 0.0008851969209168147
            }
           ]
          },
          "late": {
           "value": true,
           "probability": 0.9987533938578955,
           "distribution": [
            {
             "value": false,
             "probability": 0.001246606142104479
            },
            {
             "value": true,
             "probability": 0.9987533938578955
            }
           ]
          },
          "concerns": {
           "value": 3,
           "probability": 0.997985911354062,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006807546412708769
            },
            {
             "value": 1,
             "probability": 0.000594196203324325
            },
            {
             "value": 2,
             "probability": 0.0007391378013428453
            },
            {
             "value": 3,
             "probability": 0.997985911354062
            }
           ]
          }
         },
         "joint_probability": 0.99747070300731,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025325010681745714,
        "field_brier": 4.352975900720261e-06
       }
      },
      "timing_ms": {
       "scoring": 34.47454911656678,
       "base_joint": 0.06557907909154892,
       "constrained_joint": 0.0471300445497036,
       "constrained_total": 34.52167916111648
      },
      "input_tokens": 634,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b994372f7bdb702-0002",
      "group": "bendsynth-0b994372f7bdb702",
      "provenance": "bendsynth-v1; family=bendsynth-0b994372f7bdb702; domain=veterinary boarding kennel escalated case review; parameters=grace=91; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "quiet wing",
       "factors": 2,
       "both_flags": false
      },
      "logits": [
       [
        0.9348008632659912,
        -6.127111434936523,
        -6.117775917053223
       ],
       [
        -6.532682418823242,
        -6.374358177185059,
        0.9347066283226013,
        -6.524902820587158
       ],
       [
        0.9352335333824158,
        -5.718651294708252
       ]
      ],
      "joint_logits": [
       -8.55334758758545,
       -8.277427673339844,
       -8.066624641418457,
       -8.1444673538208,
       0.9340896606445312,
       -8.107624053955078,
       -8.472871780395508,
       -8.297643661499023,
       -8.33169937133789,
       -8.255823135375977,
       -8.203389167785645,
       -8.205870628356934,
       -7.999684810638428,
       -8.123979568481445,
       -8.34332275390625,
       -8.233118057250977,
       -8.275486946105957,
       -8.282829284667969,
       -8.19232177734375,
       -8.191511154174805,
       -8.01071834564209,
       -8.093279838562012,
       -8.311896324157715,
       -8.258564949035645
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "quiet wing",
           "probability": 0.9982806474968579,
           "distribution": [
            {
             "value": "quiet wing",
             "probability": 0.9982806474968579
            },
            {
             "value": "standard kennel",
             "probability": 0.0008556634090744692
            },
            {
             "value": "supervisor review",
             "probability": 0.0008636890940677384
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9981865521604298,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005703821139436018
            },
            {
             "value": 1,
             "probability": 0.0006682289067872284
            },
            {
             "value": 2,
             "probability": 0.9981865521604298
            },
            {
             "value": 3,
             "probability": 0.0005748368188394203
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.998712654502187,
           "distribution": [
            {
             "value": false,
             "probability": 0.998712654502187
            },
            {
             "value": true,
             "probability": 0.0012873454978130294
            }
           ]
          }
         },
         "joint_probability": 0.9975708860838445,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024320689998241947,
        "field_brier": 4.0465682250961134e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "quiet wing",
           "probability": 0.9982806474968579,
           "distribution": [
            {
             "value": "quiet wing",
             "probability": 0.9982806474968579
            },
            {
             "value": "standard kennel",
             "probability": 0.0008556634090744692
            },
            {
             "value": "supervisor review",
             "probability": 0.0008636890940677384
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9981865521604298,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005703821139436018
            },
            {
             "value": 1,
             "probability": 0.0006682289067872284
            },
            {
             "value": 2,
             "probability": 0.9981865521604298
            },
            {
             "value": 3,
             "probability": 0.0005748368188394203
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.998712654502187,
           "distribution": [
            {
             "value": false,
             "probability": 0.998712654502187
            },
            {
             "value": true,
             "probability": 0.0012873454978130294
            }
           ]
          }
         },
         "joint_probability": 0.9975708860838445,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024320689998241947,
        "field_brier": 4.0465682250961134e-06
       }
      },
      "timing_ms": {
       "scoring": 33.054254949092865,
       "base_joint": 0.0663897953927517,
       "constrained_joint": 0.04643900319933891,
       "constrained_total": 33.100693952292204
      },
      "input_tokens": 627,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b994372f7bdb702-0008",
      "group": "bendsynth-0b994372f7bdb702",
      "provenance": "bendsynth-v1; family=bendsynth-0b994372f7bdb702; domain=veterinary boarding kennel escalated case review; parameters=grace=179; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "supervisor review",
       "factors": 2,
       "both_flags": false
      },
      "logits": [
       [
        -7.110772609710693,
        0.08177721500396729,
        -6.855273246765137
       ],
       [
        -7.399989604949951,
        -7.166711807250977,
        0.08172110468149185,
        -7.5095295906066895
       ],
       [
        0.08225931227207184,
        -6.61004114151001
       ]
      ],
      "joint_logits": [
       -9.295546531677246,
       -9.241023063659668,
       -9.091806411743164,
       -9.179678916931152,
       -8.969758033752441,
       -9.063754081726074,
       -9.414456367492676,
       -9.34693717956543,
       -9.283533096313477,
       -9.201905250549316,
       -8.790336608886719,
       -8.991095542907715,
       0.08106125891208649,
       -8.928590774536133,
       -9.38180923461914,
       -9.323227882385254,
       -9.074121475219727,
       -9.07873249053955,
       -8.809174537658691,
       -8.947271347045898,
       -8.571293830871582,
       -8.799511909484863,
       -9.170814514160156,
       -9.195744514465332
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "supervisor review",
           "probability": 0.9982796659770055,
           "distribution": [
            {
             "value": "quiet wing",
             "probability": 0.0007508749249155991
            },
            {
             "value": "supervisor review",
             "probability": 0.9982796659770055
            },
            {
             "value": "standard kennel",
             "probability": 0.0009694590980789768
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9982237302102568,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005622925052987877
            },
            {
             "value": 1,
             "probability": 0.0007100247698385367
            },
            {
             "value": 2,
             "probability": 0.9982237302102568
            },
            {
             "value": 3,
             "probability": 0.0005039525146059063
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.9987611107273052,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987611107273052
            },
            {
             "value": true,
             "probability": 0.0012388892726947548
            }
           ]
          }
         },
         "joint_probability": 0.9975652667529606,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024377020298064996,
        "field_brier": 3.920772348205162e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "supervisor review",
           "probability": 0.9982796659770055,
           "distribution": [
            {
             "value": "quiet wing",
             "probability": 0.0007508749249155991
            },
            {
             "value": "supervisor review",
             "probability": 0.9982796659770055
            },
            {
             "value": "standard kennel",
             "probability": 0.0009694590980789768
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9982237302102568,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005622925052987877
            },
            {
             "value": 1,
             "probability": 0.0007100247698385367
            },
            {
             "value": 2,
             "probability": 0.9982237302102568
            },
            {
             "value": 3,
             "probability": 0.0005039525146059063
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.9987611107273052,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987611107273052
            },
            {
             "value": true,
             "probability": 0.0012388892726947548
            }
           ]
          }
         },
         "joint_probability": 0.9975652667529606,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024377020298064996,
        "field_brier": 3.920772348205162e-06
       }
      },
      "timing_ms": {
       "scoring": 34.275091951712966,
       "base_joint": 0.06546988151967525,
       "constrained_joint": 0.04768907092511654,
       "constrained_total": 34.32278102263808
      },
      "input_tokens": 630,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b994372f7bdb702-0029",
      "group": "bendsynth-0b994372f7bdb702",
      "provenance": "bendsynth-v1; family=bendsynth-0b994372f7bdb702; domain=veterinary boarding kennel escalated case review; parameters=grace=365; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "standard kennel",
       "both_flags": false,
       "factors": 0
      },
      "logits": [
       [
        -5.558974266052246,
        1.3602519035339355,
        -5.680585861206055
       ],
       [
        1.3606969118118286,
        -5.197628974914551
       ],
       [
        1.3601421117782593,
        -5.982617378234863,
        -5.940232276916504,
        -5.976314067840576
       ]
      ],
      "joint_logits": [
       -7.714787483215332,
       -7.769905090332031,
       -7.69582462310791,
       -7.719915390014648,
       -7.5951385498046875,
       -7.586618423461914,
       -7.522326469421387,
       -7.533774375915527,
       1.3595788478851318,
       -7.96047306060791,
       -8.034753799438477,
       -8.110852241516113,
       -7.914092540740967,
       -7.783498764038086,
       -7.721822738647461,
       -7.767513275146484,
       -7.854665756225586,
       -7.811387062072754,
       -7.768919467926025,
       -7.823314189910889,
       -7.617840766906738,
       -7.7701897621154785,
       -7.714974403381348,
       -7.738487243652344
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "standard kennel",
           "probability": 0.998139480426343,
           "distribution": [
            {
             "value": "supervisor review",
             "probability": 0.0009867554488433513
            },
            {
             "value": "standard kennel",
             "probability": 0.998139480426343
            },
            {
             "value": "quiet wing",
             "probability": 0.000873764124813593
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.9985837504830832,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985837504830832
            },
            {
             "value": true,
             "probability": 0.0014162495169167772
            }
           ]
          },
          "factors": {
           "value": 0,
           "probability": 0.998029986145141,
           "distribution": [
            {
             "value": 0,
             "probability": 0.998029986145141
            },
            {
             "value": 1,
             "probability": 0.0006459867602893867
            },
            {
             "value": 2,
             "probability": 0.000673955471013208
            },
            {
             "value": 3,
             "probability": 0.0006500716235564095
            }
           ]
          }
         },
         "joint_probability": 0.9974680190972367,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025351917874869706,
        "field_brier": 4.795090369798428e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "standard kennel",
           "probability": 0.998139480426343,
           "distribution": [
            {
             "value": "supervisor review",
             "probability": 0.0009867554488433513
            },
            {
             "value": "standard kennel",
             "probability": 0.998139480426343
            },
            {
             "value": "quiet wing",
             "probability": 0.000873764124813593
            }
           ]
          },
          "both_flags": {
           "value": false,
           "probability": 0.9985837504830832,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985837504830832
            },
            {
             "value": true,
             "probability": 0.0014162495169167772
            }
           ]
          },
          "factors": {
           "value": 0,
           "probability": 0.998029986145141,
           "distribution": [
            {
             "value": 0,
             "probability": 0.998029986145141
            },
            {
             "value": 1,
             "probability": 0.0006459867602893867
            },
            {
             "value": 2,
             "probability": 0.000673955471013208
            },
            {
             "value": 3,
             "probability": 0.0006500716235564095
            }
           ]
          }
         },
         "joint_probability": 0.9974680190972367,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025351917874869706,
        "field_brier": 4.795090369798428e-06
       }
      },
      "timing_ms": {
       "scoring": 34.17296288534999,
       "base_joint": 0.06751017645001411,
       "constrained_joint": 0.04826905205845833,
       "constrained_total": 34.22123193740845
      },
      "input_tokens": 630,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09414f5ab2080fe0-0002",
      "group": "bendsynth-09414f5ab2080fe0",
      "provenance": "bendsynth-v1; family=bendsynth-09414f5ab2080fe0; domain=print shop rush job audit; parameters=priority_threshold=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "violations": 2,
       "compliant": false,
       "audit_decision": "escalate audit"
      },
      "logits": [
       [
        -6.321021556854248,
        -6.033648490905762,
        1.2397313117980957,
        -6.307514190673828
       ],
       [
        1.2403141260147095,
        -5.520622253417969
       ],
       [
        1.2398877143859863,
        -5.848511219024658,
        -5.954928398132324
       ]
      ],
      "joint_logits": [
       -8.327384948730469,
       -8.085858345031738,
       -8.090262413024902,
       -8.074037551879883,
       -7.987581253051758,
       -8.143033981323242,
       -7.630069732666016,
       -7.770050525665283,
       -7.8940911293029785,
       -7.858403205871582,
       -7.905865669250488,
       -7.926887035369873,
       1.2392174005508423,
       -7.692422866821289,
       -8.017146110534668,
       -8.064237594604492,
       -7.970810413360596,
       -8.01898193359375,
       -8.228996276855469,
       -8.113058090209961,
       -8.112234115600586,
       -8.083736419677734,
       -7.973026275634766,
       -8.101268768310547
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "violations": {
           "value": 2,
           "probability": 0.9982612207160968,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000519578384388602
            },
            {
             "value": 1,
             "probability": 0.0006925569007301168
            },
            {
             "value": 2,
             "probability": 0.9982612207160968
            },
            {
             "value": 3,
             "probability": 0.0005266439987845375
            }
           ]
          },
          "compliant": {
           "value": false,
           "probability": 0.9988431959767927,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988431959767927
            },
            {
             "value": true,
             "probability": 0.0011568040232073367
            }
           ]
          },
          "audit_decision": {
           "value": "escalate audit",
           "probability": 0.998417310142235,
           "distribution": [
            {
             "value": "escalate audit",
             "probability": 0.998417310142235
            },
            {
             "value": "downgrade to warning",
             "probability": 0.0008334116577477602
            },
            {
             "value": "routine pass",
             "probability": 0.0007492782000173002
            }
           ]
          }
         },
         "joint_probability": 0.9977482682783851,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002254270681575085,
        "field_brier": 3.495865050746642e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "violations": {
           "value": 2,
           "probability": 0.9982612207160968,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000519578384388602
            },
            {
             "value": 1,
             "probability": 0.0006925569007301168
            },
            {
             "value": 2,
             "probability": 0.9982612207160968
            },
            {
             "value": 3,
             "probability": 0.0005266439987845375
            }
           ]
          },
          "compliant": {
           "value": false,
           "probability": 0.9988431959767927,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988431959767927
            },
            {
             "value": true,
             "probability": 0.0011568040232073367
            }
           ]
          },
          "audit_decision": {
           "value": "escalate audit",
           "probability": 0.998417310142235,
           "distribution": [
            {
             "value": "escalate audit",
             "probability": 0.998417310142235
            },
            {
             "value": "downgrade to warning",
             "probability": 0.0008334116577477602
            },
            {
             "value": "routine pass",
             "probability": 0.0007492782000173002
            }
           ]
          }
         },
         "joint_probability": 0.9977482682783851,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002254270681575085,
        "field_brier": 3.495865050746642e-06
       }
      },
      "timing_ms": {
       "scoring": 38.175169844180346,
       "base_joint": 0.06432016380131245,
       "constrained_joint": 0.048249028623104095,
       "constrained_total": 38.22341887280345
      },
      "input_tokens": 775,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09414f5ab2080fe0-0024",
      "group": "bendsynth-09414f5ab2080fe0",
      "provenance": "bendsynth-v1; family=bendsynth-09414f5ab2080fe0; domain=print shop rush job audit; parameters=priority_threshold=24; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "violations": 1,
       "compliant": false,
       "audit_decision": "downgrade to warning"
      },
      "logits": [
       [
        -6.268571853637695,
        1.0736422538757324,
        -6.371926307678223,
        -6.375743865966797
       ],
       [
        1.074134111404419,
        -5.5554399490356445
       ],
       [
        1.0737478733062744,
        -5.858152866363525,
        -6.146878719329834
       ]
      ],
      "joint_logits": [
       -8.202080726623535,
       -7.976548671722412,
       -8.229177474975586,
       -8.056853294372559,
       -7.797818183898926,
       -8.169639587402344,
       1.073081135749817,
       -7.806208610534668,
       -8.295825004577637,
       -8.022276878356934,
       -7.926201820373535,
       -8.127649307250977,
       -8.330567359924316,
       -8.128745079040527,
       -8.317900657653809,
       -8.182466506958008,
       -7.9127044677734375,
       -8.169326782226562,
       -8.401254653930664,
       -8.128185272216797,
       -8.333858489990234,
       -8.157706260681152,
       -7.880606174468994,
       -8.18825626373291
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "violations": {
           "value": 1,
           "probability": 0.9981898446694332,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000646442976261458
            },
            {
             "value": 1,
             "probability": 0.9981898446694332
            },
            {
             "value": 2,
             "probability": 0.0005829669427300994
            },
            {
             "value": 3,
             "probability": 0.0005807454115752885
            }
           ]
          },
          "compliant": {
           "value": false,
           "probability": 0.9986810169538282,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986810169538282
            },
            {
             "value": true,
             "probability": 0.0013189830461718208
            }
           ]
          },
          "audit_decision": {
           "value": "downgrade to warning",
           "probability": 0.9982954228498154,
           "distribution": [
            {
             "value": "downgrade to warning",
             "probability": 0.9982954228498154
            },
            {
             "value": "routine pass",
             "probability": 0.0009744798143601465
            },
            {
             "value": "escalate audit",
             "probability": 0.0007300973358244189
            }
           ]
          }
         },
         "joint_probability": 0.9976299457847115,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023728672393388156,
        "field_brier": 4.0797784577337385e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "violations": {
           "value": 1,
           "probability": 0.9981898446694332,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000646442976261458
            },
            {
             "value": 1,
             "probability": 0.9981898446694332
            },
            {
             "value": 2,
             "probability": 0.0005829669427300994
            },
            {
             "value": 3,
             "probability": 0.0005807454115752885
            }
           ]
          },
          "compliant": {
           "value": false,
           "probability": 0.9986810169538282,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986810169538282
            },
            {
             "value": true,
             "probability": 0.0013189830461718208
            }
           ]
          },
          "audit_decision": {
           "value": "downgrade to warning",
           "probability": 0.9982954228498154,
           "distribution": [
            {
             "value": "downgrade to warning",
             "probability": 0.9982954228498154
            },
            {
             "value": "routine pass",
             "probability": 0.0009744798143601465
            },
            {
             "value": "escalate audit",
             "probability": 0.0007300973358244189
            }
           ]
          }
         },
         "joint_probability": 0.9976299457847115,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023728672393388156,
        "field_brier": 4.0797784577337385e-06
       }
      },
      "timing_ms": {
       "scoring": 38.149579893797636,
       "base_joint": 0.067500164732337,
       "constrained_joint": 0.05613919347524643,
       "constrained_total": 38.20571908727288
      },
      "input_tokens": 775,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09414f5ab2080fe0-0037",
      "group": "bendsynth-09414f5ab2080fe0",
      "provenance": "bendsynth-v1; family=bendsynth-09414f5ab2080fe0; domain=print shop rush job audit; parameters=priority_threshold=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "violations": 0,
       "audit_decision": "routine pass",
       "compliant": true
      },
      "logits": [
       [
        0.5821536183357239,
        -6.706751823425293,
        -6.701108455657959,
        -6.696415901184082
       ],
       [
        -6.503961086273193,
        -6.2597432136535645,
        0.5823091864585876
       ],
       [
        -5.959648132324219,
        0.5827718377113342
       ]
      ],
      "joint_logits": [
       -8.608219146728516,
       -8.65845775604248,
       -8.342400550842285,
       -8.136756896972656,
       -8.33128833770752,
       0.5815232992172241,
       -8.533133506774902,
       -8.620314598083496,
       -8.354820251464844,
       -8.434563636779785,
       -8.5278959274292,
       -8.542359352111816,
       -8.551633834838867,
       -8.623258590698242,
       -8.339776039123535,
       -8.400460243225098,
       -8.489095687866211,
       -8.583561897277832,
       -8.491006851196289,
       -8.591939926147461,
       -8.30197525024414,
       -8.436310768127441,
       -8.52670955657959,
       -8.614538192749023
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "violations": {
           "value": 0,
           "probability": 0.997944047245263,
           "distribution": [
            {
             "value": 0,
             "probability": 0.997944047245263
            },
            {
             "value": 1,
             "probability": 0.0006816708035819837
            },
            {
             "value": 2,
             "probability": 0.0006855287594641102
            },
            {
             "value": 3,
             "probability": 0.0006887531916909503
            }
           ]
          },
          "audit_decision": {
           "value": "routine pass",
           "probability": 0.9980991987061382,
           "distribution": [
            {
             "value": "escalate audit",
             "probability": 0.0008349216187236292
            },
            {
             "value": "downgrade to warning",
             "probability": 0.0010658796751380967
            },
            {
             "value": "routine pass",
             "probability": 0.9980991987061382
            }
           ]
          },
          "compliant": {
           "value": true,
           "probability": 0.9985610757641635,
           "distribution": [
            {
             "value": false,
             "probability": 0.001438924235836384
            },
            {
             "value": true,
             "probability": 0.9985610757641635
            }
           ]
          }
         },
         "joint_probability": 0.9973151938963818,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002688416659401285,
        "field_brier": 5.0743975054233376e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "violations": {
           "value": 0,
           "probability": 0.997944047245263,
           "distribution": [
            {
             "value": 0,
             "probability": 0.997944047245263
            },
            {
             "value": 1,
             "probability": 0.0006816708035819837
            },
            {
             "value": 2,
             "probability": 0.0006855287594641102
            },
            {
             "value": 3,
             "probability": 0.0006887531916909503
            }
           ]
          },
          "audit_decision": {
           "value": "routine pass",
           "probability": 0.9980991987061382,
           "distribution": [
            {
             "value": "escalate audit",
             "probability": 0.0008349216187236292
            },
            {
             "value": "downgrade to warning",
             "probability": 0.0010658796751380967
            },
            {
             "value": "routine pass",
             "probability": 0.9980991987061382
            }
           ]
          },
          "compliant": {
           "value": true,
           "probability": 0.9985610757641635,
           "distribution": [
            {
             "value": false,
             "probability": 0.001438924235836384
            },
            {
             "value": true,
             "probability": 0.9985610757641635
            }
           ]
          }
         },
         "joint_probability": 0.9973151938963818,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002688416659401285,
        "field_brier": 5.0743975054233376e-06
       }
      },
      "timing_ms": {
       "scoring": 37.66079503111541,
       "base_joint": 0.06709899753332138,
       "constrained_joint": 0.04733004607260227,
       "constrained_total": 37.708125077188015
      },
      "input_tokens": 760,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0546afdf3eff7f9c-0017",
      "group": "bendsynth-0546afdf3eff7f9c",
      "provenance": "bendsynth-v1; family=bendsynth-0546afdf3eff7f9c; domain=vineyard harvest crew office at the counter; parameters=crew_cap=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 0,
       "senior_enough": true,
       "decision": "full harvest pass"
      },
      "logits": [
       [
        1.718870759010315,
        -5.75197696685791,
        -5.716817855834961,
        -5.747785568237305
       ],
       [
        -5.196199417114258,
        1.7196089029312134
       ],
       [
        -4.944428443908691,
        -5.612658500671387,
        1.71867036819458
       ]
      ],
      "joint_logits": [
       -7.023680686950684,
       -7.419909477233887,
       -7.968559741973877,
       -6.869898319244385,
       -7.1942949295043945,
       1.718220591545105,
       -7.178456783294678,
       -9.11441707611084,
       -7.979462146759033,
       -6.960049629211426,
       -7.356601715087891,
       -7.891935348510742,
       -7.143153190612793,
       -9.054107666015625,
       -7.942142009735107,
       -6.9330644607543945,
       -7.306227684020996,
       -7.872182369232178,
       -7.172507286071777,
       -9.094524383544922,
       -7.974787712097168,
       -6.959780216217041,
       -7.334968090057373,
       -7.916973114013672
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 0,
           "probability": 0.9982718861122106,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9982718861122106
            },
            {
             "value": 1,
             "probability": 0.0005684612376476939
            },
            {
             "value": 2,
             "probability": 0.0005888036293012984
            },
            {
             "value": 3,
             "probability": 0.0005708490208403802
            }
           ]
          },
          "senior_enough": {
           "value": true,
           "probability": 0.9990090035964598,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009909964035401408
            },
            {
             "value": true,
             "probability": 0.9990090035964598
            }
           ]
          },
          "decision": {
           "value": "limited pass",
           "probability": 0.9980718392644915,
           "distribution": [
            {
             "value": "full harvest pass",
             "probability": 0.001274719853454526
            },
            {
             "value": "counter deny",
             "probability": 0.0006534408820539812
            },
            {
             "value": "limited pass",
             "probability": 0.9980718392644915
            }
           ]
          }
         },
         "joint_probability": 0.9976230960675506,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.590498644042329,
        "field_brier": 0.6645353182511456
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 0,
           "probability": 0.9982718861122106,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9982718861122106
            },
            {
             "value": 1,
             "probability": 0.0005684612376476939
            },
            {
             "value": 2,
             "probability": 0.0005888036293012984
            },
            {
             "value": 3,
             "probability": 0.0005708490208403802
            }
           ]
          },
          "senior_enough": {
           "value": true,
           "probability": 0.9990090035964598,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009909964035401408
            },
            {
             "value": true,
             "probability": 0.9990090035964598
            }
           ]
          },
          "decision": {
           "value": "limited pass",
           "probability": 0.9980718392644915,
           "distribution": [
            {
             "value": "full harvest pass",
             "probability": 0.001274719853454526
            },
            {
             "value": "counter deny",
             "probability": 0.0006534408820539812
            },
            {
             "value": "limited pass",
             "probability": 0.9980718392644915
            }
           ]
          }
         },
         "joint_probability": 0.9976230960675506,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.590498644042329,
        "field_brier": 0.6645353182511456
       }
      },
      "timing_ms": {
       "scoring": 34.17369211092591,
       "base_joint": 0.06733881309628487,
       "constrained_joint": 0.04760012961924076,
       "constrained_total": 34.221292240545154
      },
      "input_tokens": 576,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0546afdf3eff7f9c-0018",
      "group": "bendsynth-0546afdf3eff7f9c",
      "provenance": "bendsynth-v1; family=bendsynth-0546afdf3eff7f9c; domain=vineyard harvest crew office at the counter; parameters=crew_cap=9; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "counter deny",
       "risk_factors": 3,
       "senior_enough": false
      },
      "logits": [
       [
        -0.7228527069091797,
        -7.8072991371154785,
        -7.584176063537598
       ],
       [
        -8.138381958007812,
        -8.174747467041016,
        -8.050129890441895,
        -0.7228067517280579
       ],
       [
        -0.7221589088439941,
        -7.45535135269165
       ]
      ],
      "joint_logits": [
       -10.122477531433105,
       -9.950912475585938,
       -10.017029762268066,
       -10.003734588623047,
       -9.690088272094727,
       -9.993653297424316,
       -0.7235397100448608,
       -9.943991661071777,
       -9.897492408752441,
       -10.007719039916992,
       -9.914599418640137,
       -10.094779014587402,
       -9.8172025680542,
       -10.066917419433594,
       -9.557766914367676,
       -9.844386100769043,
       -9.773369789123535,
       -9.865052223205566,
       -9.78884220123291,
       -10.008896827697754,
       -9.750982284545898,
       -9.785293579101562,
       -8.96891975402832,
       -9.778820991516113
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "counter deny",
           "probability": 0.9981179828918931,
           "distribution": [
            {
             "value": "counter deny",
             "probability": 0.9981179828918931
            },
            {
             "value": "limited pass",
             "probability": 0.0008364616003949116
            },
            {
             "value": "full harvest pass",
             "probability": 0.0010455555077119396
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.9981639258231589,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006007009955805587
            },
            {
             "value": 1,
             "probability": 0.0005792489840434206
            },
            {
             "value": 2,
             "probability": 0.000656124197217162
            },
            {
             "value": 3,
             "probability": 0.9981639258231589
            }
           ]
          },
          "senior_enough": {
           "value": false,
           "probability": 0.9988106902316878,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988106902316878
            },
            {
             "value": true,
             "probability": 0.0011893097683122647
            }
           ]
          }
         },
         "joint_probability": 0.9974325501692938,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002570751382278852,
        "field_brier": 4.220598863499344e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "counter deny",
           "probability": 0.9981179828918931,
           "distribution": [
            {
             "value": "counter deny",
             "probability": 0.9981179828918931
            },
            {
             "value": "limited pass",
             "probability": 0.0008364616003949116
            },
            {
             "value": "full harvest pass",
             "probability": 0.0010455555077119396
            }
           ]
          },
          "risk_factors": {
           "value": 3,
           "probability": 0.9981639258231589,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006007009955805587
            },
            {
             "value": 1,
             "probability": 0.0005792489840434206
            },
            {
             "value": 2,
             "probability": 0.000656124197217162
            },
            {
             "value": 3,
             "probability": 0.9981639258231589
            }
           ]
          },
          "senior_enough": {
           "value": false,
           "probability": 0.9988106902316878,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988106902316878
            },
            {
             "value": true,
             "probability": 0.0011893097683122647
            }
           ]
          }
         },
         "joint_probability": 0.9974325501692938,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002570751382278852,
        "field_brier": 4.220598863499344e-06
       }
      },
      "timing_ms": {
       "scoring": 33.322771079838276,
       "base_joint": 0.06740889512002468,
       "constrained_joint": 0.04905997775495052,
       "constrained_total": 33.371831057593226
      },
      "input_tokens": 558,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0546afdf3eff7f9c-0026",
      "group": "bendsynth-0546afdf3eff7f9c",
      "provenance": "bendsynth-v1; family=bendsynth-0546afdf3eff7f9c; domain=vineyard harvest crew office at the counter; parameters=crew_cap=11; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "limited pass",
       "senior_enough": false,
       "risk_factors": 1
      },
      "logits": [
       [
        -5.730392932891846,
        0.5467760562896729,
        -6.76570463180542
       ],
       [
        0.5478048920631409,
        -5.944601535797119
       ],
       [
        -6.752574920654297,
        0.5472992062568665,
        -6.752599716186523,
        -6.760601043701172
       ]
      ],
      "joint_logits": [
       -8.4717435836792,
       -6.45570182800293,
       -8.525152206420898,
       -8.596064567565918,
       -8.167046546936035,
       -8.344054222106934,
       -8.173517227172852,
       -8.187767028808594,
       -8.685379028320312,
       0.5459557771682739,
       -8.765098571777344,
       -8.797765731811523,
       -8.35159683227539,
       -8.432143211364746,
       -8.318968772888184,
       -8.323429107666016,
       -8.882485389709473,
       -8.975744247436523,
       -8.847196578979492,
       -8.833491325378418,
       -8.941407203674316,
       -8.696090698242188,
       -8.84790325164795,
       -8.765230178833008
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "limited pass",
           "probability": 0.9974605931306662,
           "distribution": [
            {
             "value": "full harvest pass",
             "probability": 0.0018739409458743328
            },
            {
             "value": "limited pass",
             "probability": 0.9974605931306662
            },
            {
             "value": "counter deny",
             "probability": 0.0006654659234595692
            }
           ]
          },
          "senior_enough": {
           "value": false,
           "probability": 0.9984873921191754,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984873921191754
            },
            {
             "value": true,
             "probability": 0.001512607880824641
            }
           ]
          },
          "risk_factors": {
           "value": 1,
           "probability": 0.9979826240925616,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006742610298462946
            },
            {
             "value": 1,
             "probability": 0.9979826240925616
            },
            {
             "value": 2,
             "probability": 0.0006742441338136599
            },
            {
             "value": 3,
             "probability": 0.0006688707437784502
            }
           ]
          }
         },
         "joint_probability": 0.9966427410375966,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00336290720155113,
        "field_brier": 6.801826242114392e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "limited pass",
           "probability": 0.9974605931306662,
           "distribution": [
            {
             "value": "full harvest pass",
             "probability": 0.0018739409458743328
            },
            {
             "value": "limited pass",
             "probability": 0.9974605931306662
            },
            {
             "value": "counter deny",
             "probability": 0.0006654659234595692
            }
           ]
          },
          "senior_enough": {
           "value": false,
           "probability": 0.9984873921191754,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984873921191754
            },
            {
             "value": true,
             "probability": 0.001512607880824641
            }
           ]
          },
          "risk_factors": {
           "value": 1,
           "probability": 0.9979826240925616,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006742610298462946
            },
            {
             "value": 1,
             "probability": 0.9979826240925616
            },
            {
             "value": 2,
             "probability": 0.0006742441338136599
            },
            {
             "value": 3,
             "probability": 0.0006688707437784502
            }
           ]
          }
         },
         "joint_probability": 0.9966427410375966,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00336290720155113,
        "field_brier": 6.801826242114392e-06
       }
      },
      "timing_ms": {
       "scoring": 31.551160849630833,
       "base_joint": 0.07098913192749023,
       "constrained_joint": 0.047688838094472885,
       "constrained_total": 31.598849687725306
      },
      "input_tokens": 552,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0546afdf3eff7f9c-0038",
      "group": "bendsynth-0546afdf3eff7f9c",
      "provenance": "bendsynth-v1; family=bendsynth-0546afdf3eff7f9c; domain=vineyard harvest crew office at the counter; parameters=crew_cap=7; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 0,
       "decision": "full harvest pass",
       "senior_enough": true
      },
      "logits": [
       [
        1.873180866241455,
        -5.641934394836426,
        -5.582392692565918,
        -5.5991387367248535
       ],
       [
        -5.482600688934326,
        -4.967002868652344,
        1.8731638193130493
       ],
       [
        -5.004987716674805,
        1.8738422393798828
       ]
      ],
      "joint_logits": [
       -7.3860554695129395,
       -7.277375221252441,
       -7.093733787536621,
       -6.8066205978393555,
       -7.515031337738037,
       1.8725980520248413,
       -8.196974754333496,
       -7.359457015991211,
       -7.25096321105957,
       -7.005183696746826,
       -7.583807468414307,
       -7.583665370941162,
       -8.070886611938477,
       -7.317408561706543,
       -7.168206691741943,
       -6.962367057800293,
       -7.512983798980713,
       -7.554673671722412,
       -8.102195739746094,
       -7.328773498535156,
       -7.197858810424805,
       -6.964463233947754,
       -7.5320353507995605,
       -7.575048446655273
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 0,
           "probability": 0.99831125182167,
           "distribution": [
            {
             "value": 0,
             "probability": 0.99831125182167
            },
            {
             "value": 1,
             "probability": 0.0005438671369234757
            },
            {
             "value": 2,
             "probability": 0.0005772334290958726
            },
            {
             "value": 3,
             "probability": 0.0005676476123106533
            }
           ]
          },
          "decision": {
           "value": "full harvest pass",
           "probability": 0.9982940912231533,
           "distribution": [
            {
             "value": "counter deny",
             "probability": 0.0006378089876283837
            },
            {
             "value": "limited pass",
             "probability": 0.001068099789218277
            },
            {
             "value": "full harvest pass",
             "probability": 0.9982940912231533
            }
           ]
          },
          "senior_enough": {
           "value": true,
           "probability": 0.9989717107410437,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010282892589563006
            },
            {
             "value": true,
             "probability": 0.9989717107410437
            }
           ]
          }
         },
         "joint_probability": 0.9977295960411419,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022729852336908878,
        "field_brier": 3.4585346451286755e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 0,
           "probability": 0.99831125182167,
           "distribution": [
            {
             "value": 0,
             "probability": 0.99831125182167
            },
            {
             "value": 1,
             "probability": 0.0005438671369234757
            },
            {
             "value": 2,
             "probability": 0.0005772334290958726
            },
            {
             "value": 3,
             "probability": 0.0005676476123106533
            }
           ]
          },
          "decision": {
           "value": "full harvest pass",
           "probability": 0.9982940912231533,
           "distribution": [
            {
             "value": "counter deny",
             "probability": 0.0006378089876283837
            },
            {
             "value": "limited pass",
             "probability": 0.001068099789218277
            },
            {
             "value": "full harvest pass",
             "probability": 0.9982940912231533
            }
           ]
          },
          "senior_enough": {
           "value": true,
           "probability": 0.9989717107410437,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010282892589563006
            },
            {
             "value": true,
             "probability": 0.9989717107410437
            }
           ]
          }
         },
         "joint_probability": 0.9977295960411419,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022729852336908878,
        "field_brier": 3.4585346451286755e-06
       }
      },
      "timing_ms": {
       "scoring": 34.04799313284457,
       "base_joint": 0.06749993190169334,
       "constrained_joint": 0.04746904596686363,
       "constrained_total": 34.09546217881143
      },
      "input_tokens": 579,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0cb83ecc6b11396e-0007",
      "group": "bendsynth-0cb83ecc6b11396e",
      "provenance": "bendsynth-v1; family=bendsynth-0cb83ecc6b11396e; domain=community theatre casting file audit; parameters=absence_limit=5; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "approve file",
       "attendance_ok": false,
       "concerns": 1
      },
      "logits": [
       [
        -6.850515842437744,
        -6.760310649871826,
        0.19824135303497314
       ],
       [
        0.19878947734832764,
        -6.469893932342529
       ],
       [
        -7.172922611236572,
        0.19813568890094757,
        -7.173985481262207,
        -7.114535331726074
       ]
      ],
      "joint_logits": [
       -8.951539993286133,
       -8.577632904052734,
       -8.969264030456543,
       -8.886333465576172,
       -9.138652801513672,
       -8.887934684753418,
       -9.117575645446777,
       -9.02487850189209,
       -8.834029197692871,
       -8.58481216430664,
       -8.85872745513916,
       -8.769658088684082,
       -9.042641639709473,
       -8.757314682006836,
       -9.006365776062012,
       -8.94373893737793,
       -8.848482131958008,
       0.1974470168352127,
       -8.864953994750977,
       -8.886568069458008,
       -9.006905555725098,
       -8.68647289276123,
       -9.001178741455078,
       -8.944952964782715
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve file",
           "probability": 0.9981843431953114,
           "distribution": [
            {
             "value": "flag for review",
             "probability": 0.0008669107608627806
            },
            {
             "value": "void casting",
             "probability": 0.0009487460438257973
            },
            {
             "value": "approve file",
             "probability": 0.9981843431953114
            }
           ]
          },
          "attendance_ok": {
           "value": false,
           "probability": 0.9987315408013205,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987315408013205
            },
            {
             "value": true,
             "probability": 0.0012684591986794297
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9980789291518083,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006279931721534759
            },
            {
             "value": 1,
             "probability": 0.9980789291518083
            },
            {
             "value": 2,
             "probability": 0.0006273259967144788
            },
            {
             "value": 3,
             "probability": 0.0006657516793237088
            }
           ]
          }
         },
         "joint_probability": 0.997391727810586,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002611679657680391,
        "field_brier": 4.362630755178509e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve file",
           "probability": 0.9981843431953114,
           "distribution": [
            {
             "value": "flag for review",
             "probability": 0.0008669107608627806
            },
            {
             "value": "void casting",
             "probability": 0.0009487460438257973
            },
            {
             "value": "approve file",
             "probability": 0.9981843431953114
            }
           ]
          },
          "attendance_ok": {
           "value": false,
           "probability": 0.9987315408013205,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987315408013205
            },
            {
             "value": true,
             "probability": 0.0012684591986794297
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9980789291518083,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006279931721534759
            },
            {
             "value": 1,
             "probability": 0.9980789291518083
            },
            {
             "value": 2,
             "probability": 0.0006273259967144788
            },
            {
             "value": 3,
             "probability": 0.0006657516793237088
            }
           ]
          }
         },
         "joint_probability": 0.997391727810586,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002611679657680391,
        "field_brier": 4.362630755178509e-06
       }
      },
      "timing_ms": {
       "scoring": 37.95091202482581,
       "base_joint": 0.06633903831243515,
       "constrained_joint": 0.048809219151735306,
       "constrained_total": 37.99972124397755
      },
      "input_tokens": 729,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0cb83ecc6b11396e-0014",
      "group": "bendsynth-0cb83ecc6b11396e",
      "provenance": "bendsynth-v1; family=bendsynth-0cb83ecc6b11396e; domain=community theatre casting file audit; parameters=absence_limit=5; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "void casting",
       "concerns": 3,
       "attendance_ok": false
      },
      "logits": [
       [
        -0.666127622127533,
        -7.497124671936035,
        -7.781461238861084
       ],
       [
        -7.997676849365234,
        -7.8699235916137695,
        -7.857455253601074,
        -0.6663865447044373
       ],
       [
        -0.6654762029647827,
        -7.358124732971191
       ]
      ],
      "joint_logits": [
       -9.699316024780273,
       -9.850849151611328,
       -9.47213077545166,
       -9.805107116699219,
       -9.273322105407715,
       -9.85466480255127,
       -0.667014479637146,
       -9.674959182739258,
       -9.58820629119873,
       -9.782553672790527,
       -9.434338569641113,
       -9.731526374816895,
       -9.487703323364258,
       -9.69666576385498,
       -9.241226196289062,
       -9.786962509155273,
       -9.839699745178223,
       -10.032187461853027,
       -9.67466926574707,
       -9.949358940124512,
       -9.76063346862793,
       -9.997465133666992,
       -9.67445182800293,
       -10.043437004089355
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "void casting",
           "probability": 0.9981112435645958,
           "distribution": [
            {
             "value": "void casting",
             "probability": 0.9981112435645958
            },
            {
             "value": "approve file",
             "probability": 0.001077741597926189
            },
            {
             "value": "flag for review",
             "probability": 0.0008110148374780541
            }
           ]
          },
          "concerns": {
           "value": 3,
           "probability": 0.9978526591692454,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000653322277587957
            },
            {
             "value": 1,
             "probability": 0.0007423522796230318
            },
            {
             "value": 2,
             "probability": 0.0007516662735436142
            },
            {
             "value": 3,
             "probability": 0.9978526591692454
            }
           ]
          },
          "attendance_ok": {
           "value": false,
           "probability": 0.9987615412793515,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987615412793515
            },
            {
             "value": true,
             "probability": 0.0012384587206484996
            }
           ]
          }
         },
         "joint_probability": 0.9972263990496555,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002777454508598278,
        "field_brier": 4.869408210651577e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "void casting",
           "probability": 0.9981112435645958,
           "distribution": [
            {
             "value": "void casting",
             "probability": 0.9981112435645958
            },
            {
             "value": "approve file",
             "probability": 0.001077741597926189
            },
            {
             "value": "flag for review",
             "probability": 0.0008110148374780541
            }
           ]
          },
          "concerns": {
           "value": 3,
           "probability": 0.9978526591692454,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000653322277587957
            },
            {
             "value": 1,
             "probability": 0.0007423522796230318
            },
            {
             "value": 2,
             "probability": 0.0007516662735436142
            },
            {
             "value": 3,
             "probability": 0.9978526591692454
            }
           ]
          },
          "attendance_ok": {
           "value": false,
           "probability": 0.9987615412793515,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987615412793515
            },
            {
             "value": true,
             "probability": 0.0012384587206484996
            }
           ]
          }
         },
         "joint_probability": 0.9972263990496555,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002777454508598278,
        "field_brier": 4.869408210651577e-06
       }
      },
      "timing_ms": {
       "scoring": 37.51627611927688,
       "base_joint": 0.06606895476579666,
       "constrained_joint": 0.04667998291552067,
       "constrained_total": 37.5629561021924
      },
      "input_tokens": 741,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0cb83ecc6b11396e-0036",
      "group": "bendsynth-0cb83ecc6b11396e",
      "provenance": "bendsynth-v1; family=bendsynth-0cb83ecc6b11396e; domain=community theatre casting file audit; parameters=absence_limit=2; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "flag for review",
       "attendance_ok": false,
       "concerns": 1
      },
      "logits": [
       [
        -6.547611236572266,
        -6.355016231536865,
        0.540386438369751
       ],
       [
        0.5409821271896362,
        -6.142931938171387
       ],
       [
        -6.834871292114258,
        0.5402829051017761,
        -6.736477375030518,
        -6.824431896209717
       ]
      ],
      "joint_logits": [
       -8.625177383422852,
       -8.300621032714844,
       -8.554428100585938,
       -8.62476921081543,
       -8.806780815124512,
       -8.690775871276855,
       -8.72041130065918,
       -8.789728164672852,
       -8.483580589294434,
       -8.128278732299805,
       -8.356584548950195,
       -8.456798553466797,
       -8.59953498840332,
       -8.459456443786621,
       -8.51217269897461,
       -8.55950927734375,
       -8.571477890014648,
       0.5396246314048767,
       -8.466482162475586,
       -8.617498397827148,
       -8.704113006591797,
       -8.487732887268066,
       -8.59701919555664,
       -8.680150032043457
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "flag for review",
           "probability": 0.9981559097493062,
           "distribution": [
            {
             "value": "approve file",
             "probability": 0.0008335278514032568
            },
            {
             "value": "void casting",
             "probability": 0.0010105623992905021
            },
            {
             "value": "flag for review",
             "probability": 0.9981559097493062
            }
           ]
          },
          "attendance_ok": {
           "value": false,
           "probability": 0.9987506903510946,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987506903510946
            },
            {
             "value": true,
             "probability": 0.0012493096489053434
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9980525417525071,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006254096481214124
            },
            {
             "value": 1,
             "probability": 0.9980525417525071
            },
            {
             "value": 2,
             "probability": 0.0006900756765969004
            },
            {
             "value": 3,
             "probability": 0.0006319729227745823
            }
           ]
          }
         },
         "joint_probability": 0.997395833218613,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026075635221006546,
        "field_brier": 4.4325160535222585e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "flag for review",
           "probability": 0.9981559097493062,
           "distribution": [
            {
             "value": "approve file",
             "probability": 0.0008335278514032568
            },
            {
             "value": "void casting",
             "probability": 0.0010105623992905021
            },
            {
             "value": "flag for review",
             "probability": 0.9981559097493062
            }
           ]
          },
          "attendance_ok": {
           "value": false,
           "probability": 0.9987506903510946,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987506903510946
            },
            {
             "value": true,
             "probability": 0.0012493096489053434
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9980525417525071,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006254096481214124
            },
            {
             "value": 1,
             "probability": 0.9980525417525071
            },
            {
             "value": 2,
             "probability": 0.0006900756765969004
            },
            {
             "value": 3,
             "probability": 0.0006319729227745823
            }
           ]
          }
         },
         "joint_probability": 0.997395833218613,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026075635221006546,
        "field_brier": 4.4325160535222585e-06
       }
      },
      "timing_ms": {
       "scoring": 39.328217040747404,
       "base_joint": 0.06609898991882801,
       "constrained_joint": 0.04790886305272579,
       "constrained_total": 39.37612590380013
      },
      "input_tokens": 720,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-08a722652b32bb69-0003",
      "group": "bendsynth-08a722652b32bb69",
      "provenance": "bendsynth-v1; family=bendsynth-08a722652b32bb69; domain=dental practice recall scheduling intake; parameters=planned_gap=13,tolerance=19; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flags": 3,
       "on_schedule": false,
       "decision": "block booking"
      },
      "logits": [
       [
        -6.696747779846191,
        -6.404687404632568,
        -6.181723117828369,
        0.8438692092895508
       ],
       [
        0.844679594039917,
        -5.785999774932861
       ],
       [
        0.844474196434021,
        -6.494009971618652,
        -6.918910026550293,
        -6.86190128326416
       ]
      ],
      "joint_logits": [
       -8.588262557983398,
       -8.638214111328125,
       -9.053850173950195,
       -9.0130615234375,
       -8.499431610107422,
       -8.594578742980957,
       -9.032008171081543,
       -8.99138069152832,
       -8.007079124450684,
       -8.448962211608887,
       -8.805367469787598,
       -8.791495323181152,
       -8.242469787597656,
       -8.378995895385742,
       -8.787978172302246,
       -8.745401382446289,
       -7.421167850494385,
       -8.406964302062988,
       -8.817906379699707,
       -8.761547088623047,
       -7.949463367462158,
       -8.290250778198242,
       -8.720033645629883,
       -8.660359382629395,
       0.8434476256370544,
       -8.934643745422363,
       -9.593865394592285,
       -9.339296340942383,
       -7.706066608428955,
       -9.206382751464844,
       -9.560213088989258,
       -9.545008659362793
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 3,
           "probability": 0.9978734212092244,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005299405371218149
            },
            {
             "value": 1,
             "probability": 0.0007096876335266428
            },
            {
             "value": 2,
             "probability": 0.0008869506201271859
            },
            {
             "value": 3,
             "probability": 0.9978734212092244
            }
           ]
          },
          "on_schedule": {
           "value": false,
           "probability": 0.9986824717821041,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986824717821041
            },
            {
             "value": true,
             "probability": 0.001317528217895783
            }
           ]
          },
          "decision": {
           "value": "block booking",
           "probability": 0.9984773217087546,
           "distribution": [
            {
             "value": "block booking",
             "probability": 0.9984773217087546
            },
            {
             "value": "send reminder",
             "probability": 0.0006490454053493342
            },
            {
             "value": "book directly",
             "probability": 0.00042436867836845467
            },
            {
             "value": "hygienist review",
             "probability": 0.00044926420752769235
            }
           ]
          }
         },
         "joint_probability": 0.9974527411574877,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025505086261841344,
        "field_brier": 4.229003232023524e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 3,
           "probability": 0.9978734212092244,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005299405371218149
            },
            {
             "value": 1,
             "probability": 0.0007096876335266428
            },
            {
             "value": 2,
             "probability": 0.0008869506201271859
            },
            {
             "value": 3,
             "probability": 0.9978734212092244
            }
           ]
          },
          "on_schedule": {
           "value": false,
           "probability": 0.9986824717821041,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986824717821041
            },
            {
             "value": true,
             "probability": 0.001317528217895783
            }
           ]
          },
          "decision": {
           "value": "block booking",
           "probability": 0.9984773217087546,
           "distribution": [
            {
             "value": "block booking",
             "probability": 0.9984773217087546
            },
            {
             "value": "send reminder",
             "probability": 0.0006490454053493342
            },
            {
             "value": "book directly",
             "probability": 0.00042436867836845467
            },
            {
             "value": "hygienist review",
             "probability": 0.00044926420752769235
            }
           ]
          }
         },
         "joint_probability": 0.9974527411574877,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025505086261841344,
        "field_brier": 4.229003232023524e-06
       }
      },
      "timing_ms": {
       "scoring": 35.625847056508064,
       "base_joint": 0.07771910168230534,
       "constrained_joint": 0.06212899461388588,
       "constrained_total": 35.68797605112195
      },
      "input_tokens": 636,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-08a722652b32bb69-0023",
      "group": "bendsynth-08a722652b32bb69",
      "provenance": "bendsynth-v1; family=bendsynth-08a722652b32bb69; domain=dental practice recall scheduling intake; parameters=planned_gap=12,tolerance=29; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flags": 1,
       "on_schedule": true,
       "decision": "send reminder"
      },
      "logits": [
       [
        -5.377455711364746,
        1.9402320384979248,
        -5.29511833190918,
        -5.648643970489502
       ],
       [
        -4.597773551940918,
        1.940674901008606
       ],
       [
        -5.520533561706543,
        1.9386330842971802,
        -4.171868324279785,
        -5.328761100769043
       ]
      ],
      "joint_logits": [
       -7.467613697052002,
       -7.550556182861328,
       -7.172031402587891,
       -7.241852283477783,
       -7.73259973526001,
       -7.572396278381348,
       -7.49810266494751,
       -7.5392231941223145,
       -7.539388179779053,
       -7.120816230773926,
       -7.145847797393799,
       -7.385039329528809,
       -7.407314300537109,
       1.938086748123169,
       -4.5231218338012695,
       -7.257655143737793,
       -7.396966934204102,
       -7.516498565673828,
       -7.110879421234131,
       -7.189978122711182,
       -7.71725606918335,
       -7.354292392730713,
       -7.3365159034729,
       -7.501312255859375,
       -7.691534042358398,
       -7.784928321838379,
       -7.444360256195068,
       -7.4872612953186035,
       -7.984415531158447,
       -7.96469259262085,
       -7.808950424194336,
       -7.798336982727051
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 1,
           "probability": 0.9981131671720231,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006624428124294855
            },
            {
             "value": 1,
             "probability": 0.9981131671720231
            },
            {
             "value": 2,
             "probability": 0.0007192951754529312
            },
            {
             "value": 3,
             "probability": 0.0005050948400944893
            }
           ]
          },
          "on_schedule": {
           "value": true,
           "probability": 0.9985553578973706,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014446421026294471
            },
            {
             "value": true,
             "probability": 0.9985553578973706
            }
           ]
          },
          "decision": {
           "value": "send reminder",
           "probability": 0.996518659861392,
           "distribution": [
            {
             "value": "book directly",
             "probability": 0.0005741305581730274
            },
            {
             "value": "send reminder",
             "probability": 0.996518659861392
            },
            {
             "value": "hygienist review",
             "probability": 0.002211710820930193
            },
            {
             "value": "block booking",
             "probability": 0.0006954987595047844
            }
           ]
          }
         },
         "joint_probability": 0.9959743535491707,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004033771177685319,
        "field_brier": 8.923398298391147e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 1,
           "probability": 0.9981131671720231,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006624428124294855
            },
            {
             "value": 1,
             "probability": 0.9981131671720231
            },
            {
             "value": 2,
             "probability": 0.0007192951754529312
            },
            {
             "value": 3,
             "probability": 0.0005050948400944893
            }
           ]
          },
          "on_schedule": {
           "value": true,
           "probability": 0.9985553578973706,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014446421026294471
            },
            {
             "value": true,
             "probability": 0.9985553578973706
            }
           ]
          },
          "decision": {
           "value": "send reminder",
           "probability": 0.996518659861392,
           "distribution": [
            {
             "value": "book directly",
             "probability": 0.0005741305581730274
            },
            {
             "value": "send reminder",
             "probability": 0.996518659861392
            },
            {
             "value": "hygienist review",
             "probability": 0.002211710820930193
            },
            {
             "value": "block booking",
             "probability": 0.0006954987595047844
            }
           ]
          }
         },
         "joint_probability": 0.9959743535491707,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004033771177685319,
        "field_brier": 8.923398298391147e-06
       }
      },
      "timing_ms": {
       "scoring": 34.87595496699214,
       "base_joint": 0.07408903911709785,
       "constrained_joint": 0.05754898302257061,
       "constrained_total": 34.93350395001471
      },
      "input_tokens": 633,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-08a722652b32bb69-0032",
      "group": "bendsynth-08a722652b32bb69",
      "provenance": "bendsynth-v1; family=bendsynth-08a722652b32bb69; domain=dental practice recall scheduling intake; parameters=planned_gap=18,tolerance=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "on_schedule": true,
       "decision": "send reminder",
       "flags": 1
      },
      "logits": [
       [
        -3.8394112586975098,
        1.4394192695617676
       ],
       [
        1.4342824220657349,
        -5.386982440948486,
        -3.3788890838623047,
        -5.4130859375
       ],
       [
        -5.533146381378174,
        1.4406764507293701,
        -4.729283332824707,
        -5.6800856590271
       ]
      ],
      "joint_logits": [
       -7.3887619972229,
       -5.077220916748047,
       -6.4853925704956055,
       -7.54733419418335,
       -7.361464977264404,
       -7.095613956451416,
       -6.792478561401367,
       -7.45913028717041,
       -7.133978843688965,
       -5.713759899139404,
       -6.4245829582214355,
       -7.308696746826172,
       -7.435628890991211,
       -7.257768630981445,
       -6.700213432312012,
       -7.55860710144043,
       -7.850490093231201,
       1.4316575527191162,
       -6.5918049812316895,
       -8.012127876281738,
       -8.368093490600586,
       -7.433343887329102,
       -7.834139347076416,
       -8.547280311584473,
       -7.8175530433654785,
       -3.6569132804870605,
       -6.78306770324707,
       -7.960002422332764,
       -8.151229858398438,
       -7.714800834655762,
       -7.745100498199463,
       -8.319729804992676
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "on_schedule": {
           "value": true,
           "probability": 0.9949274709113266,
           "distribution": [
            {
             "value": false,
             "probability": 0.005072529088673404
            },
            {
             "value": true,
             "probability": 0.9949274709113266
            }
           ]
          },
          "decision": {
           "value": "send reminder",
           "probability": 0.9898298473446407,
           "distribution": [
            {
             "value": "send reminder",
             "probability": 0.9898298473446407
            },
            {
             "value": "book directly",
             "probability": 0.001079252015177639
            },
            {
             "value": "hygienist review",
             "probability": 0.008039456481554508
            },
            {
             "value": "block booking",
             "probability": 0.0010514441586270957
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9961791156504802,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0009324911254761384
            },
            {
             "value": 1,
             "probability": 0.9961791156504802
            },
            {
             "value": 2,
             "probability": 0.0020833301949101666
            },
            {
             "value": 3,
             "probability": 0.0008050630291335057
            }
           ]
          }
         },
         "joint_probability": 0.9872351202301561,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.012847050866002171,
        "field_brier": 8.075112536245674e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "on_schedule": {
           "value": true,
           "probability": 0.9949274709113266,
           "distribution": [
            {
             "value": false,
             "probability": 0.005072529088673404
            },
            {
             "value": true,
             "probability": 0.9949274709113266
            }
           ]
          },
          "decision": {
           "value": "send reminder",
           "probability": 0.9898298473446407,
           "distribution": [
            {
             "value": "send reminder",
             "probability": 0.9898298473446407
            },
            {
             "value": "book directly",
             "probability": 0.001079252015177639
            },
            {
             "value": "hygienist review",
             "probability": 0.008039456481554508
            },
            {
             "value": "block booking",
             "probability": 0.0010514441586270957
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9961791156504802,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0009324911254761384
            },
            {
             "value": 1,
             "probability": 0.9961791156504802
            },
            {
             "value": 2,
             "probability": 0.0020833301949101666
            },
            {
             "value": 3,
             "probability": 0.0008050630291335057
            }
           ]
          }
         },
         "joint_probability": 0.9872351202301561,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.012847050866002171,
        "field_brier": 8.075112536245674e-05
       }
      },
      "timing_ms": {
       "scoring": 34.048022935166955,
       "base_joint": 0.07552909664809704,
       "constrained_joint": 0.05726004019379616,
       "constrained_total": 34.10528297536075
      },
      "input_tokens": 630,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03f7d21f0e08ac0d-0010",
      "group": "bendsynth-03f7d21f0e08ac0d",
      "provenance": "bendsynth-v1; family=bendsynth-03f7d21f0e08ac0d; domain=housing cooperative waiting list desk; parameters=min_days=366; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "strength": 3,
       "meets_period": true,
       "decision": "priority offer"
      },
      "logits": [
       [
        -7.367409706115723,
        -7.363382339477539,
        -7.225571632385254,
        0.21827711164951324
       ],
       [
        -6.652382850646973,
        0.2188415229320526
       ],
       [
        0.21815797686576843,
        -7.2989068031311035,
        -7.345022201538086,
        -7.107089996337891
       ]
      ],
      "joint_logits": [
       -9.442598342895508,
       -9.497540473937988,
       -9.542071342468262,
       -9.301651000976562,
       -9.429224014282227,
       -9.629807472229004,
       -9.473470687866211,
       -9.30240249633789,
       -9.443991661071777,
       -9.550201416015625,
       -9.593334197998047,
       -9.396327018737793,
       -9.310037612915039,
       -9.581554412841797,
       -9.461410522460938,
       -9.25961685180664,
       -9.36705207824707,
       -9.398958206176758,
       -9.399699211120605,
       -9.221880912780762,
       -8.89165210723877,
       -9.550634384155273,
       -9.501020431518555,
       -9.268331527709961,
       -9.226425170898438,
       -9.659381866455078,
       -9.533458709716797,
       -9.343811988830566,
       0.2176346629858017,
       -8.636343955993652,
       -9.017450332641602,
       -8.634549140930176
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "strength": {
           "value": 3,
           "probability": 0.9984001533389476,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005068539969149873
            },
            {
             "value": 1,
             "probability": 0.0005088992306530938
            },
            {
             "value": 2,
             "probability": 0.0005840934334843165
            },
            {
             "value": 3,
             "probability": 0.9984001533389476
            }
           ]
          },
          "meets_period": {
           "value": true,
           "probability": 0.9989638685875533,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010361314124467352
            },
            {
             "value": true,
             "probability": 0.9989638685875533
            }
           ]
          },
          "decision": {
           "value": "priority offer",
           "probability": 0.9982813150663612,
           "distribution": [
            {
             "value": "priority offer",
             "probability": 0.9982813150663612
            },
            {
             "value": "reject",
             "probability": 0.0005427915975609784
            },
            {
             "value": "defer",
             "probability": 0.0005183289161808764
            },
            {
             "value": "offer eligible",
             "probability": 0.0006575644198969661
            }
           ]
          }
         },
         "joint_probability": 0.9977590345560795,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002243480164618175,
        "field_brier": 3.1710823126497456e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "strength": {
           "value": 3,
           "probability": 0.9984001533389476,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005068539969149873
            },
            {
             "value": 1,
             "probability": 0.0005088992306530938
            },
            {
             "value": 2,
             "probability": 0.0005840934334843165
            },
            {
             "value": 3,
             "probability": 0.9984001533389476
            }
           ]
          },
          "meets_period": {
           "value": true,
           "probability": 0.9989638685875533,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010361314124467352
            },
            {
             "value": true,
             "probability": 0.9989638685875533
            }
           ]
          },
          "decision": {
           "value": "priority offer",
           "probability": 0.9982813150663612,
           "distribution": [
            {
             "value": "priority offer",
             "probability": 0.9982813150663612
            },
            {
             "value": "reject",
             "probability": 0.0005427915975609784
            },
            {
             "value": "defer",
             "probability": 0.0005183289161808764
            },
            {
             "value": "offer eligible",
             "probability": 0.0006575644198969661
            }
           ]
          }
         },
         "joint_probability": 0.9977590345560795,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002243480164618175,
        "field_brier": 3.1710823126497456e-06
       }
      },
      "timing_ms": {
       "scoring": 37.361867958679795,
       "base_joint": 0.0758189707994461,
       "constrained_joint": 0.05683908239006996,
       "constrained_total": 37.418707041069865
      },
      "input_tokens": 709,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-074312e3099a17a3-0031",
      "group": "bendsynth-074312e3099a17a3",
      "provenance": "bendsynth-v1; family=bendsynth-074312e3099a17a3; domain=airline standby reassignment desk for cabin crew; parameters=cutoff=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "triggers": 2,
       "below_cutoff": false,
       "decision": "release from standby"
      },
      "logits": [
       [
        -6.378600120544434,
        -6.072807312011719,
        1.0377992391586304,
        -6.289581298828125
       ],
       [
        1.0384068489074707,
        -5.486632347106934
       ],
       [
        1.0377440452575684,
        -5.646213531494141,
        -5.998536109924316
       ]
      ],
      "joint_logits": [
       -8.367751121520996,
       -8.088421821594238,
       -8.27384090423584,
       -8.181365013122559,
       -7.974606513977051,
       -8.18367862701416,
       -7.710177898406982,
       -7.834049701690674,
       -8.040582656860352,
       -7.982406139373779,
       -7.689651012420654,
       -7.9884538650512695,
       1.0369360446929932,
       -6.973416805267334,
       -7.8772077560424805,
       -7.804616451263428,
       -7.836669921875,
       -8.056248664855957,
       -8.205753326416016,
       -8.048325538635254,
       -8.172204971313477,
       -8.12000846862793,
       -7.888388633728027,
       -8.085968017578125
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "triggers": {
           "value": 2,
           "probability": 0.9979292931142396,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006000653345687757
            },
            {
             "value": 1,
             "probability": 0.0008147092928475222
            },
            {
             "value": 2,
             "probability": 0.9979292931142396
            },
            {
             "value": 3,
             "probability": 0.0006559322583440747
            }
           ]
          },
          "below_cutoff": {
           "value": false,
           "probability": 0.9985358852789548,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985358852789548
            },
            {
             "value": true,
             "probability": 0.0014641147210452002
            }
           ]
          },
          "decision": {
           "value": "release from standby",
           "probability": 0.9978743185505322,
           "distribution": [
            {
             "value": "release from standby",
             "probability": 0.9978743185505322
            },
            {
             "value": "keep on standby",
             "probability": 0.0012481588996005215
            },
            {
             "value": "route to supervisor",
             "probability": 0.000877522549867268
            }
           ]
          }
         },
         "joint_probability": 0.9970682795558076,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0029360263544498722,
        "field_brier": 5.6252118978089125e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "triggers": {
           "value": 2,
           "probability": 0.9979292931142396,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006000653345687757
            },
            {
             "value": 1,
             "probability": 0.0008147092928475222
            },
            {
             "value": 2,
             "probability": 0.9979292931142396
            },
            {
             "value": 3,
             "probability": 0.0006559322583440747
            }
           ]
          },
          "below_cutoff": {
           "value": false,
           "probability": 0.9985358852789548,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985358852789548
            },
            {
             "value": true,
             "probability": 0.0014641147210452002
            }
           ]
          },
          "decision": {
           "value": "release from standby",
           "probability": 0.9978743185505322,
           "distribution": [
            {
             "value": "release from standby",
             "probability": 0.9978743185505322
            },
            {
             "value": "keep on standby",
             "probability": 0.0012481588996005215
            },
            {
             "value": "route to supervisor",
             "probability": 0.000877522549867268
            }
           ]
          }
         },
         "joint_probability": 0.9970682795558076,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0029360263544498722,
        "field_brier": 5.6252118978089125e-06
       }
      },
      "timing_ms": {
       "scoring": 35.56813718751073,
       "base_joint": 0.06472901441156864,
       "constrained_joint": 0.04808907397091389,
       "constrained_total": 35.61622626148164
      },
      "input_tokens": 640,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-014d2dcf573da70c-0019",
      "group": "bendsynth-014d2dcf573da70c",
      "provenance": "bendsynth-v1; family=bendsynth-014d2dcf573da70c; domain=national park campground permit counter; parameters=max_group=19; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 1,
       "decision": "walk-up site",
       "oversize_group": true
      },
      "logits": [
       [
        -4.490375995635986,
        2.651132583618164,
        -4.593968868255615,
        -4.66550874710083
       ],
       [
        -3.344996690750122,
        -4.944757461547852,
        2.6495449542999268,
        -4.524295330047607
       ],
       [
        -3.9995553493499756,
        2.6520087718963623
       ]
      ],
      "joint_logits": [
       -6.717370986938477,
       -6.475916862487793,
       -7.133949279785156,
       -6.841372489929199,
       -6.5009074211120605,
       -6.075963020324707,
       -6.700825214385986,
       -6.456028461456299,
       -6.441178321838379,
       -3.659717559814453,
       -7.351846694946289,
       -6.592672348022461,
       -6.179692268371582,
       2.6486806869506836,
       -6.935299396514893,
       -6.069168567657471,
       -6.795508861541748,
       -6.487231731414795,
       -7.201574802398682,
       -6.982212543487549,
       -6.651626110076904,
       -6.189797878265381,
       -6.786120414733887,
       -6.625636577606201,
       -6.873658657073975,
       -6.502297401428223,
       -7.235527992248535,
       -7.072065353393555,
       -6.736727714538574,
       -6.306708812713623,
       -6.833102703094482,
       -6.70667839050293
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 1,
           "probability": 0.9978350883784762,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007898434063406033
            },
            {
             "value": 1,
             "probability": 0.9978350883784762
            },
            {
             "value": 2,
             "probability": 0.0007121166874580779
            },
            {
             "value": 3,
             "probability": 0.000662951527725125
            }
           ]
          },
          "decision": {
           "value": "walk-up site",
           "probability": 0.996252095971445,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0024829777426548373
            },
            {
             "value": "standard permit",
             "probability": 0.0005014246091241993
            },
            {
             "value": "walk-up site",
             "probability": 0.996252095971445
            },
            {
             "value": "supervisor review",
             "probability": 0.0007635016767760167
            }
           ]
          },
          "oversize_group": {
           "value": true,
           "probability": 0.9987096674410979,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012903325589022027
            },
            {
             "value": true,
             "probability": 0.9987096674410979
            }
           ]
          }
         },
         "joint_probability": 0.9953913512660373,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004619301297422166,
        "field_brier": 1.0211183530732213e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 1,
           "probability": 0.9978350883784762,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007898434063406033
            },
            {
             "value": 1,
             "probability": 0.9978350883784762
            },
            {
             "value": 2,
             "probability": 0.0007121166874580779
            },
            {
             "value": 3,
             "probability": 0.000662951527725125
            }
           ]
          },
          "decision": {
           "value": "walk-up site",
           "probability": 0.996252095971445,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0024829777426548373
            },
            {
             "value": "standard permit",
             "probability": 0.0005014246091241993
            },
            {
             "value": "walk-up site",
             "probability": 0.996252095971445
            },
            {
             "value": "supervisor review",
             "probability": 0.0007635016767760167
            }
           ]
          },
          "oversize_group": {
           "value": true,
           "probability": 0.9987096674410979,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012903325589022027
            },
            {
             "value": true,
             "probability": 0.9987096674410979
            }
           ]
          }
         },
         "joint_probability": 0.9953913512660373,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004619301297422166,
        "field_brier": 1.0211183530732213e-05
       }
      },
      "timing_ms": {
       "scoring": 35.58421693742275,
       "base_joint": 0.07683900184929371,
       "constrained_joint": 0.05604000762104988,
       "constrained_total": 35.6402569450438
      },
      "input_tokens": 633,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-014d2dcf573da70c-0021",
      "group": "bendsynth-014d2dcf573da70c",
      "provenance": "bendsynth-v1; family=bendsynth-014d2dcf573da70c; domain=national park campground permit counter; parameters=max_group=22; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "deny",
       "risk_factors": 2,
       "oversize_group": true
      },
      "logits": [
       [
        -6.583141803741455,
        -5.589977741241455,
        -6.720266342163086,
        0.9418322443962097
       ],
       [
        -6.590249538421631,
        -6.473677635192871,
        0.942584216594696,
        -6.5168843269348145
       ],
       [
        -5.901150703430176,
        0.9432305097579956
       ]
      ],
      "joint_logits": [
       -9.12635612487793,
       -8.753408432006836,
       -9.101370811462402,
       -8.587981224060059,
       -8.605758666992188,
       -7.9662065505981445,
       -9.014925003051758,
       -8.695455551147461,
       -8.100201606750488,
       -7.820803165435791,
       -8.103180885314941,
       -7.657076835632324,
       -7.726273059844971,
       -6.8452606201171875,
       -7.98756742477417,
       -7.799746513366699,
       -9.096927642822266,
       -8.981340408325195,
       -9.110089302062988,
       -8.853372573852539,
       -8.891536712646484,
       -8.034658432006836,
       -8.988386154174805,
       -8.958842277526855,
       -9.787443161010742,
       -9.130230903625488,
       -9.635614395141602,
       -8.794073104858398,
       -8.736173629760742,
       0.9415481090545654,
       -9.764410018920898,
       -8.956191062927246
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "deny",
           "probability": 0.9975399370684003,
           "distribution": [
            {
             "value": "standard permit",
             "probability": 0.0005381155534173385
            },
            {
             "value": "walk-up site",
             "probability": 0.001452784889235036
            },
            {
             "value": "supervisor review",
             "probability": 0.0004691624889473007
            },
            {
             "value": "deny",
             "probability": 0.9975399370684003
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9982903530990689,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005343043350247474
            },
            {
             "value": 1,
             "probability": 0.0006003649955465447
            },
            {
             "value": 2,
             "probability": 0.9982903530990689
            },
            {
             "value": 3,
             "probability": 0.0005749775703597824
            }
           ]
          },
          "oversize_group": {
           "value": true,
           "probability": 0.9989357083353178,
           "distribution": [
            {
             "value": false,
             "probability": 0.001064291664682247
            },
            {
             "value": true,
             "probability": 0.9989357083353178
            }
           ]
          }
         },
         "joint_probability": 0.9972565666591238,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027472034510668806,
        "field_brier": 4.945673276431281e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "deny",
           "probability": 0.9975399370684003,
           "distribution": [
            {
             "value": "standard permit",
             "probability": 0.0005381155534173385
            },
            {
             "value": "walk-up site",
             "probability": 0.001452784889235036
            },
            {
             "value": "supervisor review",
             "probability": 0.0004691624889473007
            },
            {
             "value": "deny",
             "probability": 0.9975399370684003
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9982903530990689,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005343043350247474
            },
            {
             "value": 1,
             "probability": 0.0006003649955465447
            },
            {
             "value": 2,
             "probability": 0.9982903530990689
            },
            {
             "value": 3,
             "probability": 0.0005749775703597824
            }
           ]
          },
          "oversize_group": {
           "value": true,
           "probability": 0.9989357083353178,
           "distribution": [
            {
             "value": false,
             "probability": 0.001064291664682247
            },
            {
             "value": true,
             "probability": 0.9989357083353178
            }
           ]
          }
         },
         "joint_probability": 0.9972565666591238,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027472034510668806,
        "field_brier": 4.945673276431281e-06
       }
      },
      "timing_ms": {
       "scoring": 35.233421018347144,
       "base_joint": 0.07294001989066601,
       "constrained_joint": 0.05842908285558224,
       "constrained_total": 35.291850101202726
      },
      "input_tokens": 648,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-014d2dcf573da70c-0033",
      "group": "bendsynth-014d2dcf573da70c",
      "provenance": "bendsynth-v1; family=bendsynth-014d2dcf573da70c; domain=national park campground permit counter; parameters=max_group=13; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 1,
       "decision": "walk-up site",
       "oversize_group": true
      },
      "logits": [
       [
        -4.699060440063477,
        2.689976930618286,
        -4.75431489944458,
        -4.825554370880127
       ],
       [
        2.6896755695343018,
        -4.745839595794678,
        -5.076920986175537,
        -4.183274745941162
       ],
       [
        -4.25728702545166,
        2.690762519836426
       ]
      ],
      "joint_logits": [
       -6.941638469696045,
       -6.489380836486816,
       -6.965095043182373,
       -6.606264114379883,
       -7.30316162109375,
       -6.898940563201904,
       -6.782521724700928,
       -6.498410224914551,
       -6.7820329666137695,
       2.689131259918213,
       -7.2416605949401855,
       -6.394337177276611,
       -7.51755428314209,
       -6.865145683288574,
       -6.7504048347473145,
       -5.087601184844971,
       -7.024392127990723,
       -6.543303966522217,
       -7.0548577308654785,
       -6.697530269622803,
       -7.360296726226807,
       -6.989875316619873,
       -6.769577980041504,
       -6.514158725738525,
       -7.135251522064209,
       -6.668725490570068,
       -7.122433662414551,
       -6.811953067779541,
       -7.437315464019775,
       -7.1015777587890625,
       -6.768481254577637,
       -6.511590480804443
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 1,
           "probability": 0.9982557264440102,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006169125386510719
            },
            {
             "value": 1,
             "probability": 0.9982557264440102
            },
            {
             "value": 2,
             "probability": 0.00058375022559176
            },
            {
             "value": 3,
             "probability": 0.0005436107917470439
            }
           ]
          },
          "decision": {
           "value": "walk-up site",
           "probability": 0.9979551944939636,
           "distribution": [
            {
             "value": "walk-up site",
             "probability": 0.9979551944939636
            },
            {
             "value": "supervisor review",
             "probability": 0.0005887185899998583
            },
            {
             "value": "standard permit",
             "probability": 0.00042278633149407164
            },
            {
             "value": "deny",
             "probability": 0.0010333005845424856
            }
           ]
          },
          "oversize_group": {
           "value": true,
           "probability": 0.9990404152959258,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009595847040741853
            },
            {
             "value": true,
             "probability": 0.9990404152959258
            }
           ]
          }
         },
         "joint_probability": 0.9974119760700636,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025913786531867604,
        "field_brier": 3.891743820560336e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 1,
           "probability": 0.9982557264440102,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006169125386510719
            },
            {
             "value": 1,
             "probability": 0.9982557264440102
            },
            {
             "value": 2,
             "probability": 0.00058375022559176
            },
            {
             "value": 3,
             "probability": 0.0005436107917470439
            }
           ]
          },
          "decision": {
           "value": "walk-up site",
           "probability": 0.9979551944939636,
           "distribution": [
            {
             "value": "walk-up site",
             "probability": 0.9979551944939636
            },
            {
             "value": "supervisor review",
             "probability": 0.0005887185899998583
            },
            {
             "value": "standard permit",
             "probability": 0.00042278633149407164
            },
            {
             "value": "deny",
             "probability": 0.0010333005845424856
            }
           ]
          },
          "oversize_group": {
           "value": true,
           "probability": 0.9990404152959258,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009595847040741853
            },
            {
             "value": true,
             "probability": 0.9990404152959258
            }
           ]
          }
         },
         "joint_probability": 0.9974119760700636,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025913786531867604,
        "field_brier": 3.891743820560336e-06
       }
      },
      "timing_ms": {
       "scoring": 36.288690054789186,
       "base_joint": 0.07711909711360931,
       "constrained_joint": 0.05669984966516495,
       "constrained_total": 36.34538990445435
      },
      "input_tokens": 639,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-038bb411ed305849-0005",
      "group": "bendsynth-038bb411ed305849",
      "provenance": "bendsynth-v1; family=bendsynth-038bb411ed305849; domain=swimming pool lane booking chatbot; parameters=max_party=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "obstacles": 2,
       "over_size": true,
       "decision": "escalate to manager"
      },
      "logits": [
       [
        -7.270495891571045,
        -7.125971794128418,
        0.029741164296865463,
        -7.220183372497559
       ],
       [
        -6.501402378082275,
        0.030449580401182175
       ],
       [
        -6.752188682556152,
        -6.706274032592773,
        0.029585720971226692
       ]
      ],
      "joint_logits": [
       -9.028100967407227,
       -8.92426586151123,
       -9.315463066101074,
       -9.06330680847168,
       -8.831565856933594,
       -9.307430267333984,
       -8.96346378326416,
       -8.790961265563965,
       -9.126898765563965,
       -8.920509338378906,
       -8.768478393554688,
       -8.97950553894043,
       -9.065092086791992,
       -8.849654197692871,
       -8.855669021606445,
       -8.072752952575684,
       -8.528164863586426,
       0.0288560688495636,
       -8.9775390625,
       -8.829911231994629,
       -9.258941650390625,
       -9.052494049072266,
       -8.8126220703125,
       -9.230880737304688
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "obstacles": {
           "value": 2,
           "probability": 0.9978386816597682,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006739188946059929
            },
            {
             "value": 1,
             "probability": 0.0007787063255379376
            },
            {
             "value": 2,
             "probability": 0.9978386816597682
            },
            {
             "value": 3,
             "probability": 0.000708693120087975
            }
           ]
          },
          "over_size": {
           "value": true,
           "probability": 0.9985458113764658,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014541886235342443
            },
            {
             "value": true,
             "probability": 0.9985458113764658
            }
           ]
          },
          "decision": {
           "value": "escalate to manager",
           "probability": 0.9976835645369158,
           "distribution": [
            {
             "value": "offer alternate slot",
             "probability": 0.0011316328186080523
            },
            {
             "value": "confirm booking",
             "probability": 0.0011848026444762588
            },
            {
             "value": "escalate to manager",
             "probability": 0.9976835645369158
            }
           ]
          }
         },
         "joint_probability": 0.9969559237283745,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003048718895861352,
        "field_brier": 6.171215208999305e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "obstacles": {
           "value": 2,
           "probability": 0.9978386816597682,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006739188946059929
            },
            {
             "value": 1,
             "probability": 0.0007787063255379376
            },
            {
             "value": 2,
             "probability": 0.9978386816597682
            },
            {
             "value": 3,
             "probability": 0.000708693120087975
            }
           ]
          },
          "over_size": {
           "value": true,
           "probability": 0.9985458113764658,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014541886235342443
            },
            {
             "value": true,
             "probability": 0.9985458113764658
            }
           ]
          },
          "decision": {
           "value": "escalate to manager",
           "probability": 0.9976835645369158,
           "distribution": [
            {
             "value": "offer alternate slot",
             "probability": 0.0011316328186080523
            },
            {
             "value": "confirm booking",
             "probability": 0.0011848026444762588
            },
            {
             "value": "escalate to manager",
             "probability": 0.9976835645369158
            }
           ]
          }
         },
         "joint_probability": 0.9969559237283745,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003048718895861352,
        "field_brier": 6.171215208999305e-06
       }
      },
      "timing_ms": {
       "scoring": 32.327611930668354,
       "base_joint": 0.06279908120632172,
       "constrained_joint": 0.048320041969418526,
       "constrained_total": 32.37593197263777
      },
      "input_tokens": 598,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-038bb411ed305849-0008",
      "group": "bendsynth-038bb411ed305849",
      "provenance": "bendsynth-v1; family=bendsynth-038bb411ed305849; domain=swimming pool lane booking chatbot; parameters=max_party=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "obstacles": 1,
       "decision": "escalate to manager",
       "over_size": true
      },
      "logits": [
       [
        -6.482188701629639,
        0.6834840774536133,
        -6.659737586975098,
        -6.620126247406006
       ],
       [
        -6.002927303314209,
        0.683509349822998,
        -6.42299222946167
       ],
       [
        -5.978947639465332,
        0.6842984557151794
       ]
      ],
      "joint_logits": [
       -8.135390281677246,
       -7.91363000869751,
       -8.460770606994629,
       -8.334138870239258,
       -8.575013160705566,
       -8.372230529785156,
       -8.257827758789062,
       -7.792052745819092,
       -8.472085952758789,
       0.6828023195266724,
       -8.765279769897461,
       -8.071149826049805,
       -8.300528526306152,
       -8.089441299438477,
       -8.661919593811035,
       -8.55331802368164,
       -8.685770988464355,
       -8.56110668182373,
       -8.242714881896973,
       -8.04016399383545,
       -8.597744941711426,
       -8.577194213867188,
       -8.64774227142334,
       -8.519991874694824
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "obstacles": {
           "value": 1,
           "probability": 0.9979116440140293,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007710454271450433
            },
            {
             "value": 1,
             "probability": 0.9979116440140293
            },
            {
             "value": 2,
             "probability": 0.0006456118962368954
            },
            {
             "value": 3,
             "probability": 0.0006716986625887242
            }
           ]
          },
          "decision": {
           "value": "escalate to manager",
           "probability": 0.9979367870819572,
           "distribution": [
            {
             "value": "confirm booking",
             "probability": 0.001245146574798881
            },
            {
             "value": "escalate to manager",
             "probability": 0.9979367870819572
            },
            {
             "value": "offer alternate slot",
             "probability": 0.0008180663432438604
            }
           ]
          },
          "over_size": {
           "value": true,
           "probability": 0.9987246342448515,
           "distribution": [
            {
             "value": false,
             "probability": 0.001275365755148579
            },
            {
             "value": true,
             "probability": 0.9987246342448515
            }
           ]
          }
         },
         "joint_probability": 0.9972315453626905,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027722938953605183,
        "field_brier": 5.18444042913954e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "obstacles": {
           "value": 1,
           "probability": 0.9979116440140293,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007710454271450433
            },
            {
             "value": 1,
             "probability": 0.9979116440140293
            },
            {
             "value": 2,
             "probability": 0.0006456118962368954
            },
            {
             "value": 3,
             "probability": 0.0006716986625887242
            }
           ]
          },
          "decision": {
           "value": "escalate to manager",
           "probability": 0.9979367870819572,
           "distribution": [
            {
             "value": "confirm booking",
             "probability": 0.001245146574798881
            },
            {
             "value": "escalate to manager",
             "probability": 0.9979367870819572
            },
            {
             "value": "offer alternate slot",
             "probability": 0.0008180663432438604
            }
           ]
          },
          "over_size": {
           "value": true,
           "probability": 0.9987246342448515,
           "distribution": [
            {
             "value": false,
             "probability": 0.001275365755148579
            },
            {
             "value": true,
             "probability": 0.9987246342448515
            }
           ]
          }
         },
         "joint_probability": 0.9972315453626905,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027722938953605183,
        "field_brier": 5.18444042913954e-06
       }
      },
      "timing_ms": {
       "scoring": 34.42472987808287,
       "base_joint": 0.0652889721095562,
       "constrained_joint": 0.0516700092703104,
       "constrained_total": 34.47639988735318
      },
      "input_tokens": 592,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-038bb411ed305849-0036",
      "group": "bendsynth-038bb411ed305849",
      "provenance": "bendsynth-v1; family=bendsynth-038bb411ed305849; domain=swimming pool lane booking chatbot; parameters=max_party=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "over_size": false,
       "decision": "offer alternate slot",
       "obstacles": 2
      },
      "logits": [
       [
        0.35103195905685425,
        -6.016500473022461
       ],
       [
        -6.18902587890625,
        0.3314840793609619,
        -3.5796566009521484
       ],
       [
        -6.788912773132324,
        -6.701547145843506,
        0.35023951530456543,
        -6.718584060668945
       ]
      ],
      "joint_logits": [
       -8.354634284973145,
       -8.38377571105957,
       -8.138056755065918,
       -8.332854270935059,
       -8.863897323608398,
       -8.504030227661133,
       0.3306190073490143,
       -8.773694038391113,
       -8.948931694030762,
       -8.610898971557617,
       -3.621262311935425,
       -8.824553489685059,
       -8.216641426086426,
       -8.232597351074219,
       -8.423738479614258,
       -8.112631797790527,
       -8.642850875854492,
       -8.657241821289062,
       -8.676802635192871,
       -8.596013069152832,
       -8.661604881286621,
       -8.645051002502441,
       -8.826095581054688,
       -8.614999771118164
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_size": {
           "value": false,
           "probability": 0.9982865515071968,
           "distribution": [
            {
             "value": false,
             "probability": 0.9982865515071968
            },
            {
             "value": true,
             "probability": 0.0017134484928031776
            }
           ]
          },
          "decision": {
           "value": "offer alternate slot",
           "probability": 0.9789615549231162,
           "distribution": [
            {
             "value": "confirm booking",
             "probability": 0.0014419303566937964
            },
            {
             "value": "offer alternate slot",
             "probability": 0.9789615549231162
            },
            {
             "value": "escalate to manager",
             "probability": 0.019596514720190056
            }
           ]
          },
          "obstacles": {
           "value": 2,
           "probability": 0.9974957676503311,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007914375665761597
            },
            {
             "value": 1,
             "probability": 0.0008636925372989546
            },
            {
             "value": 2,
             "probability": 0.9974957676503311
            },
            {
             "value": 3,
             "probability": 0.000849102245793735
            }
           ]
          }
         },
         "joint_probability": 0.9781151659346281,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.02212785930070708,
        "field_brier": 0.0002809850091886432
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_size": {
           "value": false,
           "probability": 0.9982865515071968,
           "distribution": [
            {
             "value": false,
             "probability": 0.9982865515071968
            },
            {
             "value": true,
             "probability": 0.0017134484928031776
            }
           ]
          },
          "decision": {
           "value": "offer alternate slot",
           "probability": 0.9789615549231162,
           "distribution": [
            {
             "value": "confirm booking",
             "probability": 0.0014419303566937964
            },
            {
             "value": "offer alternate slot",
             "probability": 0.9789615549231162
            },
            {
             "value": "escalate to manager",
             "probability": 0.019596514720190056
            }
           ]
          },
          "obstacles": {
           "value": 2,
           "probability": 0.9974957676503311,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007914375665761597
            },
            {
             "value": 1,
             "probability": 0.0008636925372989546
            },
            {
             "value": 2,
             "probability": 0.9974957676503311
            },
            {
             "value": 3,
             "probability": 0.000849102245793735
            }
           ]
          }
         },
         "joint_probability": 0.9781151659346281,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.02212785930070708,
        "field_brier": 0.0002809850091886432
       }
      },
      "timing_ms": {
       "scoring": 33.944305032491684,
       "base_joint": 0.07918896153569221,
       "constrained_joint": 0.047699082642793655,
       "constrained_total": 33.99200411513448
      },
      "input_tokens": 589,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-08718aa951ee7dc7-0013",
      "group": "bendsynth-08718aa951ee7dc7",
      "provenance": "bendsynth-v1; family=bendsynth-08718aa951ee7dc7; domain=cemetery plot allocation escalation desk; parameters=priority_threshold=330; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "factors": 1,
       "exception": false,
       "decision": "assign alternate plot"
      },
      "logits": [
       [
        -6.683261871337891,
        0.5406038165092468,
        -6.809091567993164,
        -6.782408237457275
       ],
       [
        0.5410793423652649,
        -5.9248576164245605
       ],
       [
        -6.797441482543945,
        0.5408565402030945,
        -6.245961666107178
       ]
      ],
      "joint_logits": [
       -8.948822975158691,
       -8.387252807617188,
       -8.38843822479248,
       -8.873993873596191,
       -8.21529769897461,
       -8.269950866699219,
       -8.409939765930176,
       0.5398184061050415,
       -8.013676643371582,
       -8.820762634277344,
       -7.8499603271484375,
       -8.266310691833496,
       -9.1400146484375,
       -8.455799102783203,
       -8.530921936035156,
       -8.980944633483887,
       -8.362706184387207,
       -8.389339447021484,
       -9.108818054199219,
       -8.528194427490234,
       -8.51555061340332,
       -8.927253723144531,
       -8.293550491333008,
       -8.327115058898926
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 1,
           "probability": 0.9979721822019951,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007275008216280081
            },
            {
             "value": 1,
             "probability": 0.9979721822019951
            },
            {
             "value": 2,
             "probability": 0.0006414849473907432
            },
            {
             "value": 3,
             "probability": 0.0006588320289861794
            }
           ]
          },
          "exception": {
           "value": false,
           "probability": 0.9984468828623438,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984468828623438
            },
            {
             "value": true,
             "probability": 0.0015531171376561006
            }
           ]
          },
          "decision": {
           "value": "assign alternate plot",
           "probability": 0.9982244480083845,
           "distribution": [
            {
             "value": "assign requested plot",
             "probability": 0.0006490018390260092
            },
            {
             "value": "assign alternate plot",
             "probability": 0.9982244480083845
            },
            {
             "value": "waitlist",
             "probability": 0.0011265501525895114
            }
           ]
          }
         },
         "joint_probability": 0.9971887129319961,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028152461573252873,
        "field_brier": 5.0513714141241305e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 1,
           "probability": 0.9979721822019951,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007275008216280081
            },
            {
             "value": 1,
             "probability": 0.9979721822019951
            },
            {
             "value": 2,
             "probability": 0.0006414849473907432
            },
            {
             "value": 3,
             "probability": 0.0006588320289861794
            }
           ]
          },
          "exception": {
           "value": false,
           "probability": 0.9984468828623438,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984468828623438
            },
            {
             "value": true,
             "probability": 0.0015531171376561006
            }
           ]
          },
          "decision": {
           "value": "assign alternate plot",
           "probability": 0.9982244480083845,
           "distribution": [
            {
             "value": "assign requested plot",
             "probability": 0.0006490018390260092
            },
            {
             "value": "assign alternate plot",
             "probability": 0.9982244480083845
            },
            {
             "value": "waitlist",
             "probability": 0.0011265501525895114
            }
           ]
          }
         },
         "joint_probability": 0.9971887129319961,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0028152461573252873,
        "field_brier": 5.0513714141241305e-06
       }
      },
      "timing_ms": {
       "scoring": 40.01640900969505,
       "base_joint": 0.06658886559307575,
       "constrained_joint": 0.056369928643107414,
       "constrained_total": 40.07277893833816
      },
      "input_tokens": 878,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-08718aa951ee7dc7-0028",
      "group": "bendsynth-08718aa951ee7dc7",
      "provenance": "bendsynth-v1; family=bendsynth-08718aa951ee7dc7; domain=cemetery plot allocation escalation desk; parameters=priority_threshold=365; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "factors": 0,
       "decision": "waitlist",
       "exception": false
      },
      "logits": [
       [
        1.084043264389038,
        -6.278094291687012,
        -6.219618797302246,
        -6.307110786437988
       ],
       [
        1.0841280221939087,
        -5.789600849151611,
        -6.040641784667969
       ],
       [
        1.0844420194625854,
        -5.40127420425415
       ]
      ],
      "joint_logits": [
       1.0834778547286987,
       -8.09839153289795,
       -7.927696704864502,
       -7.758913516998291,
       -8.353422164916992,
       -7.975623607635498,
       -8.336907386779785,
       -8.038416862487793,
       -8.099075317382812,
       -7.762539863586426,
       -8.342881202697754,
       -7.9651408195495605,
       -8.458759307861328,
       -7.990405559539795,
       -8.080172538757324,
       -7.608633995056152,
       -8.297127723693848,
       -7.864156723022461,
       -8.574502944946289,
       -8.059833526611328,
       -8.16976261138916,
       -7.698249340057373,
       -8.377938270568848,
       -7.952263832092285
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 0,
           "probability": 0.9980791034066869,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980791034066869
            },
            {
             "value": 1,
             "probability": 0.0006336207144415903
            },
            {
             "value": 2,
             "probability": 0.0006717764925605247
            },
            {
             "value": 3,
             "probability": 0.000615499386310979
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9981638474599906,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.9981638474599906
            },
            {
             "value": "assign alternate plot",
             "probability": 0.0010327123751492666
            },
            {
             "value": "assign requested plot",
             "probability": 0.0008034401648600644
            }
           ]
          },
          "exception": {
           "value": false,
           "probability": 0.9984772542096018,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984772542096018
            },
            {
             "value": true,
             "probability": 0.001522745790398268
            }
           ]
          }
         },
         "joint_probability": 0.9975150506266923,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00248804198438533,
        "field_brier": 4.8808062217693816e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 0,
           "probability": 0.9980791034066869,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980791034066869
            },
            {
             "value": 1,
             "probability": 0.0006336207144415903
            },
            {
             "value": 2,
             "probability": 0.0006717764925605247
            },
            {
             "value": 3,
             "probability": 0.000615499386310979
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9981638474599906,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.9981638474599906
            },
            {
             "value": "assign alternate plot",
             "probability": 0.0010327123751492666
            },
            {
             "value": "assign requested plot",
             "probability": 0.0008034401648600644
            }
           ]
          },
          "exception": {
           "value": false,
           "probability": 0.9984772542096018,
           "distribution": [
            {
             "value": false,
             "probability": 0.9984772542096018
            },
            {
             "value": true,
             "probability": 0.001522745790398268
            }
           ]
          }
         },
         "joint_probability": 0.9975150506266923,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00248804198438533,
        "field_brier": 4.8808062217693816e-06
       }
      },
      "timing_ms": {
       "scoring": 41.140217101201415,
       "base_joint": 0.07968908175826073,
       "constrained_joint": 0.05582999438047409,
       "constrained_total": 41.19604709558189
      },
      "input_tokens": 854,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-08718aa951ee7dc7-0035",
      "group": "bendsynth-08718aa951ee7dc7",
      "provenance": "bendsynth-v1; family=bendsynth-08718aa951ee7dc7; domain=cemetery plot allocation escalation desk; parameters=priority_threshold=295; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "exception": false,
       "factors": 0,
       "decision": "waitlist"
      },
      "logits": [
       [
        0.20247413218021393,
        -5.519195556640625
       ],
       [
        0.20055978000164032,
        -6.097094535827637,
        -6.13343620300293,
        -6.249045372009277
       ],
       [
        -0.8294832706451416,
        -0.23732474446296692,
        -5.675837993621826
       ]
      ],
      "joint_logits": [
       -0.8371018171310425,
       -0.23925645649433136,
       -7.173361778259277,
       -7.347219944000244,
       -8.186495780944824,
       -7.874816417694092,
       -7.493310451507568,
       -8.362390518188477,
       -7.8922247886657715,
       -7.63456916809082,
       -8.519964218139648,
       -8.000136375427246,
       -7.503301620483398,
       -8.383810997009277,
       -7.9192795753479,
       -7.946774482727051,
       -8.584127426147461,
       -7.810209274291992,
       -7.882152557373047,
       -8.485846519470215,
       -7.771912574768066,
       -7.974822044372559,
       -8.574751853942871,
       -7.8722968101501465
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "exception": {
           "value": false,
           "probability": 0.9967364467874605,
           "distribution": [
            {
             "value": false,
             "probability": 0.9967364467874605
            },
            {
             "value": true,
             "probability": 0.0032635532125395027
            }
           ]
          },
          "factors": {
           "value": 0,
           "probability": 0.9948301758107871,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9948301758107871
            },
            {
             "value": 1,
             "probability": 0.001831101525207075
            },
            {
             "value": 2,
             "probability": 0.0017657507903513596
            },
            {
             "value": 3,
             "probability": 0.0015729718736545218
            }
           ]
          },
          "decision": {
           "value": "assign alternate plot",
           "probability": 0.6420636496188831,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.3551459799845092
            },
            {
             "value": "assign alternate plot",
             "probability": 0.6420636496188831
            },
            {
             "value": "assign requested plot",
             "probability": 0.002790370396607786
            }
           ]
          }
         },
         "joint_probability": 0.6408246037626142,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 1.042844849216929,
        "field_brier": 0.27604906571648774
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "exception": {
           "value": false,
           "probability": 0.9967364467874605,
           "distribution": [
            {
             "value": false,
             "probability": 0.9967364467874605
            },
            {
             "value": true,
             "probability": 0.0032635532125395027
            }
           ]
          },
          "factors": {
           "value": 0,
           "probability": 0.9948301758107871,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9948301758107871
            },
            {
             "value": 1,
             "probability": 0.001831101525207075
            },
            {
             "value": 2,
             "probability": 0.0017657507903513596
            },
            {
             "value": 3,
             "probability": 0.0015729718736545218
            }
           ]
          },
          "decision": {
           "value": "assign alternate plot",
           "probability": 0.6420636496188831,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.3551459799845092
            },
            {
             "value": "assign alternate plot",
             "probability": 0.6420636496188831
            },
            {
             "value": "assign requested plot",
             "probability": 0.002790370396607786
            }
           ]
          }
         },
         "joint_probability": 0.6408246037626142,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 1.042844849216929,
        "field_brier": 0.27604906571648774
       }
      },
      "timing_ms": {
       "scoring": 41.86340980231762,
       "base_joint": 0.06362004205584526,
       "constrained_joint": 0.04741898737847805,
       "constrained_total": 41.9108287896961
      },
      "input_tokens": 884,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-093de5cd0b53c0df-0002",
      "group": "bendsynth-093de5cd0b53c0df",
      "provenance": "bendsynth-v1; family=bendsynth-093de5cd0b53c0df; domain=zoo animal transfer approvals; parameters=wait=51; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "approve",
       "eligible": true,
       "holds": 0
      },
      "logits": [
       [
        -5.696561813354492,
        1.3193399906158447,
        -5.665030479431152
       ],
       [
        -5.335764408111572,
        1.3198763132095337
       ],
       [
        1.3191620111465454,
        -5.912390232086182,
        -6.010194301605225,
        -6.06199312210083
       ]
      ],
      "joint_logits": [
       -7.840976238250732,
       -7.739319801330566,
       -7.799906253814697,
       -7.852767467498779,
       -7.671629905700684,
       -7.699895858764648,
       -7.78570032119751,
       -7.834047794342041,
       -7.808465480804443,
       -7.8061723709106445,
       -7.897329807281494,
       -7.94222354888916,
       1.3185644149780273,
       -7.585660457611084,
       -7.7265448570251465,
       -7.796209812164307,
       -7.822970390319824,
       -7.711033821105957,
       -7.803916931152344,
       -7.844097137451172,
       -7.473074436187744,
       -7.695830345153809,
       -7.805790901184082,
       -7.858913898468018
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve",
           "probability": 0.9981795782161038,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.0008958619335174077
            },
            {
             "value": "approve",
             "probability": 0.9981795782161038
            },
            {
             "value": "medical hold",
             "probability": 0.00092455985037876
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9987149100927478,
           "distribution": [
            {
             "value": false,
             "probability": 0.001285089907252275
            },
            {
             "value": true,
             "probability": 0.9987149100927478
            }
           ]
          },
          "holds": {
           "value": 0,
           "probability": 0.9980017267516782,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980017267516782
            },
            {
             "value": 1,
             "probability": 0.0007219518319612945
            },
            {
             "value": 2,
             "probability": 0.0006546850309859869
            },
            {
             "value": 3,
             "probability": 0.0006216363853744947
            }
           ]
          }
         },
         "joint_probability": 0.9974056308068102,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025977404009792098,
        "field_brier": 4.534527279878073e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve",
           "probability": 0.9981795782161038,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.0008958619335174077
            },
            {
             "value": "approve",
             "probability": 0.9981795782161038
            },
            {
             "value": "medical hold",
             "probability": 0.00092455985037876
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9987149100927478,
           "distribution": [
            {
             "value": false,
             "probability": 0.001285089907252275
            },
            {
             "value": true,
             "probability": 0.9987149100927478
            }
           ]
          },
          "holds": {
           "value": 0,
           "probability": 0.9980017267516782,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980017267516782
            },
            {
             "value": 1,
             "probability": 0.0007219518319612945
            },
            {
             "value": 2,
             "probability": 0.0006546850309859869
            },
            {
             "value": 3,
             "probability": 0.0006216363853744947
            }
           ]
          }
         },
         "joint_probability": 0.9974056308068102,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0025977404009792098,
        "field_brier": 4.534527279878073e-06
       }
      },
      "timing_ms": {
       "scoring": 36.652144975960255,
       "base_joint": 0.06705895066261292,
       "constrained_joint": 0.0463300384581089,
       "constrained_total": 36.698475014418364
      },
      "input_tokens": 724,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-093de5cd0b53c0df-0006",
      "group": "bendsynth-093de5cd0b53c0df",
      "provenance": "bendsynth-v1; family=bendsynth-093de5cd0b53c0df; domain=zoo animal transfer approvals; parameters=wait=31; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "holds": 1,
       "decision": "escalate",
       "eligible": true
      },
      "logits": [
       [
        -5.647979736328125,
        1.769851803779602,
        -5.668053150177002,
        -5.746273040771484
       ],
       [
        -5.25739860534668,
        1.769963264465332,
        -5.446533203125
       ],
       [
        -4.911614418029785,
        1.7703310251235962
       ]
      ],
      "joint_logits": [
       -7.364941596984863,
       -7.305797576904297,
       -7.3639750480651855,
       -7.426628589630127,
       -7.641175746917725,
       -7.5795207023620605,
       -7.275120735168457,
       -7.197859287261963,
       -7.07976770401001,
       1.7692387104034424,
       -7.287322521209717,
       -7.367643356323242,
       -7.38417911529541,
       -7.3683295249938965,
       -7.4452056884765625,
       -7.4784111976623535,
       -7.5229010581970215,
       -7.575798988342285,
       -7.428938865661621,
       -7.3895721435546875,
       -7.467513084411621,
       -7.683267116546631,
       -7.6335296630859375,
       -7.668278217315674
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "holds": {
           "value": 1,
           "probability": 0.9982697947201375,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005994109203358134
            },
            {
             "value": 1,
             "probability": 0.9982697947201375
            },
            {
             "value": 2,
             "probability": 0.0005874986429516208
            },
            {
             "value": 3,
             "probability": 0.000543295716574959
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.998380985164844,
           "distribution": [
            {
             "value": "medical hold",
             "probability": 0.0008858328966447135
            },
            {
             "value": "escalate",
             "probability": 0.998380985164844
            },
            {
             "value": "approve",
             "probability": 0.0007331819385113453
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9987482316035488,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012517683964511654
            },
            {
             "value": true,
             "probability": 0.9987482316035488
            }
           ]
          }
         },
         "joint_probability": 0.9976578685036581,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023448785765018535,
        "field_brier": 3.690180533966999e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "holds": {
           "value": 1,
           "probability": 0.9982697947201375,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005994109203358134
            },
            {
             "value": 1,
             "probability": 0.9982697947201375
            },
            {
             "value": 2,
             "probability": 0.0005874986429516208
            },
            {
             "value": 3,
             "probability": 0.000543295716574959
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.998380985164844,
           "distribution": [
            {
             "value": "medical hold",
             "probability": 0.0008858328966447135
            },
            {
             "value": "escalate",
             "probability": 0.998380985164844
            },
            {
             "value": "approve",
             "probability": 0.0007331819385113453
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9987482316035488,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012517683964511654
            },
            {
             "value": true,
             "probability": 0.9987482316035488
            }
           ]
          }
         },
         "joint_probability": 0.9976578685036581,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023448785765018535,
        "field_brier": 3.690180533966999e-06
       }
      },
      "timing_ms": {
       "scoring": 36.545617040246725,
       "base_joint": 0.06429897621273994,
       "constrained_joint": 0.046958914026618004,
       "constrained_total": 36.59257595427334
      },
      "input_tokens": 712,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-093de5cd0b53c0df-0009",
      "group": "bendsynth-093de5cd0b53c0df",
      "provenance": "bendsynth-v1; family=bendsynth-093de5cd0b53c0df; domain=zoo animal transfer approvals; parameters=wait=30; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "eligible": true,
       "decision": "approve",
       "holds": 1
      },
      "logits": [
       [
        -5.533872604370117,
        0.9592188596725464
       ],
       [
        0.9587651491165161,
        -6.0345964431762695,
        -5.899665355682373
       ],
       [
        -6.5109543800354,
        0.958629846572876,
        -6.05866813659668,
        -6.399051189422607
       ]
      ],
      "joint_logits": [
       -8.292744636535645,
       -7.834350109100342,
       -7.875370025634766,
       -8.2302885055542,
       -8.228751182556152,
       -8.095478057861328,
       -7.8389973640441895,
       -8.13025951385498,
       -8.177544593811035,
       -8.017253875732422,
       -7.713934898376465,
       -7.9851813316345215,
       -8.455888748168945,
       0.9579451084136963,
       -7.843362808227539,
       -8.33230972290039,
       -8.372299194335938,
       -8.086527824401855,
       -7.966190814971924,
       -8.30093002319336,
       -8.313658714294434,
       -7.712491989135742,
       -7.88179874420166,
       -8.206730842590332
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "eligible": {
           "value": true,
           "probability": 0.9984884267953481,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015115732046519078
            },
            {
             "value": true,
             "probability": 0.9984884267953481
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9980353499422115,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.9980353499422115
            },
            {
             "value": "escalate",
             "probability": 0.0009161523977272284
            },
            {
             "value": "medical hold",
             "probability": 0.0010484976600612737
            }
           ]
          },
          "holds": {
           "value": 1,
           "probability": 0.9979003331745766,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005689682084256549
            },
            {
             "value": 1,
             "probability": 0.9979003331745766
            },
            {
             "value": 2,
             "probability": 0.000894362054361073
            },
            {
             "value": 3,
             "probability": 0.000636336562636588
            }
           ]
          }
         },
         "joint_probability": 0.9972172161326209,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002786663008617838,
        "field_brier": 5.435124273279498e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "eligible": {
           "value": true,
           "probability": 0.9984884267953481,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015115732046519078
            },
            {
             "value": true,
             "probability": 0.9984884267953481
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9980353499422115,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.9980353499422115
            },
            {
             "value": "escalate",
             "probability": 0.0009161523977272284
            },
            {
             "value": "medical hold",
             "probability": 0.0010484976600612737
            }
           ]
          },
          "holds": {
           "value": 1,
           "probability": 0.9979003331745766,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005689682084256549
            },
            {
             "value": 1,
             "probability": 0.9979003331745766
            },
            {
             "value": 2,
             "probability": 0.000894362054361073
            },
            {
             "value": 3,
             "probability": 0.000636336562636588
            }
           ]
          }
         },
         "joint_probability": 0.9972172161326209,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002786663008617838,
        "field_brier": 5.435124273279498e-06
       }
      },
      "timing_ms": {
       "scoring": 37.103251088410616,
       "base_joint": 0.06568990647792816,
       "constrained_joint": 0.0467691570520401,
       "constrained_total": 37.150020245462656
      },
      "input_tokens": 715,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-093de5cd0b53c0df-0011",
      "group": "bendsynth-093de5cd0b53c0df",
      "provenance": "bendsynth-v1; family=bendsynth-093de5cd0b53c0df; domain=zoo animal transfer approvals; parameters=wait=44; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "holds": 1,
       "decision": "approve",
       "eligible": true
      },
      "logits": [
       [
        -1.9550641775131226,
        0.6936942934989929,
        -6.017702102661133,
        -5.997446060180664
       ],
       [
        0.7614153027534485,
        -5.690862655639648,
        -5.846907615661621
       ],
       [
        -5.417840003967285,
        0.7622690200805664
       ]
      ],
      "joint_logits": [
       -8.422595977783203,
       -1.966092824935913,
       -7.905171871185303,
       -7.93453311920166,
       -8.032917022705078,
       -8.176613807678223,
       -8.357353210449219,
       0.6929940581321716,
       -8.102689743041992,
       -8.003976821899414,
       -8.209761619567871,
       -8.265464782714844,
       -8.123045921325684,
       -8.134146690368652,
       -7.486822128295898,
       -7.755324363708496,
       -7.658096790313721,
       -7.8643341064453125,
       -8.104825019836426,
       -8.248560905456543,
       -7.448115348815918,
       -7.7255167961120605,
       -7.600491046905518,
       -7.831031799316406
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "holds": {
           "value": 1,
           "probability": 0.9317946691845148,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0659142063335905
            },
            {
             "value": 1,
             "probability": 0.9317946691845148
            },
            {
             "value": 2,
             "probability": 0.001133960404201409
            },
            {
             "value": 3,
             "probability": 0.0011571640776931607
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9970825181426809,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.9970825181426809
            },
            {
             "value": "medical hold",
             "probability": 0.0015723250783319575
            },
            {
             "value": "escalate",
             "probability": 0.0013451567789870575
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9979340743083542,
           "distribution": [
            {
             "value": false,
             "probability": 0.0020659256916456775
            },
            {
             "value": true,
             "probability": 0.9979340743083542
            }
           ]
          }
         },
         "joint_probability": 0.9311424183534462,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.07134303987753177,
        "field_brier": 0.0030068680314548376
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "holds": {
           "value": 1,
           "probability": 0.9317946691845148,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0659142063335905
            },
            {
             "value": 1,
             "probability": 0.9317946691845148
            },
            {
             "value": 2,
             "probability": 0.001133960404201409
            },
            {
             "value": 3,
             "probability": 0.0011571640776931607
            }
           ]
          },
          "decision": {
           "value": "approve",
           "probability": 0.9970825181426809,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.9970825181426809
            },
            {
             "value": "medical hold",
             "probability": 0.0015723250783319575
            },
            {
             "value": "escalate",
             "probability": 0.0013451567789870575
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9979340743083542,
           "distribution": [
            {
             "value": false,
             "probability": 0.0020659256916456775
            },
            {
             "value": true,
             "probability": 0.9979340743083542
            }
           ]
          }
         },
         "joint_probability": 0.9311424183534462,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.07134303987753177,
        "field_brier": 0.0030068680314548376
       }
      },
      "timing_ms": {
       "scoring": 38.02817105315626,
       "base_joint": 0.06526894867420197,
       "constrained_joint": 0.046800123527646065,
       "constrained_total": 38.0749711766839
      },
      "input_tokens": 718,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-093de5cd0b53c0df-0024",
      "group": "bendsynth-093de5cd0b53c0df",
      "provenance": "bendsynth-v1; family=bendsynth-093de5cd0b53c0df; domain=zoo animal transfer approvals; parameters=wait=30; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "eligible": true,
       "holds": 2,
       "decision": "escalate"
      },
      "logits": [
       [
        -4.449573516845703,
        2.2621798515319824
       ],
       [
        -5.379659652709961,
        -4.97917366027832,
        2.261422872543335,
        -4.898176193237305
       ],
       [
        -4.742743968963623,
        2.261784553527832,
        -4.996148586273193
       ]
      ],
      "joint_logits": [
       -7.026270866394043,
       -7.205347537994385,
       -7.408390045166016,
       -6.816501617431641,
       -6.84238862991333,
       -7.029723167419434,
       -7.119387149810791,
       -6.571872711181641,
       -7.361021041870117,
       -6.569823265075684,
       -6.79671049118042,
       -6.873539447784424,
       -7.002612590789795,
       -7.180124759674072,
       -7.262535095214844,
       -6.737898826599121,
       -6.371606349945068,
       -6.972906589508057,
       -6.899488925933838,
       2.260924816131592,
       -6.983545303344727,
       -6.559446334838867,
       -6.534671306610107,
       -6.871420860290527
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "eligible": {
           "value": true,
           "probability": 0.9987849490687015,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012150509312984859
            },
            {
             "value": true,
             "probability": 0.9987849490687015
            }
           ]
          },
          "holds": {
           "value": 2,
           "probability": 0.9980293328516955,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004793617342983046
            },
            {
             "value": 1,
             "probability": 0.0007154712238266181
            },
            {
             "value": 2,
             "probability": 0.9980293328516955
            },
            {
             "value": 3,
             "probability": 0.0007758341901795444
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9983902709001916,
           "distribution": [
            {
             "value": "medical hold",
             "probability": 0.0009063005845606161
            },
            {
             "value": "escalate",
             "probability": 0.9983902709001916
            },
            {
             "value": "approve",
             "probability": 0.000703428515247829
            }
           ]
          }
         },
         "joint_probability": 0.9975323013682289,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024707484183756263,
        "field_brier": 4.029084058732954e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "eligible": {
           "value": true,
           "probability": 0.9987849490687015,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012150509312984859
            },
            {
             "value": true,
             "probability": 0.9987849490687015
            }
           ]
          },
          "holds": {
           "value": 2,
           "probability": 0.9980293328516955,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004793617342983046
            },
            {
             "value": 1,
             "probability": 0.0007154712238266181
            },
            {
             "value": 2,
             "probability": 0.9980293328516955
            },
            {
             "value": 3,
             "probability": 0.0007758341901795444
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9983902709001916,
           "distribution": [
            {
             "value": "medical hold",
             "probability": 0.0009063005845606161
            },
            {
             "value": "escalate",
             "probability": 0.9983902709001916
            },
            {
             "value": "approve",
             "probability": 0.000703428515247829
            }
           ]
          }
         },
         "joint_probability": 0.9975323013682289,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024707484183756263,
        "field_brier": 4.029084058732954e-06
       }
      },
      "timing_ms": {
       "scoring": 36.90269310027361,
       "base_joint": 0.07384899072349072,
       "constrained_joint": 0.04992005415260792,
       "constrained_total": 36.95261315442622
      },
      "input_tokens": 712,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-093de5cd0b53c0df-0036",
      "group": "bendsynth-093de5cd0b53c0df",
      "provenance": "bendsynth-v1; family=bendsynth-093de5cd0b53c0df; domain=zoo animal transfer approvals; parameters=wait=15; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "holds": 2,
       "decision": "escalate",
       "eligible": true
      },
      "logits": [
       [
        -5.249961853027344,
        -4.775787830352783,
        2.4909605979919434,
        -4.7285027503967285
       ],
       [
        2.4912662506103516,
        -4.494501113891602,
        -4.872138500213623
       ],
       [
        -4.290272235870361,
        2.4916908740997314
       ]
      ],
      "joint_logits": [
       -7.088136672973633,
       -7.092495918273926,
       -6.8744354248046875,
       -6.830882549285889,
       -7.346527576446533,
       -7.10275411605835,
       -6.660504341125488,
       -6.2542619705200195,
       -6.511704444885254,
       -6.421431064605713,
       -6.922512054443359,
       -6.784590244293213,
       -6.554714202880859,
       2.4905126094818115,
       -6.823099136352539,
       -6.688350677490234,
       -7.225938320159912,
       -6.987767219543457,
       -6.660610675811768,
       -6.484776020050049,
       -6.330053806304932,
       -6.304215908050537,
       -6.781796932220459,
       -6.654603481292725
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "holds": {
           "value": 2,
           "probability": 0.9981382275934163,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004338613237687927
            },
            {
             "value": 1,
             "probability": 0.0006970789143176512
            },
            {
             "value": 2,
             "probability": 0.9981382275934163
            },
            {
             "value": 3,
             "probability": 0.0007308321684973049
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9984434373586317,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.9984434373586317
            },
            {
             "value": "medical hold",
             "probability": 0.0009235134838537169
            },
            {
             "value": "approve",
             "probability": 0.0006330491575146106
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9988672378628922,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011327621371077254
            },
            {
             "value": true,
             "probability": 0.9988672378628922
            }
           ]
          }
         },
         "joint_probability": 0.9976911445533292,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002311524963217697,
        "field_brier": 3.639094193066138e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "holds": {
           "value": 2,
           "probability": 0.9981382275934163,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004338613237687927
            },
            {
             "value": 1,
             "probability": 0.0006970789143176512
            },
            {
             "value": 2,
             "probability": 0.9981382275934163
            },
            {
             "value": 3,
             "probability": 0.0007308321684973049
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9984434373586317,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.9984434373586317
            },
            {
             "value": "medical hold",
             "probability": 0.0009235134838537169
            },
            {
             "value": "approve",
             "probability": 0.0006330491575146106
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9988672378628922,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011327621371077254
            },
            {
             "value": true,
             "probability": 0.9988672378628922
            }
           ]
          }
         },
         "joint_probability": 0.9976911445533292,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002311524963217697,
        "field_brier": 3.639094193066138e-06
       }
      },
      "timing_ms": {
       "scoring": 37.79121395200491,
       "base_joint": 0.06430898793041706,
       "constrained_joint": 0.04681898280978203,
       "constrained_total": 37.83803293481469
      },
      "input_tokens": 718,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-093de5cd0b53c0df-0039",
      "group": "bendsynth-093de5cd0b53c0df",
      "provenance": "bendsynth-v1; family=bendsynth-093de5cd0b53c0df; domain=zoo animal transfer approvals; parameters=wait=57; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "holds": 2,
       "decision": "medical hold",
       "eligible": false
      },
      "logits": [
       [
        -7.241101264953613,
        -6.951544761657715,
        0.2634854018688202,
        -7.080738544464111
       ],
       [
        0.26385971903800964,
        -6.87323522567749,
        -6.915420055389404
       ],
       [
        0.2642143666744232,
        -6.459349632263184
       ]
      ],
      "joint_logits": [
       -9.053712844848633,
       -8.979633331298828,
       -8.995293617248535,
       -9.046014785766602,
       -9.018665313720703,
       -9.10925006866455,
       -8.370750427246094,
       -8.696793556213379,
       -8.835899353027344,
       -8.871732711791992,
       -8.892714500427246,
       -8.907352447509766,
       0.2630464434623718,
       -8.90346622467041,
       -9.064812660217285,
       -9.148031234741211,
       -9.114423751831055,
       -9.179963111877441,
       -8.894058227539062,
       -8.846959114074707,
       -8.863041877746582,
       -8.84614372253418,
       -8.878886222839355,
       -8.907477378845215
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "holds": {
           "value": 2,
           "probability": 0.9980714098538209,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000549491490370194
            },
            {
             "value": 1,
             "probability": 0.0007340302259907649
            },
            {
             "value": 2,
             "probability": 0.9980714098538209
            },
            {
             "value": 3,
             "probability": 0.0006450684298181105
            }
           ]
          },
          "decision": {
           "value": "medical hold",
           "probability": 0.9984451464860874,
           "distribution": [
            {
             "value": "medical hold",
             "probability": 0.9984451464860874
            },
            {
             "value": "escalate",
             "probability": 0.0007938220648695745
            },
            {
             "value": "approve",
             "probability": 0.0007610314490430831
            }
           ]
          },
          "eligible": {
           "value": false,
           "probability": 0.9987991977491332,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987991977491332
            },
            {
             "value": true,
             "probability": 0.001200802250866779
            }
           ]
          }
         },
         "joint_probability": 0.9976333930328076,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002369411807638114,
        "field_brier": 3.829019459983562e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "holds": {
           "value": 2,
           "probability": 0.9980714098538209,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000549491490370194
            },
            {
             "value": 1,
             "probability": 0.0007340302259907649
            },
            {
             "value": 2,
             "probability": 0.9980714098538209
            },
            {
             "value": 3,
             "probability": 0.0006450684298181105
            }
           ]
          },
          "decision": {
           "value": "medical hold",
           "probability": 0.9984451464860874,
           "distribution": [
            {
             "value": "medical hold",
             "probability": 0.9984451464860874
            },
            {
             "value": "escalate",
             "probability": 0.0007938220648695745
            },
            {
             "value": "approve",
             "probability": 0.0007610314490430831
            }
           ]
          },
          "eligible": {
           "value": false,
           "probability": 0.9987991977491332,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987991977491332
            },
            {
             "value": true,
             "probability": 0.001200802250866779
            }
           ]
          }
         },
         "joint_probability": 0.9976333930328076,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002369411807638114,
        "field_brier": 3.829019459983562e-06
       }
      },
      "timing_ms": {
       "scoring": 36.24819009564817,
       "base_joint": 0.0755989458411932,
       "constrained_joint": 0.047168927267193794,
       "constrained_total": 36.29535902291536
      },
      "input_tokens": 709,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c25e6fc3d21cd2b-0001",
      "group": "bendsynth-0c25e6fc3d21cd2b",
      "provenance": "bendsynth-v1; family=bendsynth-0c25e6fc3d21cd2b; domain=beekeeping hive registration field inspection; parameters=limit=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "within_limit": false,
       "findings": 1,
       "decision": "conditional registration"
      },
      "logits": [
       [
        0.13694044947624207,
        -6.5200324058532715
       ],
       [
        -7.183104515075684,
        0.13623563945293427,
        -7.190500259399414,
        -7.170915603637695
       ],
       [
        -6.658466339111328,
        0.136088028550148,
        -6.752755165100098
       ]
      ],
      "joint_logits": [
       -8.805154800415039,
       -9.160746574401855,
       -8.856693267822266,
       -8.118517875671387,
       0.13547441363334656,
       -8.531702041625977,
       -8.850799560546875,
       -9.171289443969727,
       -8.843801498413086,
       -8.797724723815918,
       -9.2067289352417,
       -8.842277526855469,
       -8.903499603271484,
       -9.250917434692383,
       -8.948721885681152,
       -8.951896667480469,
       -9.182295799255371,
       -8.841747283935547,
       -8.908304214477539,
       -9.258667945861816,
       -8.935391426086426,
       -8.87260913848877,
       -9.217533111572266,
       -8.92345142364502
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_limit": {
           "value": false,
           "probability": 0.998716618841554,
           "distribution": [
            {
             "value": false,
             "probability": 0.998716618841554
            },
            {
             "value": true,
             "probability": 0.0012833811584459377
            }
           ]
          },
          "findings": {
           "value": 1,
           "probability": 0.9980129147972194,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000661282548128583
            },
            {
             "value": 1,
             "probability": 0.9980129147972194
            },
            {
             "value": 2,
             "probability": 0.000656410245622414
            },
            {
             "value": 3,
             "probability": 0.0006693924090296245
            }
           ]
          },
          "decision": {
           "value": "conditional registration",
           "probability": 0.9978656164973251,
           "distribution": [
            {
             "value": "re-inspection referral",
             "probability": 0.001117466727300259
            },
            {
             "value": "conditional registration",
             "probability": 0.9978656164973251
            },
            {
             "value": "standard registration",
             "probability": 0.0010169167753746205
            }
           ]
          }
         },
         "joint_probability": 0.9972534921560404,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027502864168004724,
        "field_brier": 5.1324472554179745e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_limit": {
           "value": false,
           "probability": 0.998716618841554,
           "distribution": [
            {
             "value": false,
             "probability": 0.998716618841554
            },
            {
             "value": true,
             "probability": 0.0012833811584459377
            }
           ]
          },
          "findings": {
           "value": 1,
           "probability": 0.9980129147972194,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000661282548128583
            },
            {
             "value": 1,
             "probability": 0.9980129147972194
            },
            {
             "value": 2,
             "probability": 0.000656410245622414
            },
            {
             "value": 3,
             "probability": 0.0006693924090296245
            }
           ]
          },
          "decision": {
           "value": "conditional registration",
           "probability": 0.9978656164973251,
           "distribution": [
            {
             "value": "re-inspection referral",
             "probability": 0.001117466727300259
            },
            {
             "value": "conditional registration",
             "probability": 0.9978656164973251
            },
            {
             "value": "standard registration",
             "probability": 0.0010169167753746205
            }
           ]
          }
         },
         "joint_probability": 0.9972534921560404,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027502864168004724,
        "field_brier": 5.1324472554179745e-06
       }
      },
      "timing_ms": {
       "scoring": 35.198400961235166,
       "base_joint": 0.07344898767769337,
       "constrained_joint": 0.04622992128133774,
       "constrained_total": 35.2446308825165
      },
      "input_tokens": 664,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-084c31b0c2770258-0000",
      "group": "bendsynth-084c31b0c2770258",
      "provenance": "bendsynth-v1; family=bendsynth-084c31b0c2770258; domain=youth football league pitch-side roster eligibility inspection; parameters=cutoff_junior=9,cutoff_senior=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "issues": 2,
       "over_cutoff": true,
       "decision": "ineligible"
      },
      "logits": [
       [
        -7.081001281738281,
        -6.7094926834106445,
        -0.15097695589065552,
        -7.052789211273193
       ],
       [
        -5.599699974060059,
        -0.15187710523605347
       ],
       [
        -6.700416088104248,
        -0.15138395130634308,
        -6.192435264587402
       ]
      ],
      "joint_logits": [
       -8.79512882232666,
       -9.211087226867676,
       -8.390095710754395,
       -9.124958992004395,
       -9.310050964355469,
       -8.721712112426758,
       -8.577042579650879,
       -8.628373146057129,
       -8.135111808776855,
       -8.899353981018066,
       -8.528157234191895,
       -8.401263236999512,
       -8.58251953125,
       -6.479275226593018,
       -7.968694686889648,
       -8.543625831604004,
       -0.15407529473304749,
       -7.831756591796875,
       -8.765250205993652,
       -9.154214859008789,
       -8.378179550170898,
       -9.143045425415039,
       -9.188815116882324,
       -8.714473724365234
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 2,
           "probability": 0.996609606736467,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0009746616237027546
            },
            {
             "value": 1,
             "probability": 0.0014131814328454493
            },
            {
             "value": 2,
             "probability": 0.996609606736467
            },
            {
             "value": 3,
             "probability": 0.0010025502069847781
            }
           ]
          },
          "over_cutoff": {
           "value": true,
           "probability": 0.9957127913330023,
           "distribution": [
            {
             "value": false,
             "probability": 0.00428720866699774
            },
            {
             "value": true,
             "probability": 0.9957127913330023
            }
           ]
          },
          "decision": {
           "value": "ineligible",
           "probability": 0.99620390758126,
           "distribution": [
            {
             "value": "provisional",
             "probability": 0.0014260665444465967
            },
            {
             "value": "ineligible",
             "probability": 0.99620390758126
            },
            {
             "value": "eligible",
             "probability": 0.00237002587429344
            }
           ]
          }
         },
         "joint_probability": 0.9935264609898152,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.006494583233406926,
        "field_brier": 2.4756080945393175e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 2,
           "probability": 0.996609606736467,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0009746616237027546
            },
            {
             "value": 1,
             "probability": 0.0014131814328454493
            },
            {
             "value": 2,
             "probability": 0.996609606736467
            },
            {
             "value": 3,
             "probability": 0.0010025502069847781
            }
           ]
          },
          "over_cutoff": {
           "value": true,
           "probability": 0.9957127913330023,
           "distribution": [
            {
             "value": false,
             "probability": 0.00428720866699774
            },
            {
             "value": true,
             "probability": 0.9957127913330023
            }
           ]
          },
          "decision": {
           "value": "ineligible",
           "probability": 0.99620390758126,
           "distribution": [
            {
             "value": "provisional",
             "probability": 0.0014260665444465967
            },
            {
             "value": "ineligible",
             "probability": 0.99620390758126
            },
            {
             "value": "eligible",
             "probability": 0.00237002587429344
            }
           ]
          }
         },
         "joint_probability": 0.9935264609898152,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.006494583233406926,
        "field_brier": 2.4756080945393175e-05
       }
      },
      "timing_ms": {
       "scoring": 33.3570409566164,
       "base_joint": 0.06554904393851757,
       "constrained_joint": 0.04824995994567871,
       "constrained_total": 33.40529091656208
      },
      "input_tokens": 654,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-084c31b0c2770258-0002",
      "group": "bendsynth-084c31b0c2770258",
      "provenance": "bendsynth-v1; family=bendsynth-084c31b0c2770258; domain=youth football league pitch-side roster eligibility inspection; parameters=cutoff_junior=10,cutoff_senior=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "issues": 1,
       "over_cutoff": false,
       "decision": "eligible"
      },
      "logits": [
       [
        -5.950943470001221,
        -3.374246120452881,
        -1.9282890558242798,
        -6.0005388259887695
       ],
       [
        -2.2573599815368652,
        -2.5245399475097656
       ],
       [
        -4.687686920166016,
        -2.407848358154297,
        -2.4591832160949707
       ]
      ],
      "joint_logits": [
       -8.627996444702148,
       -7.804171562194824,
       -8.040067672729492,
       -8.013167381286621,
       -7.205743312835693,
       -7.384729385375977,
       -6.894687652587891,
       -4.796386241912842,
       -4.930549621582031,
       -6.226426601409912,
       -4.7622389793396,
       -4.930105209350586,
       -6.111973285675049,
       -3.1859214305877686,
       -3.1506989002227783,
       -5.797381401062012,
       -3.520198106765747,
       -3.634800910949707,
       -8.716492652893066,
       -7.888082027435303,
       -8.0977783203125,
       -8.136429786682129,
       -7.204108715057373,
       -7.422007083892822
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 2,
           "probability": 0.7871134147415888,
           "distribution": [
            {
             "value": 0,
             "probability": 0.014093557160091426
            },
            {
             "value": 1,
             "probability": 0.1853813967523684
            },
            {
             "value": 2,
             "probability": 0.7871134147415888
            },
            {
             "value": 3,
             "probability": 0.013411631345951427
            }
           ]
          },
          "over_cutoff": {
           "value": false,
           "probability": 0.5664004589502619,
           "distribution": [
            {
             "value": false,
             "probability": 0.5664004589502619
            },
            {
             "value": true,
             "probability": 0.43359954104973814
            }
           ]
          },
          "decision": {
           "value": "eligible",
           "probability": 0.4628847808254252,
           "distribution": [
            {
             "value": "provisional",
             "probability": 0.049847807606421664
            },
            {
             "value": "ineligible",
             "probability": 0.4872674115681532
            },
            {
             "value": "eligible",
             "probability": 0.4628847808254252
            }
           ]
          }
         },
         "joint_probability": 0.23182030247823196,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 3.2416434870838167,
        "field_brier": 0.7293179045325348
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 2,
           "probability": 0.7871134147415888,
           "distribution": [
            {
             "value": 0,
             "probability": 0.014093557160091426
            },
            {
             "value": 1,
             "probability": 0.1853813967523684
            },
            {
             "value": 2,
             "probability": 0.7871134147415888
            },
            {
             "value": 3,
             "probability": 0.013411631345951427
            }
           ]
          },
          "over_cutoff": {
           "value": false,
           "probability": 0.5664004589502619,
           "distribution": [
            {
             "value": false,
             "probability": 0.5664004589502619
            },
            {
             "value": true,
             "probability": 0.43359954104973814
            }
           ]
          },
          "decision": {
           "value": "eligible",
           "probability": 0.4628847808254252,
           "distribution": [
            {
             "value": "provisional",
             "probability": 0.049847807606421664
            },
            {
             "value": "ineligible",
             "probability": 0.4872674115681532
            },
            {
             "value": "eligible",
             "probability": 0.4628847808254252
            }
           ]
          }
         },
         "joint_probability": 0.23182030247823196,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 3.2416434870838167,
        "field_brier": 0.7293179045325348
       }
      },
      "timing_ms": {
       "scoring": 35.784925101324916,
       "base_joint": 0.06930902600288391,
       "constrained_joint": 0.04923902451992035,
       "constrained_total": 35.834164125844836
      },
      "input_tokens": 648,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-084c31b0c2770258-0032",
      "group": "bendsynth-084c31b0c2770258",
      "provenance": "bendsynth-v1; family=bendsynth-084c31b0c2770258; domain=youth football league pitch-side roster eligibility inspection; parameters=cutoff_junior=9,cutoff_senior=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "issues": 2,
       "decision": "ineligible",
       "over_cutoff": true
      },
      "logits": [
       [
        -6.3405985832214355,
        -5.947916030883789,
        0.6942775845527649,
        -6.429101943969727
       ],
       [
        0.6936002969741821,
        -5.819708824157715,
        -5.431703090667725
       ],
       [
        -4.704636573791504,
        0.6927458047866821
       ]
      ],
      "joint_logits": [
       -8.560874938964844,
       -8.720474243164062,
       -7.78626012802124,
       -8.24038028717041,
       -7.639070987701416,
       -8.30158805847168,
       -8.004144668579102,
       -8.094019889831543,
       -7.500285625457764,
       -7.955276012420654,
       -7.312621116638184,
       -7.821444511413574,
       -5.475053310394287,
       0.6908285021781921,
       -7.819722652435303,
       -7.859302997589111,
       -7.102777004241943,
       -6.8183770179748535,
       -8.554576873779297,
       -8.72949504852295,
       -7.898801326751709,
       -8.392827987670898,
       -7.7456955909729,
       -8.380830764770508
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 2,
           "probability": 0.997018090163362,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008780013553070295
            },
            {
             "value": 1,
             "probability": 0.001300273952729189
            },
            {
             "value": 2,
             "probability": 0.997018090163362
            },
            {
             "value": 3,
             "probability": 0.0008036345286017894
            }
           ]
          },
          "decision": {
           "value": "ineligible",
           "probability": 0.9963430323834013,
           "distribution": [
            {
             "value": "ineligible",
             "probability": 0.9963430323834013
            },
            {
             "value": "eligible",
             "probability": 0.0014781374050061335
            },
            {
             "value": "provisional",
             "probability": 0.002178830211592646
            }
           ]
          },
          "over_cutoff": {
           "value": true,
           "probability": 0.9954919951309433,
           "distribution": [
            {
             "value": false,
             "probability": 0.004508004869056666
            },
            {
             "value": true,
             "probability": 0.9954919951309433
            }
           ]
          }
         },
         "joint_probability": 0.9935851560144674,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.006435507513485605,
        "field_brier": 2.431634422940762e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 2,
           "probability": 0.997018090163362,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008780013553070295
            },
            {
             "value": 1,
             "probability": 0.001300273952729189
            },
            {
             "value": 2,
             "probability": 0.997018090163362
            },
            {
             "value": 3,
             "probability": 0.0008036345286017894
            }
           ]
          },
          "decision": {
           "value": "ineligible",
           "probability": 0.9963430323834013,
           "distribution": [
            {
             "value": "ineligible",
             "probability": 0.9963430323834013
            },
            {
             "value": "eligible",
             "probability": 0.0014781374050061335
            },
            {
             "value": "provisional",
             "probability": 0.002178830211592646
            }
           ]
          },
          "over_cutoff": {
           "value": true,
           "probability": 0.9954919951309433,
           "distribution": [
            {
             "value": false,
             "probability": 0.004508004869056666
            },
            {
             "value": true,
             "probability": 0.9954919951309433
            }
           ]
          }
         },
         "joint_probability": 0.9935851560144674,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.006435507513485605,
        "field_brier": 2.431634422940762e-05
       }
      },
      "timing_ms": {
       "scoring": 34.59285805001855,
       "base_joint": 0.07109995931386948,
       "constrained_joint": 0.04842900671064854,
       "constrained_total": 34.6412870567292
      },
      "input_tokens": 627,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-084c31b0c2770258-0036",
      "group": "bendsynth-084c31b0c2770258",
      "provenance": "bendsynth-v1; family=bendsynth-084c31b0c2770258; domain=youth football league pitch-side roster eligibility inspection; parameters=cutoff_junior=8,cutoff_senior=12; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "over_cutoff": false,
       "decision": "provisional",
       "issues": 1
      },
      "logits": [
       [
        1.088579773902893,
        -4.91900634765625
       ],
       [
        -5.775982856750488,
        1.0888278484344482,
        -5.66414737701416
       ],
       [
        -6.259605407714844,
        1.088760256767273,
        -5.828744411468506,
        -6.257131576538086
       ]
      ],
      "joint_logits": [
       -8.231616973876953,
       -7.744375228881836,
       -7.901314735412598,
       -8.223458290100098,
       -8.279044151306152,
       1.0873029232025146,
       -7.650938510894775,
       -8.310935020446777,
       -8.19272518157959,
       -7.543093681335449,
       -7.848356246948242,
       -8.18076229095459,
       -7.958139896392822,
       -7.54628324508667,
       -7.545251846313477,
       -7.938220024108887,
       -7.885245323181152,
       -6.106636047363281,
       -7.447349548339844,
       -7.917041778564453,
       -7.850907802581787,
       -7.4075446128845215,
       -7.428761959075928,
       -7.820733070373535
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cutoff": {
           "value": false,
           "probability": 0.997546018233374,
           "distribution": [
            {
             "value": false,
             "probability": 0.997546018233374
            },
            {
             "value": true,
             "probability": 0.0024539817666261055
            }
           ]
          },
          "decision": {
           "value": "provisional",
           "probability": 0.9977935982174566,
           "distribution": [
            {
             "value": "ineligible",
             "probability": 0.0010415765467537386
            },
            {
             "value": "provisional",
             "probability": 0.9977935982174566
            },
            {
             "value": "eligible",
             "probability": 0.0011648252357897035
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.997726003660256,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006421798342370093
            },
            {
             "value": 1,
             "probability": 0.997726003660256
            },
            {
             "value": 2,
             "probability": 0.0009880461636424611
            },
            {
             "value": 3,
             "probability": 0.0006437703418646257
            }
           ]
          }
         },
         "joint_probability": 0.9962731426158142,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0037338194202265803,
        "field_brier": 8.776030382593316e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cutoff": {
           "value": false,
           "probability": 0.997546018233374,
           "distribution": [
            {
             "value": false,
             "probability": 0.997546018233374
            },
            {
             "value": true,
             "probability": 0.0024539817666261055
            }
           ]
          },
          "decision": {
           "value": "provisional",
           "probability": 0.9977935982174566,
           "distribution": [
            {
             "value": "ineligible",
             "probability": 0.0010415765467537386
            },
            {
             "value": "provisional",
             "probability": 0.9977935982174566
            },
            {
             "value": "eligible",
             "probability": 0.0011648252357897035
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.997726003660256,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006421798342370093
            },
            {
             "value": 1,
             "probability": 0.997726003660256
            },
            {
             "value": 2,
             "probability": 0.0009880461636424611
            },
            {
             "value": 3,
             "probability": 0.0006437703418646257
            }
           ]
          }
         },
         "joint_probability": 0.9962731426158142,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0037338194202265803,
        "field_brier": 8.776030382593316e-06
       }
      },
      "timing_ms": {
       "scoring": 34.38844089396298,
       "base_joint": 0.06595021113753319,
       "constrained_joint": 0.04681898280978203,
       "constrained_total": 34.43525987677276
      },
      "input_tokens": 627,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01f3c41ba4c4d009-0006",
      "group": "bendsynth-01f3c41ba4c4d009",
      "provenance": "bendsynth-v1; family=bendsynth-01f3c41ba4c4d009; domain=cattle auction lot intake desk; parameters=quota=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_free": false,
       "decision": "accepted",
       "issues": 0
      },
      "logits": [
       [
        0.7074869871139526,
        -5.823267936706543
       ],
       [
        -6.133069038391113,
        -5.952980995178223,
        0.7065945267677307
       ],
       [
        0.7064722180366516,
        -6.290294647216797,
        -6.42388391494751,
        -6.476294994354248
       ]
      ],
      "joint_logits": [
       -7.8821306228637695,
       -8.054256439208984,
       -8.157644271850586,
       -8.214305877685547,
       -7.615253925323486,
       -7.877405166625977,
       -7.994760036468506,
       -8.033429145812988,
       0.7056263089179993,
       -7.8533830642700195,
       -8.06130599975586,
       -8.174666404724121,
       -8.249277114868164,
       -8.331873893737793,
       -8.46597957611084,
       -8.493623733520508,
       -8.075865745544434,
       -8.202703475952148,
       -8.322933197021484,
       -8.349199295043945,
       -8.213310241699219,
       -8.277454376220703,
       -8.378847122192383,
       -8.41616439819336
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_free": {
           "value": false,
           "probability": 0.9985442175180365,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985442175180365
            },
            {
             "value": true,
             "probability": 0.001455782481963557
            }
           ]
          },
          "decision": {
           "value": "accepted",
           "probability": 0.9976533658871835,
           "distribution": [
            {
             "value": "rejected",
             "probability": 0.0010679513747629487
            },
            {
             "value": "waitlisted",
             "probability": 0.0012786827380535746
            },
            {
             "value": "accepted",
             "probability": 0.9976533658871835
            }
           ]
          },
          "issues": {
           "value": 0,
           "probability": 0.9975312771809985,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9975312771809985
            },
            {
             "value": 1,
             "probability": 0.0009125767684529824
            },
            {
             "value": 2,
             "probability": 0.0007984581053370383
            },
            {
             "value": 3,
             "probability": 0.0007576879452114827
            }
           ]
          }
         },
         "joint_probability": 0.9966878429828951,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0033176543512033556,
        "field_brier": 6.886620565382756e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_free": {
           "value": false,
           "probability": 0.9985442175180365,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985442175180365
            },
            {
             "value": true,
             "probability": 0.001455782481963557
            }
           ]
          },
          "decision": {
           "value": "accepted",
           "probability": 0.9976533658871835,
           "distribution": [
            {
             "value": "rejected",
             "probability": 0.0010679513747629487
            },
            {
             "value": "waitlisted",
             "probability": 0.0012786827380535746
            },
            {
             "value": "accepted",
             "probability": 0.9976533658871835
            }
           ]
          },
          "issues": {
           "value": 0,
           "probability": 0.9975312771809985,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9975312771809985
            },
            {
             "value": 1,
             "probability": 0.0009125767684529824
            },
            {
             "value": 2,
             "probability": 0.0007984581053370383
            },
            {
             "value": 3,
             "probability": 0.0007576879452114827
            }
           ]
          }
         },
         "joint_probability": 0.9966878429828951,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0033176543512033556,
        "field_brier": 6.886620565382756e-06
       }
      },
      "timing_ms": {
       "scoring": 35.11364315636456,
       "base_joint": 0.06474996916949749,
       "constrained_joint": 0.04760921001434326,
       "constrained_total": 35.1612523663789
      },
      "input_tokens": 621,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01f3c41ba4c4d009-0007",
      "group": "bendsynth-01f3c41ba4c4d009",
      "provenance": "bendsynth-v1; family=bendsynth-01f3c41ba4c4d009; domain=cattle auction lot intake desk; parameters=quota=7; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_free": false,
       "decision": "rejected",
       "issues": 2
      },
      "logits": [
       [
        0.505264401435852,
        -6.363475322723389
       ],
       [
        -6.786447048187256,
        -6.739316463470459,
        0.5049088001251221
       ],
       [
        -7.032705307006836,
        -6.986282825469971,
        0.5045782923698425,
        -6.855978012084961
       ]
      ],
      "joint_logits": [
       -8.746358871459961,
       -8.82176685333252,
       -8.777310371398926,
       -8.675745010375977,
       -8.812658309936523,
       -8.851476669311523,
       -8.672221183776855,
       -8.691427230834961,
       -8.698633193969727,
       -8.416971206665039,
       0.5041399598121643,
       -8.458693504333496,
       -9.17838191986084,
       -9.086708068847656,
       -8.972647666931152,
       -8.776360511779785,
       -8.936595916748047,
       -8.911312103271484,
       -9.015910148620605,
       -8.713539123535156,
       -8.661551475524902,
       -8.711258888244629,
       -8.791034698486328,
       -8.602139472961426
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_free": {
           "value": false,
           "probability": 0.9989612931846671,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989612931846671
            },
            {
             "value": true,
             "probability": 0.0010387068153329107
            }
           ]
          },
          "decision": {
           "value": "rejected",
           "probability": 0.9986062545522475,
           "distribution": [
            {
             "value": "accepted",
             "probability": 0.0006804538664545584
            },
            {
             "value": "waitlisted",
             "probability": 0.0007132915812979836
            },
            {
             "value": "rejected",
             "probability": 0.9986062545522475
            }
           ]
          },
          "issues": {
           "value": 2,
           "probability": 0.9982761270123944,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005319244753484255
            },
            {
             "value": 1,
             "probability": 0.0005571998293026436
            },
            {
             "value": 2,
             "probability": 0.9982761270123944
            },
            {
             "value": 3,
             "probability": 0.0006347486829545226
            }
           ]
          }
         },
         "joint_probability": 0.9978386402877535,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0021636988211943295,
        "field_brier": 3.0134038930168943e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_free": {
           "value": false,
           "probability": 0.9989612931846671,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989612931846671
            },
            {
             "value": true,
             "probability": 0.0010387068153329107
            }
           ]
          },
          "decision": {
           "value": "rejected",
           "probability": 0.9986062545522475,
           "distribution": [
            {
             "value": "accepted",
             "probability": 0.0006804538664545584
            },
            {
             "value": "waitlisted",
             "probability": 0.0007132915812979836
            },
            {
             "value": "rejected",
             "probability": 0.9986062545522475
            }
           ]
          },
          "issues": {
           "value": 2,
           "probability": 0.9982761270123944,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005319244753484255
            },
            {
             "value": 1,
             "probability": 0.0005571998293026436
            },
            {
             "value": 2,
             "probability": 0.9982761270123944
            },
            {
             "value": 3,
             "probability": 0.0006347486829545226
            }
           ]
          }
         },
         "joint_probability": 0.9978386402877535,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0021636988211943295,
        "field_brier": 3.0134038930168943e-06
       }
      },
      "timing_ms": {
       "scoring": 36.683085141703486,
       "base_joint": 0.06761890836060047,
       "constrained_joint": 0.04751002416014671,
       "constrained_total": 36.73059516586363
      },
      "input_tokens": 645,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01f3c41ba4c4d009-0010",
      "group": "bendsynth-01f3c41ba4c4d009",
      "provenance": "bendsynth-v1; family=bendsynth-01f3c41ba4c4d009; domain=cattle auction lot intake desk; parameters=quota=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_free": false,
       "issues": 1,
       "decision": "waitlisted"
      },
      "logits": [
       [
        0.7358258962631226,
        -5.99259614944458
       ],
       [
        -6.4727067947387695,
        0.7349795699119568,
        -6.6651611328125,
        -6.540884971618652
       ],
       [
        -6.158034324645996,
        0.7350624799728394,
        -6.22796106338501
       ]
      ],
      "joint_logits": [
       -8.103143692016602,
       -8.193777084350586,
       -8.072144508361816,
       -7.872993469238281,
       0.7343187928199768,
       -8.0783052444458,
       -8.23358154296875,
       -8.377167701721191,
       -8.298998832702637,
       -8.125089645385742,
       -8.384593963623047,
       -8.137636184692383,
       -8.404887199401855,
       -8.421032905578613,
       -8.468345642089844,
       -8.340328216552734,
       -8.416712760925293,
       -8.391796112060547,
       -8.610794067382812,
       -8.645867347717285,
       -8.667085647583008,
       -8.391731262207031,
       -8.524243354797363,
       -8.511322975158691
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_free": {
           "value": false,
           "probability": 0.9988050103825378,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988050103825378
            },
            {
             "value": true,
             "probability": 0.0011949896174621966
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.9979600921995578,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007393578925049196
            },
            {
             "value": 1,
             "probability": 0.9979600921995578
            },
            {
             "value": 2,
             "probability": 0.0006099200101749716
            },
            {
             "value": 3,
             "probability": 0.000690629897762168
            }
           ]
          },
          "decision": {
           "value": "waitlisted",
           "probability": 0.9980428402538059,
           "distribution": [
            {
             "value": "accepted",
             "probability": 0.001012780224428995
            },
            {
             "value": "waitlisted",
             "probability": 0.9980428402538059
            },
            {
             "value": "rejected",
             "probability": 0.0009443795217651445
            }
           ]
          }
         },
         "joint_probability": 0.9973008390301736,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027028102729805,
        "field_brier": 4.720299036646815e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_free": {
           "value": false,
           "probability": 0.9988050103825378,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988050103825378
            },
            {
             "value": true,
             "probability": 0.0011949896174621966
            }
           ]
          },
          "issues": {
           "value": 1,
           "probability": 0.9979600921995578,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007393578925049196
            },
            {
             "value": 1,
             "probability": 0.9979600921995578
            },
            {
             "value": 2,
             "probability": 0.0006099200101749716
            },
            {
             "value": 3,
             "probability": 0.000690629897762168
            }
           ]
          },
          "decision": {
           "value": "waitlisted",
           "probability": 0.9980428402538059,
           "distribution": [
            {
             "value": "accepted",
             "probability": 0.001012780224428995
            },
            {
             "value": "waitlisted",
             "probability": 0.9980428402538059
            },
            {
             "value": "rejected",
             "probability": 0.0009443795217651445
            }
           ]
          }
         },
         "joint_probability": 0.9973008390301736,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027028102729805,
        "field_brier": 4.720299036646815e-06
       }
      },
      "timing_ms": {
       "scoring": 34.998003859072924,
       "base_joint": 0.07025990635156631,
       "constrained_joint": 0.05547911860048771,
       "constrained_total": 35.05348297767341
      },
      "input_tokens": 621,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01f3c41ba4c4d009-0027",
      "group": "bendsynth-01f3c41ba4c4d009",
      "provenance": "bendsynth-v1; family=bendsynth-01f3c41ba4c4d009; domain=cattle auction lot intake desk; parameters=quota=3; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "issues": 0,
       "decision": "accepted",
       "quota_free": true
      },
      "logits": [
       [
        1.1668554544448853,
        -6.13283109664917,
        -6.190231800079346,
        -6.182308197021484
       ],
       [
        -5.975813388824463,
        1.1671252250671387,
        -5.8500261306762695
       ],
       [
        -5.464587688446045,
        1.1674939393997192
       ]
      ],
      "joint_logits": [
       -8.127645492553711,
       -8.298103332519531,
       -8.067105293273926,
       1.1663936376571655,
       -8.141443252563477,
       -8.00469970703125,
       -7.897093296051025,
       -7.9802470207214355,
       -7.952608108520508,
       -7.942118167877197,
       -7.767547607421875,
       -8.028600692749023,
       -8.070175170898438,
       -8.00821304321289,
       -7.9572343826293945,
       -8.078682899475098,
       -7.7681427001953125,
       -8.044981956481934,
       -8.109048843383789,
       -8.001514434814453,
       -7.914088726043701,
       -8.103164672851562,
       -7.740411758422852,
       -8.025960922241211
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 0,
           "probability": 0.9980468869603216,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980468869603216
            },
            {
             "value": 1,
             "probability": 0.0006744308655012941
            },
            {
             "value": 2,
             "probability": 0.0006368081018361676
            },
            {
             "value": 3,
             "probability": 0.0006418740723409563
            }
           ]
          },
          "decision": {
           "value": "accepted",
           "probability": 0.9983160389980997,
           "distribution": [
            {
             "value": "rejected",
             "probability": 0.0007890950234510137
            },
            {
             "value": "accepted",
             "probability": 0.9983160389980997
            },
            {
             "value": "waitlisted",
             "probability": 0.0008948659784492843
            }
           ]
          },
          "quota_free": {
           "value": true,
           "probability": 0.9986843154834997,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013156845165003728
            },
            {
             "value": true,
             "probability": 0.9986843154834997
            }
           ]
          }
         },
         "joint_probability": 0.9975860501872486,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002416868086924853,
        "field_brier": 4.269422215556637e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "issues": {
           "value": 0,
           "probability": 0.9980468869603216,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9980468869603216
            },
            {
             "value": 1,
             "probability": 0.0006744308655012941
            },
            {
             "value": 2,
             "probability": 0.0006368081018361676
            },
            {
             "value": 3,
             "probability": 0.0006418740723409563
            }
           ]
          },
          "decision": {
           "value": "accepted",
           "probability": 0.9983160389980997,
           "distribution": [
            {
             "value": "rejected",
             "probability": 0.0007890950234510137
            },
            {
             "value": "accepted",
             "probability": 0.9983160389980997
            },
            {
             "value": "waitlisted",
             "probability": 0.0008948659784492843
            }
           ]
          },
          "quota_free": {
           "value": true,
           "probability": 0.9986843154834997,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013156845165003728
            },
            {
             "value": true,
             "probability": 0.9986843154834997
            }
           ]
          }
         },
         "joint_probability": 0.9975860501872486,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002416868086924853,
        "field_brier": 4.269422215556637e-06
       }
      },
      "timing_ms": {
       "scoring": 37.3478289693594,
       "base_joint": 0.06374018266797066,
       "constrained_joint": 0.046459026634693146,
       "constrained_total": 37.39428799599409
      },
      "input_tokens": 672,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05dbaa9e77d169c5-0005",
      "group": "bendsynth-05dbaa9e77d169c5",
      "provenance": "bendsynth-v1; family=bendsynth-05dbaa9e77d169c5; domain=blasting permit counter at a mine office; parameters=standby_days=18; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "wait_satisfied": true,
       "concerns": 1,
       "decision": "approved"
      },
      "logits": [
       [
        -0.6188269853591919,
        -6.926041603088379
       ],
       [
        -7.733669281005859,
        -6.645774841308594,
        -0.6211559176445007,
        -7.606155872344971
       ],
       [
        -0.6193533539772034,
        -7.395224094390869,
        -7.336816787719727
       ]
      ],
      "joint_logits": [
       -9.437670707702637,
       -9.585227966308594,
       -9.67741584777832,
       -7.110924243927002,
       -9.312790870666504,
       -9.153532028198242,
       -0.6218499541282654,
       -9.592079162597656,
       -9.234309196472168,
       -9.354135513305664,
       -9.462911605834961,
       -9.537313461303711,
       -9.442770004272461,
       -9.599637031555176,
       -9.436800003051758,
       -9.172295570373535,
       -9.21778678894043,
       -9.383302688598633,
       -9.375993728637695,
       -9.673940658569336,
       -9.73359489440918,
       -9.294533729553223,
       -9.442023277282715,
       -9.318613052368164
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_satisfied": {
           "value": false,
           "probability": 0.9981802138426512,
           "distribution": [
            {
             "value": false,
             "probability": 0.9981802138426512
            },
            {
             "value": true,
             "probability": 0.0018197861573487505
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.9958582450113759,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000811469696964559
            },
            {
             "value": 1,
             "probability": 0.002408455579914943
            },
            {
             "value": 2,
             "probability": 0.9958582450113759
            },
            {
             "value": 3,
             "probability": 0.0009218297117445254
            }
           ]
          },
          "decision": {
           "value": "approved",
           "probability": 0.9976549332403968,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.9976549332403968
            },
            {
             "value": "conditional approval",
             "probability": 0.0011383009966660772
            },
            {
             "value": "denied",
             "probability": 0.0012067657629371991
            }
           ]
          }
         },
         "joint_probability": 0.9951673210956482,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 8.555290010301269,
        "field_brier": 1.3265532572911563
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_satisfied": {
           "value": false,
           "probability": 0.9981802138426512,
           "distribution": [
            {
             "value": false,
             "probability": 0.9981802138426512
            },
            {
             "value": true,
             "probability": 0.0018197861573487505
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.9958582450113759,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000811469696964559
            },
            {
             "value": 1,
             "probability": 0.002408455579914943
            },
            {
             "value": 2,
             "probability": 0.9958582450113759
            },
            {
             "value": 3,
             "probability": 0.0009218297117445254
            }
           ]
          },
          "decision": {
           "value": "approved",
           "probability": 0.9976549332403968,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.9976549332403968
            },
            {
             "value": "conditional approval",
             "probability": 0.0011383009966660772
            },
            {
             "value": "denied",
             "probability": 0.0012067657629371991
            }
           ]
          }
         },
         "joint_probability": 0.9951673210956482,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 8.555290010301269,
        "field_brier": 1.3265532572911563
       }
      },
      "timing_ms": {
       "scoring": 36.69242491014302,
       "base_joint": 0.06796908564865589,
       "constrained_joint": 0.04761992022395134,
       "constrained_total": 36.74004483036697
      },
      "input_tokens": 717,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05dbaa9e77d169c5-0006",
      "group": "bendsynth-05dbaa9e77d169c5",
      "provenance": "bendsynth-v1; family=bendsynth-05dbaa9e77d169c5; domain=blasting permit counter at a mine office; parameters=standby_days=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "wait_satisfied": true,
       "decision": "denied",
       "concerns": 0
      },
      "logits": [
       [
        -5.807878494262695,
        0.8346034288406372
       ],
       [
        -6.058994293212891,
        -6.219627380371094,
        0.8340290784835815
       ],
       [
        0.8340281248092651,
        -6.476060390472412,
        -6.495317459106445,
        -6.661918640136719
       ]
      ],
      "joint_logits": [
       -8.194148063659668,
       -8.079559326171875,
       -8.066165924072266,
       -8.166751861572266,
       -8.273225784301758,
       -8.199310302734375,
       -8.175480842590332,
       -8.446778297424316,
       -8.406891822814941,
       -8.503604888916016,
       -8.513885498046875,
       -8.698039054870605,
       -7.906148910522461,
       -8.198561668395996,
       -8.230799674987793,
       -8.322818756103516,
       -8.305647850036621,
       -8.355563163757324,
       -8.278565406799316,
       -8.387402534484863,
       0.8334327340126038,
       -8.325150489807129,
       -8.546867370605469,
       -8.855327606201172
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_satisfied": {
           "value": true,
           "probability": 0.9986979100891876,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013020899108123739
            },
            {
             "value": true,
             "probability": 0.9986979100891876
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9981244401657206,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.0010129375981036105
            },
            {
             "value": "conditional approval",
             "probability": 0.0008626222361758475
            },
            {
             "value": "denied",
             "probability": 0.9981244401657206
            }
           ]
          },
          "concerns": {
           "value": 0,
           "probability": 0.9981234370092487,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981234370092487
            },
            {
             "value": 1,
             "probability": 0.0006675029106338073
            },
            {
             "value": 2,
             "probability": 0.000654771622511197
            },
            {
             "value": 3,
             "probability": 0.0005542884576063536
            }
           ]
          }
         },
         "joint_probability": 0.9975293780502325,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002473678972379667,
        "field_brier": 4.460590343160614e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_satisfied": {
           "value": true,
           "probability": 0.9986979100891876,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013020899108123739
            },
            {
             "value": true,
             "probability": 0.9986979100891876
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9981244401657206,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.0010129375981036105
            },
            {
             "value": "conditional approval",
             "probability": 0.0008626222361758475
            },
            {
             "value": "denied",
             "probability": 0.9981244401657206
            }
           ]
          },
          "concerns": {
           "value": 0,
           "probability": 0.9981234370092487,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981234370092487
            },
            {
             "value": 1,
             "probability": 0.0006675029106338073
            },
            {
             "value": 2,
             "probability": 0.000654771622511197
            },
            {
             "value": 3,
             "probability": 0.0005542884576063536
            }
           ]
          }
         },
         "joint_probability": 0.9975293780502325,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002473678972379667,
        "field_brier": 4.460590343160614e-06
       }
      },
      "timing_ms": {
       "scoring": 38.24957809410989,
       "base_joint": 0.06567896343767643,
       "constrained_joint": 0.04892004653811455,
       "constrained_total": 38.29849814064801
      },
      "input_tokens": 735,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05dbaa9e77d169c5-0019",
      "group": "bendsynth-05dbaa9e77d169c5",
      "provenance": "bendsynth-v1; family=bendsynth-05dbaa9e77d169c5; domain=blasting permit counter at a mine office; parameters=standby_days=10; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "concerns": 0,
       "decision": "denied",
       "wait_satisfied": true
      },
      "logits": [
       [
        1.1170505285263062,
        -5.866756439208984,
        -6.076014041900635,
        -6.2936201095581055
       ],
       [
        1.1163886785507202,
        -5.154962539672852,
        -5.735715389251709
       ],
       [
        -5.349244594573975,
        1.1177786588668823
       ]
      ],
      "joint_logits": [
       -8.140506744384766,
       1.1159034967422485,
       -7.337411880493164,
       -6.512920379638672,
       -7.721482276916504,
       -7.363733768463135,
       -8.619447708129883,
       -8.04080581665039,
       -7.163407802581787,
       -7.256764888763428,
       -7.837819576263428,
       -7.694972991943359,
       -8.766042709350586,
       -8.368828773498535,
       -7.357494354248047,
       -7.482393741607666,
       -8.035003662109375,
       -7.862884044647217,
       -8.923646926879883,
       -8.701635360717773,
       -7.602375507354736,
       -7.692884922027588,
       -8.229419708251953,
       -8.026998519897461
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 0,
           "probability": 0.9977218869204506,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9977218869204506
            },
            {
             "value": 1,
             "probability": 0.0009246569068519199
            },
            {
             "value": 2,
             "probability": 0.0007500689773013147
            },
            {
             "value": 3,
             "probability": 0.0006033871953961404
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9970617564434796,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.9970617564434796
            },
            {
             "value": "approved",
             "probability": 0.0018841211056114101
            },
            {
             "value": "conditional approval",
             "probability": 0.0010541224509090536
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9984485667086647,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015514332913353082
            },
            {
             "value": true,
             "probability": 0.9984485667086647
            }
           ]
          }
         },
         "joint_probability": 0.9965780363775697,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0034278319311976035,
        "field_brier": 8.359907123787472e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 0,
           "probability": 0.9977218869204506,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9977218869204506
            },
            {
             "value": 1,
             "probability": 0.0009246569068519199
            },
            {
             "value": 2,
             "probability": 0.0007500689773013147
            },
            {
             "value": 3,
             "probability": 0.0006033871953961404
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9970617564434796,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.9970617564434796
            },
            {
             "value": "approved",
             "probability": 0.0018841211056114101
            },
            {
             "value": "conditional approval",
             "probability": 0.0010541224509090536
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9984485667086647,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015514332913353082
            },
            {
             "value": true,
             "probability": 0.9984485667086647
            }
           ]
          }
         },
         "joint_probability": 0.9965780363775697,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0034278319311976035,
        "field_brier": 8.359907123787472e-06
       }
      },
      "timing_ms": {
       "scoring": 36.65010607801378,
       "base_joint": 0.06463914178311825,
       "constrained_joint": 0.04718010313808918,
       "constrained_total": 36.69728618115187
      },
      "input_tokens": 729,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05dbaa9e77d169c5-0031",
      "group": "bendsynth-05dbaa9e77d169c5",
      "provenance": "bendsynth-v1; family=bendsynth-05dbaa9e77d169c5; domain=blasting permit counter at a mine office; parameters=standby_days=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "conditional approval",
       "concerns": 1,
       "wait_satisfied": true
      },
      "logits": [
       [
        -4.381749629974365,
        2.1305248737335205,
        -3.0077500343322754
       ],
       [
        -4.992925643920898,
        2.134617328643799,
        -4.2513580322265625,
        -5.059046268463135
       ],
       [
        -4.091748237609863,
        2.1358790397644043
       ]
      ],
      "joint_logits": [
       -6.758222579956055,
       -6.951354026794434,
       -6.548775672912598,
       -5.788425445556641,
       -6.146806716918945,
       -6.421064376831055,
       -6.7545857429504395,
       -6.951274871826172,
       -6.988719463348389,
       -6.326474189758301,
       -5.806489944458008,
       2.1288321018218994,
       -6.438440799713135,
       -5.406304836273193,
       -7.005058765411377,
       -6.483295917510986,
       -7.434393405914307,
       -6.595861434936523,
       -6.118999481201172,
       -3.242598533630371,
       -6.929866313934326,
       -5.680331707000732,
       -7.409857273101807,
       -6.736008167266846
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "conditional approval",
           "probability": 0.9927007709422335,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.001474257684403142
            },
            {
             "value": "conditional approval",
             "probability": 0.9927007709422335
            },
            {
             "value": "denied",
             "probability": 0.005824971373363342
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9967714121939022,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008000974868900145
            },
            {
             "value": 1,
             "probability": 0.9967714121939022
            },
            {
             "value": 2,
             "probability": 0.0016795844159490842
            },
            {
             "value": 3,
             "probability": 0.0007489059032586786
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9980297589052829,
           "distribution": [
            {
             "value": false,
             "probability": 0.0019702410947171557
            },
            {
             "value": true,
             "probability": 0.9980297589052829
            }
           ]
          }
         },
         "joint_probability": 0.9910215659981019,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.009018983034139741,
        "field_brier": 3.719732369111227e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "conditional approval",
           "probability": 0.9927007709422335,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.001474257684403142
            },
            {
             "value": "conditional approval",
             "probability": 0.9927007709422335
            },
            {
             "value": "denied",
             "probability": 0.005824971373363342
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9967714121939022,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008000974868900145
            },
            {
             "value": 1,
             "probability": 0.9967714121939022
            },
            {
             "value": 2,
             "probability": 0.0016795844159490842
            },
            {
             "value": 3,
             "probability": 0.0007489059032586786
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9980297589052829,
           "distribution": [
            {
             "value": false,
             "probability": 0.0019702410947171557
            },
            {
             "value": true,
             "probability": 0.9980297589052829
            }
           ]
          }
         },
         "joint_probability": 0.9910215659981019,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.009018983034139741,
        "field_brier": 3.719732369111227e-05
       }
      },
      "timing_ms": {
       "scoring": 36.62114613689482,
       "base_joint": 0.0635890755802393,
       "constrained_joint": 0.04730001091957092,
       "constrained_total": 36.66844614781439
      },
      "input_tokens": 729,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05dbaa9e77d169c5-0032",
      "group": "bendsynth-05dbaa9e77d169c5",
      "provenance": "bendsynth-v1; family=bendsynth-05dbaa9e77d169c5; domain=blasting permit counter at a mine office; parameters=standby_days=15; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "wait_satisfied": true,
       "concerns": 1,
       "decision": "conditional approval"
      },
      "logits": [
       [
        -4.899632453918457,
        -0.9033955931663513
       ],
       [
        -6.679112434387207,
        -0.9113186597824097,
        -4.789838790893555,
        -6.834214687347412
       ],
       [
        -6.683804512023926,
        -0.8919941782951355,
        -6.46815299987793
       ]
      ],
      "joint_logits": [
       -9.072770118713379,
       -8.490447044372559,
       -9.013883590698242,
       -8.592814445495605,
       -5.2729644775390625,
       -8.51799488067627,
       -9.010019302368164,
       -7.173585414886475,
       -8.764078140258789,
       -8.898560523986816,
       -8.684111595153809,
       -8.853012084960938,
       -8.996066093444824,
       -7.544144630432129,
       -8.728412628173828,
       -8.316924095153809,
       -0.9267587661743164,
       -7.821235656738281,
       -8.632369041442871,
       -4.9788899421691895,
       -8.47382640838623,
       -8.834794998168945,
       -8.03938102722168,
       -8.74386978149414
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_satisfied": {
           "value": true,
           "probability": 0.981947200983313,
           "distribution": [
            {
             "value": false,
             "probability": 0.01805279901668696
            },
            {
             "value": true,
             "probability": 0.981947200983313
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9741978859935484,
           "distribution": [
            {
             "value": 0,
             "probability": 0.003045974335146713
            },
            {
             "value": 1,
             "probability": 0.9741978859935484
            },
            {
             "value": 2,
             "probability": 0.020147787616038203
            },
            {
             "value": 3,
             "probability": 0.0026083520552666987
            }
           ]
          },
          "decision": {
           "value": "conditional approval",
           "probability": 0.9932069264662712,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.0030317158215045757
            },
            {
             "value": "conditional approval",
             "probability": 0.9932069264662712
            },
            {
             "value": "approved",
             "probability": 0.0037613577122242
            }
           ]
          }
         },
         "joint_probability": 0.9592717545156956,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.04158087144067078,
        "field_brier": 0.0006030186528292
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "wait_satisfied": {
           "value": true,
           "probability": 0.981947200983313,
           "distribution": [
            {
             "value": false,
             "probability": 0.01805279901668696
            },
            {
             "value": true,
             "probability": 0.981947200983313
            }
           ]
          },
          "concerns": {
           "value": 1,
           "probability": 0.9741978859935484,
           "distribution": [
            {
             "value": 0,
             "probability": 0.003045974335146713
            },
            {
             "value": 1,
             "probability": 0.9741978859935484
            },
            {
             "value": 2,
             "probability": 0.020147787616038203
            },
            {
             "value": 3,
             "probability": 0.0026083520552666987
            }
           ]
          },
          "decision": {
           "value": "conditional approval",
           "probability": 0.9932069264662712,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.0030317158215045757
            },
            {
             "value": "conditional approval",
             "probability": 0.9932069264662712
            },
            {
             "value": "approved",
             "probability": 0.0037613577122242
            }
           ]
          }
         },
         "joint_probability": 0.9592717545156956,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.04158087144067078,
        "field_brier": 0.0006030186528292
       }
      },
      "timing_ms": {
       "scoring": 37.51573688350618,
       "base_joint": 0.0666491687297821,
       "constrained_joint": 0.047939131036400795,
       "constrained_total": 37.56367601454258
      },
      "input_tokens": 708,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05dbaa9e77d169c5-0035",
      "group": "bendsynth-05dbaa9e77d169c5",
      "provenance": "bendsynth-v1; family=bendsynth-05dbaa9e77d169c5; domain=blasting permit counter at a mine office; parameters=standby_days=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "conditional approval",
       "concerns": 2,
       "wait_satisfied": true
      },
      "logits": [
       [
        -6.374074935913086,
        -2.3106319904327393,
        -7.355257034301758
       ],
       [
        -7.843042373657227,
        -8.065542221069336,
        -2.301919460296631,
        -7.1695051193237305
       ],
       [
        -2.477214813232422,
        -4.041780471801758
       ]
      ],
      "joint_logits": [
       -9.710916519165039,
       -9.735934257507324,
       -9.951981544494629,
       -9.998013496398926,
       -7.160377025604248,
       -7.600043296813965,
       -9.124308586120605,
       -9.150569915771484,
       -9.175731658935547,
       -9.394509315490723,
       -9.426952362060547,
       -9.535123825073242,
       -2.4980907440185547,
       -4.121505260467529,
       -8.470294952392578,
       -8.581815719604492,
       -10.043773651123047,
       -10.064168930053711,
       -10.265504837036133,
       -10.305424690246582,
       -8.571915626525879,
       -8.8875150680542,
       -9.437884330749512,
       -9.461057662963867
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "conditional approval",
           "probability": 0.976912036427654,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.016792861277194926
            },
            {
             "value": "conditional approval",
             "probability": 0.976912036427654
            },
            {
             "value": "approved",
             "probability": 0.006295102295151029
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.9854607585192796,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0038650952141650527
            },
            {
             "value": 1,
             "probability": 0.0030940664143781645
            },
            {
             "value": 2,
             "probability": 0.9854607585192796
            },
            {
             "value": 3,
             "probability": 0.007580079852177171
            }
           ]
          },
          "wait_satisfied": {
           "value": false,
           "probability": 0.8270075056644797,
           "distribution": [
            {
             "value": false,
             "probability": 0.8270075056644797
            },
            {
             "value": true,
             "probability": 0.17299249433552027
            }
           ]
          }
         },
         "joint_probability": 0.8099219957588826,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 1.834231853934161,
        "field_brier": 0.4563436235922865
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "conditional approval",
           "probability": 0.976912036427654,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.016792861277194926
            },
            {
             "value": "conditional approval",
             "probability": 0.976912036427654
            },
            {
             "value": "approved",
             "probability": 0.006295102295151029
            }
           ]
          },
          "concerns": {
           "value": 2,
           "probability": 0.9854607585192796,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0038650952141650527
            },
            {
             "value": 1,
             "probability": 0.0030940664143781645
            },
            {
             "value": 2,
             "probability": 0.9854607585192796
            },
            {
             "value": 3,
             "probability": 0.007580079852177171
            }
           ]
          },
          "wait_satisfied": {
           "value": false,
           "probability": 0.8270075056644797,
           "distribution": [
            {
             "value": false,
             "probability": 0.8270075056644797
            },
            {
             "value": true,
             "probability": 0.17299249433552027
            }
           ]
          }
         },
         "joint_probability": 0.8099219957588826,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 1.834231853934161,
        "field_brier": 0.4563436235922865
       }
      },
      "timing_ms": {
       "scoring": 39.38668593764305,
       "base_joint": 0.06580888293683529,
       "constrained_joint": 0.04691001959145069,
       "constrained_total": 39.4335959572345
      },
      "input_tokens": 705,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05dbaa9e77d169c5-0037",
      "group": "bendsynth-05dbaa9e77d169c5",
      "provenance": "bendsynth-v1; family=bendsynth-05dbaa9e77d169c5; domain=blasting permit counter at a mine office; parameters=standby_days=20; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "concerns": 1,
       "wait_satisfied": true,
       "decision": "conditional approval"
      },
      "logits": [
       [
        -5.944486618041992,
        1.285581111907959,
        -5.551427841186523,
        -5.982884407043457
       ],
       [
        -5.011711597442627,
        1.2862346172332764
       ],
       [
        1.2834635972976685,
        -4.467069149017334,
        -5.260333061218262
       ]
      ],
      "joint_logits": [
       -9.047200202941895,
       -7.369295597076416,
       -7.795866966247559,
       -7.9988884925842285,
       -7.275516510009766,
       -7.745545387268066,
       -7.129055500030518,
       -6.7087554931640625,
       -7.206925392150879,
       1.282790184020996,
       -5.161922454833984,
       -6.446033954620361,
       -8.695804595947266,
       -7.133842945098877,
       -7.431514263153076,
       -7.415250301361084,
       -6.737091064453125,
       -7.530272960662842,
       -9.064176559448242,
       -7.355405330657959,
       -7.763777732849121,
       -8.146607398986816,
       -7.444580554962158,
       -7.668091297149658
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 1,
           "probability": 0.9975112478990933,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007226688559828494
            },
            {
             "value": 1,
             "probability": 0.9975112478990933
            },
            {
             "value": 2,
             "probability": 0.001070637472330397
            },
            {
             "value": 3,
             "probability": 0.0006954457725934696
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9981632999071945,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018367000928054195
            },
            {
             "value": true,
             "probability": 0.9981632999071945
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9954011465709148,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.9954011465709148
            },
            {
             "value": "conditional approval",
             "probability": 0.0031664569734102654
            },
            {
             "value": "approved",
             "probability": 0.001432396455674884
            }
           ]
          }
         },
         "joint_probability": 0.9947310104844747,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 6.449995558448934,
        "field_brier": 0.6615058999461396
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "concerns": {
           "value": 1,
           "probability": 0.9975112478990933,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007226688559828494
            },
            {
             "value": 1,
             "probability": 0.9975112478990933
            },
            {
             "value": 2,
             "probability": 0.001070637472330397
            },
            {
             "value": 3,
             "probability": 0.0006954457725934696
            }
           ]
          },
          "wait_satisfied": {
           "value": true,
           "probability": 0.9981632999071945,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018367000928054195
            },
            {
             "value": true,
             "probability": 0.9981632999071945
            }
           ]
          },
          "decision": {
           "value": "denied",
           "probability": 0.9954011465709148,
           "distribution": [
            {
             "value": "denied",
             "probability": 0.9954011465709148
            },
            {
             "value": "conditional approval",
             "probability": 0.0031664569734102654
            },
            {
             "value": "approved",
             "probability": 0.001432396455674884
            }
           ]
          }
         },
         "joint_probability": 0.9947310104844747,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 6.449995558448934,
        "field_brier": 0.6615058999461396
       }
      },
      "timing_ms": {
       "scoring": 37.356289103627205,
       "base_joint": 0.06711995229125023,
       "constrained_joint": 0.04753889515995979,
       "constrained_total": 37.403827998787165
      },
      "input_tokens": 759,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aa8b3ddbb653901-0007",
      "group": "bendsynth-0aa8b3ddbb653901",
      "provenance": "bendsynth-v1; family=bendsynth-0aa8b3ddbb653901; domain=regional archive document request desk; parameters=quota=101; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "exempt": false,
       "decision": "partial approval",
       "extra_steps": 3
      },
      "logits": [
       [
        1.8882955312728882,
        -4.49484395980835
       ],
       [
        1.8862175941467285,
        -4.665894985198975,
        -4.168622970581055
       ],
       [
        -5.485533237457275,
        -5.370466232299805,
        -4.626373767852783,
        1.887171745300293
       ]
      ],
      "joint_logits": [
       -7.180638313293457,
       -6.9823760986328125,
       -5.90353536605835,
       1.8849791288375854,
       -7.187561988830566,
       -7.072556495666504,
       -6.493422031402588,
       -5.783942222595215,
       -6.96925163269043,
       -6.842400074005127,
       -6.034641265869141,
       -4.981536865234375,
       -7.479648590087891,
       -7.438816070556641,
       -7.002937316894531,
       -6.388914585113525,
       -7.608506679534912,
       -7.505157470703125,
       -6.909383296966553,
       -6.7489752769470215,
       -7.372671604156494,
       -7.308165073394775,
       -6.688521862030029,
       -6.398265838623047
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "exempt": {
           "value": false,
           "probability": 0.9983130409939976,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983130409939976
            },
            {
             "value": true,
             "probability": 0.0016869590060023913
            }
           ]
          },
          "decision": {
           "value": "partial approval",
           "probability": 0.9962406121749373,
           "distribution": [
            {
             "value": "partial approval",
             "probability": 0.9962406121749373
            },
            {
             "value": "full approval",
             "probability": 0.0014217326860132278
            },
            {
             "value": "reject",
             "probability": 0.0023376551390495064
            }
           ]
          },
          "extra_steps": {
           "value": 3,
           "probability": 0.9971917593077839,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006264027109755889
            },
            {
             "value": 1,
             "probability": 0.0007027915172029903
            },
            {
             "value": 2,
             "probability": 0.0014790464640374037
            },
            {
             "value": 3,
             "probability": 0.9971917593077839
            }
           ]
          }
         },
         "joint_probability": 0.9950077289891183,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0050047740254541145,
        "field_brier": 1.2756901358622461e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "exempt": {
           "value": false,
           "probability": 0.9983130409939976,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983130409939976
            },
            {
             "value": true,
             "probability": 0.0016869590060023913
            }
           ]
          },
          "decision": {
           "value": "partial approval",
           "probability": 0.9962406121749373,
           "distribution": [
            {
             "value": "partial approval",
             "probability": 0.9962406121749373
            },
            {
             "value": "full approval",
             "probability": 0.0014217326860132278
            },
            {
             "value": "reject",
             "probability": 0.0023376551390495064
            }
           ]
          },
          "extra_steps": {
           "value": 3,
           "probability": 0.9971917593077839,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006264027109755889
            },
            {
             "value": 1,
             "probability": 0.0007027915172029903
            },
            {
             "value": 2,
             "probability": 0.0014790464640374037
            },
            {
             "value": 3,
             "probability": 0.9971917593077839
            }
           ]
          }
         },
         "joint_probability": 0.9950077289891183,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0050047740254541145,
        "field_brier": 1.2756901358622461e-05
       }
      },
      "timing_ms": {
       "scoring": 35.36871913820505,
       "base_joint": 0.07057888433337212,
       "constrained_joint": 0.04805997014045715,
       "constrained_total": 35.41677910834551
      },
      "input_tokens": 615,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aa8b3ddbb653901-0033",
      "group": "bendsynth-0aa8b3ddbb653901",
      "provenance": "bendsynth-v1; family=bendsynth-0aa8b3ddbb653901; domain=regional archive document request desk; parameters=quota=32; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "exempt": false,
       "decision": "reject",
       "extra_steps": 3
      },
      "logits": [
       [
        1.2229279279708862,
        -4.994844436645508
       ],
       [
        -4.929073333740234,
        -2.521606683731079,
        1.1988584995269775
       ],
       [
        -5.861227512359619,
        -5.7339324951171875,
        -5.253529071807861,
        1.2215912342071533
       ]
      ],
      "joint_logits": [
       -7.488430500030518,
       -7.374332427978516,
       -6.9815778732299805,
       -5.930114269256592,
       -7.704867839813232,
       -7.605169296264648,
       -6.835073947906494,
       -2.5715603828430176,
       -7.426041126251221,
       -7.197728157043457,
       -6.483649253845215,
       1.1972604990005493,
       -7.578209400177002,
       -7.546605110168457,
       -7.177031993865967,
       -7.329020023345947,
       -8.034687995910645,
       -7.9044904708862305,
       -7.795907974243164,
       -7.148787975311279,
       -7.8077898025512695,
       -7.675513744354248,
       -7.559304714202881,
       -6.877161026000977
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "exempt": {
           "value": false,
           "probability": 0.998010285272532,
           "distribution": [
            {
             "value": false,
             "probability": 0.998010285272532
            },
            {
             "value": true,
             "probability": 0.0019897147274680247
            }
           ]
          },
          "decision": {
           "value": "partial approval",
           "probability": 0.9742754379625027,
           "distribution": [
            {
             "value": "full approval",
             "probability": 0.0021249801538417417
            },
            {
             "value": "reject",
             "probability": 0.023599581883655543
            },
            {
             "value": "partial approval",
             "probability": 0.9742754379625027
            }
           ]
          },
          "extra_steps": {
           "value": 3,
           "probability": 0.9966770064089839,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008366147546397904
            },
            {
             "value": 1,
             "probability": 0.0009501868822994607
            },
            {
             "value": 2,
             "probability": 0.0015361919540768038
            },
            {
             "value": 3,
             "probability": 0.9966770064089839
            }
           ]
          }
         },
         "joint_probability": 0.9727198775881725,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 3.7964800156824245,
        "field_brier": 0.6341992813121103
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "exempt": {
           "value": false,
           "probability": 0.998010285272532,
           "distribution": [
            {
             "value": false,
             "probability": 0.998010285272532
            },
            {
             "value": true,
             "probability": 0.0019897147274680247
            }
           ]
          },
          "decision": {
           "value": "partial approval",
           "probability": 0.9742754379625027,
           "distribution": [
            {
             "value": "full approval",
             "probability": 0.0021249801538417417
            },
            {
             "value": "reject",
             "probability": 0.023599581883655543
            },
            {
             "value": "partial approval",
             "probability": 0.9742754379625027
            }
           ]
          },
          "extra_steps": {
           "value": 3,
           "probability": 0.9966770064089839,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008366147546397904
            },
            {
             "value": 1,
             "probability": 0.0009501868822994607
            },
            {
             "value": 2,
             "probability": 0.0015361919540768038
            },
            {
             "value": 3,
             "probability": 0.9966770064089839
            }
           ]
          }
         },
         "joint_probability": 0.9727198775881725,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 3.7964800156824245,
        "field_brier": 0.6341992813121103
       }
      },
      "timing_ms": {
       "scoring": 35.31318996101618,
       "base_joint": 0.07582898251712322,
       "constrained_joint": 0.06668898276984692,
       "constrained_total": 35.379878943786025
      },
      "input_tokens": 660,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aa8b3ddbb653901-0038",
      "group": "bendsynth-0aa8b3ddbb653901",
      "provenance": "bendsynth-v1; family=bendsynth-0aa8b3ddbb653901; domain=regional archive document request desk; parameters=quota=89; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "exempt": false,
       "decision": "partial approval",
       "extra_steps": 3
      },
      "logits": [
       [
        1.3956966400146484,
        -5.34678840637207
       ],
       [
        -5.463744163513184,
        -5.614931106567383,
        1.394924283027649
       ],
       [
        -5.997422218322754,
        -6.0318450927734375,
        -5.692876815795898,
        1.3948314189910889
       ]
      ],
      "joint_logits": [
       -7.600200653076172,
       -7.641643047332764,
       -7.4030842781066895,
       -7.076817035675049,
       -7.663773059844971,
       -7.650491237640381,
       -7.492515563964844,
       -7.328843116760254,
       -7.789657115936279,
       -7.726027488708496,
       -7.0382280349731445,
       1.3941287994384766,
       -7.71936559677124,
       -7.809090614318848,
       -7.635592460632324,
       -7.662094593048096,
       -8.06142807006836,
       -8.221678733825684,
       -7.691511154174805,
       -7.721444606781006,
       -7.9817118644714355,
       -8.018763542175293,
       -7.8597917556762695,
       -7.777637958526611
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "exempt": {
           "value": false,
           "probability": 0.9988216780416066,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988216780416066
            },
            {
             "value": true,
             "probability": 0.0011783219583933837
            }
           ]
          },
          "decision": {
           "value": "partial approval",
           "probability": 0.9980505568671026,
           "distribution": [
            {
             "value": "full approval",
             "probability": 0.0010482640950964075
            },
            {
             "value": "reject",
             "probability": 0.0009011790378009718
            },
            {
             "value": "partial approval",
             "probability": 0.9980505568671026
            }
           ]
          },
          "extra_steps": {
           "value": 3,
           "probability": 0.997957701291359,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006147482326606397
            },
            {
             "value": 1,
             "probability": 0.0005939468925642374
            },
            {
             "value": 2,
             "probability": 0.0008336035834161488
            },
            {
             "value": 3,
             "probability": 0.997957701291359
            }
           ]
          }
         },
         "joint_probability": 0.9972568241638604,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027469452379730213,
        "field_brier": 4.694920775084026e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "exempt": {
           "value": false,
           "probability": 0.9988216780416066,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988216780416066
            },
            {
             "value": true,
             "probability": 0.0011783219583933837
            }
           ]
          },
          "decision": {
           "value": "partial approval",
           "probability": 0.9980505568671026,
           "distribution": [
            {
             "value": "full approval",
             "probability": 0.0010482640950964075
            },
            {
             "value": "reject",
             "probability": 0.0009011790378009718
            },
            {
             "value": "partial approval",
             "probability": 0.9980505568671026
            }
           ]
          },
          "extra_steps": {
           "value": 3,
           "probability": 0.997957701291359,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006147482326606397
            },
            {
             "value": 1,
             "probability": 0.0005939468925642374
            },
            {
             "value": 2,
             "probability": 0.0008336035834161488
            },
            {
             "value": 3,
             "probability": 0.997957701291359
            }
           ]
          }
         },
         "joint_probability": 0.9972568241638604,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027469452379730213,
        "field_brier": 4.694920775084026e-06
       }
      },
      "timing_ms": {
       "scoring": 34.14793289266527,
       "base_joint": 0.07621897384524345,
       "constrained_joint": 0.04752003587782383,
       "constrained_total": 34.19545292854309
      },
      "input_tokens": 612,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-04788a5547abfc18-0015",
      "group": "bendsynth-04788a5547abfc18",
      "provenance": "bendsynth-v1; family=bendsynth-04788a5547abfc18; domain=swimming pool lane booking chatbot; parameters=max_group=7,min_notice=7; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "short_notice": false,
       "flags": 1,
       "decision": "confirmed",
       "over_cap": false
      },
      "logits": [
       [
        1.731694221496582,
        -5.604689121246338
       ],
       [
        -6.2195234298706055,
        1.731382966041565,
        -6.382892608642578,
        -6.343057632446289
       ],
       [
        -6.348954200744629,
        -6.330383777618408,
        1.7312225103378296,
        -5.872414588928223
       ],
       [
        1.7317005395889282,
        -5.614584445953369
       ]
      ],
      "joint_logits": [
       -8.903159141540527,
       -9.220096588134766,
       -8.886154174804688,
       -9.188701629638672,
       -8.534622192382812,
       -8.766668319702148,
       -8.572599411010742,
       -8.949766159057617,
       -8.194185256958008,
       -8.930217742919922,
       -8.488274574279785,
       -8.793242454528809,
       1.7307788133621216,
       -8.065632820129395,
       -7.146780490875244,
       -8.565970420837402,
       -9.080838203430176,
       -9.398544311523438,
       -9.075974464416504,
       -9.328448295593262,
       -8.694457054138184,
       -8.948999404907227,
       -8.740104675292969,
       -9.109652519226074,
       -9.032726287841797,
       -9.354341506958008,
       -9.030624389648438,
       -9.303901672363281,
       -8.724059104919434,
       -8.90732479095459,
       -8.704168319702148,
       -9.057982444763184,
       -9.265643119812012,
       -9.509984970092773,
       -9.195549964904785,
       -9.387589454650879,
       -8.81700325012207,
       -9.094525337219238,
       -8.979575157165527,
       -9.181252479553223,
       -8.949976921081543,
       -9.195867538452148,
       -8.794293403625488,
       -9.135933876037598,
       -7.856431484222412,
       -8.86108684539795,
       -8.558526039123535,
       -8.862035751342773,
       -9.422202110290527,
       -9.643174171447754,
       -9.346158027648926,
       -9.55604362487793,
       -8.963953971862793,
       -9.255976676940918,
       -9.149348258972168,
       -9.330340385437012,
       -9.363260269165039,
       -9.555845260620117,
       -9.304487228393555,
       -9.490091323852539,
       -8.94757080078125,
       -9.213075637817383,
       -9.084744453430176,
       -9.257597923278809
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "short_notice": {
           "value": false,
           "probability": 0.9993490218485608,
           "distribution": [
            {
             "value": false,
             "probability": 0.9993490218485608
            },
            {
             "value": true,
             "probability": 0.0006509781514391636
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9990379496321263,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003520036845731151
            },
            {
             "value": 1,
             "probability": 0.9990379496321263
            },
            {
             "value": 2,
             "probability": 0.0002989489485337093
            },
            {
             "value": 3,
             "probability": 0.00031109773476679177
            }
           ]
          },
          "decision": {
           "value": "confirmed",
           "probability": 0.998877590395504,
           "distribution": [
            {
             "value": "refer to staff",
             "probability": 0.00030926881746783536
            },
            {
             "value": "deny",
             "probability": 0.0003150657709503589
            },
            {
             "value": "confirmed",
             "probability": 0.998877590395504
            },
            {
             "value": "waitlist",
             "probability": 0.0004980750160779182
            }
           ]
          },
          "over_cap": {
           "value": false,
           "probability": 0.9993554317851717,
           "distribution": [
            {
             "value": false,
             "probability": 0.9993554317851717
            },
            {
             "value": true,
             "probability": 0.0006445682148282244
            }
           ]
          }
         },
         "joint_probability": 0.9984345829810384,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0015666435643871214,
        "field_brier": 1.1542192341274965e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "short_notice": {
           "value": false,
           "probability": 0.9993490218485608,
           "distribution": [
            {
             "value": false,
             "probability": 0.9993490218485608
            },
            {
             "value": true,
             "probability": 0.0006509781514391636
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9990379496321263,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003520036845731151
            },
            {
             "value": 1,
             "probability": 0.9990379496321263
            },
            {
             "value": 2,
             "probability": 0.0002989489485337093
            },
            {
             "value": 3,
             "probability": 0.00031109773476679177
            }
           ]
          },
          "decision": {
           "value": "confirmed",
           "probability": 0.998877590395504,
           "distribution": [
            {
             "value": "refer to staff",
             "probability": 0.00030926881746783536
            },
            {
             "value": "deny",
             "probability": 0.0003150657709503589
            },
            {
             "value": "confirmed",
             "probability": 0.998877590395504
            },
            {
             "value": "waitlist",
             "probability": 0.0004980750160779182
            }
           ]
          },
          "over_cap": {
           "value": false,
           "probability": 0.9993554317851717,
           "distribution": [
            {
             "value": false,
             "probability": 0.9993554317851717
            },
            {
             "value": true,
             "probability": 0.0006445682148282244
            }
           ]
          }
         },
         "joint_probability": 0.9984345829810384,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0015666435643871214,
        "field_brier": 1.1542192341274965e-06
       }
      },
      "timing_ms": {
       "scoring": 38.71961310505867,
       "base_joint": 0.1210290938615799,
       "constrained_joint": 0.10745902545750141,
       "constrained_total": 38.82707213051617
      },
      "input_tokens": 841,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-04788a5547abfc18-0036",
      "group": "bendsynth-04788a5547abfc18",
      "provenance": "bendsynth-v1; family=bendsynth-04788a5547abfc18; domain=swimming pool lane booking chatbot; parameters=max_group=10,min_notice=3; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "refer to staff",
       "short_notice": false,
       "flags": 1,
       "over_cap": false
      },
      "logits": [
       [
        -6.501741886138916,
        -0.6466560959815979,
        -6.776577472686768,
        0.20442494750022888
       ],
       [
        0.5601651072502136,
        -6.0778303146362305
       ],
       [
        -6.7867536544799805,
        0.5596187710762024,
        -6.852400302886963,
        -6.845839500427246
       ],
       [
        0.5603010654449463,
        -6.187541961669922
       ]
      ],
      "joint_logits": [
       -9.45837688446045,
       -9.692880630493164,
       -7.562416076660156,
       -9.553023338317871,
       -9.522253036499023,
       -9.80129337310791,
       -9.532831192016602,
       -9.74331283569336,
       -9.58730411529541,
       -9.873722076416016,
       -9.310049057006836,
       -9.645722389221191,
       -9.646584510803223,
       -9.86667251586914,
       -9.621557235717773,
       -9.852084159851074,
       -9.41579818725586,
       -9.656329154968262,
       -0.648776113986969,
       -9.426466941833496,
       -9.434621810913086,
       -9.723274230957031,
       -9.52717113494873,
       -9.692278861999512,
       -9.47171401977539,
       -9.58134937286377,
       -8.998884201049805,
       -9.581735610961914,
       -9.584569931030273,
       -9.649080276489258,
       -9.543904304504395,
       -9.633813858032227,
       -9.575146675109863,
       -9.68521785736084,
       -8.389387130737305,
       -9.683088302612305,
       -9.646052360534668,
       -9.762675285339355,
       -9.631780624389648,
       -9.71901798248291,
       -9.5861177444458,
       -9.931922912597656,
       -9.513832092285156,
       -9.690011978149414,
       -9.699783325195312,
       -9.987054824829102,
       -9.65603256225586,
       -9.972965240478516,
       -9.256783485412598,
       -9.548126220703125,
       0.20337645709514618,
       -9.20622444152832,
       -9.26181411743164,
       -9.616520881652832,
       -9.356401443481445,
       -9.59919261932373,
       -9.408210754394531,
       -9.438867568969727,
       -8.694243431091309,
       -9.50849723815918,
       -9.493622779846191,
       -9.534695625305176,
       -9.486544609069824,
       -9.489625930786133
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.6997382036813206,
           "distribution": [
            {
             "value": "confirmed",
             "probability": 0.0008560210901811462
            },
            {
             "value": "refer to staff",
             "probability": 0.298755458490585
            },
            {
             "value": "deny",
             "probability": 0.00065031673791339
            },
            {
             "value": "waitlist",
             "probability": 0.6997382036813206
            }
           ]
          },
          "short_notice": {
           "value": false,
           "probability": 0.9986920633135644,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986920633135644
            },
            {
             "value": true,
             "probability": 0.0013079366864354829
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9981466382934382,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006437323032655177
            },
            {
             "value": 1,
             "probability": 0.9981466382934382
            },
            {
             "value": 2,
             "probability": 0.0006028307799971521
            },
            {
             "value": 3,
             "probability": 0.0006067986232990457
            }
           ]
          },
          "over_cap": {
           "value": false,
           "probability": 0.9988279669137855,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988279669137855
            },
            {
             "value": true,
             "probability": 0.001172033086214471
            }
           ]
          }
         },
         "joint_probability": 0.6990049860321509,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 3,
        "joint_nll": 1.2102499747626765,
        "field_brier": 0.24534734151053786
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.6997382036813206,
           "distribution": [
            {
             "value": "confirmed",
             "probability": 0.0008560210901811462
            },
            {
             "value": "refer to staff",
             "probability": 0.298755458490585
            },
            {
             "value": "deny",
             "probability": 0.00065031673791339
            },
            {
             "value": "waitlist",
             "probability": 0.6997382036813206
            }
           ]
          },
          "short_notice": {
           "value": false,
           "probability": 0.9986920633135644,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986920633135644
            },
            {
             "value": true,
             "probability": 0.0013079366864354829
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9981466382934382,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006437323032655177
            },
            {
             "value": 1,
             "probability": 0.9981466382934382
            },
            {
             "value": 2,
             "probability": 0.0006028307799971521
            },
            {
             "value": 3,
             "probability": 0.0006067986232990457
            }
           ]
          },
          "over_cap": {
           "value": false,
           "probability": 0.9988279669137855,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988279669137855
            },
            {
             "value": true,
             "probability": 0.001172033086214471
            }
           ]
          }
         },
         "joint_probability": 0.6990049860321509,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 64,
         "valid_assignments": 64,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 3,
        "joint_nll": 1.2102499747626765,
        "field_brier": 0.24534734151053786
       }
      },
      "timing_ms": {
       "scoring": 36.7032450158149,
       "base_joint": 0.13755797408521175,
       "constrained_joint": 0.1040289644151926,
       "constrained_total": 36.80727398023009
      },
      "input_tokens": 773,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-049e3537fb5c059c-0015",
      "group": "bendsynth-049e3537fb5c059c",
      "provenance": "bendsynth-v1; family=bendsynth-049e3537fb5c059c; domain=horse sale consignment intake desk at a regional auction barn; parameters=min_bidders=13; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "both_flags": false,
       "decision": "standard",
       "low_interest": false,
       "flags_count": 0
      },
      "logits": [
       [
        0.24525675177574158,
        -6.7931013107299805
       ],
       [
        0.24486403167247772,
        -7.161360740661621,
        -7.073598384857178
       ],
       [
        0.24524317681789398,
        -6.777885437011719
       ],
       [
        0.24496041238307953,
        -7.2168779373168945,
        -7.174309253692627
       ]
      ],
      "joint_logits": [
       0.244246244430542,
       -9.52140998840332,
       -9.48603343963623,
       -9.228150367736816,
       -9.589376449584961,
       -9.584489822387695,
       -9.2565279006958,
       -9.557876586914062,
       -9.481130599975586,
       -9.383113861083984,
       -9.841894149780273,
       -9.789249420166016,
       -9.275261878967285,
       -9.458154678344727,
       -9.391194343566895,
       -9.305570602416992,
       -9.70700454711914,
       -9.66720199584961,
       -9.436463356018066,
       -9.552102088928223,
       -9.523491859436035,
       -9.491423606872559,
       -9.874160766601562,
       -9.83616828918457,
       -9.436142921447754,
       -9.804058074951172,
       -9.780190467834473,
       -9.73104476928711,
       -10.035319328308105,
       -9.999541282653809,
       -9.35116195678711,
       -9.708277702331543,
       -9.66301441192627,
       -9.591517448425293,
       -9.959476470947266,
       -9.902658462524414
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "both_flags": {
           "value": false,
           "probability": 0.9991232033884764,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991232033884764
            },
            {
             "value": true,
             "probability": 0.0008767966115235357
            }
           ]
          },
          "decision": {
           "value": "standard",
           "probability": 0.9987309718593994,
           "distribution": [
            {
             "value": "standard",
             "probability": 0.9987309718593994
            },
            {
             "value": "downgrade",
             "probability": 0.0006066886831903782
            },
            {
             "value": "escalate",
             "probability": 0.000662339457410271
            }
           ]
          },
          "low_interest": {
           "value": false,
           "probability": 0.9991097601380861,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991097601380861
            },
            {
             "value": true,
             "probability": 0.0008902398619140105
            }
           ]
          },
          "flags_count": {
           "value": 0,
           "probability": 0.9988271913143378,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9988271913143378
            },
            {
             "value": 1,
             "probability": 0.0005739250094745342
            },
            {
             "value": 2,
             "probability": 0.0005988836761876535
            }
           ]
          }
         },
         "joint_probability": 0.9981140871604078,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0018876934122335092,
        "field_brier": 1.900831885874673e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "both_flags": {
           "value": false,
           "probability": 0.9991232033884764,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991232033884764
            },
            {
             "value": true,
             "probability": 0.0008767966115235357
            }
           ]
          },
          "decision": {
           "value": "standard",
           "probability": 0.9987309718593994,
           "distribution": [
            {
             "value": "standard",
             "probability": 0.9987309718593994
            },
            {
             "value": "downgrade",
             "probability": 0.0006066886831903782
            },
            {
             "value": "escalate",
             "probability": 0.000662339457410271
            }
           ]
          },
          "low_interest": {
           "value": false,
           "probability": 0.9991097601380861,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991097601380861
            },
            {
             "value": true,
             "probability": 0.0008902398619140105
            }
           ]
          },
          "flags_count": {
           "value": 0,
           "probability": 0.9988271913143378,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9988271913143378
            },
            {
             "value": 1,
             "probability": 0.0005739250094745342
            },
            {
             "value": 2,
             "probability": 0.0005988836761876535
            }
           ]
          }
         },
         "joint_probability": 0.9981140871604078,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0018876934122335092,
        "field_brier": 1.900831885874673e-06
       }
      },
      "timing_ms": {
       "scoring": 35.39620013907552,
       "base_joint": 0.08384906686842442,
       "constrained_joint": 0.06756908260285854,
       "constrained_total": 35.463769221678376
      },
      "input_tokens": 781,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-049e3537fb5c059c-0034",
      "group": "bendsynth-049e3537fb5c059c",
      "provenance": "bendsynth-v1; family=bendsynth-049e3537fb5c059c; domain=horse sale consignment intake desk at a regional auction barn; parameters=min_bidders=21; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "low_interest": false,
       "decision": "escalate",
       "both_flags": true,
       "flags_count": 2
      },
      "logits": [
       [
        -0.5804721713066101,
        -7.707231044769287
       ],
       [
        -0.5808765888214111,
        -8.039688110351562,
        -7.947751998901367
       ],
       [
        -7.513998985290527,
        -0.5806431770324707
       ],
       [
        -7.967923164367676,
        -7.915884971618652,
        -0.580939769744873
       ]
      ],
      "joint_logits": [
       -10.069331169128418,
       -10.045701026916504,
       -9.760077476501465,
       -10.280146598815918,
       -9.89665699005127,
       -0.5816045999526978,
       -10.412209510803223,
       -10.46957015991211,
       -10.269990921020508,
       -10.36323356628418,
       -10.291858673095703,
       -10.267440795898438,
       -10.3212251663208,
       -10.37942123413086,
       -10.1973295211792,
       -10.327973365783691,
       -10.221824645996094,
       -10.085040092468262,
       -10.53665542602539,
       -10.571541786193848,
       -10.398185729980469,
       -10.398967742919922,
       -10.404983520507812,
       -10.241657257080078,
       -10.869674682617188,
       -10.869564056396484,
       -10.720122337341309,
       -10.72334098815918,
       -10.729984283447266,
       -10.595824241638184,
       -10.809490203857422,
       -10.811503410339355,
       -10.621232032775879,
       -10.629307746887207,
       -10.618156433105469,
       -10.483570098876953
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "low_interest": {
           "value": false,
           "probability": 0.9991973261113523,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991973261113523
            },
            {
             "value": true,
             "probability": 0.0008026738886477639
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9987932776752337,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.9987932776752337
            },
            {
             "value": "standard",
             "probability": 0.0005756454610326378
            },
            {
             "value": "downgrade",
             "probability": 0.0006310768637335991
            }
           ]
          },
          "both_flags": {
           "value": true,
           "probability": 0.9990262246507104,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009737753492895997
            },
            {
             "value": true,
             "probability": 0.9990262246507104
            }
           ]
          },
          "flags_count": {
           "value": 2,
           "probability": 0.9987300137359368,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006184748989621082
            },
            {
             "value": 1,
             "probability": 0.0006515113651010602
            },
            {
             "value": 2,
             "probability": 0.9987300137359368
            }
           ]
          }
         },
         "joint_probability": 0.9980663179656841,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0019355540110151948,
        "field_brier": 1.9476738622624077e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "low_interest": {
           "value": false,
           "probability": 0.9991973261113523,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991973261113523
            },
            {
             "value": true,
             "probability": 0.0008026738886477639
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9987932776752337,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.9987932776752337
            },
            {
             "value": "standard",
             "probability": 0.0005756454610326378
            },
            {
             "value": "downgrade",
             "probability": 0.0006310768637335991
            }
           ]
          },
          "both_flags": {
           "value": true,
           "probability": 0.9990262246507104,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009737753492895997
            },
            {
             "value": true,
             "probability": 0.9990262246507104
            }
           ]
          },
          "flags_count": {
           "value": 2,
           "probability": 0.9987300137359368,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006184748989621082
            },
            {
             "value": 1,
             "probability": 0.0006515113651010602
            },
            {
             "value": 2,
             "probability": 0.9987300137359368
            }
           ]
          }
         },
         "joint_probability": 0.9980663179656841,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0019355540110151948,
        "field_brier": 1.9476738622624077e-06
       }
      },
      "timing_ms": {
       "scoring": 36.5854068659246,
       "base_joint": 0.08834898471832275,
       "constrained_joint": 0.06833905354142189,
       "constrained_total": 36.65374591946602
      },
      "input_tokens": 753,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03a7ea76a90ca87f-0004",
      "group": "bendsynth-03a7ea76a90ca87f",
      "provenance": "bendsynth-v1; family=bendsynth-03a7ea76a90ca87f; domain=print shop rush job escalation review; parameters=max_copies=361,min_lead=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "accept rush",
       "problems": 0,
       "deposit_cleared": true
      },
      "logits": [
       [
        -5.781008243560791,
        1.3060276508331299,
        -5.580123424530029
       ],
       [
        1.305776834487915,
        -5.890982151031494,
        -6.004365921020508,
        -5.971245288848877
       ],
       [
        -5.4890947341918945,
        1.3067657947540283
       ]
      ],
      "joint_logits": [
       -8.230168342590332,
       -7.851848125457764,
       -7.9693603515625,
       -7.621439456939697,
       -8.028646469116211,
       -7.705731391906738,
       -8.027022361755371,
       -7.61785364151001,
       -8.140256881713867,
       1.3052922487258911,
       -7.907346725463867,
       -7.501949310302734,
       -8.036896705627441,
       -7.750748157501221,
       -7.992661952972412,
       -7.79837703704834,
       -8.040390968322754,
       -7.5639495849609375,
       -7.773384094238281,
       -7.439292907714844,
       -7.823545932769775,
       -7.528034687042236,
       -7.817373275756836,
       -7.449150085449219
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "accept rush",
           "probability": 0.9981457336824877,
           "distribution": [
            {
             "value": "schedule standard",
             "probability": 0.0008343215732791555
            },
            {
             "value": "accept rush",
             "probability": 0.9981457336824877
            },
            {
             "value": "reject",
             "probability": 0.0010199447442331233
            }
           ]
          },
          "problems": {
           "value": 0,
           "probability": 0.9978954663707527,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978954663707527
            },
            {
             "value": 1,
             "probability": 0.0007474332194947616
            },
            {
             "value": 2,
             "probability": 0.0006673142231740927
            },
            {
             "value": 3,
             "probability": 0.0006897861865784686
            }
           ]
          },
          "deposit_cleared": {
           "value": true,
           "probability": 0.9988828540118931,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011171459881068253
            },
            {
             "value": true,
             "probability": 0.9988828540118931
            }
           ]
          }
         },
         "joint_probability": 0.9974119813739797,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002591373335508363,
        "field_brier": 4.52651504418566e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "accept rush",
           "probability": 0.9981457336824877,
           "distribution": [
            {
             "value": "schedule standard",
             "probability": 0.0008343215732791555
            },
            {
             "value": "accept rush",
             "probability": 0.9981457336824877
            },
            {
             "value": "reject",
             "probability": 0.0010199447442331233
            }
           ]
          },
          "problems": {
           "value": 0,
           "probability": 0.9978954663707527,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978954663707527
            },
            {
             "value": 1,
             "probability": 0.0007474332194947616
            },
            {
             "value": 2,
             "probability": 0.0006673142231740927
            },
            {
             "value": 3,
             "probability": 0.0006897861865784686
            }
           ]
          },
          "deposit_cleared": {
           "value": true,
           "probability": 0.9988828540118931,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011171459881068253
            },
            {
             "value": true,
             "probability": 0.9988828540118931
            }
           ]
          }
         },
         "joint_probability": 0.9974119813739797,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002591373335508363,
        "field_brier": 4.52651504418566e-06
       }
      },
      "timing_ms": {
       "scoring": 38.76481298357248,
       "base_joint": 0.06784917786717415,
       "constrained_joint": 0.04664994776248932,
       "constrained_total": 38.81146293133497
      },
      "input_tokens": 705,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03a7ea76a90ca87f-0017",
      "group": "bendsynth-03a7ea76a90ca87f",
      "provenance": "bendsynth-v1; family=bendsynth-03a7ea76a90ca87f; domain=print shop rush job escalation review; parameters=max_copies=462,min_lead=7; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "deposit_cleared": false,
       "decision": "reject",
       "problems": 2
      },
      "logits": [
       [
        0.5649022459983826,
        -6.000436782836914
       ],
       [
        -6.564924716949463,
        -6.304017543792725,
        0.5644701719284058
       ],
       [
        -7.029849052429199,
        0.5640851855278015,
        -6.29744291305542,
        -6.73500919342041
       ]
      ],
      "joint_logits": [
       -9.013071060180664,
       -8.685225486755371,
       -8.2828369140625,
       -8.648469924926758,
       -8.824789047241211,
       -8.055198669433594,
       -8.139771461486816,
       -8.550768852233887,
       -8.68595027923584,
       0.5634976625442505,
       -7.760657787322998,
       -8.488958358764648,
       -9.036657333374023,
       -8.766654968261719,
       -8.303528785705566,
       -8.69517993927002,
       -8.69281005859375,
       -8.68958854675293,
       -8.066193580627441,
       -8.380659103393555,
       -8.73711109161377,
       -8.413414001464844,
       -8.083173751831055,
       -8.433789253234863
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "deposit_cleared": {
           "value": false,
           "probability": 0.9985936342677183,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985936342677183
            },
            {
             "value": true,
             "probability": 0.0014063657322817196
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9981621309719347,
           "distribution": [
            {
             "value": "schedule standard",
             "probability": 0.0007997318039861186
            },
            {
             "value": "accept rush",
             "probability": 0.0010381372240791849
            },
            {
             "value": "reject",
             "probability": 0.9981621309719347
            }
           ]
          },
          "problems": {
           "value": 1,
           "probability": 0.9977779890918406,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005023773510774739
            },
            {
             "value": 1,
             "probability": 0.9977779890918406
            },
            {
             "value": 2,
             "probability": 0.001044985068691888
            },
            {
             "value": 3,
             "probability": 0.0006746484883900272
            }
           ]
          }
         },
         "joint_probability": 0.9971918929642498,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.326967507032217,
        "field_brier": 0.6644939318992392
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "deposit_cleared": {
           "value": false,
           "probability": 0.9985936342677183,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985936342677183
            },
            {
             "value": true,
             "probability": 0.0014063657322817196
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9981621309719347,
           "distribution": [
            {
             "value": "schedule standard",
             "probability": 0.0007997318039861186
            },
            {
             "value": "accept rush",
             "probability": 0.0010381372240791849
            },
            {
             "value": "reject",
             "probability": 0.9981621309719347
            }
           ]
          },
          "problems": {
           "value": 1,
           "probability": 0.9977779890918406,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005023773510774739
            },
            {
             "value": 1,
             "probability": 0.9977779890918406
            },
            {
             "value": 2,
             "probability": 0.001044985068691888
            },
            {
             "value": 3,
             "probability": 0.0006746484883900272
            }
           ]
          }
         },
         "joint_probability": 0.9971918929642498,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.326967507032217,
        "field_brier": 0.6644939318992392
       }
      },
      "timing_ms": {
       "scoring": 37.823243997991085,
       "base_joint": 0.06690993905067444,
       "constrained_joint": 0.04813913255929947,
       "constrained_total": 37.871383130550385
      },
      "input_tokens": 759,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-00182ea4182590cd-0016",
      "group": "bendsynth-00182ea4182590cd",
      "provenance": "bendsynth-v1; family=bendsynth-00182ea4182590cd; domain=postal parcel redirection file audit; parameters=grace=21; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "within_grace": true,
       "decision": "reject",
       "flags": 1
      },
      "logits": [
       [
        -5.114322662353516,
        1.8301737308502197
       ],
       [
        -5.886446952819824,
        1.8297042846679688,
        -5.792408466339111,
        -5.7736358642578125
       ],
       [
        -5.709743499755859,
        1.829654335975647,
        -5.825239181518555,
        -5.815282821655273
       ]
      ],
      "joint_logits": [
       -7.914553642272949,
       -8.021921157836914,
       -8.033451080322266,
       -7.975233554840088,
       -7.669416427612305,
       -7.6588335037231445,
       -7.7911906242370605,
       -7.775488376617432,
       -7.918788909912109,
       -7.9155592918396,
       -8.030411720275879,
       -8.013387680053711,
       -7.843479633331299,
       -7.910992622375488,
       -7.934981346130371,
       -7.89154577255249,
       -7.854069232940674,
       -7.985408306121826,
       -8.00322437286377,
       -7.951869010925293,
       -7.617213249206543,
       1.8292022943496704,
       -7.6796159744262695,
       -7.819144248962402,
       -7.767283916473389,
       -7.604531288146973,
       -7.910778999328613,
       -7.8831892013549805,
       -7.771313667297363,
       -7.714449882507324,
       -7.908529758453369,
       -7.870169162750244
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_grace": {
           "value": true,
           "probability": 0.9990370025639944,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009629974360055753
            },
            {
             "value": true,
             "probability": 0.9990370025639944
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9985681914335498,
           "distribution": [
            {
             "value": "resubmit",
             "probability": 0.00044493432850654057
            },
            {
             "value": "reject",
             "probability": 0.9985681914335498
            },
            {
             "value": "approve",
             "probability": 0.0004888056254608133
            },
            {
             "value": "approve priority",
             "probability": 0.0004980686124828774
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9985183003389116,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005309297759486123
            },
            {
             "value": 1,
             "probability": 0.9985183003389116
            },
            {
             "value": 2,
             "probability": 0.0004730184199416743
            },
            {
             "value": 3,
             "probability": 0.0004777514651980876
            }
           ]
          }
         },
         "joint_probability": 0.9980669818099251,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019348888808446748,
        "field_brier": 2.5063623113781215e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_grace": {
           "value": true,
           "probability": 0.9990370025639944,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009629974360055753
            },
            {
             "value": true,
             "probability": 0.9990370025639944
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9985681914335498,
           "distribution": [
            {
             "value": "resubmit",
             "probability": 0.00044493432850654057
            },
            {
             "value": "reject",
             "probability": 0.9985681914335498
            },
            {
             "value": "approve",
             "probability": 0.0004888056254608133
            },
            {
             "value": "approve priority",
             "probability": 0.0004980686124828774
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9985183003389116,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005309297759486123
            },
            {
             "value": 1,
             "probability": 0.9985183003389116
            },
            {
             "value": 2,
             "probability": 0.0004730184199416743
            },
            {
             "value": 3,
             "probability": 0.0004777514651980876
            }
           ]
          }
         },
         "joint_probability": 0.9980669818099251,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019348888808446748,
        "field_brier": 2.5063623113781215e-06
       }
      },
      "timing_ms": {
       "scoring": 33.88954605907202,
       "base_joint": 0.07774890400469303,
       "constrained_joint": 0.05747005343437195,
       "constrained_total": 33.94701611250639
      },
      "input_tokens": 527,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-00182ea4182590cd-0019",
      "group": "bendsynth-00182ea4182590cd",
      "provenance": "bendsynth-v1; family=bendsynth-00182ea4182590cd; domain=postal parcel redirection file audit; parameters=grace=9; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flags": 1,
       "decision": "resubmit",
       "within_grace": false
      },
      "logits": [
       [
        -6.481213569641113,
        1.1194695234298706,
        -6.48770809173584,
        -6.5685715675354
       ],
       [
        -6.446230888366699,
        1.1193768978118896,
        -6.323156356811523,
        -6.599666118621826
       ],
       [
        1.1199320554733276,
        -5.795662879943848
       ]
      ],
      "joint_logits": [
       -8.456918716430664,
       -8.650899887084961,
       -8.489922523498535,
       -8.572335243225098,
       -8.336430549621582,
       -8.498870849609375,
       -8.743942260742188,
       -8.825028419494629,
       -8.261635780334473,
       -8.494863510131836,
       1.118903398513794,
       -8.163378715515137,
       -8.147527694702148,
       -8.356919288635254,
       -8.163996696472168,
       -8.639835357666016,
       -8.513822555541992,
       -8.625560760498047,
       -8.51494312286377,
       -8.492551803588867,
       -8.421927452087402,
       -8.484400749206543,
       -8.73499870300293,
       -8.811934471130371,
       -8.572408676147461,
       -8.697799682617188,
       -8.625273704528809,
       -8.594849586486816,
       -8.490337371826172,
       -8.544412612915039,
       -8.8389253616333,
       -8.88845157623291
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 1,
           "probability": 0.9985468576093655,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004993828858210488
            },
            {
             "value": 1,
             "probability": 0.9985468576093655
            },
            {
             "value": 2,
             "probability": 0.0004961502813063888
            },
            {
             "value": 3,
             "probability": 0.00045760922350709547
            }
           ]
          },
          "decision": {
           "value": "resubmit",
           "probability": 0.9984543448030958,
           "distribution": [
            {
             "value": "approve priority",
             "probability": 0.0005171620969463374
            },
            {
             "value": "resubmit",
             "probability": 0.9984543448030958
            },
            {
             "value": "reject",
             "probability": 0.0005848941566027418
            },
            {
             "value": "approve",
             "probability": 0.00044359894335506113
            }
           ]
          },
          "within_grace": {
           "value": false,
           "probability": 0.999008792158835,
           "distribution": [
            {
             "value": false,
             "probability": 0.999008792158835
            },
            {
             "value": true,
             "probability": 0.0009912078411650517
            }
           ]
          }
         },
         "joint_probability": 0.9979817720388784,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002020267327572682,
        "field_brier": 2.6589837217239824e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 1,
           "probability": 0.9985468576093655,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004993828858210488
            },
            {
             "value": 1,
             "probability": 0.9985468576093655
            },
            {
             "value": 2,
             "probability": 0.0004961502813063888
            },
            {
             "value": 3,
             "probability": 0.00045760922350709547
            }
           ]
          },
          "decision": {
           "value": "resubmit",
           "probability": 0.9984543448030958,
           "distribution": [
            {
             "value": "approve priority",
             "probability": 0.0005171620969463374
            },
            {
             "value": "resubmit",
             "probability": 0.9984543448030958
            },
            {
             "value": "reject",
             "probability": 0.0005848941566027418
            },
            {
             "value": "approve",
             "probability": 0.00044359894335506113
            }
           ]
          },
          "within_grace": {
           "value": false,
           "probability": 0.999008792158835,
           "distribution": [
            {
             "value": false,
             "probability": 0.999008792158835
            },
            {
             "value": true,
             "probability": 0.0009912078411650517
            }
           ]
          }
         },
         "joint_probability": 0.9979817720388784,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002020267327572682,
        "field_brier": 2.6589837217239824e-06
       }
      },
      "timing_ms": {
       "scoring": 31.55969991348684,
       "base_joint": 0.0804790761321783,
       "constrained_joint": 0.059379031881690025,
       "constrained_total": 31.61907894536853
      },
      "input_tokens": 563,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-00182ea4182590cd-0026",
      "group": "bendsynth-00182ea4182590cd",
      "provenance": "bendsynth-v1; family=bendsynth-00182ea4182590cd; domain=postal parcel redirection file audit; parameters=grace=21; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "approve priority",
       "within_grace": true,
       "flags": 1
      },
      "logits": [
       [
        -6.277364253997803,
        -6.4934587478637695,
        -6.359443664550781,
        1.2379734516143799
       ],
       [
        -5.66630220413208,
        1.2384551763534546
       ],
       [
        -6.401054859161377,
        1.2380194664001465,
        -6.431708335876465,
        -6.380670547485352
       ]
      ],
      "joint_logits": [
       -8.33248233795166,
       -8.405000686645508,
       -8.364802360534668,
       -8.28521728515625,
       -8.455255508422852,
       -8.111145973205566,
       -8.497970581054688,
       -8.458694458007812,
       -8.541717529296875,
       -8.55542278289795,
       -8.557701110839844,
       -8.479185104370117,
       -8.661903381347656,
       -8.471315383911133,
       -8.69920539855957,
       -8.641159057617188,
       -8.412623405456543,
       -8.432350158691406,
       -8.438899993896484,
       -8.368273735046387,
       -8.488152503967285,
       -8.3729887008667,
       -8.529723167419434,
       -8.47921085357666,
       -8.532172203063965,
       -8.302444458007812,
       -8.570533752441406,
       -8.512163162231445,
       -8.452919960021973,
       1.2375489473342896,
       -8.465046882629395,
       -8.495320320129395
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve priority",
           "probability": 0.998516975813283,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0005438582759290523
            },
            {
             "value": "resubmit",
             "probability": 0.00043816430707840573
            },
            {
             "value": "reject",
             "probability": 0.0005010016037096076
            },
            {
             "value": "approve priority",
             "probability": 0.998516975813283
            }
           ]
          },
          "within_grace": {
           "value": true,
           "probability": 0.9989980026214348,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010019973785652537
            },
            {
             "value": true,
             "probability": 0.9989980026214348
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9985628650057811,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004805821835957828
            },
            {
             "value": 1,
             "probability": 0.9985628650057811
            },
            {
             "value": 2,
             "probability": 0.00046607397768235804
            },
            {
             "value": 3,
             "probability": 0.0004904788329408349
            }
           ]
          }
         },
         "joint_probability": 0.998093098481582,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019087219697675275,
        "field_brier": 2.5667470959794566e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve priority",
           "probability": 0.998516975813283,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0005438582759290523
            },
            {
             "value": "resubmit",
             "probability": 0.00043816430707840573
            },
            {
             "value": "reject",
             "probability": 0.0005010016037096076
            },
            {
             "value": "approve priority",
             "probability": 0.998516975813283
            }
           ]
          },
          "within_grace": {
           "value": true,
           "probability": 0.9989980026214348,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010019973785652537
            },
            {
             "value": true,
             "probability": 0.9989980026214348
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9985628650057811,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004805821835957828
            },
            {
             "value": 1,
             "probability": 0.9985628650057811
            },
            {
             "value": 2,
             "probability": 0.00046607397768235804
            },
            {
             "value": 3,
             "probability": 0.0004904788329408349
            }
           ]
          }
         },
         "joint_probability": 0.998093098481582,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019087219697675275,
        "field_brier": 2.5667470959794566e-06
       }
      },
      "timing_ms": {
       "scoring": 35.28701979666948,
       "base_joint": 0.07768906652927399,
       "constrained_joint": 0.06539910100400448,
       "constrained_total": 35.35241889767349
      },
      "input_tokens": 560,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-00182ea4182590cd-0035",
      "group": "bendsynth-00182ea4182590cd",
      "provenance": "bendsynth-v1; family=bendsynth-00182ea4182590cd; domain=postal parcel redirection file audit; parameters=grace=15; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "within_grace": true,
       "decision": "approve priority",
       "flags": 1
      },
      "logits": [
       [
        -5.228623390197754,
        1.6243524551391602
       ],
       [
        1.6239241361618042,
        -6.112319469451904,
        -5.91444730758667,
        -5.945277690887451
       ],
       [
        -6.012524604797363,
        1.623928427696228,
        -5.984925270080566,
        -5.972467422485352
       ]
      ],
      "joint_logits": [
       -8.072673797607422,
       -7.844761848449707,
       -8.06381893157959,
       -8.043370246887207,
       -8.141624450683594,
       -8.163687705993652,
       -8.10843276977539,
       -8.062407493591309,
       -7.957464218139648,
       -8.004365921020508,
       -7.9508466720581055,
       -7.869350910186768,
       -7.978182315826416,
       -7.987078666687012,
       -7.940305709838867,
       -7.897680759429932,
       -8.013835906982422,
       1.6234639883041382,
       -7.96560001373291,
       -8.057297706604004,
       -8.311325073242188,
       -8.188886642456055,
       -8.298844337463379,
       -8.289939880371094,
       -8.168329238891602,
       -7.804281711578369,
       -8.123894691467285,
       -8.133232116699219,
       -8.139360427856445,
       -8.04758358001709,
       -8.112232208251953,
       -8.12434196472168
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_grace": {
           "value": true,
           "probability": 0.9989448067214669,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010551932785331764
            },
            {
             "value": true,
             "probability": 0.9989448067214669
            }
           ]
          },
          "decision": {
           "value": "approve priority",
           "probability": 0.9985171253072447,
           "distribution": [
            {
             "value": "approve priority",
             "probability": 0.9985171253072447
            },
            {
             "value": "resubmit",
             "probability": 0.00043606129167081745
            },
            {
             "value": "approve",
             "probability": 0.0005314745026463503
            },
            {
             "value": "reject",
             "probability": 0.000515338898438094
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9985213541503704,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004818233479862542
            },
            {
             "value": 1,
             "probability": 0.9985213541503704
            },
            {
             "value": 2,
             "probability": 0.0004953067910031633
            },
            {
             "value": 3,
             "probability": 0.0005015157106401353
            }
           ]
          }
         },
         "joint_probability": 0.9980576162848238,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019442725887658622,
        "field_brier": 2.6931219847872623e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_grace": {
           "value": true,
           "probability": 0.9989448067214669,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010551932785331764
            },
            {
             "value": true,
             "probability": 0.9989448067214669
            }
           ]
          },
          "decision": {
           "value": "approve priority",
           "probability": 0.9985171253072447,
           "distribution": [
            {
             "value": "approve priority",
             "probability": 0.9985171253072447
            },
            {
             "value": "resubmit",
             "probability": 0.00043606129167081745
            },
            {
             "value": "approve",
             "probability": 0.0005314745026463503
            },
            {
             "value": "reject",
             "probability": 0.000515338898438094
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9985213541503704,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004818233479862542
            },
            {
             "value": 1,
             "probability": 0.9985213541503704
            },
            {
             "value": 2,
             "probability": 0.0004953067910031633
            },
            {
             "value": 3,
             "probability": 0.0005015157106401353
            }
           ]
          }
         },
         "joint_probability": 0.9980576162848238,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0019442725887658622,
        "field_brier": 2.6931219847872623e-06
       }
      },
      "timing_ms": {
       "scoring": 33.24270294979215,
       "base_joint": 0.08372892625629902,
       "constrained_joint": 0.05777995102107525,
       "constrained_total": 33.30048290081322
      },
      "input_tokens": 566,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01691b55a7b123e5-0000",
      "group": "bendsynth-01691b55a7b123e5",
      "provenance": "bendsynth-v1; family=bendsynth-01691b55a7b123e5; domain=musical instrument insurance claims chatbot; parameters=auto_limit=1000; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "needs_review": true,
       "decision": "escalate to adjuster",
       "risk_flags": 3
      },
      "logits": [
       [
        -6.235321044921875,
        0.21038351953029633
       ],
       [
        0.21025249361991882,
        -6.797757148742676,
        -6.903432846069336
       ],
       [
        -7.1405110359191895,
        -7.203117847442627,
        -7.094542980194092,
        0.2100534290075302
       ]
      ],
      "joint_logits": [
       -8.817123413085938,
       -8.833691596984863,
       -8.815934181213379,
       -8.670021057128906,
       -8.491167068481445,
       -8.570596694946289,
       -8.631702423095703,
       -8.785547256469727,
       -8.746861457824707,
       -8.99168872833252,
       -8.627217292785645,
       -8.76335620880127,
       -9.388052940368652,
       -9.3065824508667,
       -8.909605979919434,
       0.20950378477573395,
       -9.250231742858887,
       -9.288031578063965,
       -9.268866539001465,
       -9.140951156616211,
       -9.209017753601074,
       -9.199065208435059,
       -9.268540382385254,
       -9.317938804626465
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_review": {
           "value": true,
           "probability": 0.9984151891414786,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015848108585213357
            },
            {
             "value": true,
             "probability": 0.9984151891414786
            }
           ]
          },
          "decision": {
           "value": "escalate to adjuster",
           "probability": 0.9982844505947209,
           "distribution": [
            {
             "value": "escalate to adjuster",
             "probability": 0.9982844505947209
            },
            {
             "value": "partial payout",
             "probability": 0.0009030556152512986
            },
            {
             "value": "paid in full",
             "probability": 0.0008124937900277608
            }
           ]
          },
          "risk_flags": {
           "value": 3,
           "probability": 0.998085745992147,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006410003040216587
            },
            {
             "value": 1,
             "probability": 0.0006020998845845767
            },
            {
             "value": 2,
             "probability": 0.0006711538192467184
            },
            {
             "value": 3,
             "probability": 0.998085745992147
            }
           ]
          }
         },
         "joint_probability": 0.9975373110799135,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002465726326260855,
        "field_brier": 4.776745932013808e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_review": {
           "value": true,
           "probability": 0.9984151891414786,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015848108585213357
            },
            {
             "value": true,
             "probability": 0.9984151891414786
            }
           ]
          },
          "decision": {
           "value": "escalate to adjuster",
           "probability": 0.9982844505947209,
           "distribution": [
            {
             "value": "escalate to adjuster",
             "probability": 0.9982844505947209
            },
            {
             "value": "partial payout",
             "probability": 0.0009030556152512986
            },
            {
             "value": "paid in full",
             "probability": 0.0008124937900277608
            }
           ]
          },
          "risk_flags": {
           "value": 3,
           "probability": 0.998085745992147,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006410003040216587
            },
            {
             "value": 1,
             "probability": 0.0006020998845845767
            },
            {
             "value": 2,
             "probability": 0.0006711538192467184
            },
            {
             "value": 3,
             "probability": 0.998085745992147
            }
           ]
          }
         },
         "joint_probability": 0.9975373110799135,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002465726326260855,
        "field_brier": 4.776745932013808e-06
       }
      },
      "timing_ms": {
       "scoring": 34.5661579631269,
       "base_joint": 0.06851903162896633,
       "constrained_joint": 0.04739989526569843,
       "constrained_total": 34.613557858392596
      },
      "input_tokens": 656,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-01691b55a7b123e5-0029",
      "group": "bendsynth-01691b55a7b123e5",
      "provenance": "bendsynth-v1; family=bendsynth-01691b55a7b123e5; domain=musical instrument insurance claims chatbot; parameters=auto_limit=16000; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_flags": 0,
       "decision": "paid in full",
       "needs_review": false
      },
      "logits": [
       [
        0.6521323323249817,
        -6.415849685668945,
        -6.63520622253418,
        -6.7024946212768555
       ],
       [
        -6.321054935455322,
        0.6522112488746643,
        -6.107053756713867
       ],
       [
        0.652889609336853,
        -5.9063615798950195
       ]
      ],
      "joint_logits": [
       -8.391432762145996,
       -8.575994491577148,
       0.6514014601707458,
       -8.437328338623047,
       -7.457589626312256,
       -8.550313949584961,
       -8.236557960510254,
       -8.285046577453613,
       -8.139507293701172,
       -8.270968437194824,
       -8.229950904846191,
       -8.097800254821777,
       -8.43066692352295,
       -8.406428337097168,
       -8.540002822875977,
       -8.43606185913086,
       -8.3928861618042,
       -8.364727973937988,
       -8.478324890136719,
       -8.4393949508667,
       -8.6749849319458,
       -8.485065460205078,
       -8.462197303771973,
       -8.444429397583008
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 0,
           "probability": 0.9978289999064929,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978289999064929
            },
            {
             "value": 1,
             "probability": 0.0008501013328104265
            },
            {
             "value": 2,
             "probability": 0.00068266115916543
            },
            {
             "value": 3,
             "probability": 0.0006382376015312784
            }
           ]
          },
          "decision": {
           "value": "paid in full",
           "probability": 0.9979077164869242,
           "distribution": [
            {
             "value": "escalate to adjuster",
             "probability": 0.0009346294033912083
            },
            {
             "value": "paid in full",
             "probability": 0.9979077164869242
            },
            {
             "value": "partial payout",
             "probability": 0.0011576541096846142
            }
           ]
          },
          "needs_review": {
           "value": false,
           "probability": 0.9985850588638907,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985850588638907
            },
            {
             "value": true,
             "probability": 0.001414941136109327
            }
           ]
          }
         },
         "joint_probability": 0.9970999562804364,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0029042569941099133,
        "field_brier": 5.634916490681968e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_flags": {
           "value": 0,
           "probability": 0.9978289999064929,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9978289999064929
            },
            {
             "value": 1,
             "probability": 0.0008501013328104265
            },
            {
             "value": 2,
             "probability": 0.00068266115916543
            },
            {
             "value": 3,
             "probability": 0.0006382376015312784
            }
           ]
          },
          "decision": {
           "value": "paid in full",
           "probability": 0.9979077164869242,
           "distribution": [
            {
             "value": "escalate to adjuster",
             "probability": 0.0009346294033912083
            },
            {
             "value": "paid in full",
             "probability": 0.9979077164869242
            },
            {
             "value": "partial payout",
             "probability": 0.0011576541096846142
            }
           ]
          },
          "needs_review": {
           "value": false,
           "probability": 0.9985850588638907,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985850588638907
            },
            {
             "value": true,
             "probability": 0.001414941136109327
            }
           ]
          }
         },
         "joint_probability": 0.9970999562804364,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0029042569941099133,
        "field_brier": 5.634916490681968e-06
       }
      },
      "timing_ms": {
       "scoring": 36.911112954840064,
       "base_joint": 0.0630989670753479,
       "constrained_joint": 0.04766997881233692,
       "constrained_total": 36.9587829336524
      },
      "input_tokens": 684,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c738801142900c5-0013",
      "group": "bendsynth-0c738801142900c5",
      "provenance": "bendsynth-v1; family=bendsynth-0c738801142900c5; domain=cemetery plot allocation counter at a municipal records office; parameters=quota=9; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_used": 2,
       "decision": "place on waitlist",
       "exempt": false,
       "fee_waived": false
      },
      "logits": [
       [
        -8.122065544128418,
        -7.987164497375488,
        -0.6916923522949219
       ],
       [
        -8.04015064239502,
        -0.6916859149932861,
        -8.074670791625977
       ],
       [
        -0.6913037896156311,
        -7.7239990234375
       ],
       [
        -0.6912878155708313,
        -7.741985321044922
       ]
      ],
      "joint_logits": [
       -10.453099250793457,
       -10.606796264648438,
       -10.587943077087402,
       -11.014267921447754,
       -10.176854133605957,
       -10.554404258728027,
       -10.590384483337402,
       -10.67239761352539,
       -10.508904457092285,
       -10.642184257507324,
       -10.712740898132324,
       -11.059619903564453,
       -10.317724227905273,
       -10.529327392578125,
       -10.424102783203125,
       -10.928067207336426,
       -10.125248908996582,
       -10.460275650024414,
       -10.44098949432373,
       -10.531571388244629,
       -10.329526901245117,
       -10.485989570617676,
       -10.455475807189941,
       -10.903703689575195,
       -10.104205131530762,
       -10.53443431854248,
       -10.525968551635742,
       -10.589021682739258,
       -0.6923477649688721,
       -10.357244491577148,
       -10.17522144317627,
       -10.624049186706543,
       -10.140591621398926,
       -10.55576229095459,
       -10.594877243041992,
       -10.643304824829102
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_used": {
           "value": 2,
           "probability": 0.9987300443232203,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005922130185021083
            },
            {
             "value": 1,
             "probability": 0.0006777426582776343
            },
            {
             "value": 2,
             "probability": 0.9987300443232203
            }
           ]
          },
          "decision": {
           "value": "place on waitlist",
           "probability": 0.9987362766456581,
           "distribution": [
            {
             "value": "assign plot",
             "probability": 0.0006427664505450261
            },
            {
             "value": "place on waitlist",
             "probability": 0.9987362766456581
            },
            {
             "value": "deny",
             "probability": 0.0006209569037968043
            }
           ]
          },
          "exempt": {
           "value": false,
           "probability": 0.9991182281856874,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991182281856874
            },
            {
             "value": true,
             "probability": 0.0008817718143126202
            }
           ]
          },
          "fee_waived": {
           "value": false,
           "probability": 0.9991339464330796,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991339464330796
            },
            {
             "value": true,
             "probability": 0.0008660535669203643
            }
           ]
          }
         },
         "joint_probability": 0.9980756019214919,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0019262521114705297,
        "field_brier": 1.9684280800822734e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_used": {
           "value": 2,
           "probability": 0.9987300443232203,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005922130185021083
            },
            {
             "value": 1,
             "probability": 0.0006777426582776343
            },
            {
             "value": 2,
             "probability": 0.9987300443232203
            }
           ]
          },
          "decision": {
           "value": "place on waitlist",
           "probability": 0.9987362766456581,
           "distribution": [
            {
             "value": "assign plot",
             "probability": 0.0006427664505450261
            },
            {
             "value": "place on waitlist",
             "probability": 0.9987362766456581
            },
            {
             "value": "deny",
             "probability": 0.0006209569037968043
            }
           ]
          },
          "exempt": {
           "value": false,
           "probability": 0.9991182281856874,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991182281856874
            },
            {
             "value": true,
             "probability": 0.0008817718143126202
            }
           ]
          },
          "fee_waived": {
           "value": false,
           "probability": 0.9991339464330796,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991339464330796
            },
            {
             "value": true,
             "probability": 0.0008660535669203643
            }
           ]
          }
         },
         "joint_probability": 0.9980756019214919,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0019262521114705297,
        "field_brier": 1.9684280800822734e-06
       }
      },
      "timing_ms": {
       "scoring": 45.91085691936314,
       "base_joint": 0.09505893103778362,
       "constrained_joint": 0.06792903877794743,
       "constrained_total": 45.97878595814109
      },
      "input_tokens": 1010,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c738801142900c5-0014",
      "group": "bendsynth-0c738801142900c5",
      "provenance": "bendsynth-v1; family=bendsynth-0c738801142900c5; domain=cemetery plot allocation counter at a municipal records office; parameters=quota=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_used": 1,
       "decision": "assign plot",
       "exempt": true,
       "fee_waived": true
      },
      "logits": [
       [
        -1.2102452516555786,
        -7.541693210601807,
        -8.489446640014648
       ],
       [
        -8.384641647338867,
        -1.209146499633789,
        -8.622476577758789
       ],
       [
        -8.023808479309082,
        -1.2088757753372192
       ],
       [
        -7.997215747833252,
        -1.2089051008224487
       ]
      ],
      "joint_logits": [
       -10.74203109741211,
       -10.899587631225586,
       -10.764636993408203,
       -10.293965339660645,
       -10.77584171295166,
       -9.897640228271484,
       -9.975143432617188,
       -1.211207389831543,
       -11.013595581054688,
       -11.171316146850586,
       -11.068074226379395,
       -10.491525650024414,
       -11.124981880187988,
       -11.20056438446045,
       -11.081744194030762,
       -10.773014068603516,
       -11.183614730834961,
       -10.50304889678955,
       -10.468255996704102,
       -7.973085403442383,
       -11.347750663757324,
       -11.51141357421875,
       -11.345108032226562,
       -10.941113471984863,
       -11.108778953552246,
       -10.957096099853516,
       -10.910173416137695,
       -10.933255195617676,
       -10.859147071838379,
       -11.036151885986328,
       -10.916912078857422,
       -10.45296573638916,
       -11.300053596496582,
       -11.172551155090332,
       -11.104199409484863,
       -11.230524063110352
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_used": {
           "value": 0,
           "probability": 0.997536890631056,
           "distribution": [
            {
             "value": 0,
             "probability": 0.997536890631056
            },
            {
             "value": 1,
             "probability": 0.0017750723471002012
            },
            {
             "value": 2,
             "probability": 0.0006880370218437882
            }
           ]
          },
          "decision": {
           "value": "assign plot",
           "probability": 0.9986336046541991,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.000764060997231193
            },
            {
             "value": "assign plot",
             "probability": 0.9986336046541991
            },
            {
             "value": "place on waitlist",
             "probability": 0.0006023343485696849
            }
           ]
          },
          "exempt": {
           "value": true,
           "probability": 0.9989039355439153,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010960644560846645
            },
            {
             "value": true,
             "probability": 0.9989039355439153
            }
           ]
          },
          "fee_waived": {
           "value": true,
           "probability": 0.9988743970798007,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011256029201992912
            },
            {
             "value": true,
             "probability": 0.9988743970798007
            }
           ]
          }
         },
         "joint_probability": 0.9965775492120138,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 3,
        "joint_nll": 6.7653063343805,
        "field_brier": 0.4978852695157559
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_used": {
           "value": 0,
           "probability": 0.997536890631056,
           "distribution": [
            {
             "value": 0,
             "probability": 0.997536890631056
            },
            {
             "value": 1,
             "probability": 0.0017750723471002012
            },
            {
             "value": 2,
             "probability": 0.0006880370218437882
            }
           ]
          },
          "decision": {
           "value": "assign plot",
           "probability": 0.9986336046541991,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.000764060997231193
            },
            {
             "value": "assign plot",
             "probability": 0.9986336046541991
            },
            {
             "value": "place on waitlist",
             "probability": 0.0006023343485696849
            }
           ]
          },
          "exempt": {
           "value": true,
           "probability": 0.9989039355439153,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010960644560846645
            },
            {
             "value": true,
             "probability": 0.9989039355439153
            }
           ]
          },
          "fee_waived": {
           "value": true,
           "probability": 0.9988743970798007,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011256029201992912
            },
            {
             "value": true,
             "probability": 0.9988743970798007
            }
           ]
          }
         },
         "joint_probability": 0.9965775492120138,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 3,
        "joint_nll": 6.7653063343805,
        "field_brier": 0.4978852695157559
       }
      },
      "timing_ms": {
       "scoring": 44.90746697410941,
       "base_joint": 0.08542905561625957,
       "constrained_joint": 0.06882892921566963,
       "constrained_total": 44.97629590332508
      },
      "input_tokens": 1006,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c738801142900c5-0025",
      "group": "bendsynth-0c738801142900c5",
      "provenance": "bendsynth-v1; family=bendsynth-0c738801142900c5; domain=cemetery plot allocation counter at a municipal records office; parameters=quota=20; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "fee_waived": false,
       "decision": "place on waitlist",
       "exempt": false,
       "quota_used": 2
      },
      "logits": [
       [
        -6.784421443939209,
        -0.2954980134963989
       ],
       [
        -7.080177307128906,
        -7.28665828704834,
        -0.29602888226509094
       ],
       [
        -0.29524698853492737,
        -6.965072154998779
       ],
       [
        -5.963148593902588,
        -6.379509925842285,
        -0.29972174763679504
       ]
      ],
      "joint_logits": [
       -9.8092622756958,
       -9.41217041015625,
       -9.846405982971191,
       -9.970013618469238,
       -9.603829383850098,
       -9.985164642333984,
       -9.937756538391113,
       -9.508455276489258,
       -10.103425979614258,
       -10.161245346069336,
       -9.688791275024414,
       -10.157705307006836,
       -9.412367820739746,
       -8.814603805541992,
       -8.991329193115234,
       -10.28402328491211,
       -9.90880012512207,
       -10.131241798400879,
       -9.341658592224121,
       -8.886395454406738,
       -9.051248550415039,
       -10.051871299743652,
       -9.627593040466309,
       -10.139034271240234,
       -9.621377944946289,
       -9.252930641174316,
       -9.304688453674316,
       -10.165178298950195,
       -9.72110652923584,
       -10.409918785095215,
       -6.226154804229736,
       -7.19830846786499,
       -0.3006743788719177,
       -9.815106391906738,
       -9.16675090789795,
       -9.376947402954102
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "fee_waived": {
           "value": true,
           "probability": 0.9984821222860293,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015178777139707063
            },
            {
             "value": true,
             "probability": 0.9984821222860293
            }
           ]
          },
          "decision": {
           "value": "assign plot",
           "probability": 0.9979521638075547,
           "distribution": [
            {
             "value": "place on waitlist",
             "probability": 0.0011292539411333808
            },
            {
             "value": "deny",
             "probability": 0.0009185822513119269
            },
            {
             "value": "assign plot",
             "probability": 0.9979521638075547
            }
           ]
          },
          "exempt": {
           "value": false,
           "probability": 0.998732986790936,
           "distribution": [
            {
             "value": false,
             "probability": 0.998732986790936
            },
            {
             "value": true,
             "probability": 0.0012670132090639944
            }
           ]
          },
          "quota_used": {
           "value": 2,
           "probability": 0.994273715406882,
           "distribution": [
            {
             "value": 0,
             "probability": 0.003450729039162905
            },
            {
             "value": 1,
             "probability": 0.0022755555539550484
            },
            {
             "value": 2,
             "probability": 0.994273715406882
            }
           ]
          }
         },
         "joint_probability": 0.9933270198448315,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 9.552426948131176,
        "field_brier": 0.9969095790218805
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "fee_waived": {
           "value": true,
           "probability": 0.9984821222860293,
           "distribution": [
            {
             "value": false,
             "probability": 0.0015178777139707063
            },
            {
             "value": true,
             "probability": 0.9984821222860293
            }
           ]
          },
          "decision": {
           "value": "assign plot",
           "probability": 0.9979521638075547,
           "distribution": [
            {
             "value": "place on waitlist",
             "probability": 0.0011292539411333808
            },
            {
             "value": "deny",
             "probability": 0.0009185822513119269
            },
            {
             "value": "assign plot",
             "probability": 0.9979521638075547
            }
           ]
          },
          "exempt": {
           "value": false,
           "probability": 0.998732986790936,
           "distribution": [
            {
             "value": false,
             "probability": 0.998732986790936
            },
            {
             "value": true,
             "probability": 0.0012670132090639944
            }
           ]
          },
          "quota_used": {
           "value": 2,
           "probability": 0.994273715406882,
           "distribution": [
            {
             "value": 0,
             "probability": 0.003450729039162905
            },
            {
             "value": 1,
             "probability": 0.0022755555539550484
            },
            {
             "value": 2,
             "probability": 0.994273715406882
            }
           ]
          }
         },
         "joint_probability": 0.9933270198448315,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 9.552426948131176,
        "field_brier": 0.9969095790218805
       }
      },
      "timing_ms": {
       "scoring": 46.96383490227163,
       "base_joint": 0.08930894546210766,
       "constrained_joint": 0.06796885281801224,
       "constrained_total": 47.03180375508964
      },
      "input_tokens": 1010,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c738801142900c5-0026",
      "group": "bendsynth-0c738801142900c5",
      "provenance": "bendsynth-v1; family=bendsynth-0c738801142900c5; domain=cemetery plot allocation counter at a municipal records office; parameters=quota=16; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "exempt": false,
       "fee_waived": false,
       "decision": "deny",
       "quota_used": 1
      },
      "logits": [
       [
        0.6371961236000061,
        -5.9915032386779785
       ],
       [
        0.6371870636940002,
        -5.9847412109375
       ],
       [
        -6.178689002990723,
        0.6363232135772705,
        -6.176953315734863
       ],
       [
        -3.8035333156585693,
        0.6256778836250305,
        -6.283945083618164
       ]
      ],
      "joint_logits": [
       -7.83922815322876,
       -8.155912399291992,
       -8.690741539001465,
       -3.917870283126831,
       0.6247584819793701,
       -7.998722553253174,
       -7.855410575866699,
       -8.193487167358398,
       -8.694845199584961,
       -8.669586181640625,
       -9.140660285949707,
       -8.751385688781738,
       -8.267624855041504,
       -8.416509628295898,
       -9.021893501281738,
       -8.655586242675781,
       -9.154557228088379,
       -8.739298820495605,
       -8.699691772460938,
       -9.156610488891602,
       -8.779773712158203,
       -8.270895004272461,
       -8.39117431640625,
       -9.0508451461792,
       -8.683634757995605,
       -9.169919967651367,
       -8.743720054626465,
       -8.783102989196777,
       -9.213386535644531,
       -9.134804725646973,
       -9.018780708312988,
       -9.420767784118652,
       -9.153996467590332,
       -8.752699851989746,
       -9.180829048156738,
       -9.104187965393066
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "exempt": {
           "value": false,
           "probability": 0.9986798636512462,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986798636512462
            },
            {
             "value": true,
             "probability": 0.0013201363487537628
            }
           ]
          },
          "fee_waived": {
           "value": false,
           "probability": 0.9986709064255489,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986709064255489
            },
            {
             "value": true,
             "probability": 0.0013290935744511554
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.997808547455167,
           "distribution": [
            {
             "value": "assign plot",
             "probability": 0.0010947753544819558
            },
            {
             "value": "deny",
             "probability": 0.997808547455167
            },
            {
             "value": "place on waitlist",
             "probability": 0.0010966771903511393
            }
           ]
          },
          "quota_used": {
           "value": 1,
           "probability": 0.9872428203725641,
           "distribution": [
            {
             "value": 0,
             "probability": 0.011771778744556283
            },
            {
             "value": 1,
             "probability": 0.9872428203725641
            },
            {
             "value": 2,
             "probability": 0.0009854008828796214
            }
           ]
          }
         },
         "joint_probability": 0.9863356718696866,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.013758544313658852,
        "field_brier": 7.912840484205072e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "exempt": {
           "value": false,
           "probability": 0.9986798636512462,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986798636512462
            },
            {
             "value": true,
             "probability": 0.0013201363487537628
            }
           ]
          },
          "fee_waived": {
           "value": false,
           "probability": 0.9986709064255489,
           "distribution": [
            {
             "value": false,
             "probability": 0.9986709064255489
            },
            {
             "value": true,
             "probability": 0.0013290935744511554
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.997808547455167,
           "distribution": [
            {
             "value": "assign plot",
             "probability": 0.0010947753544819558
            },
            {
             "value": "deny",
             "probability": 0.997808547455167
            },
            {
             "value": "place on waitlist",
             "probability": 0.0010966771903511393
            }
           ]
          },
          "quota_used": {
           "value": 1,
           "probability": 0.9872428203725641,
           "distribution": [
            {
             "value": 0,
             "probability": 0.011771778744556283
            },
            {
             "value": 1,
             "probability": 0.9872428203725641
            },
            {
             "value": 2,
             "probability": 0.0009854008828796214
            }
           ]
          }
         },
         "joint_probability": 0.9863356718696866,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.013758544313658852,
        "field_brier": 7.912840484205072e-05
       }
      },
      "timing_ms": {
       "scoring": 44.52001093886793,
       "base_joint": 0.08509890176355839,
       "constrained_joint": 0.06787898018956184,
       "constrained_total": 44.58788991905749
      },
      "input_tokens": 1010,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c738801142900c5-0029",
      "group": "bendsynth-0c738801142900c5",
      "provenance": "bendsynth-v1; family=bendsynth-0c738801142900c5; domain=cemetery plot allocation counter at a municipal records office; parameters=quota=19; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_used": 2,
       "exempt": false,
       "decision": "place on waitlist",
       "fee_waived": false
      },
      "logits": [
       [
        -7.728763580322266,
        -8.052799224853516,
        -1.6927869319915771
       ],
       [
        -1.6907469034194946,
        -7.868838310241699
       ],
       [
        -8.174053192138672,
        -1.6923151016235352,
        -7.850663185119629
       ],
       [
        -6.5249342918396,
        -1.6966431140899658
       ]
      ],
      "joint_logits": [
       -10.929387092590332,
       -10.39389705657959,
       -10.047765731811523,
       -8.626276016235352,
       -10.878070831298828,
       -10.361433029174805,
       -10.919800758361816,
       -10.784635543823242,
       -10.978970527648926,
       -10.52114486694336,
       -11.030381202697754,
       -10.8551664352417,
       -10.929015159606934,
       -10.498747825622559,
       -10.578364372253418,
       -9.59286117553711,
       -10.858357429504395,
       -10.370448112487793,
       -10.970993995666504,
       -10.539834976196289,
       -10.990561485290527,
       -10.531059265136719,
       -10.84217357635498,
       -10.739141464233398,
       -10.765010833740234,
       -9.90748405456543,
       -6.798038005828857,
       -1.7009539604187012,
       -10.261734962463379,
       -8.879971504211426,
       -11.185354232788086,
       -10.79211711883545,
       -10.908158302307129,
       -10.000429153442383,
       -11.13967514038086,
       -10.677938461303711
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_used": {
           "value": 2,
           "probability": 0.9958964035483004,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0023813475230328425
            },
            {
             "value": 1,
             "probability": 0.001722248928666694
            },
            {
             "value": 2,
             "probability": 0.9958964035483004
            }
           ]
          },
          "exempt": {
           "value": false,
           "probability": 0.9979299104528164,
           "distribution": [
            {
             "value": false,
             "probability": 0.9979299104528164
            },
            {
             "value": true,
             "probability": 0.002070089547183642
            }
           ]
          },
          "decision": {
           "value": "assign plot",
           "probability": 0.9963663596153242,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0015255831796077719
            },
            {
             "value": "assign plot",
             "probability": 0.9963663596153242
            },
            {
             "value": "place on waitlist",
             "probability": 0.0021080572050680784
            }
           ]
          },
          "fee_waived": {
           "value": true,
           "probability": 0.9920633150157303,
           "distribution": [
            {
             "value": false,
             "probability": 0.007936684984269717
            },
            {
             "value": true,
             "probability": 0.9920633150157303
            }
           ]
          }
         },
         "joint_probability": 0.98779591687972,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.573060166478419,
        "field_brier": 0.9892374671187195
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_used": {
           "value": 2,
           "probability": 0.9958964035483004,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0023813475230328425
            },
            {
             "value": 1,
             "probability": 0.001722248928666694
            },
            {
             "value": 2,
             "probability": 0.9958964035483004
            }
           ]
          },
          "exempt": {
           "value": false,
           "probability": 0.9979299104528164,
           "distribution": [
            {
             "value": false,
             "probability": 0.9979299104528164
            },
            {
             "value": true,
             "probability": 0.002070089547183642
            }
           ]
          },
          "decision": {
           "value": "assign plot",
           "probability": 0.9963663596153242,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0015255831796077719
            },
            {
             "value": "assign plot",
             "probability": 0.9963663596153242
            },
            {
             "value": "place on waitlist",
             "probability": 0.0021080572050680784
            }
           ]
          },
          "fee_waived": {
           "value": true,
           "probability": 0.9920633150157303,
           "distribution": [
            {
             "value": false,
             "probability": 0.007936684984269717
            },
            {
             "value": true,
             "probability": 0.9920633150157303
            }
           ]
          }
         },
         "joint_probability": 0.98779591687972,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 36,
         "valid_assignments": 36,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.573060166478419,
        "field_brier": 0.9892374671187195
       }
      },
      "timing_ms": {
       "scoring": 45.581419952213764,
       "base_joint": 0.09309896267950535,
       "constrained_joint": 0.06822892464697361,
       "constrained_total": 45.64964887686074
      },
      "input_tokens": 1026,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-06d297d6ccec26a5-0014",
      "group": "bendsynth-06d297d6ccec26a5",
      "provenance": "bendsynth-v1; family=bendsynth-06d297d6ccec26a5; domain=art gallery outgoing loan escalation review; parameters=high_value=1500,max_risks=1; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "deny",
       "dual_courier": false,
       "risk_count": 2
      },
      "logits": [
       [
        -8.177529335021973,
        -8.411338806152344,
        -1.7074530124664307
       ],
       [
        -1.7065515518188477,
        -7.987459182739258
       ],
       [
        -8.74336051940918,
        -8.811208724975586,
        -1.7070317268371582,
        -9.042678833007812
       ]
      ],
      "joint_logits": [
       -10.305917739868164,
       -10.401865005493164,
       -9.677349090576172,
       -10.655142784118652,
       -10.365693092346191,
       -10.48975658416748,
       -9.958942413330078,
       -10.629136085510254,
       -10.46377182006836,
       -10.570176124572754,
       -10.026314735412598,
       -10.828662872314453,
       -10.484979629516602,
       -10.611741065979004,
       -10.388784408569336,
       -10.784826278686523,
       -10.792012214660645,
       -10.603836059570312,
       -1.7083836793899536,
       -10.898651123046875,
       -10.95221996307373,
       -11.059232711791992,
       -9.719560623168945,
       -11.376984596252441
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "deny",
           "probability": 0.9972324351957758,
           "distribution": [
            {
             "value": "approve with conditions",
             "probability": 0.0015448199497764968
            },
            {
             "value": "approve",
             "probability": 0.0012227448544477116
            },
            {
             "value": "deny",
             "probability": 0.9972324351957758
            }
           ]
          },
          "dual_courier": {
           "value": false,
           "probability": 0.9981317955768947,
           "distribution": [
            {
             "value": false,
             "probability": 0.9981317955768947
            },
            {
             "value": true,
             "probability": 0.001868204423105221
            }
           ]
          },
          "risk_count": {
           "value": 2,
           "probability": 0.9976526265921907,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008772851138596218
            },
            {
             "value": 1,
             "probability": 0.0008197368511424444
            },
            {
             "value": 2,
             "probability": 0.9976526265921907
            },
            {
             "value": 3,
             "probability": 0.0006503514428071914
            }
           ]
          }
         },
         "joint_probability": 0.9963047589248369,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003702085344483923,
        "field_brier": 8.632026908626382e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "deny",
           "probability": 0.9972324351957758,
           "distribution": [
            {
             "value": "approve with conditions",
             "probability": 0.0015448199497764968
            },
            {
             "value": "approve",
             "probability": 0.0012227448544477116
            },
            {
             "value": "deny",
             "probability": 0.9972324351957758
            }
           ]
          },
          "dual_courier": {
           "value": false,
           "probability": 0.9981317955768947,
           "distribution": [
            {
             "value": false,
             "probability": 0.9981317955768947
            },
            {
             "value": true,
             "probability": 0.001868204423105221
            }
           ]
          },
          "risk_count": {
           "value": 2,
           "probability": 0.9976526265921907,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008772851138596218
            },
            {
             "value": 1,
             "probability": 0.0008197368511424444
            },
            {
             "value": 2,
             "probability": 0.9976526265921907
            },
            {
             "value": 3,
             "probability": 0.0006503514428071914
            }
           ]
          }
         },
         "joint_probability": 0.9963047589248369,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003702085344483923,
        "field_brier": 8.632026908626382e-06
       }
      },
      "timing_ms": {
       "scoring": 35.66719684749842,
       "base_joint": 0.0642300583422184,
       "constrained_joint": 0.04628906026482582,
       "constrained_total": 35.71348590776324
      },
      "input_tokens": 704,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05ccdac9b999599e-0007",
      "group": "bendsynth-05ccdac9b999599e",
      "provenance": "bendsynth-v1; family=bendsynth-05ccdac9b999599e; domain=translation agency job intake file audit; parameters=cap=2500; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 3,
       "decision": "escalate",
       "hot_combo": true
      },
      "logits": [
       [
        -8.448042869567871,
        -8.735629081726074,
        -8.131356239318848,
        -0.6357971429824829,
        -8.267631530761719
       ],
       [
        -8.05735969543457,
        -7.805559158325195,
        -0.6354162693023682
       ],
       [
        -7.526860237121582,
        -0.6350653767585754
       ]
      ],
      "joint_logits": [
       -10.489535331726074,
       -10.266983985900879,
       -10.329047203063965,
       -10.153582572937012,
       -10.180513381958008,
       -10.072546005249023,
       -10.931870460510254,
       -10.633707046508789,
       -10.582574844360352,
       -10.373296737670898,
       -10.370524406433105,
       -10.386933326721191,
       -10.370292663574219,
       -10.135583877563477,
       -10.099272727966309,
       -9.879350662231445,
       -9.978758811950684,
       -9.379115104675293,
       -10.352194786071777,
       -10.120391845703125,
       -10.084762573242188,
       -9.717546463012695,
       -9.770184516906738,
       -0.6362338066101074,
       -10.37131404876709,
       -10.185330390930176,
       -10.167046546936035,
       -9.974498748779297,
       -9.984232902526855,
       -9.780467987060547
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 3,
           "probability": 0.9982543971077108,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00040404147473663526
            },
            {
             "value": 1,
             "probability": 0.00030306026019947456
            },
            {
             "value": 2,
             "probability": 0.0005545764123112367
            },
            {
             "value": 3,
             "probability": 0.9982543971077108
            },
            {
             "value": 4,
             "probability": 0.0004839247450419018
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9986346682347694,
           "distribution": [
            {
             "value": "downgrade",
             "probability": 0.0005971690835632368
            },
            {
             "value": "standard",
             "probability": 0.0007681626816673547
            },
            {
             "value": "escalate",
             "probability": 0.9986346682347694
            }
           ]
          },
          "hot_combo": {
           "value": true,
           "probability": 0.9989849427246837,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010150572753162884
            },
            {
             "value": true,
             "probability": 0.9989849427246837
            }
           ]
          }
         },
         "joint_probability": 0.9978184740814406,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0021839089125655995,
        "field_brier": 2.9051536138355242e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 3,
           "probability": 0.9982543971077108,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00040404147473663526
            },
            {
             "value": 1,
             "probability": 0.00030306026019947456
            },
            {
             "value": 2,
             "probability": 0.0005545764123112367
            },
            {
             "value": 3,
             "probability": 0.9982543971077108
            },
            {
             "value": 4,
             "probability": 0.0004839247450419018
            }
           ]
          },
          "decision": {
           "value": "escalate",
           "probability": 0.9986346682347694,
           "distribution": [
            {
             "value": "downgrade",
             "probability": 0.0005971690835632368
            },
            {
             "value": "standard",
             "probability": 0.0007681626816673547
            },
            {
             "value": "escalate",
             "probability": 0.9986346682347694
            }
           ]
          },
          "hot_combo": {
           "value": true,
           "probability": 0.9989849427246837,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010150572753162884
            },
            {
             "value": true,
             "probability": 0.9989849427246837
            }
           ]
          }
         },
         "joint_probability": 0.9978184740814406,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0021839089125655995,
        "field_brier": 2.9051536138355242e-06
       }
      },
      "timing_ms": {
       "scoring": 34.27363187074661,
       "base_joint": 0.0754399225115776,
       "constrained_joint": 0.05544908344745636,
       "constrained_total": 34.32908095419407
      },
      "input_tokens": 621,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05ccdac9b999599e-0014",
      "group": "bendsynth-05ccdac9b999599e",
      "provenance": "bendsynth-v1; family=bendsynth-05ccdac9b999599e; domain=translation agency job intake file audit; parameters=cap=30000; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 2,
       "hot_combo": false,
       "decision": "downgrade"
      },
      "logits": [
       [
        -6.746236324310303,
        -6.782833576202393,
        1.084249496459961,
        -6.6836628913879395,
        -6.665946960449219
       ],
       [
        1.0848195552825928,
        -5.761170387268066
       ],
       [
        -6.233198165893555,
        1.084618330001831,
        -6.331315040588379
       ]
      ],
      "joint_logits": [
       -8.624667167663574,
       -8.466719627380371,
       -8.748749732971191,
       -8.512469291687012,
       -8.342347145080566,
       -8.581721305847168,
       -8.694268226623535,
       -8.249293327331543,
       -8.838759422302246,
       -8.635047912597656,
       -8.437112808227539,
       -8.71180534362793,
       -8.480376243591309,
       1.0838819742202759,
       -8.40146255493164,
       -8.469820022583008,
       -8.232320785522461,
       -8.625249862670898,
       -8.551383018493652,
       -8.272306442260742,
       -8.676835060119629,
       -8.476999282836914,
       -8.296408653259277,
       -8.654671669006348,
       -8.536650657653809,
       -8.38476276397705,
       -8.653868675231934,
       -8.410772323608398,
       -8.287738800048828,
       -8.51423454284668
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 2,
           "probability": 0.9983683306327139,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00039678385574096577
            },
            {
             "value": 1,
             "probability": 0.0003825251304334495
            },
            {
             "value": 2,
             "probability": 0.9983683306327139
            },
            {
             "value": 3,
             "probability": 0.0004224052502139262
            },
            {
             "value": 4,
             "probability": 0.000429955130897699
            }
           ]
          },
          "hot_combo": {
           "value": false,
           "probability": 0.9989374174656249,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989374174656249
            },
            {
             "value": true,
             "probability": 0.0010625825343750837
            }
           ]
          },
          "decision": {
           "value": "downgrade",
           "probability": 0.9987363983127444,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.0006627709800862028
            },
            {
             "value": "downgrade",
             "probability": 0.9987363983127444
            },
            {
             "value": "standard",
             "probability": 0.0006008307071694631
            }
           ]
          }
         },
         "joint_probability": 0.9980012961174739,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002000703956612703,
        "field_brier": 2.6615036190661448e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 2,
           "probability": 0.9983683306327139,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00039678385574096577
            },
            {
             "value": 1,
             "probability": 0.0003825251304334495
            },
            {
             "value": 2,
             "probability": 0.9983683306327139
            },
            {
             "value": 3,
             "probability": 0.0004224052502139262
            },
            {
             "value": 4,
             "probability": 0.000429955130897699
            }
           ]
          },
          "hot_combo": {
           "value": false,
           "probability": 0.9989374174656249,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989374174656249
            },
            {
             "value": true,
             "probability": 0.0010625825343750837
            }
           ]
          },
          "decision": {
           "value": "downgrade",
           "probability": 0.9987363983127444,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.0006627709800862028
            },
            {
             "value": "downgrade",
             "probability": 0.9987363983127444
            },
            {
             "value": "standard",
             "probability": 0.0006008307071694631
            }
           ]
          }
         },
         "joint_probability": 0.9980012961174739,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002000703956612703,
        "field_brier": 2.6615036190661448e-06
       }
      },
      "timing_ms": {
       "scoring": 33.77966699190438,
       "base_joint": 0.07239915430545807,
       "constrained_joint": 0.05690893158316612,
       "constrained_total": 33.836575923487544
      },
      "input_tokens": 601,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05ccdac9b999599e-0039",
      "group": "bendsynth-05ccdac9b999599e",
      "provenance": "bendsynth-v1; family=bendsynth-05ccdac9b999599e; domain=translation agency job intake file audit; parameters=cap=12001; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 1,
       "hot_combo": false,
       "decision": "downgrade"
      },
      "logits": [
       [
        -5.914869785308838,
        1.8192497491836548,
        -5.964020252227783,
        -5.936399936676025,
        -5.943624973297119
       ],
       [
        1.8198636770248413,
        -4.9989495277404785
       ],
       [
        -5.434162139892578,
        -5.464957237243652,
        1.819563388824463
       ]
      ],
      "joint_logits": [
       -7.703416347503662,
       -7.802310943603516,
       -7.737272262573242,
       -7.7125420570373535,
       -7.68234920501709,
       -7.611749649047852,
       -7.901452541351318,
       -7.685118675231934,
       1.8189178705215454,
       -7.784773826599121,
       -7.804466247558594,
       -7.843254566192627,
       -7.777220249176025,
       -7.85830020904541,
       -7.718955039978027,
       -7.719854354858398,
       -7.786619663238525,
       -7.6835222244262695,
       -7.718682289123535,
       -7.822254657745361,
       -7.863063335418701,
       -7.671199798583984,
       -7.705509185791016,
       -7.610412120819092,
       -7.722671985626221,
       -7.807982444763184,
       -7.879240989685059,
       -7.675808429718018,
       -7.737867832183838,
       -7.611119747161865
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 1,
           "probability": 0.9982950781734248,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00043689151768918125
            },
            {
             "value": 1,
             "probability": 0.9982950781734248
            },
            {
             "value": 2,
             "probability": 0.00041593716047756506
            },
            {
             "value": 3,
             "probability": 0.0004275855883316854
            },
            {
             "value": 4,
             "probability": 0.00042450756007682955
            }
           ]
          },
          "hot_combo": {
           "value": false,
           "probability": 0.998908176042705,
           "distribution": [
            {
             "value": false,
             "probability": 0.998908176042705
            },
            {
             "value": true,
             "probability": 0.0010918239572950323
            }
           ]
          },
          "decision": {
           "value": "downgrade",
           "probability": 0.9986083287795932,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.0007065486846373723
            },
            {
             "value": "standard",
             "probability": 0.0006851225357694351
            },
            {
             "value": "downgrade",
             "probability": 0.9986083287795932
            }
           ]
          }
         },
         "joint_probability": 0.9979638672868306,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002038208449510549,
        "field_brier": 2.974394761425389e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 1,
           "probability": 0.9982950781734248,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00043689151768918125
            },
            {
             "value": 1,
             "probability": 0.9982950781734248
            },
            {
             "value": 2,
             "probability": 0.00041593716047756506
            },
            {
             "value": 3,
             "probability": 0.0004275855883316854
            },
            {
             "value": 4,
             "probability": 0.00042450756007682955
            }
           ]
          },
          "hot_combo": {
           "value": false,
           "probability": 0.998908176042705,
           "distribution": [
            {
             "value": false,
             "probability": 0.998908176042705
            },
            {
             "value": true,
             "probability": 0.0010918239572950323
            }
           ]
          },
          "decision": {
           "value": "downgrade",
           "probability": 0.9986083287795932,
           "distribution": [
            {
             "value": "escalate",
             "probability": 0.0007065486846373723
            },
            {
             "value": "standard",
             "probability": 0.0006851225357694351
            },
            {
             "value": "downgrade",
             "probability": 0.9986083287795932
            }
           ]
          }
         },
         "joint_probability": 0.9979638672868306,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002038208449510549,
        "field_brier": 2.974394761425389e-06
       }
      },
      "timing_ms": {
       "scoring": 34.99450394883752,
       "base_joint": 0.07245899178087711,
       "constrained_joint": 0.054448843002319336,
       "constrained_total": 35.04895279183984
      },
      "input_tokens": 634,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0371b8ffaf895730-0027",
      "group": "bendsynth-0371b8ffaf895730",
      "provenance": "bendsynth-v1; family=bendsynth-0371b8ffaf895730; domain=car rental return desk damage claim processing; parameters=waiver_discount=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "light_damage": false,
       "decision": "minor damage fee",
       "factors": 2
      },
      "logits": [
       [
        0.6884979605674744,
        -5.534677505493164
       ],
       [
        -5.6243181228637695,
        0.6862837672233582,
        -5.351449012756348
       ],
       [
        -6.302326679229736,
        -5.211746692657471,
        0.6852970123291016,
        -6.389919757843018,
        -6.610111713409424
       ]
      ],
      "joint_logits": [
       -8.281756401062012,
       -7.305570602416992,
       -6.814272880554199,
       -8.408177375793457,
       -8.547534942626953,
       -7.643291473388672,
       -5.8820481300354,
       0.6834463477134705,
       -7.39420747756958,
       -8.169254302978516,
       -8.064253807067871,
       -7.051756858825684,
       -6.357947826385498,
       -8.180354118347168,
       -8.373147964477539,
       -8.183069229125977,
       -8.14271068572998,
       -8.541363716125488,
       -8.60417652130127,
       -8.420327186584473,
       -8.50993824005127,
       -7.586537837982178,
       -7.847479820251465,
       -8.554862976074219,
       -8.6823148727417,
       -8.097458839416504,
       -8.024567604064941,
       -8.285616874694824,
       -8.597840309143066,
       -8.299899101257324
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "light_damage": {
           "value": false,
           "probability": 0.9980209853789361,
           "distribution": [
            {
             "value": false,
             "probability": 0.9980209853789361
            },
            {
             "value": true,
             "probability": 0.0019790146210638283
            }
           ]
          },
          "decision": {
           "value": "minor damage fee",
           "probability": 0.9958136934941421,
           "distribution": [
            {
             "value": "no charge",
             "probability": 0.0018093336721096346
            },
            {
             "value": "minor damage fee",
             "probability": 0.9958136934941421
            },
            {
             "value": "full damage claim",
             "probability": 0.002376972833748361
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9948316006563863,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0009184662630927251
            },
            {
             "value": 1,
             "probability": 0.002733355333197557
            },
            {
             "value": 2,
             "probability": 0.9948316006563863
            },
            {
             "value": 3,
             "probability": 0.0008414377485590521
            },
            {
             "value": 4,
             "probability": 0.0006751399987643769
            }
           ]
          }
         },
         "joint_probability": 0.9929922055153251,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0070324643984611,
        "field_brier": 2.3490947674274006e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "light_damage": {
           "value": false,
           "probability": 0.9980209853789361,
           "distribution": [
            {
             "value": false,
             "probability": 0.9980209853789361
            },
            {
             "value": true,
             "probability": 0.0019790146210638283
            }
           ]
          },
          "decision": {
           "value": "minor damage fee",
           "probability": 0.9958136934941421,
           "distribution": [
            {
             "value": "no charge",
             "probability": 0.0018093336721096346
            },
            {
             "value": "minor damage fee",
             "probability": 0.9958136934941421
            },
            {
             "value": "full damage claim",
             "probability": 0.002376972833748361
            }
           ]
          },
          "factors": {
           "value": 2,
           "probability": 0.9948316006563863,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0009184662630927251
            },
            {
             "value": 1,
             "probability": 0.002733355333197557
            },
            {
             "value": 2,
             "probability": 0.9948316006563863
            },
            {
             "value": 3,
             "probability": 0.0008414377485590521
            },
            {
             "value": 4,
             "probability": 0.0006751399987643769
            }
           ]
          }
         },
         "joint_probability": 0.9929922055153251,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 30,
         "valid_assignments": 30,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0070324643984611,
        "field_brier": 2.3490947674274006e-05
       }
      },
      "timing_ms": {
       "scoring": 41.03611898608506,
       "base_joint": 0.07417891174554825,
       "constrained_joint": 0.05512009374797344,
       "constrained_total": 41.09123907983303
      },
      "input_tokens": 836,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07dcbac851eabe78-0005",
      "group": "bendsynth-07dcbac851eabe78",
      "provenance": "bendsynth-v1; family=bendsynth-07dcbac851eabe78; domain=mining site blast clearance review; parameters=min_crew=4; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "suspended",
       "shutdown_report": true,
       "quota_met": false
      },
      "logits": [
       [
        -6.8828630447387695,
        -7.029628753662109,
        -0.296354204416275
       ],
       [
        -6.430296897888184,
        -0.295952707529068
       ],
       [
        -0.2954755425453186,
        -6.6791534423828125
       ]
      ],
      "joint_logits": [
       -7.974095821380615,
       -8.425888061523438,
       -8.362983703613281,
       -8.384100914001465,
       -8.15627384185791,
       -8.509770393371582,
       -8.487279891967773,
       -8.564453125,
       -8.151049613952637,
       -8.210689544677734,
       -0.29730337858200073,
       -8.836996078491211
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "suspended",
           "probability": 0.9974371117020001,
           "distribution": [
            {
             "value": "cleared with restrictions",
             "probability": 0.001375311731653001
            },
            {
             "value": "cleared",
             "probability": 0.0011875765663468775
            },
            {
             "value": "suspended",
             "probability": 0.9974371117020001
            }
           ]
          },
          "shutdown_report": {
           "value": true,
           "probability": 0.9978375403313212,
           "distribution": [
            {
             "value": false,
             "probability": 0.0021624596686788396
            },
            {
             "value": true,
             "probability": 0.9978375403313212
            }
           ]
          },
          "quota_met": {
           "value": false,
           "probability": 0.9983139479028992,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983139479028992
            },
            {
             "value": true,
             "probability": 0.0016860520971008629
            }
           ]
          }
         },
         "joint_probability": 0.9964907855425738,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0035153861933405113,
        "field_brier": 8.302741291261542e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "suspended",
           "probability": 0.9974371117020001,
           "distribution": [
            {
             "value": "cleared with restrictions",
             "probability": 0.001375311731653001
            },
            {
             "value": "cleared",
             "probability": 0.0011875765663468775
            },
            {
             "value": "suspended",
             "probability": 0.9974371117020001
            }
           ]
          },
          "shutdown_report": {
           "value": true,
           "probability": 0.9978375403313212,
           "distribution": [
            {
             "value": false,
             "probability": 0.0021624596686788396
            },
            {
             "value": true,
             "probability": 0.9978375403313212
            }
           ]
          },
          "quota_met": {
           "value": false,
           "probability": 0.9983139479028992,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983139479028992
            },
            {
             "value": true,
             "probability": 0.0016860520971008629
            }
           ]
          }
         },
         "joint_probability": 0.9964907855425738,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0035153861933405113,
        "field_brier": 8.302741291261542e-06
       }
      },
      "timing_ms": {
       "scoring": 33.05039484985173,
       "base_joint": 0.04982016980648041,
       "constrained_joint": 0.03365008160471916,
       "constrained_total": 33.08404493145645
      },
      "input_tokens": 582,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07dcbac851eabe78-0009",
      "group": "bendsynth-07dcbac851eabe78",
      "provenance": "bendsynth-v1; family=bendsynth-07dcbac851eabe78; domain=mining site blast clearance review; parameters=min_crew=2; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "suspended",
       "shutdown_report": true,
       "quota_met": true
      },
      "logits": [
       [
        -0.809067964553833,
        -7.271993637084961,
        -7.412546634674072
       ],
       [
        -6.767838478088379,
        -0.8087351322174072
       ],
       [
        -6.350631237030029,
        -0.8100730180740356
       ]
      ],
      "joint_logits": [
       -8.646174430847168,
       -9.052268028259277,
       -7.106779098510742,
       -0.8115693926811218,
       -8.2371187210083,
       -8.545784950256348,
       -8.899577140808105,
       -9.217658042907715,
       -8.37946605682373,
       -8.687304496765137,
       -9.030591011047363,
       -9.364736557006836
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "suspended",
           "probability": 0.9970926086503226,
           "distribution": [
            {
             "value": "suspended",
             "probability": 0.9970926086503226
            },
            {
             "value": "cleared with restrictions",
             "probability": 0.0015556885528498006
            },
            {
             "value": "cleared",
             "probability": 0.0013517027968276535
            }
           ]
          },
          "shutdown_report": {
           "value": true,
           "probability": 0.9974244242439919,
           "distribution": [
            {
             "value": false,
             "probability": 0.0025755757560080657
            },
            {
             "value": true,
             "probability": 0.9974244242439919
            }
           ]
          },
          "quota_met": {
           "value": true,
           "probability": 0.9960910047857402,
           "distribution": [
            {
             "value": false,
             "probability": 0.003908995214259813
            },
            {
             "value": true,
             "probability": 0.9960910047857402
            }
           ]
          }
         },
         "joint_probability": 0.9946015089399008,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.005413115570233786,
        "field_brier": 1.884261996822803e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "suspended",
           "probability": 0.9970926086503226,
           "distribution": [
            {
             "value": "suspended",
             "probability": 0.9970926086503226
            },
            {
             "value": "cleared with restrictions",
             "probability": 0.0015556885528498006
            },
            {
             "value": "cleared",
             "probability": 0.0013517027968276535
            }
           ]
          },
          "shutdown_report": {
           "value": true,
           "probability": 0.9974244242439919,
           "distribution": [
            {
             "value": false,
             "probability": 0.0025755757560080657
            },
            {
             "value": true,
             "probability": 0.9974244242439919
            }
           ]
          },
          "quota_met": {
           "value": true,
           "probability": 0.9960910047857402,
           "distribution": [
            {
             "value": false,
             "probability": 0.003908995214259813
            },
            {
             "value": true,
             "probability": 0.9960910047857402
            }
           ]
          }
         },
         "joint_probability": 0.9946015089399008,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.005413115570233786,
        "field_brier": 1.884261996822803e-05
       }
      },
      "timing_ms": {
       "scoring": 32.32267219573259,
       "base_joint": 0.0487789511680603,
       "constrained_joint": 0.031949952244758606,
       "constrained_total": 32.35462214797735
      },
      "input_tokens": 600,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07dcbac851eabe78-0024",
      "group": "bendsynth-07dcbac851eabe78",
      "provenance": "bendsynth-v1; family=bendsynth-07dcbac851eabe78; domain=mining site blast clearance review; parameters=min_crew=3; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_met": true,
       "decision": "cleared",
       "shutdown_report": false
      },
      "logits": [
       [
        -5.749717712402344,
        0.4284779727458954
       ],
       [
        0.42776769399642944,
        -6.180531024932861,
        -6.117353916168213
       ],
       [
        0.4284604787826538,
        -5.741343975067139
       ]
      ],
      "joint_logits": [
       -7.526845455169678,
       -7.383353233337402,
       -7.458208084106445,
       -7.719349384307861,
       -7.448207855224609,
       -7.774280548095703,
       0.4266737103462219,
       -7.567333698272705,
       -7.701838970184326,
       -7.424439430236816,
       -7.434416770935059,
       -7.400719165802002
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_met": {
           "value": true,
           "probability": 0.9979301260539818,
           "distribution": [
            {
             "value": false,
             "probability": 0.0020698739460182827
            },
            {
             "value": true,
             "probability": 0.9979301260539818
            }
           ]
          },
          "decision": {
           "value": "cleared",
           "probability": 0.9972215067918856,
           "distribution": [
            {
             "value": "cleared",
             "probability": 0.9972215067918856
            },
            {
             "value": "suspended",
             "probability": 0.0013453768044172825
            },
            {
             "value": "cleared with restrictions",
             "probability": 0.0014331164036970935
            }
           ]
          },
          "shutdown_report": {
           "value": false,
           "probability": 0.9979127211957803,
           "distribution": [
            {
             "value": false,
             "probability": 0.9979127211957803
            },
            {
             "value": true,
             "probability": 0.002087278804219739
            }
           ]
          }
         },
         "joint_probability": 0.9961311920518842,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003876311144120563,
        "field_brier": 9.622035932615856e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_met": {
           "value": true,
           "probability": 0.9979301260539818,
           "distribution": [
            {
             "value": false,
             "probability": 0.0020698739460182827
            },
            {
             "value": true,
             "probability": 0.9979301260539818
            }
           ]
          },
          "decision": {
           "value": "cleared",
           "probability": 0.9972215067918856,
           "distribution": [
            {
             "value": "cleared",
             "probability": 0.9972215067918856
            },
            {
             "value": "suspended",
             "probability": 0.0013453768044172825
            },
            {
             "value": "cleared with restrictions",
             "probability": 0.0014331164036970935
            }
           ]
          },
          "shutdown_report": {
           "value": false,
           "probability": 0.9979127211957803,
           "distribution": [
            {
             "value": false,
             "probability": 0.9979127211957803
            },
            {
             "value": true,
             "probability": 0.002087278804219739
            }
           ]
          }
         },
         "joint_probability": 0.9961311920518842,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003876311144120563,
        "field_brier": 9.622035932615856e-06
       }
      },
      "timing_ms": {
       "scoring": 31.3150929287076,
       "base_joint": 0.04868907853960991,
       "constrained_joint": 0.032120151445269585,
       "constrained_total": 31.34721308015287
      },
      "input_tokens": 603,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07dcbac851eabe78-0025",
      "group": "bendsynth-07dcbac851eabe78",
      "provenance": "bendsynth-v1; family=bendsynth-07dcbac851eabe78; domain=mining site blast clearance review; parameters=min_crew=2; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "cleared",
       "shutdown_report": false,
       "quota_met": true
      },
      "logits": [
       [
        0.9255182147026062,
        -5.69389009475708,
        -5.522070407867432
       ],
       [
        0.9262470006942749,
        -5.198591709136963
       ],
       [
        -5.304018974304199,
        0.9264659881591797
       ]
      ],
      "joint_logits": [
       -7.235490322113037,
       0.9245300889015198,
       -7.010638236999512,
       -7.048148155212402,
       -7.04417085647583,
       -7.298954486846924,
       -7.20357084274292,
       -6.836531639099121,
       -6.929561138153076,
       -6.827291011810303,
       -7.190179824829102,
       -6.741064071655273
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "cleared",
           "probability": 0.9970899347305725,
           "distribution": [
            {
             "value": "cleared",
             "probability": 0.9970899347305725
            },
            {
             "value": "suspended",
             "probability": 0.001330337506890001
            },
            {
             "value": "cleared with restrictions",
             "probability": 0.001579727762537576
            }
           ]
          },
          "shutdown_report": {
           "value": false,
           "probability": 0.9978169324725352,
           "distribution": [
            {
             "value": false,
             "probability": 0.9978169324725352
            },
            {
             "value": true,
             "probability": 0.0021830675274647346
            }
           ]
          },
          "quota_met": {
           "value": true,
           "probability": 0.9980353694856046,
           "distribution": [
            {
             "value": false,
             "probability": 0.0019646305143954574
            },
            {
             "value": true,
             "probability": 0.9980353694856046
            }
           ]
          }
         },
         "joint_probability": 0.9961052421765116,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0039023621438193383,
        "field_brier": 9.994977111142673e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "cleared",
           "probability": 0.9970899347305725,
           "distribution": [
            {
             "value": "cleared",
             "probability": 0.9970899347305725
            },
            {
             "value": "suspended",
             "probability": 0.001330337506890001
            },
            {
             "value": "cleared with restrictions",
             "probability": 0.001579727762537576
            }
           ]
          },
          "shutdown_report": {
           "value": false,
           "probability": 0.9978169324725352,
           "distribution": [
            {
             "value": false,
             "probability": 0.9978169324725352
            },
            {
             "value": true,
             "probability": 0.0021830675274647346
            }
           ]
          },
          "quota_met": {
           "value": true,
           "probability": 0.9980353694856046,
           "distribution": [
            {
             "value": false,
             "probability": 0.0019646305143954574
            },
            {
             "value": true,
             "probability": 0.9980353694856046
            }
           ]
          }
         },
         "joint_probability": 0.9961052421765116,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0039023621438193383,
        "field_brier": 9.994977111142673e-06
       }
      },
      "timing_ms": {
       "scoring": 30.021297046914697,
       "base_joint": 0.048469053581357,
       "constrained_joint": 0.031709903851151466,
       "constrained_total": 30.053006950765848
      },
      "input_tokens": 555,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07dcbac851eabe78-0037",
      "group": "bendsynth-07dcbac851eabe78",
      "provenance": "bendsynth-v1; family=bendsynth-07dcbac851eabe78; domain=mining site blast clearance review; parameters=min_crew=2; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "cleared with restrictions",
       "shutdown_report": false,
       "quota_met": false
      },
      "logits": [
       [
        -6.10383939743042,
        -6.054078578948975,
        0.4209785759449005
       ],
       [
        0.4218462109565735,
        -5.726073741912842
       ],
       [
        0.42180588841438293,
        -5.707364082336426
       ]
      ],
      "joint_logits": [
       -7.416378021240234,
       -7.492886543273926,
       -7.484812259674072,
       -7.57257080078125,
       -7.454159259796143,
       -7.425647258758545,
       -7.44913387298584,
       -7.432821750640869,
       0.41996294260025024,
       -7.531637191772461,
       -7.634233474731445,
       -7.548569679260254
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "cleared with restrictions",
           "probability": 0.9970010237254033,
           "distribution": [
            {
             "value": "cleared",
             "probability": 0.0014621879348045669
            },
            {
             "value": "suspended",
             "probability": 0.0015367883397921225
            },
            {
             "value": "cleared with restrictions",
             "probability": 0.9970010237254033
            }
           ]
          },
          "shutdown_report": {
           "value": false,
           "probability": 0.997866637289471,
           "distribution": [
            {
             "value": false,
             "probability": 0.997866637289471
            },
            {
             "value": true,
             "probability": 0.0021333627105289453
            }
           ]
          },
          "quota_met": {
           "value": false,
           "probability": 0.9978263470135806,
           "distribution": [
            {
             "value": false,
             "probability": 0.9978263470135806
            },
            {
             "value": true,
             "probability": 0.00217365298641938
            }
           ]
          }
         },
         "joint_probability": 0.9959890485024512,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004019016937456267,
        "field_brier": 1.0681859391231165e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "cleared with restrictions",
           "probability": 0.9970010237254033,
           "distribution": [
            {
             "value": "cleared",
             "probability": 0.0014621879348045669
            },
            {
             "value": "suspended",
             "probability": 0.0015367883397921225
            },
            {
             "value": "cleared with restrictions",
             "probability": 0.9970010237254033
            }
           ]
          },
          "shutdown_report": {
           "value": false,
           "probability": 0.997866637289471,
           "distribution": [
            {
             "value": false,
             "probability": 0.997866637289471
            },
            {
             "value": true,
             "probability": 0.0021333627105289453
            }
           ]
          },
          "quota_met": {
           "value": false,
           "probability": 0.9978263470135806,
           "distribution": [
            {
             "value": false,
             "probability": 0.9978263470135806
            },
            {
             "value": true,
             "probability": 0.00217365298641938
            }
           ]
          }
         },
         "joint_probability": 0.9959890485024512,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 12,
         "valid_assignments": 12,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004019016937456267,
        "field_brier": 1.0681859391231165e-05
       }
      },
      "timing_ms": {
       "scoring": 31.709518982097507,
       "base_joint": 0.05676993168890476,
       "constrained_joint": 0.032939016819000244,
       "constrained_total": 31.742457998916507
      },
      "input_tokens": 579,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07b1cabc406fff3a-0002",
      "group": "bendsynth-07b1cabc406fff3a",
      "provenance": "bendsynth-v1; family=bendsynth-07b1cabc406fff3a; domain=walk-in counter at a phone repair shop processing warranty claims; parameters=cover_days=60; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "in_cover": true,
       "complications": 2,
       "decision": "decline claim"
      },
      "logits": [
       [
        -5.342742919921875,
        1.3552134037017822
       ],
       [
        -5.858893871307373,
        -5.930365562438965,
        1.3542817831039429,
        -5.849201679229736
       ],
       [
        -5.576098442077637,
        1.3546878099441528,
        -5.798068523406982
       ]
      ],
      "joint_logits": [
       -7.774433135986328,
       -7.6505303382873535,
       -7.914813041687012,
       -7.88097620010376,
       -7.775660514831543,
       -8.069971084594727,
       -8.029606819152832,
       -7.574734687805176,
       -8.262596130371094,
       -7.715270042419434,
       -7.623100757598877,
       -7.878209590911865,
       -7.59092903137207,
       -7.383363246917725,
       -7.670356750488281,
       -7.736503601074219,
       -7.252702236175537,
       -7.818634986877441,
       -7.1536335945129395,
       1.353699803352356,
       -7.88936710357666,
       -7.613301753997803,
       -7.419639587402344,
       -7.6519927978515625
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "in_cover": {
           "value": true,
           "probability": 0.9987680889776644,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012319110223355213
            },
            {
             "value": true,
             "probability": 0.9987680889776644
            }
           ]
          },
          "complications": {
           "value": 2,
           "probability": 0.9978378913331079,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007352206694527267
            },
            {
             "value": 1,
             "probability": 0.0006845068967650389
            },
            {
             "value": 2,
             "probability": 0.9978378913331079
            },
            {
             "value": 3,
             "probability": 0.0007423811006743942
            }
           ]
          },
          "decision": {
           "value": "discounted repair",
           "probability": 0.9982431556119968,
           "distribution": [
            {
             "value": "decline claim",
             "probability": 0.0009755156343261843
            },
            {
             "value": "discounted repair",
             "probability": 0.9982431556119968
            },
            {
             "value": "free warranty repair",
             "probability": 0.0007813287536770103
            }
           ]
          }
         },
         "joint_probability": 0.9972574097313408,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.510079755925227,
        "field_brier": 0.6648497329050546
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "in_cover": {
           "value": true,
           "probability": 0.9987680889776644,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012319110223355213
            },
            {
             "value": true,
             "probability": 0.9987680889776644
            }
           ]
          },
          "complications": {
           "value": 2,
           "probability": 0.9978378913331079,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007352206694527267
            },
            {
             "value": 1,
             "probability": 0.0006845068967650389
            },
            {
             "value": 2,
             "probability": 0.9978378913331079
            },
            {
             "value": 3,
             "probability": 0.0007423811006743942
            }
           ]
          },
          "decision": {
           "value": "discounted repair",
           "probability": 0.9982431556119968,
           "distribution": [
            {
             "value": "decline claim",
             "probability": 0.0009755156343261843
            },
            {
             "value": "discounted repair",
             "probability": 0.9982431556119968
            },
            {
             "value": "free warranty repair",
             "probability": 0.0007813287536770103
            }
           ]
          }
         },
         "joint_probability": 0.9972574097313408,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.510079755925227,
        "field_brier": 0.6648497329050546
       }
      },
      "timing_ms": {
       "scoring": 32.46853104792535,
       "base_joint": 0.06487895734608173,
       "constrained_joint": 0.04770001396536827,
       "constrained_total": 32.51623106189072
      },
      "input_tokens": 567,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07b1cabc406fff3a-0008",
      "group": "bendsynth-07b1cabc406fff3a",
      "provenance": "bendsynth-v1; family=bendsynth-07b1cabc406fff3a; domain=walk-in counter at a phone repair shop processing warranty claims; parameters=cover_days=119; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "complications": 2,
       "decision": "decline claim",
       "in_cover": false
      },
      "logits": [
       [
        -6.914555549621582,
        -6.591752052307129,
        0.4036831259727478,
        -6.844827175140381
       ],
       [
        -6.690637111663818,
        -5.285403728485107,
        0.4017593562602997
       ],
       [
        0.4045073688030243,
        -6.121590614318848
       ]
      ],
      "joint_logits": [
       -8.798380851745605,
       -8.852221488952637,
       -8.980260848999023,
       -8.768366813659668,
       -8.431471824645996,
       -8.518022537231445,
       -8.635392189025879,
       -8.619982719421387,
       -8.559709548950195,
       -8.619030952453613,
       -7.829977035522461,
       -8.32029914855957,
       -8.705181121826172,
       -9.072308540344238,
       -5.527092456817627,
       -8.919471740722656,
       0.4005345106124878,
       -8.062314987182617,
       -8.771265983581543,
       -8.7744779586792,
       -8.913742065429688,
       -8.70710563659668,
       -8.305365562438965,
       -8.477622032165527
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "complications": {
           "value": 2,
           "probability": 0.99771461764135,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006618138056471813
            },
            {
             "value": 1,
             "probability": 0.0009139607907894105
            },
            {
             "value": 2,
             "probability": 0.99771461764135
            },
            {
             "value": 3,
             "probability": 0.0007096077622133223
            }
           ]
          },
          "decision": {
           "value": "discounted repair",
           "probability": 0.9957971414145779,
           "distribution": [
            {
             "value": "free warranty repair",
             "probability": 0.0008279082980876376
            },
            {
             "value": "decline claim",
             "probability": 0.0033749502873344946
            },
            {
             "value": "discounted repair",
             "probability": 0.9957971414145779
            }
           ]
          },
          "in_cover": {
           "value": false,
           "probability": 0.9985374321791162,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985374321791162
            },
            {
             "value": true,
             "probability": 0.0014625678208838905
            }
           ]
          }
         },
         "joint_probability": 0.9945781926356739,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 5.933063526135435,
        "field_brier": 0.661628466681057
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "complications": {
           "value": 2,
           "probability": 0.99771461764135,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006618138056471813
            },
            {
             "value": 1,
             "probability": 0.0009139607907894105
            },
            {
             "value": 2,
             "probability": 0.99771461764135
            },
            {
             "value": 3,
             "probability": 0.0007096077622133223
            }
           ]
          },
          "decision": {
           "value": "discounted repair",
           "probability": 0.9957971414145779,
           "distribution": [
            {
             "value": "free warranty repair",
             "probability": 0.0008279082980876376
            },
            {
             "value": "decline claim",
             "probability": 0.0033749502873344946
            },
            {
             "value": "discounted repair",
             "probability": 0.9957971414145779
            }
           ]
          },
          "in_cover": {
           "value": false,
           "probability": 0.9985374321791162,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985374321791162
            },
            {
             "value": true,
             "probability": 0.0014625678208838905
            }
           ]
          }
         },
         "joint_probability": 0.9945781926356739,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 5.933063526135435,
        "field_brier": 0.661628466681057
       }
      },
      "timing_ms": {
       "scoring": 32.69350901246071,
       "base_joint": 0.06561912596225739,
       "constrained_joint": 0.046798959374427795,
       "constrained_total": 32.740307971835136
      },
      "input_tokens": 585,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07b1cabc406fff3a-0015",
      "group": "bendsynth-07b1cabc406fff3a",
      "provenance": "bendsynth-v1; family=bendsynth-07b1cabc406fff3a; domain=walk-in counter at a phone repair shop processing warranty claims; parameters=cover_days=56; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "decline claim",
       "complications": 3,
       "in_cover": false
      },
      "logits": [
       [
        -6.231227397918701,
        0.8944004774093628,
        -6.453557014465332
       ],
       [
        -6.4987616539001465,
        -6.600974082946777,
        -6.5060224533081055,
        0.8940663933753967
       ],
       [
        0.8947131037712097,
        -5.886271953582764
       ]
      ],
      "joint_logits": [
       -8.20348072052002,
       -8.244673728942871,
       -8.37304401397705,
       -8.366479873657227,
       -8.26477336883545,
       -8.418292045593262,
       -8.234599113464355,
       -8.405619621276855,
       -8.315631866455078,
       -8.203560829162598,
       -8.240819931030273,
       -8.33473014831543,
       -7.917285919189453,
       -8.2394380569458,
       0.8936119675636292,
       -8.190288543701172,
       -8.38772201538086,
       -8.408498764038086,
       -8.554098129272461,
       -8.52198314666748,
       -8.485675811767578,
       -8.614988327026367,
       -8.706787109375,
       -8.626599311828613
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "decline claim",
           "probability": 0.9985539597818726,
           "distribution": [
            {
             "value": "discounted repair",
             "probability": 0.0008030650063449813
            },
            {
             "value": "decline claim",
             "probability": 0.9985539597818726
            },
            {
             "value": "free warranty repair",
             "probability": 0.0006429752117825648
            }
           ]
          },
          "complications": {
           "value": 3,
           "probability": 0.9982204872464377,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006145568273328887
            },
            {
             "value": 1,
             "probability": 0.000554845132209712
            },
            {
             "value": 2,
             "probability": 0.0006101107940196843
            },
            {
             "value": 3,
             "probability": 0.9982204872464377
            }
           ]
          },
          "in_cover": {
           "value": false,
           "probability": 0.9988661307871937,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988661307871937
            },
            {
             "value": true,
             "probability": 0.0011338692128063004
            }
           ]
          }
         },
         "joint_probability": 0.9977669142571527,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022355827969313194,
        "field_brier": 3.315038553051039e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "decline claim",
           "probability": 0.9985539597818726,
           "distribution": [
            {
             "value": "discounted repair",
             "probability": 0.0008030650063449813
            },
            {
             "value": "decline claim",
             "probability": 0.9985539597818726
            },
            {
             "value": "free warranty repair",
             "probability": 0.0006429752117825648
            }
           ]
          },
          "complications": {
           "value": 3,
           "probability": 0.9982204872464377,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006145568273328887
            },
            {
             "value": 1,
             "probability": 0.000554845132209712
            },
            {
             "value": 2,
             "probability": 0.0006101107940196843
            },
            {
             "value": 3,
             "probability": 0.9982204872464377
            }
           ]
          },
          "in_cover": {
           "value": false,
           "probability": 0.9988661307871937,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988661307871937
            },
            {
             "value": true,
             "probability": 0.0011338692128063004
            }
           ]
          }
         },
         "joint_probability": 0.9977669142571527,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022355827969313194,
        "field_brier": 3.315038553051039e-06
       }
      },
      "timing_ms": {
       "scoring": 31.020625960081816,
       "base_joint": 0.06434903480112553,
       "constrained_joint": 0.04863017238676548,
       "constrained_total": 31.06925613246858
      },
      "input_tokens": 594,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07b1cabc406fff3a-0034",
      "group": "bendsynth-07b1cabc406fff3a",
      "provenance": "bendsynth-v1; family=bendsynth-07b1cabc406fff3a; domain=walk-in counter at a phone repair shop processing warranty claims; parameters=cover_days=96; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "free warranty repair",
       "in_cover": true,
       "complications": 0
      },
      "logits": [
       [
        -6.034182548522949,
        -6.049245834350586,
        1.1330444812774658
       ],
       [
        -5.776206970214844,
        1.1335771083831787
       ],
       [
        1.132907748222351,
        -6.393552303314209,
        -6.340638160705566,
        -6.351820945739746
       ]
      ],
      "joint_logits": [
       -8.296308517456055,
       -8.18796443939209,
       -8.09447193145752,
       -8.142308235168457,
       -8.11901569366455,
       -8.051321983337402,
       -8.042214393615723,
       -8.005105972290039,
       -8.281615257263184,
       -8.46815013885498,
       -8.299402236938477,
       -8.411717414855957,
       -7.942920207977295,
       -7.962900161743164,
       -7.920461654663086,
       -7.932181358337402,
       -8.206399917602539,
       -8.29561710357666,
       -8.273788452148438,
       -8.2352876663208,
       1.132447600364685,
       -8.225112915039062,
       -8.219528198242188,
       -8.207982063293457
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "free warranty repair",
           "probability": 0.9984709568982353,
           "distribution": [
            {
             "value": "decline claim",
             "probability": 0.0007702794826675883
            },
            {
             "value": "discounted repair",
             "probability": 0.0007587636190971443
            },
            {
             "value": "free warranty repair",
             "probability": 0.9984709568982353
            }
           ]
          },
          "in_cover": {
           "value": true,
           "probability": 0.999003021583719,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009969784162809547
            },
            {
             "value": true,
             "probability": 0.999003021583719
            }
           ]
          },
          "complications": {
           "value": 0,
           "probability": 0.9983346297790217,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9983346297790217
            },
            {
             "value": 1,
             "probability": 0.0005377445958767104
            },
            {
             "value": 2,
             "probability": 0.0005669652114877124
            },
            {
             "value": 3,
             "probability": 0.0005606604136139365
            }
           ]
          }
         },
         "joint_probability": 0.9978752974298034,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0021269629530308795,
        "field_brier": 3.0644581056485566e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "free warranty repair",
           "probability": 0.9984709568982353,
           "distribution": [
            {
             "value": "decline claim",
             "probability": 0.0007702794826675883
            },
            {
             "value": "discounted repair",
             "probability": 0.0007587636190971443
            },
            {
             "value": "free warranty repair",
             "probability": 0.9984709568982353
            }
           ]
          },
          "in_cover": {
           "value": true,
           "probability": 0.999003021583719,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009969784162809547
            },
            {
             "value": true,
             "probability": 0.999003021583719
            }
           ]
          },
          "complications": {
           "value": 0,
           "probability": 0.9983346297790217,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9983346297790217
            },
            {
             "value": 1,
             "probability": 0.0005377445958767104
            },
            {
             "value": 2,
             "probability": 0.0005669652114877124
            },
            {
             "value": 3,
             "probability": 0.0005606604136139365
            }
           ]
          }
         },
         "joint_probability": 0.9978752974298034,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0021269629530308795,
        "field_brier": 3.0644581056485566e-06
       }
      },
      "timing_ms": {
       "scoring": 31.658570049330592,
       "base_joint": 0.06463914178311825,
       "constrained_joint": 0.04652887582778931,
       "constrained_total": 31.70509892515838
      },
      "input_tokens": 564,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-07b1cabc406fff3a-0037",
      "group": "bendsynth-07b1cabc406fff3a",
      "provenance": "bendsynth-v1; family=bendsynth-07b1cabc406fff3a; domain=walk-in counter at a phone repair shop processing warranty claims; parameters=cover_days=99; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "free warranty repair",
       "in_cover": true,
       "complications": 0
      },
      "logits": [
       [
        -5.989617347717285,
        -6.070591926574707,
        1.1315475702285767
       ],
       [
        -5.750932216644287,
        1.1320745944976807
       ],
       [
        1.1314127445220947,
        -6.3765716552734375,
        -6.342548370361328,
        -6.336171627044678
       ]
      ],
      "joint_logits": [
       -8.223687171936035,
       -8.099849700927734,
       -8.052836418151855,
       -8.083415031433105,
       -8.11689567565918,
       -8.04138469696045,
       -8.00054931640625,
       -7.9564208984375,
       -8.234882354736328,
       -8.393741607666016,
       -8.325745582580566,
       -8.47422981262207,
       -7.964060306549072,
       -8.009771347045898,
       -7.988162517547607,
       -7.963559150695801,
       -8.293641090393066,
       -8.278100967407227,
       -8.253080368041992,
       -8.201946258544922,
       1.1309515237808228,
       -8.243364334106445,
       -8.238718032836914,
       -8.176310539245605
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "free warranty repair",
           "probability": 0.9984495916739289,
           "distribution": [
            {
             "value": "decline claim",
             "probability": 0.0008065728521578684
            },
            {
             "value": "discounted repair",
             "probability": 0.0007438354739132989
            },
            {
             "value": "free warranty repair",
             "probability": 0.9984495916739289
            }
           ]
          },
          "in_cover": {
           "value": true,
           "probability": 0.9989759921853384,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010240078146616953
            },
            {
             "value": true,
             "probability": 0.9989759921853384
            }
           ]
          },
          "complications": {
           "value": 0,
           "probability": 0.9983151755887661,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9983151755887661
            },
            {
             "value": 1,
             "probability": 0.0005477614547364588
            },
            {
             "value": 2,
             "probability": 0.0005667187095923292
            },
            {
             "value": 3,
             "probability": 0.0005703442469050417
            }
           ]
          }
         },
         "joint_probability": 0.9978547457720647,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002147558581993249,
        "field_brier": 3.1633131761269963e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "free warranty repair",
           "probability": 0.9984495916739289,
           "distribution": [
            {
             "value": "decline claim",
             "probability": 0.0008065728521578684
            },
            {
             "value": "discounted repair",
             "probability": 0.0007438354739132989
            },
            {
             "value": "free warranty repair",
             "probability": 0.9984495916739289
            }
           ]
          },
          "in_cover": {
           "value": true,
           "probability": 0.9989759921853384,
           "distribution": [
            {
             "value": false,
             "probability": 0.0010240078146616953
            },
            {
             "value": true,
             "probability": 0.9989759921853384
            }
           ]
          },
          "complications": {
           "value": 0,
           "probability": 0.9983151755887661,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9983151755887661
            },
            {
             "value": 1,
             "probability": 0.0005477614547364588
            },
            {
             "value": 2,
             "probability": 0.0005667187095923292
            },
            {
             "value": 3,
             "probability": 0.0005703442469050417
            }
           ]
          }
         },
         "joint_probability": 0.9978547457720647,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002147558581993249,
        "field_brier": 3.1633131761269963e-06
       }
      },
      "timing_ms": {
       "scoring": 30.95435700379312,
       "base_joint": 0.08648913353681564,
       "constrained_joint": 0.049519119784235954,
       "constrained_total": 31.003876123577356
      },
      "input_tokens": 564,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aacc7383f3cd150-0008",
      "group": "bendsynth-0aacc7383f3cd150",
      "provenance": "bendsynth-v1; family=bendsynth-0aacc7383f3cd150; domain=film festival submission screening escalation desk; parameters=cap=40,grace=13; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "factors": 0,
       "decision": "fast-track accept",
       "over_cap": false
      },
      "logits": [
       [
        1.4111695289611816,
        -6.211491107940674,
        -6.19069242477417,
        -6.239553928375244
       ],
       [
        -6.1345062255859375,
        -6.391185760498047,
        -6.149186134338379,
        1.411176085472107
       ],
       [
        1.4116517305374146,
        -5.51454496383667
       ]
      ],
      "joint_logits": [
       -8.093557357788086,
       -8.310502052307129,
       -8.397786140441895,
       -8.386547088623047,
       -8.372672080993652,
       -8.249815940856934,
       1.4107460975646973,
       -8.338958740234375,
       -8.314111709594727,
       -8.134581565856934,
       -8.347744941711426,
       -8.71314811706543,
       -8.236096382141113,
       -8.164384841918945,
       -8.275557518005371,
       -8.245437622070312,
       -8.29659366607666,
       -8.109455108642578,
       -8.331180572509766,
       -8.605053901672363,
       -8.238504409790039,
       -8.127038955688477,
       -8.313356399536133,
       -8.219481468200684,
       -8.34632682800293,
       -8.146306037902832,
       -8.373702049255371,
       -8.699169158935547,
       -8.270551681518555,
       -8.189249992370605,
       -8.393163681030273,
       -8.231654167175293
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 0,
           "probability": 0.9985376827861207,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9985376827861207
            },
            {
             "value": 1,
             "probability": 0.0004885229400586513
            },
            {
             "value": 2,
             "probability": 0.0004987900100871882
            },
            {
             "value": 3,
             "probability": 0.0004750042637335855
            }
           ]
          },
          "decision": {
           "value": "fast-track accept",
           "probability": 0.9985442805017458,
           "distribution": [
            {
             "value": "standard review",
             "probability": 0.0005276175684297104
            },
            {
             "value": "senior panel",
             "probability": 0.00040817331894320806
            },
            {
             "value": "reject",
             "probability": 0.0005199286108813247
            },
            {
             "value": "fast-track accept",
             "probability": 0.9985442805017458
            }
           ]
          },
          "over_cap": {
           "value": false,
           "probability": 0.9990192350422457,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990192350422457
            },
            {
             "value": true,
             "probability": 0.0009807649577543215
            }
           ]
          }
         },
         "joint_probability": 0.9981149340342582,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018868449385950781,
        "field_brier": 2.5365591337467246e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 0,
           "probability": 0.9985376827861207,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9985376827861207
            },
            {
             "value": 1,
             "probability": 0.0004885229400586513
            },
            {
             "value": 2,
             "probability": 0.0004987900100871882
            },
            {
             "value": 3,
             "probability": 0.0004750042637335855
            }
           ]
          },
          "decision": {
           "value": "fast-track accept",
           "probability": 0.9985442805017458,
           "distribution": [
            {
             "value": "standard review",
             "probability": 0.0005276175684297104
            },
            {
             "value": "senior panel",
             "probability": 0.00040817331894320806
            },
            {
             "value": "reject",
             "probability": 0.0005199286108813247
            },
            {
             "value": "fast-track accept",
             "probability": 0.9985442805017458
            }
           ]
          },
          "over_cap": {
           "value": false,
           "probability": 0.9990192350422457,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990192350422457
            },
            {
             "value": true,
             "probability": 0.0009807649577543215
            }
           ]
          }
         },
         "joint_probability": 0.9981149340342582,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018868449385950781,
        "field_brier": 2.5365591337467246e-06
       }
      },
      "timing_ms": {
       "scoring": 38.34701795130968,
       "base_joint": 0.07600989192724228,
       "constrained_joint": 0.05981908179819584,
       "constrained_total": 38.40683703310788
      },
      "input_tokens": 748,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aacc7383f3cd150-0013",
      "group": "bendsynth-0aacc7383f3cd150",
      "provenance": "bendsynth-v1; family=bendsynth-0aacc7383f3cd150; domain=film festival submission screening escalation desk; parameters=cap=60,grace=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "factors": 1,
       "over_cap": false,
       "decision": "standard review"
      },
      "logits": [
       [
        -5.386812210083008,
        1.9218809604644775,
        -5.81315803527832,
        -5.753384590148926
       ],
       [
        1.9224750995635986,
        -5.009369850158691
       ],
       [
        -5.4275922775268555,
        -5.554795742034912,
        1.921878695487976,
        -5.995993614196777
       ]
      ],
      "joint_logits": [
       -7.3170318603515625,
       -7.401376724243164,
       -7.25245475769043,
       -7.660090446472168,
       -7.425787448883057,
       -7.326245307922363,
       -7.587899208068848,
       -7.923802375793457,
       -7.372445583343506,
       -7.619154453277588,
       1.92142653465271,
       -7.901102542877197,
       -7.676863670349121,
       -7.727470397949219,
       -7.784761905670166,
       -8.129462242126465,
       -7.647170066833496,
       -7.781486988067627,
       -7.956207752227783,
       -7.997452259063721,
       -7.598876953125,
       -7.8468475341796875,
       -7.899698734283447,
       -8.848855018615723,
       -7.569926738739014,
       -7.731266498565674,
       -8.021402359008789,
       -7.936735153198242,
       -7.509331226348877,
       -7.771377086639404,
       -7.801845073699951,
       -8.7843017578125
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 1,
           "probability": 0.9984313700904305,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000668641154145121
            },
            {
             "value": 1,
             "probability": 0.9984313700904305
            },
            {
             "value": 2,
             "probability": 0.00043654963544062064
            },
            {
             "value": 3,
             "probability": 0.00046343911998374744
            }
           ]
          },
          "over_cap": {
           "value": false,
           "probability": 0.9990247535980017,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990247535980017
            },
            {
             "value": true,
             "probability": 0.0009752464019982819
            }
           ]
          },
          "decision": {
           "value": "standard review",
           "probability": 0.9984292252906312,
           "distribution": [
            {
             "value": "fast-track accept",
             "probability": 0.0006419225646786253
            },
            {
             "value": "senior panel",
             "probability": 0.0005652477924126961
            },
            {
             "value": "standard review",
             "probability": 0.9984292252906312
            },
            {
             "value": "reject",
             "probability": 0.0003636043522775367
            }
           ]
          }
         },
         "joint_probability": 0.9979778174459893,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0020242299257233667,
        "field_brier": 2.8487847453424742e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "factors": {
           "value": 1,
           "probability": 0.9984313700904305,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000668641154145121
            },
            {
             "value": 1,
             "probability": 0.9984313700904305
            },
            {
             "value": 2,
             "probability": 0.00043654963544062064
            },
            {
             "value": 3,
             "probability": 0.00046343911998374744
            }
           ]
          },
          "over_cap": {
           "value": false,
           "probability": 0.9990247535980017,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990247535980017
            },
            {
             "value": true,
             "probability": 0.0009752464019982819
            }
           ]
          },
          "decision": {
           "value": "standard review",
           "probability": 0.9984292252906312,
           "distribution": [
            {
             "value": "fast-track accept",
             "probability": 0.0006419225646786253
            },
            {
             "value": "senior panel",
             "probability": 0.0005652477924126961
            },
            {
             "value": "standard review",
             "probability": 0.9984292252906312
            },
            {
             "value": "reject",
             "probability": 0.0003636043522775367
            }
           ]
          }
         },
         "joint_probability": 0.9979778174459893,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0020242299257233667,
        "field_brier": 2.8487847453424742e-06
       }
      },
      "timing_ms": {
       "scoring": 37.93542203493416,
       "base_joint": 0.07875892333686352,
       "constrained_joint": 0.0641189981251955,
       "constrained_total": 37.99954103305936
      },
      "input_tokens": 763,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0aacc7383f3cd150-0033",
      "group": "bendsynth-0aacc7383f3cd150",
      "provenance": "bendsynth-v1; family=bendsynth-0aacc7383f3cd150; domain=film festival submission screening escalation desk; parameters=cap=90,grace=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "over_cap": false,
       "decision": "fast-track accept",
       "factors": 1
      },
      "logits": [
       [
        1.6982223987579346,
        -5.157153606414795
       ],
       [
        -5.606167793273926,
        -5.720195770263672,
        1.697569727897644,
        -6.04548978805542
       ],
       [
        -5.534183025360107,
        1.697595477104187,
        -5.994773864746094,
        -5.899420738220215
       ]
      ],
      "joint_logits": [
       -7.5376105308532715,
       -7.057150840759277,
       -7.919378757476807,
       -7.832169532775879,
       -7.660730838775635,
       -7.700773239135742,
       -7.919038772583008,
       -7.813968181610107,
       -7.237530708312988,
       1.6970360279083252,
       -8.077191352844238,
       -8.06674861907959,
       -7.787899494171143,
       -8.086108207702637,
       -8.197251319885254,
       -8.140669822692871,
       -7.5850934982299805,
       -7.969498157501221,
       -8.02833366394043,
       -7.9838433265686035,
       -7.641210556030273,
       -7.8551716804504395,
       -8.023418426513672,
       -7.843894004821777,
       -7.796777725219727,
       -7.8051018714904785,
       -7.943996906280518,
       -7.8322038650512695,
       -7.792736053466797,
       -8.14065170288086,
       -8.680686950683594,
       -8.491734504699707
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cap": {
           "value": false,
           "probability": 0.9989473337790402,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989473337790402
            },
            {
             "value": true,
             "probability": 0.0010526662209598455
            }
           ]
          },
          "decision": {
           "value": "standard review",
           "probability": 0.9982956590885183,
           "distribution": [
            {
             "value": "fast-track accept",
             "probability": 0.0006718716698717763
            },
            {
             "value": "senior panel",
             "probability": 0.0005994659722131462
            },
            {
             "value": "standard review",
             "probability": 0.9982956590885183
            },
            {
             "value": "reject",
             "probability": 0.00043300326939676466
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9983213465127354,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007220193701038005
            },
            {
             "value": 1,
             "probability": 0.9983213465127354
            },
            {
             "value": 2,
             "probability": 0.0004555298302304478
            },
            {
             "value": 3,
             "probability": 0.0005011042869304295
            }
           ]
          }
         },
         "joint_probability": 0.9977629934405295,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.756426381063994,
        "field_brier": 0.6650858306309462
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cap": {
           "value": false,
           "probability": 0.9989473337790402,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989473337790402
            },
            {
             "value": true,
             "probability": 0.0010526662209598455
            }
           ]
          },
          "decision": {
           "value": "standard review",
           "probability": 0.9982956590885183,
           "distribution": [
            {
             "value": "fast-track accept",
             "probability": 0.0006718716698717763
            },
            {
             "value": "senior panel",
             "probability": 0.0005994659722131462
            },
            {
             "value": "standard review",
             "probability": 0.9982956590885183
            },
            {
             "value": "reject",
             "probability": 0.00043300326939676466
            }
           ]
          },
          "factors": {
           "value": 1,
           "probability": 0.9983213465127354,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007220193701038005
            },
            {
             "value": 1,
             "probability": 0.9983213465127354
            },
            {
             "value": 2,
             "probability": 0.0004555298302304478
            },
            {
             "value": 3,
             "probability": 0.0005011042869304295
            }
           ]
          }
         },
         "joint_probability": 0.9977629934405295,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.756426381063994,
        "field_brier": 0.6650858306309462
       }
      },
      "timing_ms": {
       "scoring": 36.424747901037335,
       "base_joint": 0.07237889803946018,
       "constrained_joint": 0.05540996789932251,
       "constrained_total": 36.48015786893666
      },
      "input_tokens": 757,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0b416e8e68635558-0038",
      "group": "bendsynth-0b416e8e68635558",
      "provenance": "bendsynth-v1; family=bendsynth-0b416e8e68635558; domain=backcountry campsite exception desk, ranger supervisor review; parameters=cap=11; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "approve as filed",
       "risk_flags": 1,
       "pair_present": false
      },
      "logits": [
       [
        -5.450540542602539,
        -5.471579074859619,
        1.6080572605133057
       ],
       [
        -5.815066337585449,
        1.6080414056777954,
        -5.891632080078125,
        -5.866011619567871
       ],
       [
        1.6085344552993774,
        -5.096822738647461
       ]
      ],
      "joint_logits": [
       -7.530737400054932,
       -7.507628917694092,
       -7.46120548248291,
       -7.400209903717041,
       -7.561223030090332,
       -7.7038397789001465,
       -7.550625324249268,
       -7.551045894622803,
       -7.677810192108154,
       -7.499469757080078,
       -7.314727306365967,
       -7.516146659851074,
       -7.667779445648193,
       -7.604149341583252,
       -7.674111366271973,
       -7.50997257232666,
       -7.752167224884033,
       -7.704301834106445,
       1.6074589490890503,
       -7.579356670379639,
       -7.849982738494873,
       -7.739379405975342,
       -8.01123332977295,
       -7.731472015380859
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve as filed",
           "probability": 0.9983008299834176,
           "distribution": [
            {
             "value": "escalate to district ranger",
             "probability": 0.0008585216399353634
            },
            {
             "value": "approve with conditions",
             "probability": 0.0008406483766470359
            },
            {
             "value": "approve as filed",
             "probability": 0.9983008299834176
            }
           ]
          },
          "risk_flags": {
           "value": 1,
           "probability": 0.9982847688642761,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000596265582691029
            },
            {
             "value": 1,
             "probability": 0.9982847688642761
            },
            {
             "value": 2,
             "probability": 0.000552316182195757
            },
            {
             "value": 3,
             "probability": 0.0005666493708371686
            }
           ]
          },
          "pair_present": {
           "value": false,
           "probability": 0.9987771617746036,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987771617746036
            },
            {
             "value": true,
             "probability": 0.001222838225396328
            }
           ]
          }
         },
         "joint_probability": 0.9977035818630975,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002299058948745668,
        "field_brier": 3.7484298880914067e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "approve as filed",
           "probability": 0.9983008299834176,
           "distribution": [
            {
             "value": "escalate to district ranger",
             "probability": 0.0008585216399353634
            },
            {
             "value": "approve with conditions",
             "probability": 0.0008406483766470359
            },
            {
             "value": "approve as filed",
             "probability": 0.9983008299834176
            }
           ]
          },
          "risk_flags": {
           "value": 1,
           "probability": 0.9982847688642761,
           "distribution": [
            {
             "value": 0,
             "probability": 0.000596265582691029
            },
            {
             "value": 1,
             "probability": 0.9982847688642761
            },
            {
             "value": 2,
             "probability": 0.000552316182195757
            },
            {
             "value": 3,
             "probability": 0.0005666493708371686
            }
           ]
          },
          "pair_present": {
           "value": false,
           "probability": 0.9987771617746036,
           "distribution": [
            {
             "value": false,
             "probability": 0.9987771617746036
            },
            {
             "value": true,
             "probability": 0.001222838225396328
            }
           ]
          }
         },
         "joint_probability": 0.9977035818630975,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002299058948745668,
        "field_brier": 3.7484298880914067e-06
       }
      },
      "timing_ms": {
       "scoring": 33.29086187295616,
       "base_joint": 0.06575905717909336,
       "constrained_joint": 0.046140048652887344,
       "constrained_total": 33.337001921609044
      },
      "input_tokens": 593,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-00b4c75a0b6107ac-0007",
      "group": "bendsynth-00b4c75a0b6107ac",
      "provenance": "bendsynth-v1; family=bendsynth-00b4c75a0b6107ac; domain=notary appointment desk, escalated case supervisor review; parameters=rush_fee_pct=29; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "id_holds": true,
       "risk_flags": 1,
       "decision": "reschedule"
      },
      "logits": [
       [
        -5.182170391082764,
        1.9685064554214478
       ],
       [
        -5.770293712615967,
        1.9678218364715576,
        -6.174127101898193,
        -5.9567179679870605,
        -5.9049859046936035
       ],
       [
        -4.8770575523376465,
        1.9673397541046143,
        -5.735565185546875,
        -5.771337509155273
       ]
      ],
      "joint_logits": [
       -7.830545425415039,
       -8.221487998962402,
       -8.097625732421875,
       -8.089264869689941,
       -7.900026798248291,
       -7.990050792694092,
       -8.252907752990723,
       -8.238279342651367,
       -8.204489707946777,
       -8.577835083007812,
       -8.454904556274414,
       -8.539557456970215,
       -7.9924235343933105,
       -8.34049129486084,
       -8.229482650756836,
       -8.348180770874023,
       -7.939475059509277,
       -8.269916534423828,
       -8.166891098022461,
       -8.264642715454102,
       -7.4995269775390625,
       -7.65045690536499,
       -7.819706439971924,
       -7.797817230224609,
       -5.51118803024292,
       1.966968059539795,
       -7.719614505767822,
       -7.858060359954834,
       -7.93630838394165,
       -8.21558666229248,
       -8.145033836364746,
       -8.130453109741211,
       -7.689342975616455,
       -8.10962200164795,
       -7.8845720291137695,
       -7.892848968505859,
       -7.655198097229004,
       -8.045656204223633,
       -7.853432655334473,
       -7.8507080078125
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "id_holds": {
           "value": true,
           "probability": 0.9992162816065522,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007837183934478329
            },
            {
             "value": true,
             "probability": 0.9992162816065522
            }
           ]
          },
          "risk_flags": {
           "value": 1,
           "probability": 0.9985324756222326,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00043525261853970433
            },
            {
             "value": 1,
             "probability": 0.9985324756222326
            },
            {
             "value": 2,
             "probability": 0.00029064224289567794
            },
            {
             "value": 3,
             "probability": 0.0003612254231203448
            },
            {
             "value": 4,
             "probability": 0.0003804040932116995
            }
           ]
          },
          "decision": {
           "value": "reschedule",
           "probability": 0.9980512360236152,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0010633319224066175
            },
            {
             "value": "reschedule",
             "probability": 0.9980512360236152
            },
            {
             "value": "split booking",
             "probability": 0.00045063367773873664
            },
            {
             "value": "decline",
             "probability": 0.00043479837623942765
            }
           ]
          }
         },
         "joint_probability": 0.9976802991824605,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023223954915124316,
        "field_brier": 3.08388060929902e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "id_holds": {
           "value": true,
           "probability": 0.9992162816065522,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007837183934478329
            },
            {
             "value": true,
             "probability": 0.9992162816065522
            }
           ]
          },
          "risk_flags": {
           "value": 1,
           "probability": 0.9985324756222326,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00043525261853970433
            },
            {
             "value": 1,
             "probability": 0.9985324756222326
            },
            {
             "value": 2,
             "probability": 0.00029064224289567794
            },
            {
             "value": 3,
             "probability": 0.0003612254231203448
            },
            {
             "value": 4,
             "probability": 0.0003804040932116995
            }
           ]
          },
          "decision": {
           "value": "reschedule",
           "probability": 0.9980512360236152,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0010633319224066175
            },
            {
             "value": "reschedule",
             "probability": 0.9980512360236152
            },
            {
             "value": "split booking",
             "probability": 0.00045063367773873664
            },
            {
             "value": "decline",
             "probability": 0.00043479837623942765
            }
           ]
          }
         },
         "joint_probability": 0.9976802991824605,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023223954915124316,
        "field_brier": 3.08388060929902e-06
       }
      },
      "timing_ms": {
       "scoring": 38.17404014989734,
       "base_joint": 0.08466909639537334,
       "constrained_joint": 0.06600003689527512,
       "constrained_total": 38.24004018679261
      },
      "input_tokens": 740,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-00b4c75a0b6107ac-0014",
      "group": "bendsynth-00b4c75a0b6107ac",
      "provenance": "bendsynth-v1; family=bendsynth-00b4c75a0b6107ac; domain=notary appointment desk, escalated case supervisor review; parameters=rush_fee_pct=21; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "decline",
       "risk_flags": 3,
       "id_holds": false
      },
      "logits": [
       [
        -6.317983627319336,
        -6.499378204345703,
        -6.5517120361328125,
        1.4100341796875
       ],
       [
        -6.646435260772705,
        -6.5774688720703125,
        -6.29036808013916,
        1.40971040725708,
        -6.490481853485107
       ],
       [
        1.410331130027771,
        -5.649336814880371
       ]
      ],
      "joint_logits": [
       -8.722786903381348,
       -8.839699745178223,
       -8.662642478942871,
       -8.770048141479492,
       -8.456494331359863,
       -8.623241424560547,
       -8.297463417053223,
       -8.624013900756836,
       -8.5923490524292,
       -8.735111236572266,
       -8.898453712463379,
       -9.012872695922852,
       -8.843938827514648,
       -8.99368667602539,
       -8.591039657592773,
       -8.741731643676758,
       -8.550749778747559,
       -8.865530014038086,
       -8.784270286560059,
       -8.843870162963867,
       -8.901408195495605,
       -9.018484115600586,
       -8.895218849182129,
       -8.9977445602417,
       -8.708200454711914,
       -8.876018524169922,
       -8.568880081176758,
       -8.856112480163574,
       -8.804821014404297,
       -9.009644508361816,
       -8.331384658813477,
       -8.358789443969727,
       -8.117409706115723,
       -8.35317325592041,
       -7.584892749786377,
       -8.101279258728027,
       1.4093601703643799,
       -7.991314888000488,
       -7.944394111633301,
       -8.30533218383789
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "decline",
           "probability": 0.9988452048334661,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.00043980768274611194
            },
            {
             "value": "split booking",
             "probability": 0.0003668461733328197
            },
            {
             "value": "reschedule",
             "probability": 0.000348141310455091
            },
            {
             "value": "decline",
             "probability": 0.9988452048334661
            }
           ]
          },
          "risk_flags": {
           "value": 3,
           "probability": 0.9985217866606078,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00031667799136978587
            },
            {
             "value": 1,
             "probability": 0.00033928888872428794
            },
            {
             "value": 2,
             "probability": 0.0004521221382013332
            },
            {
             "value": 3,
             "probability": 0.9985217866606078
            },
            {
             "value": 4,
             "probability": 0.000370124321096788
            }
           ]
          },
          "id_holds": {
           "value": false,
           "probability": 0.9991416739452502,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991416739452502
            },
            {
             "value": true,
             "probability": 0.0008583260547496978
            }
           ]
          }
         },
         "joint_probability": 0.9981720332950117,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001829639474948312,
        "field_brier": 1.9993771372725043e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "decline",
           "probability": 0.9988452048334661,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.00043980768274611194
            },
            {
             "value": "split booking",
             "probability": 0.0003668461733328197
            },
            {
             "value": "reschedule",
             "probability": 0.000348141310455091
            },
            {
             "value": "decline",
             "probability": 0.9988452048334661
            }
           ]
          },
          "risk_flags": {
           "value": 3,
           "probability": 0.9985217866606078,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00031667799136978587
            },
            {
             "value": 1,
             "probability": 0.00033928888872428794
            },
            {
             "value": 2,
             "probability": 0.0004521221382013332
            },
            {
             "value": 3,
             "probability": 0.9985217866606078
            },
            {
             "value": 4,
             "probability": 0.000370124321096788
            }
           ]
          },
          "id_holds": {
           "value": false,
           "probability": 0.9991416739452502,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991416739452502
            },
            {
             "value": true,
             "probability": 0.0008583260547496978
            }
           ]
          }
         },
         "joint_probability": 0.9981720332950117,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 40,
         "valid_assignments": 40,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001829639474948312,
        "field_brier": 1.9993771372725043e-06
       }
      },
      "timing_ms": {
       "scoring": 38.6359840631485,
       "base_joint": 0.08953898213803768,
       "constrained_joint": 0.0762490089982748,
       "constrained_total": 38.71223307214677
      },
      "input_tokens": 743,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-027756daa50a5d65-0004",
      "group": "bendsynth-027756daa50a5d65",
      "provenance": "bendsynth-v1; family=bendsynth-027756daa50a5d65; domain=fishing quota transfer review at a regional fisheries office; parameters=grace_days=16,cap_pct=60; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "needs_signoff": false,
       "decision": "approve",
       "risk_factors": 1
      },
      "logits": [
       [
        0.36399221420288086,
        -5.500262260437012
       ],
       [
        -5.883884429931641,
        0.02807220257818699,
        -0.8869873285293579
       ],
       [
        -6.252429008483887,
        0.3631102740764618,
        -6.421505928039551,
        -6.318674564361572
       ]
      ],
      "joint_logits": [
       -8.042617797851562,
       -8.031469345092773,
       -8.197477340698242,
       -8.059183120727539,
       -8.225945472717285,
       0.026083623990416527,
       -8.414695739746094,
       -8.424053192138672,
       -8.294450759887695,
       -0.891520082950592,
       -8.500360488891602,
       -8.49690055847168,
       -7.743124008178711,
       -8.129145622253418,
       -7.864259719848633,
       -7.744510650634766,
       -7.98887825012207,
       -7.861795902252197,
       -8.170327186584473,
       -8.025148391723633,
       -8.066574096679688,
       -8.014864921569824,
       -8.25977611541748,
       -8.10416316986084
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_signoff": {
           "value": false,
           "probability": 0.9971688994040807,
           "distribution": [
            {
             "value": false,
             "probability": 0.9971688994040807
            },
            {
             "value": true,
             "probability": 0.002831100595919262
            }
           ]
          },
          "decision": {
           "value": "approve with conditions",
           "probability": 0.7126569732314058,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0019290808931633834
            },
            {
             "value": "approve with conditions",
             "probability": 0.7126569732314058
            },
            {
             "value": "reject",
             "probability": 0.28541394587543095
            }
           ]
          },
          "risk_factors": {
           "value": 1,
           "probability": 0.9962898436270673,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001334423055429941
            },
            {
             "value": 1,
             "probability": 0.9962898436270673
            },
            {
             "value": 2,
             "probability": 0.0011268452526055012
            },
            {
             "value": 3,
             "probability": 0.0012488880648973471
            }
           ]
          }
         },
         "joint_probability": 0.7112411925709227,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.398296645730037,
        "field_brier": 0.5285070157567635
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_signoff": {
           "value": false,
           "probability": 0.9971688994040807,
           "distribution": [
            {
             "value": false,
             "probability": 0.9971688994040807
            },
            {
             "value": true,
             "probability": 0.002831100595919262
            }
           ]
          },
          "decision": {
           "value": "approve with conditions",
           "probability": 0.7126569732314058,
           "distribution": [
            {
             "value": "approve",
             "probability": 0.0019290808931633834
            },
            {
             "value": "approve with conditions",
             "probability": 0.7126569732314058
            },
            {
             "value": "reject",
             "probability": 0.28541394587543095
            }
           ]
          },
          "risk_factors": {
           "value": 1,
           "probability": 0.9962898436270673,
           "distribution": [
            {
             "value": 0,
             "probability": 0.001334423055429941
            },
            {
             "value": 1,
             "probability": 0.9962898436270673
            },
            {
             "value": 2,
             "probability": 0.0011268452526055012
            },
            {
             "value": 3,
             "probability": 0.0012488880648973471
            }
           ]
          }
         },
         "joint_probability": 0.7112411925709227,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.398296645730037,
        "field_brier": 0.5285070157567635
       }
      },
      "timing_ms": {
       "scoring": 33.9650041423738,
       "base_joint": 0.06295903585851192,
       "constrained_joint": 0.046269968152046204,
       "constrained_total": 34.01127411052585
      },
      "input_tokens": 678,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-027756daa50a5d65-0007",
      "group": "bendsynth-027756daa50a5d65",
      "provenance": "bendsynth-v1; family=bendsynth-027756daa50a5d65; domain=fishing quota transfer review at a regional fisheries office; parameters=grace_days=10,cap_pct=31; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "needs_signoff": true,
       "risk_factors": 1,
       "decision": "approve"
      },
      "logits": [
       [
        -4.579615592956543,
        1.6242785453796387
       ],
       [
        -5.472352027893066,
        1.624204158782959,
        -5.757033824920654,
        -5.721571922302246
       ],
       [
        -5.340773582458496,
        -5.221961498260498,
        1.624292254447937
       ]
      ],
      "joint_logits": [
       -7.14899206161499,
       -6.972596645355225,
       -7.101461887359619,
       -7.0952653884887695,
       -7.13032865524292,
       -6.183468818664551,
       -7.43138313293457,
       -7.206417083740234,
       -7.3861236572265625,
       -7.391063213348389,
       -7.149548530578613,
       -7.318130970001221,
       -7.704639434814453,
       -7.448361396789551,
       -7.383469104766846,
       -7.23178243637085,
       -7.293490409851074,
       1.6231993436813354,
       -7.8695549964904785,
       -7.745237350463867,
       -7.845531940460205,
       -7.793973922729492,
       -7.735832691192627,
       -7.929129600524902
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_signoff": {
           "value": true,
           "probability": 0.9979825354335298,
           "distribution": [
            {
             "value": false,
             "probability": 0.002017464566470293
            },
            {
             "value": true,
             "probability": 0.9979825354335298
            }
           ]
          },
          "risk_factors": {
           "value": 1,
           "probability": 0.9979082912974432,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008262195627788081
            },
            {
             "value": 1,
             "probability": 0.9979082912974432
            },
            {
             "value": 2,
             "probability": 0.0006215266380011769
            },
            {
             "value": 3,
             "probability": 0.0006439625017769155
            }
           ]
          },
          "decision": {
           "value": "approve with conditions",
           "probability": 0.9979962899542958,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.0009424085226514714
            },
            {
             "value": "approve",
             "probability": 0.001061301523052628
            },
            {
             "value": "approve with conditions",
             "probability": 0.9979962899542958
            }
           ]
          }
         },
         "joint_probability": 0.9969061474490899,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.919788401939458,
        "field_brier": 0.6646300018016206
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_signoff": {
           "value": true,
           "probability": 0.9979825354335298,
           "distribution": [
            {
             "value": false,
             "probability": 0.002017464566470293
            },
            {
             "value": true,
             "probability": 0.9979825354335298
            }
           ]
          },
          "risk_factors": {
           "value": 1,
           "probability": 0.9979082912974432,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008262195627788081
            },
            {
             "value": 1,
             "probability": 0.9979082912974432
            },
            {
             "value": 2,
             "probability": 0.0006215266380011769
            },
            {
             "value": 3,
             "probability": 0.0006439625017769155
            }
           ]
          },
          "decision": {
           "value": "approve with conditions",
           "probability": 0.9979962899542958,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.0009424085226514714
            },
            {
             "value": "approve",
             "probability": 0.001061301523052628
            },
            {
             "value": "approve with conditions",
             "probability": 0.9979962899542958
            }
           ]
          }
         },
         "joint_probability": 0.9969061474490899,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.919788401939458,
        "field_brier": 0.6646300018016206
       }
      },
      "timing_ms": {
       "scoring": 36.65732592344284,
       "base_joint": 0.06829993799328804,
       "constrained_joint": 0.046718865633010864,
       "constrained_total": 36.70404478907585
      },
      "input_tokens": 684,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-027756daa50a5d65-0038",
      "group": "bendsynth-027756daa50a5d65",
      "provenance": "bendsynth-v1; family=bendsynth-027756daa50a5d65; domain=fishing quota transfer review at a regional fisheries office; parameters=grace_days=10,cap_pct=60; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 1,
       "decision": "approve with conditions",
       "needs_signoff": false
      },
      "logits": [
       [
        -6.011261940002441,
        1.1920456886291504,
        -6.214869976043701,
        -6.13514518737793
       ],
       [
        -5.556210517883301,
        1.191928744316101,
        -5.764182090759277
       ],
       [
        1.1925632953643799,
        -5.316379547119141
       ]
      ],
      "joint_logits": [
       -7.710747241973877,
       -7.624088764190674,
       -7.998024940490723,
       -7.950835227966309,
       -7.824484348297119,
       -7.760722637176514,
       -7.110070705413818,
       -7.6150360107421875,
       1.1912213563919067,
       -7.500036716461182,
       -7.686527729034424,
       -7.843578338623047,
       -7.905226707458496,
       -7.81615686416626,
       -8.31675910949707,
       -8.152865409851074,
       -8.004740715026855,
       -7.9257307052612305,
       -7.797500133514404,
       -7.741770267486572,
       -8.332249641418457,
       -8.065593719482422,
       -7.925835132598877,
       -7.813436985015869
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 1,
           "probability": 0.9979954475872465,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007426289400586613
            },
            {
             "value": 1,
             "probability": 0.9979954475872465
            },
            {
             "value": 2,
             "probability": 0.0006058233514721817
            },
            {
             "value": 3,
             "probability": 0.0006561001212227137
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9978786542351663,
           "distribution": [
            {
             "value": "approve with conditions",
             "probability": 0.0011705718577739244
            },
            {
             "value": "reject",
             "probability": 0.9978786542351663
            },
            {
             "value": "approve",
             "probability": 0.0009507739070597828
            }
           ]
          },
          "needs_signoff": {
           "value": false,
           "probability": 0.9985121621575566,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985121621575566
            },
            {
             "value": true,
             "probability": 0.0014878378424435356
            }
           ]
          }
         },
         "joint_probability": 0.9971730948846957,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.304122970163599,
        "field_brier": 0.6644775778708155
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 1,
           "probability": 0.9979954475872465,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007426289400586613
            },
            {
             "value": 1,
             "probability": 0.9979954475872465
            },
            {
             "value": 2,
             "probability": 0.0006058233514721817
            },
            {
             "value": 3,
             "probability": 0.0006561001212227137
            }
           ]
          },
          "decision": {
           "value": "reject",
           "probability": 0.9978786542351663,
           "distribution": [
            {
             "value": "approve with conditions",
             "probability": 0.0011705718577739244
            },
            {
             "value": "reject",
             "probability": 0.9978786542351663
            },
            {
             "value": "approve",
             "probability": 0.0009507739070597828
            }
           ]
          },
          "needs_signoff": {
           "value": false,
           "probability": 0.9985121621575566,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985121621575566
            },
            {
             "value": true,
             "probability": 0.0014878378424435356
            }
           ]
          }
         },
         "joint_probability": 0.9971730948846957,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.304122970163599,
        "field_brier": 0.6644775778708155
       }
      },
      "timing_ms": {
       "scoring": 36.09874309040606,
       "base_joint": 0.07267016917467117,
       "constrained_joint": 0.0541189219802618,
       "constrained_total": 36.15286201238632
      },
      "input_tokens": 684,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-061b2cc09529c1b6-0000",
      "group": "bendsynth-061b2cc09529c1b6",
      "provenance": "bendsynth-v1; family=bendsynth-061b2cc09529c1b6; domain=university chemistry stockroom request audit; parameters=cooldown=91; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "release",
       "eligible": true,
       "flags": 2
      },
      "logits": [
       [
        -5.82611083984375,
        1.2343961000442505,
        -5.8464155197143555
       ],
       [
        -5.57626485824585,
        1.2349933385849
       ],
       [
        -6.206952095031738,
        -5.998809814453125,
        1.234136700630188,
        -6.103788375854492
       ]
      ],
      "joint_logits": [
       -8.296103477478027,
       -8.097127914428711,
       -8.056890487670898,
       -8.047830581665039,
       -7.857924938201904,
       -7.736415386199951,
       -7.555904388427734,
       -7.793034553527832,
       -7.999728679656982,
       -7.915912628173828,
       -7.890965938568115,
       -7.938222408294678,
       -7.819550514221191,
       -7.323420524597168,
       1.23354172706604,
       -7.724827766418457,
       -8.222962379455566,
       -8.130701065063477,
       -8.141684532165527,
       -8.076494216918945,
       -7.892770767211914,
       -7.77168083190918,
       -7.527752876281738,
       -7.842536449432373
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "release",
           "probability": 0.9983034501385848,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.000856886651493902
            },
            {
             "value": "release",
             "probability": 0.9983034501385848
            },
            {
             "value": "limited release",
             "probability": 0.0008396632099212081
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9988999052832667,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011000947167332804
            },
            {
             "value": true,
             "probability": 0.9988999052832667
            }
           ]
          },
          "flags": {
           "value": 2,
           "probability": 0.9980443977700976,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005854988457723226
            },
            {
             "value": 1,
             "probability": 0.0007209764903831037
            },
            {
             "value": 2,
             "probability": 0.9980443977700976
            },
            {
             "value": 3,
             "probability": 0.0006491268937470743
            }
           ]
          }
         },
         "joint_probability": 0.9974508361613862,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002552418489019798,
        "field_brier": 3.948783015770534e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "release",
           "probability": 0.9983034501385848,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.000856886651493902
            },
            {
             "value": "release",
             "probability": 0.9983034501385848
            },
            {
             "value": "limited release",
             "probability": 0.0008396632099212081
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9988999052832667,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011000947167332804
            },
            {
             "value": true,
             "probability": 0.9988999052832667
            }
           ]
          },
          "flags": {
           "value": 2,
           "probability": 0.9980443977700976,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005854988457723226
            },
            {
             "value": 1,
             "probability": 0.0007209764903831037
            },
            {
             "value": 2,
             "probability": 0.9980443977700976
            },
            {
             "value": 3,
             "probability": 0.0006491268937470743
            }
           ]
          }
         },
         "joint_probability": 0.9974508361613862,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002552418489019798,
        "field_brier": 3.948783015770534e-06
       }
      },
      "timing_ms": {
       "scoring": 34.027034882456064,
       "base_joint": 0.06546988151967525,
       "constrained_joint": 0.04722899757325649,
       "constrained_total": 34.07426388002932
      },
      "input_tokens": 608,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-061b2cc09529c1b6-0021",
      "group": "bendsynth-061b2cc09529c1b6",
      "provenance": "bendsynth-v1; family=bendsynth-061b2cc09529c1b6; domain=university chemistry stockroom request audit; parameters=cooldown=120; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "deny",
       "flags": 2,
       "eligible": true
      },
      "logits": [
       [
        0.2195008397102356,
        -6.930363655090332,
        -6.703141212463379
       ],
       [
        -7.161163330078125,
        -6.992856979370117,
        0.2192704677581787,
        -7.134524345397949
       ],
       [
        -6.42266845703125,
        0.21996690332889557
       ]
      ],
      "joint_logits": [
       -8.921390533447266,
       -8.873481750488281,
       -8.838310241699219,
       -8.470283508300781,
       -8.527009010314941,
       0.21860776841640472,
       -8.935821533203125,
       -8.778549194335938,
       -9.090134620666504,
       -9.104866981506348,
       -9.039422988891602,
       -8.932381629943848,
       -9.114825248718262,
       -8.715924263000488,
       -9.103259086608887,
       -9.046631813049316,
       -8.867389678955078,
       -8.88979434967041,
       -8.816492080688477,
       -8.710803985595703,
       -8.90483283996582,
       -8.4349946975708,
       -8.884903907775879,
       -8.845968246459961
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "deny",
           "probability": 0.9982329344326555,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.9982329344326555
            },
            {
             "value": "limited release",
             "probability": 0.0007835832046509647
            },
            {
             "value": "release",
             "probability": 0.0009834823626935967
            }
           ]
          },
          "flags": {
           "value": 2,
           "probability": 0.9980029209346699,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006220855376829702
            },
            {
             "value": 1,
             "probability": 0.0007361132770076606
            },
            {
             "value": 2,
             "probability": 0.9980029209346699
            },
            {
             "value": 3,
             "probability": 0.0006388802506395559
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9986981098477689,
           "distribution": [
            {
             "value": false,
             "probability": 0.001301890152231245
            },
            {
             "value": true,
             "probability": 0.9986981098477689
            }
           ]
          }
         },
         "joint_probability": 0.9973417721486032,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002661767212729135,
        "field_brier": 4.472980931068802e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "deny",
           "probability": 0.9982329344326555,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.9982329344326555
            },
            {
             "value": "limited release",
             "probability": 0.0007835832046509647
            },
            {
             "value": "release",
             "probability": 0.0009834823626935967
            }
           ]
          },
          "flags": {
           "value": 2,
           "probability": 0.9980029209346699,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006220855376829702
            },
            {
             "value": 1,
             "probability": 0.0007361132770076606
            },
            {
             "value": 2,
             "probability": 0.9980029209346699
            },
            {
             "value": 3,
             "probability": 0.0006388802506395559
            }
           ]
          },
          "eligible": {
           "value": true,
           "probability": 0.9986981098477689,
           "distribution": [
            {
             "value": false,
             "probability": 0.001301890152231245
            },
            {
             "value": true,
             "probability": 0.9986981098477689
            }
           ]
          }
         },
         "joint_probability": 0.9973417721486032,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002661767212729135,
        "field_brier": 4.472980931068802e-06
       }
      },
      "timing_ms": {
       "scoring": 33.580299001187086,
       "base_joint": 0.06518908776342869,
       "constrained_joint": 0.048138899728655815,
       "constrained_total": 33.62843790091574
      },
      "input_tokens": 605,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-061b2cc09529c1b6-0031",
      "group": "bendsynth-061b2cc09529c1b6",
      "provenance": "bendsynth-v1; family=bendsynth-061b2cc09529c1b6; domain=university chemistry stockroom request audit; parameters=cooldown=180; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "eligible": true,
       "decision": "release",
       "flags": 1
      },
      "logits": [
       [
        -5.571105003356934,
        0.9658166170120239
       ],
       [
        0.9627769589424133,
        -4.6542067527771,
        -6.093485355377197
       ],
       [
        -6.242593288421631,
        0.9652715921401978,
        -6.456988334655762,
        -6.363605499267578
       ]
      ],
      "joint_logits": [
       -7.990489959716797,
       -8.281022071838379,
       -8.176682472229004,
       -8.110838890075684,
       -7.751538276672363,
       -8.1553316116333,
       -7.9432501792907715,
       -7.8511786460876465,
       -7.9722795486450195,
       -8.357828140258789,
       -8.238141059875488,
       -8.024073600769043,
       -8.312260627746582,
       0.9620818495750427,
       -8.506118774414062,
       -8.523847579956055,
       -8.066025733947754,
       -4.929805278778076,
       -8.294052124023438,
       -8.224884986877441,
       -8.211004257202148,
       -7.934074878692627,
       -8.436859130859375,
       -8.336861610412598
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "eligible": {
           "value": true,
           "probability": 0.9985531536102187,
           "distribution": [
            {
             "value": false,
             "probability": 0.001446846389781418
            },
            {
             "value": true,
             "probability": 0.9985531536102187
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.9955225517789685,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.9955225517789685
            },
            {
             "value": "release",
             "probability": 0.003619313599847492
            },
            {
             "value": "limited release",
             "probability": 0.0008581346211838684
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9980091277171681,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007392623312029302
            },
            {
             "value": 1,
             "probability": 0.9980091277171681
            },
            {
             "value": 2,
             "probability": 0.0005966064634753484
            },
            {
             "value": 3,
             "probability": 0.0006550034881536685
            }
           ]
          }
         },
         "joint_probability": 0.9948308312730689,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 5.897069703452537,
        "field_brier": 0.6612832805019367
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "eligible": {
           "value": true,
           "probability": 0.9985531536102187,
           "distribution": [
            {
             "value": false,
             "probability": 0.001446846389781418
            },
            {
             "value": true,
             "probability": 0.9985531536102187
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.9955225517789685,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.9955225517789685
            },
            {
             "value": "release",
             "probability": 0.003619313599847492
            },
            {
             "value": "limited release",
             "probability": 0.0008581346211838684
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9980091277171681,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007392623312029302
            },
            {
             "value": 1,
             "probability": 0.9980091277171681
            },
            {
             "value": 2,
             "probability": 0.0005966064634753484
            },
            {
             "value": 3,
             "probability": 0.0006550034881536685
            }
           ]
          }
         },
         "joint_probability": 0.9948308312730689,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 5.897069703452537,
        "field_brier": 0.6612832805019367
       }
      },
      "timing_ms": {
       "scoring": 33.74109696596861,
       "base_joint": 0.0638989731669426,
       "constrained_joint": 0.04784995689988136,
       "constrained_total": 33.78894692286849
      },
      "input_tokens": 605,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0462845a7f1dd43e-0014",
      "group": "bendsynth-0462845a7f1dd43e",
      "provenance": "bendsynth-v1; family=bendsynth-0462845a7f1dd43e; domain=student visa document check chatbot; parameters=min_funds=24182,min_english=67; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "weak_file": true,
       "problems": 1,
       "decision": "documents requested"
      },
      "logits": [
       [
        -4.047767162322998,
        2.479741334915161
       ],
       [
        -5.311225414276123,
        2.4790103435516357,
        -4.1529130935668945,
        -5.197488784790039
       ],
       [
        -4.587427139282227,
        2.479071855545044,
        -4.1818413734436035
       ]
      ],
      "joint_logits": [
       -7.238377571105957,
       -7.4021782875061035,
       -6.852268695831299,
       -6.778460502624512,
       -5.837363243103027,
       -6.342284202575684,
       -6.248414516448975,
       -6.1784539222717285,
       -5.950531959533691,
       -7.140113353729248,
       -7.280947208404541,
       -6.752252578735352,
       -7.295208930969238,
       -7.021729469299316,
       -6.928614616394043,
       -6.185039520263672,
       2.4780685901641846,
       -5.697812080383301,
       -6.171088695526123,
       -5.57536506652832,
       -5.735921382904053,
       -7.1709771156311035,
       -6.915133953094482,
       -6.794628143310547
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "weak_file": {
           "value": true,
           "probability": 0.9985394902891338,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014605097108661429
            },
            {
             "value": true,
             "probability": 0.9985394902891338
            }
           ]
          },
          "problems": {
           "value": 1,
           "probability": 0.9978098303317664,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004128491999822767
            },
            {
             "value": 1,
             "probability": 0.9978098303317664
            },
            {
             "value": 2,
             "probability": 0.0013147407733678559
            },
            {
             "value": 3,
             "probability": 0.00046257969488351945
            }
           ]
          },
          "decision": {
           "value": "documents requested",
           "probability": 0.997871348736378,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.0008513989962880026
            },
            {
             "value": "documents requested",
             "probability": 0.997871348736378
            },
            {
             "value": "rejected",
             "probability": 0.0012772522673340188
            }
           ]
          }
         },
         "joint_probability": 0.9968706800328974,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003134226527636652,
        "field_brier": 6.021132650443182e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "weak_file": {
           "value": true,
           "probability": 0.9985394902891338,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014605097108661429
            },
            {
             "value": true,
             "probability": 0.9985394902891338
            }
           ]
          },
          "problems": {
           "value": 1,
           "probability": 0.9978098303317664,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004128491999822767
            },
            {
             "value": 1,
             "probability": 0.9978098303317664
            },
            {
             "value": 2,
             "probability": 0.0013147407733678559
            },
            {
             "value": 3,
             "probability": 0.00046257969488351945
            }
           ]
          },
          "decision": {
           "value": "documents requested",
           "probability": 0.997871348736378,
           "distribution": [
            {
             "value": "approved",
             "probability": 0.0008513989962880026
            },
            {
             "value": "documents requested",
             "probability": 0.997871348736378
            },
            {
             "value": "rejected",
             "probability": 0.0012772522673340188
            }
           ]
          }
         },
         "joint_probability": 0.9968706800328974,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.003134226527636652,
        "field_brier": 6.021132650443182e-06
       }
      },
      "timing_ms": {
       "scoring": 33.27618190087378,
       "base_joint": 0.06455904804170132,
       "constrained_joint": 0.0475000124424696,
       "constrained_total": 33.32368191331625
      },
      "input_tokens": 615,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0462845a7f1dd43e-0015",
      "group": "bendsynth-0462845a7f1dd43e",
      "provenance": "bendsynth-v1; family=bendsynth-0462845a7f1dd43e; domain=student visa document check chatbot; parameters=min_funds=26824,min_english=54; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "weak_file": true,
       "decision": "documents requested",
       "problems": 1
      },
      "logits": [
       [
        -4.289334297180176,
        -0.5212365984916687
       ],
       [
        -0.5118181109428406,
        -4.949084281921387,
        -6.9026288986206055
       ],
       [
        -6.935558795928955,
        -0.5024347305297852,
        -7.16909122467041,
        -7.262304782867432
       ]
      ],
      "joint_logits": [
       -8.700104713439941,
       -4.476502418518066,
       -9.09520435333252,
       -9.238773345947266,
       -8.532530784606934,
       -7.043496131896973,
       -8.676667213439941,
       -8.760638236999512,
       -8.673418998718262,
       -9.178677558898926,
       -8.793760299682617,
       -8.862044334411621,
       -8.803722381591797,
       -0.5322071313858032,
       -9.180648803710938,
       -9.2590970993042,
       -8.684480667114258,
       -5.239679336547852,
       -9.0292387008667,
       -9.194777488708496,
       -9.039605140686035,
       -9.23978042602539,
       -9.09027099609375,
       -9.123857498168945
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "weak_file": {
           "value": true,
           "probability": 0.9774254268287539,
           "distribution": [
            {
             "value": false,
             "probability": 0.022574573171246212
            },
            {
             "value": true,
             "probability": 0.9774254268287539
            }
           ]
          },
          "decision": {
           "value": "documents requested",
           "probability": 0.986674832419184,
           "distribution": [
            {
             "value": "documents requested",
             "probability": 0.986674832419184
            },
            {
             "value": "approved",
             "probability": 0.01167061699577326
            },
            {
             "value": "rejected",
             "probability": 0.0016545505850428107
            }
           ]
          },
          "problems": {
           "value": 1,
           "probability": 0.9959768032886747,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0016009542482817783
            },
            {
             "value": 1,
             "probability": 0.9959768032886747
            },
            {
             "value": 2,
             "probability": 0.0012675270383496237
            },
            {
             "value": 3,
             "probability": 0.0011547154246940386
            }
           ]
          }
         },
         "joint_probability": 0.9667612172591299,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.03380374550114063,
        "field_brier": 0.000452470932091102
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "weak_file": {
           "value": true,
           "probability": 0.9774254268287539,
           "distribution": [
            {
             "value": false,
             "probability": 0.022574573171246212
            },
            {
             "value": true,
             "probability": 0.9774254268287539
            }
           ]
          },
          "decision": {
           "value": "documents requested",
           "probability": 0.986674832419184,
           "distribution": [
            {
             "value": "documents requested",
             "probability": 0.986674832419184
            },
            {
             "value": "approved",
             "probability": 0.01167061699577326
            },
            {
             "value": "rejected",
             "probability": 0.0016545505850428107
            }
           ]
          },
          "problems": {
           "value": 1,
           "probability": 0.9959768032886747,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0016009542482817783
            },
            {
             "value": 1,
             "probability": 0.9959768032886747
            },
            {
             "value": 2,
             "probability": 0.0012675270383496237
            },
            {
             "value": 3,
             "probability": 0.0011547154246940386
            }
           ]
          }
         },
         "joint_probability": 0.9667612172591299,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.03380374550114063,
        "field_brier": 0.000452470932091102
       }
      },
      "timing_ms": {
       "scoring": 35.24870099499822,
       "base_joint": 0.06379885599017143,
       "constrained_joint": 0.049560097977519035,
       "constrained_total": 35.298261092975736
      },
      "input_tokens": 651,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a9295bd8d8b158a-0022",
      "group": "bendsynth-0a9295bd8d8b158a",
      "provenance": "bendsynth-v1; family=bendsynth-0a9295bd8d8b158a; domain=driveway curb-cut site inspection; parameters=max_slope=11; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "too_steep": true,
       "defects": 1,
       "decision": "pass with correction"
      },
      "logits": [
       [
        -5.147451877593994,
        1.84048330783844
       ],
       [
        -5.865268707275391,
        1.839949131011963,
        -5.699326038360596,
        -5.811062335968018
       ],
       [
        -5.655601978302002,
        1.8398841619491577,
        -5.6748833656311035,
        -5.929962635040283
       ]
      ],
      "joint_logits": [
       -7.935804843902588,
       -8.066143035888672,
       -7.937702178955078,
       -8.180540084838867,
       -7.839498996734619,
       -7.608137130737305,
       -7.906174659729004,
       -8.115212440490723,
       -7.78354024887085,
       -7.914449214935303,
       -7.779904842376709,
       -8.006471633911133,
       -7.845126152038574,
       -7.999746799468994,
       -7.8588786125183105,
       -8.108641624450684,
       -7.823883533477783,
       -7.725477695465088,
       -7.878162384033203,
       -8.088472366333008,
       -7.365225315093994,
       1.839424967765808,
       -7.317174434661865,
       -7.718597412109375,
       -7.639496326446533,
       -7.554778099060059,
       -7.704370021820068,
       -7.93111515045166,
       -7.7655720710754395,
       -7.7898993492126465,
       -7.805699348449707,
       -8.003108978271484
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "too_steep": {
           "value": true,
           "probability": 0.9990779007875062,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009220992124938293
            },
            {
             "value": true,
             "probability": 0.9990779007875062
            }
           ]
          },
          "defects": {
           "value": 1,
           "probability": 0.998544305714841,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004498148059166026
            },
            {
             "value": 1,
             "probability": 0.998544305714841
            },
            {
             "value": 2,
             "probability": 0.0005310089138366534
            },
            {
             "value": 3,
             "probability": 0.0004748705654056603
            }
           ]
          },
          "decision": {
           "value": "pass with correction",
           "probability": 0.9984794741089038,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.000554741780110151
            },
            {
             "value": "pass with correction",
             "probability": 0.9984794741089038
            },
            {
             "value": "pass",
             "probability": 0.000544148135467664
            },
            {
             "value": "re-inspection",
             "probability": 0.00042163597551845606
            }
           ]
          }
         },
         "joint_probability": 0.9980210900525677,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001980870576755223,
        "field_brier": 2.5409990548568014e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "too_steep": {
           "value": true,
           "probability": 0.9990779007875062,
           "distribution": [
            {
             "value": false,
             "probability": 0.0009220992124938293
            },
            {
             "value": true,
             "probability": 0.9990779007875062
            }
           ]
          },
          "defects": {
           "value": 1,
           "probability": 0.998544305714841,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004498148059166026
            },
            {
             "value": 1,
             "probability": 0.998544305714841
            },
            {
             "value": 2,
             "probability": 0.0005310089138366534
            },
            {
             "value": 3,
             "probability": 0.0004748705654056603
            }
           ]
          },
          "decision": {
           "value": "pass with correction",
           "probability": 0.9984794741089038,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.000554741780110151
            },
            {
             "value": "pass with correction",
             "probability": 0.9984794741089038
            },
            {
             "value": "pass",
             "probability": 0.000544148135467664
            },
            {
             "value": "re-inspection",
             "probability": 0.00042163597551845606
            }
           ]
          }
         },
         "joint_probability": 0.9980210900525677,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.001980870576755223,
        "field_brier": 2.5409990548568014e-06
       }
      },
      "timing_ms": {
       "scoring": 34.41683994606137,
       "base_joint": 0.07506902329623699,
       "constrained_joint": 0.05715014412999153,
       "constrained_total": 34.473990090191364
      },
      "input_tokens": 596,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a9295bd8d8b158a-0024",
      "group": "bendsynth-0a9295bd8d8b158a",
      "provenance": "bendsynth-v1; family=bendsynth-0a9295bd8d8b158a; domain=driveway curb-cut site inspection; parameters=max_slope=5; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "too_steep": false,
       "decision": "re-inspection",
       "defects": 1
      },
      "logits": [
       [
        2.158733367919922,
        -4.763501167297363
       ],
       [
        -5.3064188957214355,
        -5.259083271026611,
        2.158115863800049,
        -5.5935564041137695
       ],
       [
        -5.578970909118652,
        2.158254623413086,
        -5.353795051574707,
        -5.479169845581055
       ]
      ],
      "joint_logits": [
       -7.50602388381958,
       -7.127669811248779,
       -7.370121479034424,
       -7.468308925628662,
       -7.3978681564331055,
       -7.041898250579834,
       -7.221374988555908,
       -7.367353439331055,
       -7.705857753753662,
       2.1577093601226807,
       -7.444221496582031,
       -7.654142379760742,
       -7.783349990844727,
       -7.357611656188965,
       -7.647445201873779,
       -7.7753448486328125,
       -7.614582061767578,
       -7.366203784942627,
       -7.300888538360596,
       -7.408278942108154,
       -7.739126205444336,
       -7.29290771484375,
       -7.33495569229126,
       -7.448041915893555,
       -7.691274642944336,
       -7.367102146148682,
       -7.634852409362793,
       -7.737113952636719,
       -7.9242024421691895,
       -7.683828353881836,
       -7.607073783874512,
       -7.703602313995361
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "too_steep": {
           "value": false,
           "probability": 0.9990153448516765,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990153448516765
            },
            {
             "value": true,
             "probability": 0.0009846551483234298
            }
           ]
          },
          "decision": {
           "value": "re-inspection",
           "probability": 0.9983986640279462,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.0005721341250521056
            },
            {
             "value": "pass with correction",
             "probability": 0.0005998675032763596
            },
            {
             "value": "re-inspection",
             "probability": 0.9983986640279462
            },
            {
             "value": "pass",
             "probability": 0.00042933434372539707
            }
           ]
          },
          "defects": {
           "value": 1,
           "probability": 0.9985373340365264,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00043564208313928937
            },
            {
             "value": 1,
             "probability": 0.9985373340365264
            },
            {
             "value": 2,
             "probability": 0.0005456606943216875
            },
            {
             "value": 3,
             "probability": 0.0004813631860125358
            }
           ]
          }
         },
         "joint_probability": 0.997992928241184,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002009088626455742,
        "field_brier": 2.744502243451082e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "too_steep": {
           "value": false,
           "probability": 0.9990153448516765,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990153448516765
            },
            {
             "value": true,
             "probability": 0.0009846551483234298
            }
           ]
          },
          "decision": {
           "value": "re-inspection",
           "probability": 0.9983986640279462,
           "distribution": [
            {
             "value": "reject",
             "probability": 0.0005721341250521056
            },
            {
             "value": "pass with correction",
             "probability": 0.0005998675032763596
            },
            {
             "value": "re-inspection",
             "probability": 0.9983986640279462
            },
            {
             "value": "pass",
             "probability": 0.00042933434372539707
            }
           ]
          },
          "defects": {
           "value": 1,
           "probability": 0.9985373340365264,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00043564208313928937
            },
            {
             "value": 1,
             "probability": 0.9985373340365264
            },
            {
             "value": 2,
             "probability": 0.0005456606943216875
            },
            {
             "value": 3,
             "probability": 0.0004813631860125358
            }
           ]
          }
         },
         "joint_probability": 0.997992928241184,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002009088626455742,
        "field_brier": 2.744502243451082e-06
       }
      },
      "timing_ms": {
       "scoring": 33.40326086618006,
       "base_joint": 0.07590907625854015,
       "constrained_joint": 0.0574490986764431,
       "constrained_total": 33.460709964856505
      },
      "input_tokens": 587,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0a9295bd8d8b158a-0031",
      "group": "bendsynth-0a9295bd8d8b158a",
      "provenance": "bendsynth-v1; family=bendsynth-0a9295bd8d8b158a; domain=driveway curb-cut site inspection; parameters=max_slope=9; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "reject",
       "too_steep": true,
       "defects": 3
      },
      "logits": [
       [
        1.5629048347473145,
        -6.322887420654297,
        -5.642263412475586,
        -5.824306011199951
       ],
       [
        -5.501655101776123,
        1.5637874603271484
       ],
       [
        -6.008435249328613,
        -5.91404914855957,
        -5.502623081207275,
        1.562707543373108
       ]
      ],
      "joint_logits": [
       -8.362287521362305,
       -8.22883129119873,
       -7.996322154998779,
       -8.358611106872559,
       -7.922099590301514,
       -7.728491306304932,
       -6.931380748748779,
       1.5623072385787964,
       -8.825927734375,
       -9.094853401184082,
       -8.1754150390625,
       -9.210988998413086,
       -8.180083274841309,
       -8.163206100463867,
       -7.834319114685059,
       -8.551916122436523,
       -7.946122646331787,
       -7.893537998199463,
       -7.68682336807251,
       -8.76677417755127,
       -7.650148868560791,
       -7.494243621826172,
       -7.301616191864014,
       -7.608330726623535,
       -8.306841850280762,
       -8.205284118652344,
       -7.906033039093018,
       -8.8906831741333,
       -7.935256481170654,
       -7.838013172149658,
       -7.446433067321777,
       -7.430695056915283
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "re-inspection",
           "probability": 0.9982651085570481,
           "distribution": [
            {
             "value": "re-inspection",
             "probability": 0.9982651085570481
            },
            {
             "value": "pass",
             "probability": 0.0003753962909666632
            },
            {
             "value": "reject",
             "probability": 0.0007414487797092208
            },
            {
             "value": "pass with correction",
             "probability": 0.0006180463722761029
            }
           ]
          },
          "too_steep": {
           "value": true,
           "probability": 0.9991466119071948,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008533880928051381
            },
            {
             "value": true,
             "probability": 0.9991466119071948
            }
           ]
          },
          "defects": {
           "value": 3,
           "probability": 0.9980683322026195,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005141084537560433
            },
            {
             "value": 1,
             "probability": 0.0005649969875305
            },
            {
             "value": 2,
             "probability": 0.0008525623560940617
            },
            {
             "value": 3,
             "probability": 0.9980683322026195
            }
           ]
          }
         },
         "joint_probability": 0.9976688212669041,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 9.172971865362815,
        "field_brier": 0.6650193001092018
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "re-inspection",
           "probability": 0.9982651085570481,
           "distribution": [
            {
             "value": "re-inspection",
             "probability": 0.9982651085570481
            },
            {
             "value": "pass",
             "probability": 0.0003753962909666632
            },
            {
             "value": "reject",
             "probability": 0.0007414487797092208
            },
            {
             "value": "pass with correction",
             "probability": 0.0006180463722761029
            }
           ]
          },
          "too_steep": {
           "value": true,
           "probability": 0.9991466119071948,
           "distribution": [
            {
             "value": false,
             "probability": 0.0008533880928051381
            },
            {
             "value": true,
             "probability": 0.9991466119071948
            }
           ]
          },
          "defects": {
           "value": 3,
           "probability": 0.9980683322026195,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005141084537560433
            },
            {
             "value": 1,
             "probability": 0.0005649969875305
            },
            {
             "value": 2,
             "probability": 0.0008525623560940617
            },
            {
             "value": 3,
             "probability": 0.9980683322026195
            }
           ]
          }
         },
         "joint_probability": 0.9976688212669041,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 9.172971865362815,
        "field_brier": 0.6650193001092018
       }
      },
      "timing_ms": {
       "scoring": 34.80919590219855,
       "base_joint": 0.09049894288182259,
       "constrained_joint": 0.05815899930894375,
       "constrained_total": 34.8673549015075
      },
      "input_tokens": 596,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c6f72abda597b1b-0012",
      "group": "bendsynth-0c6f72abda597b1b",
      "provenance": "bendsynth-v1; family=bendsynth-0c6f72abda597b1b; domain=car rental damage claim escalations; parameters=cap=3000; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "needs_escalation": true,
       "decision": "claimant pays in full",
       "severity": 2,
       "over_cap": true
      },
      "logits": [
       [
        -8.09843921661377,
        -0.8528383374214172
       ],
       [
        -0.853081464767456,
        -8.576308250427246,
        -8.426156044006348
       ],
       [
        -8.82052993774414,
        -8.66717529296875,
        -0.8532494306564331,
        -8.74443244934082
       ],
       [
        -8.103211402893066,
        -0.8528349995613098
       ]
      ],
      "joint_logits": [
       -11.309276580810547,
       -11.115522384643555,
       -11.264945983886719,
       -11.007711410522461,
       -11.078688621520996,
       -10.228025436401367,
       -11.299483299255371,
       -11.108985900878906,
       -11.866687774658203,
       -11.457755088806152,
       -11.802129745483398,
       -11.375561714172363,
       -11.470010757446289,
       -11.268417358398438,
       -11.799445152282715,
       -11.384282112121582,
       -11.733623504638672,
       -11.34256362915039,
       -11.643020629882812,
       -11.290350914001465,
       -11.314223289489746,
       -11.094630241394043,
       -11.63144588470459,
       -11.290816307067871,
       -11.088488578796387,
       -10.91873836517334,
       -10.971644401550293,
       -10.421600341796875,
       -10.384279251098633,
       -0.8537495136260986,
       -11.065536499023438,
       -10.716716766357422,
       -11.467432022094727,
       -11.302033424377441,
       -11.361690521240234,
       -11.131192207336426,
       -11.245100975036621,
       -10.708280563354492,
       -11.356240272521973,
       -11.201007843017578,
       -11.310526847839355,
       -11.133949279785156,
       -11.256257057189941,
       -11.021191596984863,
       -11.08880615234375,
       -10.442336082458496,
       -11.250905990600586,
       -11.069974899291992
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_escalation": {
           "value": true,
           "probability": 0.999287203072655,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007127969273450616
            },
            {
             "value": true,
             "probability": 0.999287203072655
            }
           ]
          },
          "decision": {
           "value": "claimant pays in full",
           "probability": 0.9990443738129755,
           "distribution": [
            {
             "value": "claimant pays in full",
             "probability": 0.9990443738129755
            },
            {
             "value": "waiver covers in full",
             "probability": 0.00044200795523132173
            },
            {
             "value": "partial waiver payout",
             "probability": 0.0005136182317930669
            }
           ]
          },
          "severity": {
           "value": 2,
           "probability": 0.998876547823183,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003462310420909023
            },
            {
             "value": 1,
             "probability": 0.0004036145068506673
            },
            {
             "value": 2,
             "probability": 0.998876547823183
            },
            {
             "value": 3,
             "probability": 0.0003736066278754715
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9992905963228313,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007094036771686783
            },
            {
             "value": true,
             "probability": 0.9992905963228313
            }
           ]
          }
         },
         "joint_probability": 0.9983771907716961,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0016241274094970777,
        "field_brier": 1.269892378526469e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "needs_escalation": {
           "value": true,
           "probability": 0.999287203072655,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007127969273450616
            },
            {
             "value": true,
             "probability": 0.999287203072655
            }
           ]
          },
          "decision": {
           "value": "claimant pays in full",
           "probability": 0.9990443738129755,
           "distribution": [
            {
             "value": "claimant pays in full",
             "probability": 0.9990443738129755
            },
            {
             "value": "waiver covers in full",
             "probability": 0.00044200795523132173
            },
            {
             "value": "partial waiver payout",
             "probability": 0.0005136182317930669
            }
           ]
          },
          "severity": {
           "value": 2,
           "probability": 0.998876547823183,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0003462310420909023
            },
            {
             "value": 1,
             "probability": 0.0004036145068506673
            },
            {
             "value": 2,
             "probability": 0.998876547823183
            },
            {
             "value": 3,
             "probability": 0.0003736066278754715
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9992905963228313,
           "distribution": [
            {
             "value": false,
             "probability": 0.0007094036771686783
            },
            {
             "value": true,
             "probability": 0.9992905963228313
            }
           ]
          }
         },
         "joint_probability": 0.9983771907716961,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.0016241274094970777,
        "field_brier": 1.269892378526469e-06
       }
      },
      "timing_ms": {
       "scoring": 44.46391295641661,
       "base_joint": 0.10659894905984402,
       "constrained_joint": 0.09226915426552296,
       "constrained_total": 44.55618211068213
      },
      "input_tokens": 955,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0c6f72abda597b1b-0032",
      "group": "bendsynth-0c6f72abda597b1b",
      "provenance": "bendsynth-v1; family=bendsynth-0c6f72abda597b1b; domain=car rental damage claim escalations; parameters=cap=3884; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "over_cap": true,
       "needs_escalation": false,
       "severity": 2,
       "decision": "partial waiver payout"
      },
      "logits": [
       [
        -4.2384443283081055,
        1.9164717197418213
       ],
       [
        1.9177619218826294,
        -5.17512321472168
       ],
       [
        -6.004335403442383,
        -5.200095176696777,
        1.9170074462890625,
        -5.877104759216309
       ],
       [
        1.9172202348709106,
        -5.247313022613525,
        -5.502141952514648
       ]
      ],
      "joint_logits": [
       -8.141398429870605,
       -8.084769248962402,
       -8.246773719787598,
       -7.087255477905273,
       -7.453721523284912,
       -7.617851257324219,
       -5.01740026473999,
       -7.4525065422058105,
       -7.795957565307617,
       -7.974687576293945,
       -7.957475185394287,
       -8.122746467590332,
       -8.431164741516113,
       -8.552329063415527,
       -8.68159294128418,
       -7.866669654846191,
       -7.924099445343018,
       -8.126115798950195,
       -8.010056495666504,
       -8.31214427947998,
       -8.496953964233398,
       -8.305558204650879,
       -8.424935340881348,
       -8.578265190124512,
       -8.199617385864258,
       -8.492704391479492,
       -8.696396827697754,
       -6.932317733764648,
       -7.661313533782959,
       -7.972192287445068,
       1.9154517650604248,
       -7.213179588317871,
       -7.732088088989258,
       -8.088434219360352,
       -8.394003868103027,
       -8.597945213317871,
       -9.037652969360352,
       -8.842361450195312,
       -9.066045761108398,
       -8.191587448120117,
       -8.210041046142578,
       -8.384839057922363,
       -7.323757648468018,
       -8.522293090820312,
       -8.851812362670898,
       -8.925619125366211,
       -8.67837142944336,
       -8.899063110351562
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cap": {
           "value": true,
           "probability": 0.9978814786403635,
           "distribution": [
            {
             "value": false,
             "probability": 0.002118521359636527
            },
            {
             "value": true,
             "probability": 0.9978814786403635
            }
           ]
          },
          "needs_escalation": {
           "value": false,
           "probability": 0.9991696934883662,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991696934883662
            },
            {
             "value": true,
             "probability": 0.0008303065116338172
            }
           ]
          },
          "severity": {
           "value": 2,
           "probability": 0.9984163293307933,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00036234003843795215
            },
            {
             "value": 1,
             "probability": 0.0008098287779135839
            },
            {
             "value": 2,
             "probability": 0.9984163293307933
            },
            {
             "value": 3,
             "probability": 0.0004115018528552462
            }
           ]
          },
          "decision": {
           "value": "partial waiver payout",
           "probability": 0.9986288115449797,
           "distribution": [
            {
             "value": "partial waiver payout",
             "probability": 0.9986288115449797
            },
            {
             "value": "waiver covers in full",
             "probability": 0.0007724791432851909
            },
            {
             "value": "claimant pays in full",
             "probability": 0.0005987093117351549
            }
           ]
          }
         },
         "joint_probability": 0.9968642271889097,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.003140699648977255,
        "field_brier": 4.1637193678366425e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cap": {
           "value": true,
           "probability": 0.9978814786403635,
           "distribution": [
            {
             "value": false,
             "probability": 0.002118521359636527
            },
            {
             "value": true,
             "probability": 0.9978814786403635
            }
           ]
          },
          "needs_escalation": {
           "value": false,
           "probability": 0.9991696934883662,
           "distribution": [
            {
             "value": false,
             "probability": 0.9991696934883662
            },
            {
             "value": true,
             "probability": 0.0008303065116338172
            }
           ]
          },
          "severity": {
           "value": 2,
           "probability": 0.9984163293307933,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00036234003843795215
            },
            {
             "value": 1,
             "probability": 0.0008098287779135839
            },
            {
             "value": 2,
             "probability": 0.9984163293307933
            },
            {
             "value": 3,
             "probability": 0.0004115018528552462
            }
           ]
          },
          "decision": {
           "value": "partial waiver payout",
           "probability": 0.9986288115449797,
           "distribution": [
            {
             "value": "partial waiver payout",
             "probability": 0.9986288115449797
            },
            {
             "value": "waiver covers in full",
             "probability": 0.0007724791432851909
            },
            {
             "value": "claimant pays in full",
             "probability": 0.0005987093117351549
            }
           ]
          }
         },
         "joint_probability": 0.9968642271889097,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 48,
         "valid_assignments": 48,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 4,
        "joint_nll": 0.003140699648977255,
        "field_brier": 4.1637193678366425e-06
       }
      },
      "timing_ms": {
       "scoring": 43.82013902068138,
       "base_joint": 0.10040891356766224,
       "constrained_joint": 0.08289888501167297,
       "constrained_total": 43.903037905693054
      },
      "input_tokens": 933,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05eb04ef601996df-0011",
      "group": "bendsynth-05eb04ef601996df",
      "provenance": "bendsynth-v1; family=bendsynth-05eb04ef601996df; domain=municipal street tree removal permit intake; parameters=grace=39; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "flags": 2,
       "in_grace": false,
       "decision": "deny"
      },
      "logits": [
       [
        -5.413733005523682,
        2.255221128463745,
        -5.085184574127197,
        -5.45051908493042
       ],
       [
        -4.409528732299805,
        2.255512237548828
       ],
       [
        -5.521328449249268,
        2.2552573680877686,
        -5.328157424926758,
        -5.16053581237793
       ]
      ],
      "joint_logits": [
       -7.639707565307617,
       -7.269671440124512,
       -7.491276741027832,
       -7.321024417877197,
       -7.834039688110352,
       -7.25483512878418,
       -7.729898929595947,
       -7.569411754608154,
       -7.459089279174805,
       -6.178047180175781,
       -7.275874137878418,
       -7.107433319091797,
       -7.431792736053467,
       2.25451922416687,
       -7.067590236663818,
       -6.824494361877441,
       -7.352275848388672,
       -6.943686485290527,
       -7.170863151550293,
       -7.0263261795043945,
       -7.677221298217773,
       -6.671686172485352,
       -7.492384910583496,
       -7.356820106506348,
       -7.677644729614258,
       -7.306121349334717,
       -7.488109588623047,
       -7.345275402069092,
       -7.8550801277160645,
       -7.326898097991943,
       -7.762020587921143,
       -7.634923458099365
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 1,
           "probability": 0.9984363202993956,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00046637578668803323
            },
            {
             "value": 1,
             "probability": 0.9984363202993956
            },
            {
             "value": 2,
             "probability": 0.0006477727272251364
            },
            {
             "value": 3,
             "probability": 0.00044953118669131034
            }
           ]
          },
          "in_grace": {
           "value": true,
           "probability": 0.9987269180613496,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012730819386504713
            },
            {
             "value": true,
             "probability": 0.9987269180613496
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.9984723993936434,
           "distribution": [
            {
             "value": "replant required",
             "probability": 0.0004188011028726643
            },
            {
             "value": "deny",
             "probability": 0.9984723993936434
            },
            {
             "value": "standard review",
             "probability": 0.0005080435963519609
            },
            {
             "value": "fast-track approval",
             "probability": 0.000600755907131993
            }
           ]
          }
         },
         "joint_probability": 0.9977356626667764,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 9.200472614278906,
        "field_brier": 1.3301648070424303
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "flags": {
           "value": 1,
           "probability": 0.9984363202993956,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00046637578668803323
            },
            {
             "value": 1,
             "probability": 0.9984363202993956
            },
            {
             "value": 2,
             "probability": 0.0006477727272251364
            },
            {
             "value": 3,
             "probability": 0.00044953118669131034
            }
           ]
          },
          "in_grace": {
           "value": true,
           "probability": 0.9987269180613496,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012730819386504713
            },
            {
             "value": true,
             "probability": 0.9987269180613496
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.9984723993936434,
           "distribution": [
            {
             "value": "replant required",
             "probability": 0.0004188011028726643
            },
            {
             "value": "deny",
             "probability": 0.9984723993936434
            },
            {
             "value": "standard review",
             "probability": 0.0005080435963519609
            },
            {
             "value": "fast-track approval",
             "probability": 0.000600755907131993
            }
           ]
          }
         },
         "joint_probability": 0.9977356626667764,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 9.200472614278906,
        "field_brier": 1.3301648070424303
       }
      },
      "timing_ms": {
       "scoring": 39.041630923748016,
       "base_joint": 0.07836893200874329,
       "constrained_joint": 0.05811918526887894,
       "constrained_total": 39.099750109016895
      },
      "input_tokens": 699,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05eb04ef601996df-0019",
      "group": "bendsynth-05eb04ef601996df",
      "provenance": "bendsynth-v1; family=bendsynth-05eb04ef601996df; domain=municipal street tree removal permit intake; parameters=grace=60; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "in_grace": true,
       "decision": "fast-track approval",
       "flags": 0
      },
      "logits": [
       [
        -5.344431400299072,
        1.3473790884017944
       ],
       [
        -6.24975061416626,
        1.346907615661621,
        -6.202538967132568,
        -5.939822673797607
       ],
       [
        1.346967101097107,
        -6.001686096191406,
        -6.180669784545898,
        -6.311427116394043
       ]
      ],
      "joint_logits": [
       -8.251211166381836,
       -8.137231826782227,
       -8.26175594329834,
       -8.337784767150879,
       -7.8482866287231445,
       -8.073372840881348,
       -8.213268280029297,
       -8.358912467956543,
       -8.180124282836914,
       -8.017236709594727,
       -8.216300964355469,
       -8.339016914367676,
       -7.941459655761719,
       -7.81226921081543,
       -7.990288257598877,
       -8.118387222290039,
       -8.262041091918945,
       -8.366425514221191,
       -8.49413013458252,
       -8.596790313720703,
       1.3463878631591797,
       -7.98480224609375,
       -8.317167282104492,
       -8.487855911254883,
       -8.235182762145996,
       -8.30333423614502,
       -8.462645530700684,
       -8.616423606872559,
       -7.783680438995361,
       -8.062629699707031,
       -8.214544296264648,
       -8.364194869995117
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "in_grace": {
           "value": true,
           "probability": 0.9987605042078505,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012394957921494935
            },
            {
             "value": true,
             "probability": 0.9987605042078505
            }
           ]
          },
          "decision": {
           "value": "fast-track approval",
           "probability": 0.9982898388714653,
           "distribution": [
            {
             "value": "replant required",
             "probability": 0.0005012679273147349
            },
            {
             "value": "fast-track approval",
             "probability": 0.9982898388714653
            },
            {
             "value": "deny",
             "probability": 0.0005255010959126794
            },
            {
             "value": "standard review",
             "probability": 0.0006833921053073536
            }
           ]
          },
          "flags": {
           "value": 0,
           "probability": 0.998349198480007,
           "distribution": [
            {
             "value": 0,
             "probability": 0.998349198480007
            },
            {
             "value": 1,
             "probability": 0.0006423960942038738
            },
            {
             "value": 2,
             "probability": 0.0005371199151539912
            },
            {
             "value": 3,
             "probability": 0.0004712855106351214
            }
           ]
          }
         },
         "joint_probability": 0.9977710770684978,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022314106775697925,
        "field_brier": 3.546740888489766e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "in_grace": {
           "value": true,
           "probability": 0.9987605042078505,
           "distribution": [
            {
             "value": false,
             "probability": 0.0012394957921494935
            },
            {
             "value": true,
             "probability": 0.9987605042078505
            }
           ]
          },
          "decision": {
           "value": "fast-track approval",
           "probability": 0.9982898388714653,
           "distribution": [
            {
             "value": "replant required",
             "probability": 0.0005012679273147349
            },
            {
             "value": "fast-track approval",
             "probability": 0.9982898388714653
            },
            {
             "value": "deny",
             "probability": 0.0005255010959126794
            },
            {
             "value": "standard review",
             "probability": 0.0006833921053073536
            }
           ]
          },
          "flags": {
           "value": 0,
           "probability": 0.998349198480007,
           "distribution": [
            {
             "value": 0,
             "probability": 0.998349198480007
            },
            {
             "value": 1,
             "probability": 0.0006423960942038738
            },
            {
             "value": 2,
             "probability": 0.0005371199151539912
            },
            {
             "value": 3,
             "probability": 0.0004712855106351214
            }
           ]
          }
         },
         "joint_probability": 0.9977710770684978,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022314106775697925,
        "field_brier": 3.546740888489766e-06
       }
      },
      "timing_ms": {
       "scoring": 37.591325817629695,
       "base_joint": 0.07449882104992867,
       "constrained_joint": 0.05659996531903744,
       "constrained_total": 37.64792578294873
      },
      "input_tokens": 669,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-05eb04ef601996df-0037",
      "group": "bendsynth-05eb04ef601996df",
      "provenance": "bendsynth-v1; family=bendsynth-05eb04ef601996df; domain=municipal street tree removal permit intake; parameters=grace=30; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "replant required",
       "in_grace": true,
       "flags": 1
      },
      "logits": [
       [
        -5.9374542236328125,
        -6.148144245147705,
        -6.175562858581543,
        1.3426125049591064
       ],
       [
        -5.478991508483887,
        1.3433128595352173
       ],
       [
        -6.250606536865234,
        1.342782974243164,
        -5.921388149261475,
        -6.442740440368652
       ]
      ],
      "joint_logits": [
       -8.180194854736328,
       -8.040834426879883,
       -7.822710037231445,
       -8.349215507507324,
       -8.145501136779785,
       -7.604003429412842,
       -7.892917633056641,
       -8.345064163208008,
       -8.360403060913086,
       -8.236831665039062,
       -8.011714935302734,
       -8.544048309326172,
       -8.303715705871582,
       -7.908898830413818,
       -8.133825302124023,
       -8.497295379638672,
       -8.378920555114746,
       -8.246540069580078,
       -8.034902572631836,
       -8.523463249206543,
       -8.356561660766602,
       -8.032366752624512,
       -8.069547653198242,
       -8.555135726928711,
       -8.5311861038208,
       -8.139975547790527,
       -8.3157377243042,
       -8.73239803314209,
       -8.441044807434082,
       1.3421741724014282,
       -7.821434497833252,
       -8.699969291687012
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "replant required",
           "probability": 0.9982127176083707,
           "distribution": [
            {
             "value": "standard review",
             "probability": 0.0006879077238154749
            },
            {
             "value": "fast-track approval",
             "probability": 0.000557222682551678
            },
            {
             "value": "deny",
             "probability": 0.0005421519852621785
            },
            {
             "value": "replant required",
             "probability": 0.9982127176083707
            }
           ]
          },
          "in_grace": {
           "value": true,
           "probability": 0.9989119769374647,
           "distribution": [
            {
             "value": false,
             "probability": 0.001088023062535367
            },
            {
             "value": true,
             "probability": 0.9989119769374647
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9983829574866475,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005029558804085519
            },
            {
             "value": 1,
             "probability": 0.9983829574866475
            },
            {
             "value": 2,
             "probability": 0.000699049201267428
            },
            {
             "value": 3,
             "probability": 0.00041503743167653246
            }
           ]
          }
         },
         "joint_probability": 0.997775208103138,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002227270423173309,
        "field_brier": 3.389442203105527e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "replant required",
           "probability": 0.9982127176083707,
           "distribution": [
            {
             "value": "standard review",
             "probability": 0.0006879077238154749
            },
            {
             "value": "fast-track approval",
             "probability": 0.000557222682551678
            },
            {
             "value": "deny",
             "probability": 0.0005421519852621785
            },
            {
             "value": "replant required",
             "probability": 0.9982127176083707
            }
           ]
          },
          "in_grace": {
           "value": true,
           "probability": 0.9989119769374647,
           "distribution": [
            {
             "value": false,
             "probability": 0.001088023062535367
            },
            {
             "value": true,
             "probability": 0.9989119769374647
            }
           ]
          },
          "flags": {
           "value": 1,
           "probability": 0.9983829574866475,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005029558804085519
            },
            {
             "value": 1,
             "probability": 0.9983829574866475
            },
            {
             "value": 2,
             "probability": 0.000699049201267428
            },
            {
             "value": 3,
             "probability": 0.00041503743167653246
            }
           ]
          }
         },
         "joint_probability": 0.997775208103138,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002227270423173309,
        "field_brier": 3.389442203105527e-06
       }
      },
      "timing_ms": {
       "scoring": 36.46992798894644,
       "base_joint": 0.08076895028352737,
       "constrained_joint": 0.0595401506870985,
       "constrained_total": 36.529468139633536
      },
      "input_tokens": 711,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-04b122985c24df28-0004",
      "group": "bendsynth-04b122985c24df28",
      "provenance": "bendsynth-v1; family=bendsynth-04b122985c24df28; domain=dialysis chair scheduling counter; parameters=min_notice=72; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "too_late": false,
       "red_flags": 2,
       "decision": "deny"
      },
      "logits": [
       [
        1.2993762493133545,
        -5.686740398406982
       ],
       [
        -6.4176716804504395,
        -5.965641975402832,
        1.298626184463501,
        -6.24408483505249
       ],
       [
        1.2985377311706543,
        -6.354806423187256,
        -6.35561466217041,
        -5.812858581542969
       ]
      ],
      "joint_logits": [
       -8.283573150634766,
       -8.562612533569336,
       -8.560418128967285,
       -8.039007186889648,
       -7.568066120147705,
       -8.121272087097168,
       -8.162694931030273,
       -7.611823081970215,
       1.2980008125305176,
       -7.888591766357422,
       -7.981311321258545,
       -7.318954944610596,
       -8.206170082092285,
       -8.407561302185059,
       -8.380842208862305,
       -7.864755153656006,
       -8.663989067077637,
       -8.908401489257812,
       -8.869325637817383,
       -8.395184516906738,
       -8.248416900634766,
       -8.551356315612793,
       -8.518293380737305,
       -8.056107521057129,
       -8.193986892700195,
       -8.73691463470459,
       -8.645242691040039,
       -8.115498542785645,
       -8.449931144714355,
       -8.72266674041748,
       -8.662524223327637,
       -8.170790672302246
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "too_late": {
           "value": false,
           "probability": 0.9990762240259363,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990762240259363
            },
            {
             "value": true,
             "probability": 0.0009237759740637541
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.9983272237184091,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00044476155871090475
            },
            {
             "value": 1,
             "probability": 0.0006989424068884244
            },
            {
             "value": 2,
             "probability": 0.9983272237184091
            },
            {
             "value": 3,
             "probability": 0.0005290723159914662
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.9982388260361149,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.9982388260361149
            },
            {
             "value": "escalate",
             "probability": 0.0004736192686025734
            },
            {
             "value": "waitlist",
             "probability": 0.0004732363991233334
            },
            {
             "value": "confirm",
             "probability": 0.0008143182961591062
            }
           ]
          }
         },
         "joint_probability": 0.9977029499024036,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022996923642265616,
        "field_brier": 3.2280904520477923e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "too_late": {
           "value": false,
           "probability": 0.9990762240259363,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990762240259363
            },
            {
             "value": true,
             "probability": 0.0009237759740637541
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.9983272237184091,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00044476155871090475
            },
            {
             "value": 1,
             "probability": 0.0006989424068884244
            },
            {
             "value": 2,
             "probability": 0.9983272237184091
            },
            {
             "value": 3,
             "probability": 0.0005290723159914662
            }
           ]
          },
          "decision": {
           "value": "deny",
           "probability": 0.9982388260361149,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.9982388260361149
            },
            {
             "value": "escalate",
             "probability": 0.0004736192686025734
            },
            {
             "value": "waitlist",
             "probability": 0.0004732363991233334
            },
            {
             "value": "confirm",
             "probability": 0.0008143182961591062
            }
           ]
          }
         },
         "joint_probability": 0.9977029499024036,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0022996923642265616,
        "field_brier": 3.2280904520477923e-06
       }
      },
      "timing_ms": {
       "scoring": 40.43718590401113,
       "base_joint": 0.07482897490262985,
       "constrained_joint": 0.056718941777944565,
       "constrained_total": 40.493904845789075
      },
      "input_tokens": 675,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-04b122985c24df28-0018",
      "group": "bendsynth-04b122985c24df28",
      "provenance": "bendsynth-v1; family=bendsynth-04b122985c24df28; domain=dialysis chair scheduling counter; parameters=min_notice=44; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "waitlist",
       "too_late": false,
       "red_flags": 1
      },
      "logits": [
       [
        -5.505399227142334,
        -5.394360542297363,
        -5.394114971160889,
        2.005485773086548
       ],
       [
        2.006305694580078,
        -4.953675746917725
       ],
       [
        -5.624457359313965,
        2.005739212036133,
        -5.461154937744141,
        -5.680939674377441
       ]
      ],
      "joint_logits": [
       -7.66792106628418,
       -7.040249824523926,
       -7.481149673461914,
       -7.717304229736328,
       -7.850133419036865,
       -7.656386375427246,
       -7.693107604980469,
       -7.839340686798096,
       -7.513428688049316,
       -7.036393642425537,
       -7.348480701446533,
       -7.565301418304443,
       -7.6891889572143555,
       -7.58546257019043,
       -7.518793106079102,
       -7.712686538696289,
       -7.698455810546875,
       -6.4360833168029785,
       -7.589303493499756,
       -7.772719860076904,
       -7.923389911651611,
       -7.762504577636719,
       -7.757415771484375,
       -7.971909523010254,
       -7.586440086364746,
       2.0050272941589355,
       -7.380198001861572,
       -7.715360641479492,
       -7.76391077041626,
       -7.553097248077393,
       -7.6296067237854,
       -7.838869094848633
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9982331868066521,
           "distribution": [
            {
             "value": "confirm",
             "probability": 0.0005461300850665504
            },
            {
             "value": "deny",
             "probability": 0.0006102665920718427
            },
            {
             "value": "escalate",
             "probability": 0.0006104165162096421
            },
            {
             "value": "waitlist",
             "probability": 0.9982331868066521
            }
           ]
          },
          "too_late": {
           "value": false,
           "probability": 0.9990517856234019,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990517856234019
            },
            {
             "value": true,
             "probability": 0.0009482143765980153
            }
           ]
          },
          "red_flags": {
           "value": 1,
           "probability": 0.9984861288335503,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00048483026810985447
            },
            {
             "value": 1,
             "probability": 0.9984861288335503
            },
            {
             "value": 2,
             "probability": 0.0005708358862680789
            },
            {
             "value": 3,
             "probability": 0.00045820501207176947
            }
           ]
          }
         },
         "joint_probability": 0.9977755763258338,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002226901379500164,
        "field_brier": 3.0086044380868633e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9982331868066521,
           "distribution": [
            {
             "value": "confirm",
             "probability": 0.0005461300850665504
            },
            {
             "value": "deny",
             "probability": 0.0006102665920718427
            },
            {
             "value": "escalate",
             "probability": 0.0006104165162096421
            },
            {
             "value": "waitlist",
             "probability": 0.9982331868066521
            }
           ]
          },
          "too_late": {
           "value": false,
           "probability": 0.9990517856234019,
           "distribution": [
            {
             "value": false,
             "probability": 0.9990517856234019
            },
            {
             "value": true,
             "probability": 0.0009482143765980153
            }
           ]
          },
          "red_flags": {
           "value": 1,
           "probability": 0.9984861288335503,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00048483026810985447
            },
            {
             "value": 1,
             "probability": 0.9984861288335503
            },
            {
             "value": 2,
             "probability": 0.0005708358862680789
            },
            {
             "value": 3,
             "probability": 0.00045820501207176947
            }
           ]
          }
         },
         "joint_probability": 0.9977755763258338,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002226901379500164,
        "field_brier": 3.0086044380868633e-06
       }
      },
      "timing_ms": {
       "scoring": 35.41481913998723,
       "base_joint": 0.07694889791309834,
       "constrained_joint": 0.0630600843578577,
       "constrained_total": 35.47787922434509
      },
      "input_tokens": 684,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-04b122985c24df28-0026",
      "group": "bendsynth-04b122985c24df28",
      "provenance": "bendsynth-v1; family=bendsynth-04b122985c24df28; domain=dialysis chair scheduling counter; parameters=min_notice=48; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "too_late": false,
       "red_flags": 1,
       "decision": "waitlist"
      },
      "logits": [
       [
        1.2238363027572632,
        -5.190746784210205
       ],
       [
        -5.897952079772949,
        1.222847819328308,
        -5.60000467300415,
        -5.997575759887695
       ],
       [
        1.0870873928070068,
        -0.8357604146003723,
        -5.7111101150512695,
        -5.714615821838379
       ]
      ],
      "joint_logits": [
       -8.050315856933594,
       -8.1707763671875,
       -7.73587703704834,
       -7.871856212615967,
       1.086253046989441,
       -0.840915322303772,
       -7.625483512878418,
       -7.405290603637695,
       -7.5781354904174805,
       -7.66912841796875,
       -7.576436996459961,
       -7.644009113311768,
       -8.181384086608887,
       -8.284014701843262,
       -7.863121509552002,
       -7.990597724914551,
       -8.020424842834473,
       -8.123086929321289,
       -8.011405944824219,
       -7.907110214233398,
       -8.009698867797852,
       -8.108098983764648,
       -8.002873420715332,
       -8.054039001464844,
       -7.813828468322754,
       -7.919872760772705,
       -7.613199710845947,
       -7.670093059539795,
       -8.117515563964844,
       -8.233243942260742,
       -8.025039672851562,
       -7.992401599884033
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "too_late": {
           "value": false,
           "probability": 0.9983651744527386,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983651744527386
            },
            {
             "value": true,
             "probability": 0.0016348255472613672
            }
           ]
          },
          "red_flags": {
           "value": 1,
           "probability": 0.9973786651771652,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008060019253539033
            },
            {
             "value": 1,
             "probability": 0.9973786651771652
            },
            {
             "value": 2,
             "probability": 0.0010857577819925106
            },
            {
             "value": 3,
             "probability": 0.0007295751154883627
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.8707633043711004,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.8707633043711004
            },
            {
             "value": "escalate",
             "probability": 0.12729692720981306
            },
            {
             "value": "deny",
             "probability": 0.0009715842233351756
            },
            {
             "value": "confirm",
             "probability": 0.0009681841957514374
            }
           ]
          }
         },
         "joint_probability": 0.8700370052814067,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.13921953343187227,
        "field_brier": 0.01097436334166152
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "too_late": {
           "value": false,
           "probability": 0.9983651744527386,
           "distribution": [
            {
             "value": false,
             "probability": 0.9983651744527386
            },
            {
             "value": true,
             "probability": 0.0016348255472613672
            }
           ]
          },
          "red_flags": {
           "value": 1,
           "probability": 0.9973786651771652,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008060019253539033
            },
            {
             "value": 1,
             "probability": 0.9973786651771652
            },
            {
             "value": 2,
             "probability": 0.0010857577819925106
            },
            {
             "value": 3,
             "probability": 0.0007295751154883627
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.8707633043711004,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.8707633043711004
            },
            {
             "value": "escalate",
             "probability": 0.12729692720981306
            },
            {
             "value": "deny",
             "probability": 0.0009715842233351756
            },
            {
             "value": "confirm",
             "probability": 0.0009681841957514374
            }
           ]
          }
         },
         "joint_probability": 0.8700370052814067,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.13921953343187227,
        "field_brier": 0.01097436334166152
       }
      },
      "timing_ms": {
       "scoring": 35.52269795909524,
       "base_joint": 0.07332884706556797,
       "constrained_joint": 0.05941884592175484,
       "constrained_total": 35.582116805016994
      },
      "input_tokens": 654,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-04b122985c24df28-0033",
      "group": "bendsynth-04b122985c24df28",
      "provenance": "bendsynth-v1; family=bendsynth-04b122985c24df28; domain=dialysis chair scheduling counter; parameters=min_notice=14; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "waitlist",
       "too_late": false,
       "red_flags": 0
      },
      "logits": [
       [
        1.4779003858566284,
        -6.179980754852295,
        -5.925961971282959,
        -5.617415428161621
       ],
       [
        1.4786945581436157,
        -5.320314407348633
       ],
       [
        1.477997064590454,
        -5.667018413543701,
        -6.036701679229736,
        -6.164850234985352
       ]
      ],
      "joint_logits": [
       1.477312445640564,
       -7.328289031982422,
       -7.945548057556152,
       -8.104951858520508,
       -7.909982204437256,
       -7.910939693450928,
       -8.171477317810059,
       -8.305749893188477,
       -8.017047882080078,
       -8.028952598571777,
       -8.28273868560791,
       -8.415271759033203,
       -8.247699737548828,
       -8.142598152160645,
       -8.488399505615234,
       -8.612052917480469,
       -7.574478626251221,
       -7.771937847137451,
       -8.111088752746582,
       -8.227360725402832,
       -8.040020942687988,
       -7.9325032234191895,
       -8.257802963256836,
       -8.392302513122559,
       -7.113569736480713,
       -7.460155010223389,
       -7.866700649261475,
       -7.9838457107543945,
       -7.843862533569336,
       -7.679651260375977,
       -7.954187870025635,
       -8.063664436340332
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9980934594602959,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.9980934594602959
            },
            {
             "value": "escalate",
             "probability": 0.0004714066839905079
            },
            {
             "value": "confirm",
             "probability": 0.0006077355058567656
            },
            {
             "value": "deny",
             "probability": 0.0008273983498568482
            }
           ]
          },
          "too_late": {
           "value": false,
           "probability": 0.9988863620387409,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988863620387409
            },
            {
             "value": true,
             "probability": 0.0011136379612590609
            }
           ]
          },
          "red_flags": {
           "value": 0,
           "probability": 0.9981900200765571,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981900200765571
            },
            {
             "value": 1,
             "probability": 0.0007873585243547016
            },
            {
             "value": 2,
             "probability": 0.0005440277373092781
            },
            {
             "value": 3,
             "probability": 0.00047859366177887847
            }
           ]
          }
         },
         "joint_probability": 0.9975068535905915,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024962594742117747,
        "field_brier": 3.9374698081274875e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "waitlist",
           "probability": 0.9980934594602959,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.9980934594602959
            },
            {
             "value": "escalate",
             "probability": 0.0004714066839905079
            },
            {
             "value": "confirm",
             "probability": 0.0006077355058567656
            },
            {
             "value": "deny",
             "probability": 0.0008273983498568482
            }
           ]
          },
          "too_late": {
           "value": false,
           "probability": 0.9988863620387409,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988863620387409
            },
            {
             "value": true,
             "probability": 0.0011136379612590609
            }
           ]
          },
          "red_flags": {
           "value": 0,
           "probability": 0.9981900200765571,
           "distribution": [
            {
             "value": 0,
             "probability": 0.9981900200765571
            },
            {
             "value": 1,
             "probability": 0.0007873585243547016
            },
            {
             "value": 2,
             "probability": 0.0005440277373092781
            },
            {
             "value": 3,
             "probability": 0.00047859366177887847
            }
           ]
          }
         },
         "joint_probability": 0.9975068535905915,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024962594742117747,
        "field_brier": 3.9374698081274875e-06
       }
      },
      "timing_ms": {
       "scoring": 38.98209100589156,
       "base_joint": 0.07687904872000217,
       "constrained_joint": 0.057818833738565445,
       "constrained_total": 39.03990983963013
      },
      "input_tokens": 675,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-04b122985c24df28-0037",
      "group": "bendsynth-04b122985c24df28",
      "provenance": "bendsynth-v1; family=bendsynth-04b122985c24df28; domain=dialysis chair scheduling counter; parameters=min_notice=48; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "escalate",
       "red_flags": 2,
       "too_late": true
      },
      "logits": [
       [
        -6.675710678100586,
        -6.6456217765808105,
        0.97639000415802,
        -6.667792320251465
       ],
       [
        -6.6976118087768555,
        -6.75881814956665,
        0.9764529466629028,
        -6.668169021606445
       ],
       [
        -6.108299255371094,
        0.9769954085350037
       ]
      ],
      "joint_logits": [
       -8.909954071044922,
       -8.664957046508789,
       -9.016195297241211,
       -8.7800931930542,
       -8.826884269714355,
       -8.40562915802002,
       -8.877171516418457,
       -8.688326835632324,
       -8.864566802978516,
       -8.693613052368164,
       -8.977245330810547,
       -8.727852821350098,
       -8.896886825561523,
       -8.34219741821289,
       -8.812447547912598,
       -8.630654335021973,
       -8.886053085327148,
       -8.619183540344238,
       -8.94872760772705,
       -8.575211524963379,
       -8.614009857177734,
       0.9759602546691895,
       -8.884099006652832,
       -8.605379104614258,
       -8.904399871826172,
       -8.724056243896484,
       -8.997379302978516,
       -8.773465156555176,
       -8.888875961303711,
       -8.323203086853027,
       -8.855554580688477,
       -8.674159049987793
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "escalate",
           "probability": 0.9985586577781709,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0004743603933824642
            },
            {
             "value": "confirm",
             "probability": 0.0004888503816781706
            },
            {
             "value": "escalate",
             "probability": 0.9985586577781709
            },
            {
             "value": "waitlist",
             "probability": 0.0004781314467685294
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.9986214329590036,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004640842904741818
            },
            {
             "value": 1,
             "probability": 0.0004365312143288775
            },
            {
             "value": 2,
             "probability": 0.9986214329590036
            },
            {
             "value": 3,
             "probability": 0.0004779515361932908
            }
           ]
          },
          "too_late": {
           "value": true,
           "probability": 0.9991633722107939,
           "distribution": [
            {
             "value": false,
             "probability": 0.000836627789206078
            },
            {
             "value": true,
             "probability": 0.9991633722107939
            }
           ]
          }
         },
         "joint_probability": 0.9981295818714166,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018721695448324445,
        "field_brier": 2.234926720618766e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "escalate",
           "probability": 0.9985586577781709,
           "distribution": [
            {
             "value": "deny",
             "probability": 0.0004743603933824642
            },
            {
             "value": "confirm",
             "probability": 0.0004888503816781706
            },
            {
             "value": "escalate",
             "probability": 0.9985586577781709
            },
            {
             "value": "waitlist",
             "probability": 0.0004781314467685294
            }
           ]
          },
          "red_flags": {
           "value": 2,
           "probability": 0.9986214329590036,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004640842904741818
            },
            {
             "value": 1,
             "probability": 0.0004365312143288775
            },
            {
             "value": 2,
             "probability": 0.9986214329590036
            },
            {
             "value": 3,
             "probability": 0.0004779515361932908
            }
           ]
          },
          "too_late": {
           "value": true,
           "probability": 0.9991633722107939,
           "distribution": [
            {
             "value": false,
             "probability": 0.000836627789206078
            },
            {
             "value": true,
             "probability": 0.9991633722107939
            }
           ]
          }
         },
         "joint_probability": 0.9981295818714166,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0018721695448324445,
        "field_brier": 2.234926720618766e-06
       }
      },
      "timing_ms": {
       "scoring": 36.65817598812282,
       "base_joint": 0.07464992813766003,
       "constrained_joint": 0.057279132306575775,
       "constrained_total": 36.7154551204294
      },
      "input_tokens": 669,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0552f72baccc6d63-0010",
      "group": "bendsynth-0552f72baccc6d63",
      "provenance": "bendsynth-v1; family=bendsynth-0552f72baccc6d63; domain=alpine hut overnight booking \u2014 supervisor reviewing an escalated group request; parameters=max_group=5; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "confirm booking",
       "clearance": 3,
       "over_quota": false
      },
      "logits": [
       [
        1.9274296760559082,
        -5.34758186340332,
        -5.721565246582031,
        -5.531435489654541
       ],
       [
        -5.78035831451416,
        -5.444515228271484,
        -5.183438301086426,
        1.9272810220718384
       ],
       [
        1.9281585216522217,
        -4.964052200317383
       ]
      ],
      "joint_logits": [
       -7.575538635253906,
       -8.04543399810791,
       -7.09773588180542,
       -7.703568935394287,
       -6.72851037979126,
       -7.464261531829834,
       1.9267935752868652,
       -7.622797966003418,
       -7.589926242828369,
       -7.876533508300781,
       -7.2851715087890625,
       -7.590395927429199,
       -7.058406352996826,
       -7.376948356628418,
       -7.2219038009643555,
       -7.66788911819458,
       -7.99673318862915,
       -8.12255859375,
       -7.702584743499756,
       -7.815106391906738,
       -7.459067344665527,
       -7.6032843589782715,
       -7.868283748626709,
       -8.02155876159668,
       -7.778045654296875,
       -8.056673049926758,
       -7.448265552520752,
       -7.791743755340576,
       -7.202768802642822,
       -7.535139083862305,
       -7.470746994018555,
       -7.872982025146484
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "confirm booking",
           "probability": 0.9982575768552244,
           "distribution": [
            {
             "value": "confirm booking",
             "probability": 0.9982575768552244
            },
            {
             "value": "overflow dorm",
             "probability": 0.0006914250427694924
            },
            {
             "value": "reject",
             "probability": 0.0004756924479059115
            },
            {
             "value": "waitlist",
             "probability": 0.0005753056541002418
            }
           ]
          },
          "clearance": {
           "value": 3,
           "probability": 0.9981091556204128,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00044853141827908217
            },
            {
             "value": 1,
             "probability": 0.0006275489245341246
            },
            {
             "value": 2,
             "probability": 0.0008147640367740742
            },
            {
             "value": 3,
             "probability": 0.9981091556204128
            }
           ]
          },
          "over_quota": {
           "value": false,
           "probability": 0.9989853645362998,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989853645362998
            },
            {
             "value": true,
             "probability": 0.0010146354637002135
            }
           ]
          }
         },
         "joint_probability": 0.9976227129560032,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023801172772491956,
        "field_brier": 3.6548227145177656e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "confirm booking",
           "probability": 0.9982575768552244,
           "distribution": [
            {
             "value": "confirm booking",
             "probability": 0.9982575768552244
            },
            {
             "value": "overflow dorm",
             "probability": 0.0006914250427694924
            },
            {
             "value": "reject",
             "probability": 0.0004756924479059115
            },
            {
             "value": "waitlist",
             "probability": 0.0005753056541002418
            }
           ]
          },
          "clearance": {
           "value": 3,
           "probability": 0.9981091556204128,
           "distribution": [
            {
             "value": 0,
             "probability": 0.00044853141827908217
            },
            {
             "value": 1,
             "probability": 0.0006275489245341246
            },
            {
             "value": 2,
             "probability": 0.0008147640367740742
            },
            {
             "value": 3,
             "probability": 0.9981091556204128
            }
           ]
          },
          "over_quota": {
           "value": false,
           "probability": 0.9989853645362998,
           "distribution": [
            {
             "value": false,
             "probability": 0.9989853645362998
            },
            {
             "value": true,
             "probability": 0.0010146354637002135
            }
           ]
          }
         },
         "joint_probability": 0.9976227129560032,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0023801172772491956,
        "field_brier": 3.6548227145177656e-06
       }
      },
      "timing_ms": {
       "scoring": 38.85367210023105,
       "base_joint": 0.07823901250958443,
       "constrained_joint": 0.057859113439917564,
       "constrained_total": 38.91153121367097
      },
      "input_tokens": 746,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0552f72baccc6d63-0025",
      "group": "bendsynth-0552f72baccc6d63",
      "provenance": "bendsynth-v1; family=bendsynth-0552f72baccc6d63; domain=alpine hut overnight booking \u2014 supervisor reviewing an escalated group request; parameters=max_group=6; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "clearance": 1,
       "over_quota": true,
       "decision": "waitlist"
      },
      "logits": [
       [
        -4.805950164794922,
        2.3518049716949463,
        -4.344635009765625,
        -5.040822982788086
       ],
       [
        -4.418832778930664,
        2.3532867431640625
       ],
       [
        -5.206172943115234,
        2.3520963191986084,
        -4.495244026184082,
        -4.838479042053223
       ]
      ],
      "joint_logits": [
       -7.408751964569092,
       -7.247000694274902,
       -6.891097545623779,
       -7.149840354919434,
       -7.0267438888549805,
       -6.356470108032227,
       -6.557471752166748,
       -6.889786243438721,
       -7.533613681793213,
       -6.937720775604248,
       -7.103901386260986,
       -7.423120498657227,
       -7.162118434906006,
       2.3510186672210693,
       -5.8786211013793945,
       -6.34402322769165,
       -7.364050388336182,
       -7.060870170593262,
       -6.907193660736084,
       -7.082263946533203,
       -7.1951003074646,
       -5.227419376373291,
       -6.168315410614014,
       -6.548836708068848,
       -7.573822498321533,
       -7.493122577667236,
       -7.001473903656006,
       -7.278008460998535,
       -7.154865741729736,
       -6.797028064727783,
       -6.81308650970459,
       -7.131824970245361
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "clearance": {
           "value": 1,
           "probability": 0.9973770198782218,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007767579715620713
            },
            {
             "value": 1,
             "probability": 0.9973770198782218
            },
            {
             "value": 2,
             "probability": 0.00123206162170203
            },
            {
             "value": 3,
             "probability": 0.0006141605285140914
            }
           ]
          },
          "over_quota": {
           "value": true,
           "probability": 0.9988560451231133,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011439548768866264
            },
            {
             "value": true,
             "probability": 0.9988560451231133
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9976677418520489,
           "distribution": [
            {
             "value": "confirm booking",
             "probability": 0.0005205605748201198
            },
            {
             "value": "waitlist",
             "probability": 0.9976677418520489
            },
            {
             "value": "reject",
             "probability": 0.0010597998664440703
            },
            {
             "value": "overflow dorm",
             "probability": 0.0007518977066869827
            }
           ]
          }
         },
         "joint_probability": 0.9965931594674251,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0034126570281213372,
        "field_brier": 6.464916493255841e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "clearance": {
           "value": 1,
           "probability": 0.9973770198782218,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007767579715620713
            },
            {
             "value": 1,
             "probability": 0.9973770198782218
            },
            {
             "value": 2,
             "probability": 0.00123206162170203
            },
            {
             "value": 3,
             "probability": 0.0006141605285140914
            }
           ]
          },
          "over_quota": {
           "value": true,
           "probability": 0.9988560451231133,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011439548768866264
            },
            {
             "value": true,
             "probability": 0.9988560451231133
            }
           ]
          },
          "decision": {
           "value": "waitlist",
           "probability": 0.9976677418520489,
           "distribution": [
            {
             "value": "confirm booking",
             "probability": 0.0005205605748201198
            },
            {
             "value": "waitlist",
             "probability": 0.9976677418520489
            },
            {
             "value": "reject",
             "probability": 0.0010597998664440703
            },
            {
             "value": "overflow dorm",
             "probability": 0.0007518977066869827
            }
           ]
          }
         },
         "joint_probability": 0.9965931594674251,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0034126570281213372,
        "field_brier": 6.464916493255841e-06
       }
      },
      "timing_ms": {
       "scoring": 36.4302780944854,
       "base_joint": 0.07643899880349636,
       "constrained_joint": 0.061789993196725845,
       "constrained_total": 36.49206808768213
      },
      "input_tokens": 707,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-0552f72baccc6d63-0034",
      "group": "bendsynth-0552f72baccc6d63",
      "provenance": "bendsynth-v1; family=bendsynth-0552f72baccc6d63; domain=alpine hut overnight booking \u2014 supervisor reviewing an escalated group request; parameters=max_group=8; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "clearance": 0,
       "decision": "reject",
       "over_quota": true
      },
      "logits": [
       [
        -2.962899684906006,
        -2.8584561347961426,
        0.7260323762893677,
        -4.4552435874938965
       ],
       [
        -3.252173900604248,
        -1.667610764503479,
        0.6676393151283264,
        -4.557271480560303
       ],
       [
        -3.6096837520599365,
        0.7703151702880859
       ]
      ],
      "joint_logits": [
       -7.4176836013793945,
       -5.9147725105285645,
       -7.254881858825684,
       -4.650298118591309,
       -6.464221477508545,
       -3.3477067947387695,
       -7.556180953979492,
       -7.061348915100098,
       -6.545577526092529,
       -5.315384387969971,
       -6.629644393920898,
       -4.5649871826171875,
       -5.821878433227539,
       -3.406101942062378,
       -6.531143665313721,
       -6.385914325714111,
       -6.078503131866455,
       -3.7106423377990723,
       -5.716130256652832,
       -1.8260424137115479,
       -4.991178035736084,
       0.6229511499404907,
       -6.891023635864258,
       -5.646008014678955,
       -7.060840606689453,
       -6.552257537841797,
       -7.623447418212891,
       -6.281646251678467,
       -6.9390645027160645,
       -5.384973049163818,
       -7.154966831207275,
       -7.3567705154418945
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "clearance": {
           "value": 2,
           "probability": 0.9448488233716529,
           "distribution": [
            {
             "value": 0,
             "probability": 0.02361997771395768
            },
            {
             "value": 1,
             "probability": 0.02622036352468533
            },
            {
             "value": 2,
             "probability": 0.9448488233716529
            },
            {
             "value": 3,
             "probability": 0.005310835389704143
            }
           ]
          },
          "decision": {
           "value": "overflow dorm",
           "probability": 0.891256115439952,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.017686803504725773
            },
            {
             "value": "reject",
             "probability": 0.08626137377167341
            },
            {
             "value": "overflow dorm",
             "probability": 0.891256115439952
            },
            {
             "value": "confirm booking",
             "probability": 0.004795707283648857
            }
           ]
          },
          "over_quota": {
           "value": true,
           "probability": 0.9876295707691934,
           "distribution": [
            {
             "value": false,
             "probability": 0.012370429230806508
            },
            {
             "value": true,
             "probability": 0.9876295707691934
            }
           ]
          }
         },
         "joint_probability": 0.8523043917876401,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 5.433060817133021,
        "field_brier": 1.158890192206003
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "clearance": {
           "value": 2,
           "probability": 0.9448488233716529,
           "distribution": [
            {
             "value": 0,
             "probability": 0.02361997771395768
            },
            {
             "value": 1,
             "probability": 0.02622036352468533
            },
            {
             "value": 2,
             "probability": 0.9448488233716529
            },
            {
             "value": 3,
             "probability": 0.005310835389704143
            }
           ]
          },
          "decision": {
           "value": "overflow dorm",
           "probability": 0.891256115439952,
           "distribution": [
            {
             "value": "waitlist",
             "probability": 0.017686803504725773
            },
            {
             "value": "reject",
             "probability": 0.08626137377167341
            },
            {
             "value": "overflow dorm",
             "probability": 0.891256115439952
            },
            {
             "value": "confirm booking",
             "probability": 0.004795707283648857
            }
           ]
          },
          "over_quota": {
           "value": true,
           "probability": 0.9876295707691934,
           "distribution": [
            {
             "value": false,
             "probability": 0.012370429230806508
            },
            {
             "value": true,
             "probability": 0.9876295707691934
            }
           ]
          }
         },
         "joint_probability": 0.8523043917876401,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 32,
         "valid_assignments": 32,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 1,
        "joint_nll": 5.433060817133021,
        "field_brier": 1.158890192206003
       }
      },
      "timing_ms": {
       "scoring": 37.79294388368726,
       "base_joint": 0.07560919038951397,
       "constrained_joint": 0.05686911754310131,
       "constrained_total": 37.84981300123036
      },
      "input_tokens": 713,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-06e83966f8a9bbf0-0010",
      "group": "bendsynth-06e83966f8a9bbf0",
      "provenance": "bendsynth-v1; family=bendsynth-06e83966f8a9bbf0; domain=harbour berth assignment \u2014 dock inspector filing a site report; parameters=max_draft=39; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "hazard_count": 3,
       "draft_ok": false,
       "decision": "harbourmaster review"
      },
      "logits": [
       [
        -7.124056339263916,
        -7.075955390930176,
        -7.0382490158081055,
        0.3122423589229584
       ],
       [
        0.31298118829727173,
        -6.489978790283203
       ],
       [
        -6.622879981994629,
        -7.010878562927246,
        0.31245943903923035
       ]
      ],
      "joint_logits": [
       -8.66535758972168,
       -9.025333404541016,
       -9.011316299438477,
       -8.718354225158691,
       -9.116241455078125,
       -9.051048278808594,
       -8.659037590026855,
       -8.99454402923584,
       -8.805642127990723,
       -8.715513229370117,
       -9.116181373596191,
       -8.997096061706543,
       -8.661412239074707,
       -8.978400230407715,
       -8.545357704162598,
       -8.814144134521484,
       -9.16873550415039,
       -8.939489364624023,
       -8.528515815734863,
       -9.108410835266113,
       0.31176313757896423,
       -8.899145126342773,
       -9.243023872375488,
       -9.088441848754883
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "hazard_count": {
           "value": 3,
           "probability": 0.9981531677506713,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005883742536817158
            },
            {
             "value": 1,
             "probability": 0.0006173674724445055
            },
            {
             "value": 2,
             "probability": 0.000641090523202555
            },
            {
             "value": 3,
             "probability": 0.9981531677506713
            }
           ]
          },
          "draft_ok": {
           "value": false,
           "probability": 0.9988907485829027,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988907485829027
            },
            {
             "value": true,
             "probability": 0.001109251417097347
            }
           ]
          },
          "decision": {
           "value": "harbourmaster review",
           "probability": 0.9983699132706889,
           "distribution": [
            {
             "value": "outer basin assignment",
             "probability": 0.0009712070603535149
            },
            {
             "value": "inner berth approved",
             "probability": 0.0006588796689576611
            },
            {
             "value": "harbourmaster review",
             "probability": 0.9983699132706889
            }
           ]
          }
         },
         "joint_probability": 0.9976749164756513,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002327790728177069,
        "field_brier": 3.6815130015052694e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "hazard_count": {
           "value": 3,
           "probability": 0.9981531677506713,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005883742536817158
            },
            {
             "value": 1,
             "probability": 0.0006173674724445055
            },
            {
             "value": 2,
             "probability": 0.000641090523202555
            },
            {
             "value": 3,
             "probability": 0.9981531677506713
            }
           ]
          },
          "draft_ok": {
           "value": false,
           "probability": 0.9988907485829027,
           "distribution": [
            {
             "value": false,
             "probability": 0.9988907485829027
            },
            {
             "value": true,
             "probability": 0.001109251417097347
            }
           ]
          },
          "decision": {
           "value": "harbourmaster review",
           "probability": 0.9983699132706889,
           "distribution": [
            {
             "value": "outer basin assignment",
             "probability": 0.0009712070603535149
            },
            {
             "value": "inner berth approved",
             "probability": 0.0006588796689576611
            },
            {
             "value": "harbourmaster review",
             "probability": 0.9983699132706889
            }
           ]
          }
         },
         "joint_probability": 0.9976749164756513,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002327790728177069,
        "field_brier": 3.6815130015052694e-06
       }
      },
      "timing_ms": {
       "scoring": 36.84566402807832,
       "base_joint": 0.06776885129511356,
       "constrained_joint": 0.047969864681363106,
       "constrained_total": 36.89363389275968
      },
      "input_tokens": 679,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-092d9cb09aa4011a-0009",
      "group": "bendsynth-092d9cb09aa4011a",
      "provenance": "bendsynth-v1; family=bendsynth-092d9cb09aa4011a; domain=airline crew rostering chatbot handling leave-adjacent duty swap requests; parameters=quota=3; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "quota_exempt": false,
       "blockers": 2,
       "decision": "deferred"
      },
      "logits": [
       [
        -6.383176803588867,
        0.0389140285551548
       ],
       [
        -7.359238624572754,
        -7.324774742126465,
        0.038591839373111725,
        -7.223948001861572
       ],
       [
        -7.012845039367676,
        0.03883077949285507,
        -7.040155410766602
       ]
      ],
      "joint_logits": [
       -8.97998046875,
       -8.985918998718262,
       -9.025280952453613,
       -8.997653007507324,
       -8.947502136230469,
       -9.069594383239746,
       -8.984578132629395,
       -8.194154739379883,
       -9.032679557800293,
       -8.79920482635498,
       -8.921954154968262,
       -8.844557762145996,
       -9.3566255569458,
       -9.365019798278809,
       -9.280455589294434,
       -9.397561073303223,
       -9.059361457824707,
       -9.305636405944824,
       -9.1556978225708,
       0.03790578618645668,
       -9.359036445617676,
       -9.215107917785645,
       -9.268709182739258,
       -9.146241188049316
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_exempt": {
           "value": true,
           "probability": 0.9983773823648332,
           "distribution": [
            {
             "value": false,
             "probability": 0.0016226176351667463
            },
            {
             "value": true,
             "probability": 0.9983773823648332
            }
           ]
          },
          "blockers": {
           "value": 2,
           "probability": 0.9980558222414131,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006113892046208621
            },
            {
             "value": 1,
             "probability": 0.0006328276989650754
            },
            {
             "value": 2,
             "probability": 0.9980558222414131
            },
            {
             "value": 3,
             "probability": 0.0006999608550009232
            }
           ]
          },
          "decision": {
           "value": "deferred",
           "probability": 0.9982943308660107,
           "distribution": [
            {
             "value": "waitlisted",
             "probability": 0.0008644794971583665
            },
            {
             "value": "deferred",
             "probability": 0.9982943308660107
            },
            {
             "value": "approved",
             "probability": 0.0008411896368309226
            }
           ]
          }
         },
         "joint_probability": 0.9973713082709669,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.234692678372209,
        "field_brier": 0.664508067835775
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "quota_exempt": {
           "value": true,
           "probability": 0.9983773823648332,
           "distribution": [
            {
             "value": false,
             "probability": 0.0016226176351667463
            },
            {
             "value": true,
             "probability": 0.9983773823648332
            }
           ]
          },
          "blockers": {
           "value": 2,
           "probability": 0.9980558222414131,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006113892046208621
            },
            {
             "value": 1,
             "probability": 0.0006328276989650754
            },
            {
             "value": 2,
             "probability": 0.9980558222414131
            },
            {
             "value": 3,
             "probability": 0.0006999608550009232
            }
           ]
          },
          "decision": {
           "value": "deferred",
           "probability": 0.9982943308660107,
           "distribution": [
            {
             "value": "waitlisted",
             "probability": 0.0008644794971583665
            },
            {
             "value": "deferred",
             "probability": 0.9982943308660107
            },
            {
             "value": "approved",
             "probability": 0.0008411896368309226
            }
           ]
          }
         },
         "joint_probability": 0.9973713082709669,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 8.234692678372209,
        "field_brier": 0.664508067835775
       }
      },
      "timing_ms": {
       "scoring": 38.76608284190297,
       "base_joint": 0.06498908624053001,
       "constrained_joint": 0.04784995689988136,
       "constrained_total": 38.81393279880285
      },
      "input_tokens": 695,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-063673af06a05b83-0006",
      "group": "bendsynth-063673af06a05b83",
      "provenance": "bendsynth-v1; family=bendsynth-063673af06a05b83; domain=museum photography permit compliance audit; parameters=audit_window=89; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "no action",
       "within_audit_window": false,
       "violations": 2
      },
      "logits": [
       [
        -7.077497959136963,
        -0.3476550281047821,
        -7.134983539581299
       ],
       [
        -0.34675106406211853,
        -6.9059624671936035
       ],
       [
        -7.581429481506348,
        -0.34932559728622437,
        -6.349119186401367,
        -7.484968662261963
       ]
      ],
      "joint_logits": [
       -9.385648727416992,
       -8.409396171569824,
       -9.137528419494629,
       -9.307523727416992,
       -9.340787887573242,
       -9.404363632202148,
       -9.544629096984863,
       -9.238872528076172,
       -9.238142967224121,
       -0.3502260446548462,
       -6.631855487823486,
       -9.227575302124023,
       -9.682541847229004,
       -9.169543266296387,
       -9.646710395812988,
       -9.578055381774902,
       -9.254570007324219,
       -8.9414644241333,
       -9.227090835571289,
       -9.120636940002441,
       -9.398508071899414,
       -9.266255378723145,
       -9.333006858825684,
       -9.24465274810791
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "no action",
           "probability": 0.9976826836344547,
           "distribution": [
            {
             "value": "escalated",
             "probability": 0.0011919520111908363
            },
            {
             "value": "no action",
             "probability": 0.9976826836344547
            },
            {
             "value": "advisory note",
             "probability": 0.001125364354354483
            }
           ]
          },
          "within_audit_window": {
           "value": false,
           "probability": 0.9985850023119155,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985850023119155
            },
            {
             "value": true,
             "probability": 0.0014149976880845237
            }
           ]
          },
          "violations": {
           "value": 1,
           "probability": 0.9960174492965748,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007201187376277121
            },
            {
             "value": 1,
             "probability": 0.9960174492965748
            },
            {
             "value": 2,
             "probability": 0.0024693894033367405
            },
            {
             "value": 3,
             "probability": 0.0007930425624607996
            }
           ]
          }
         },
         "joint_probability": 0.9951209540146383,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 6.286520430556467,
        "field_brier": 0.6623770958182295
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "no action",
           "probability": 0.9976826836344547,
           "distribution": [
            {
             "value": "escalated",
             "probability": 0.0011919520111908363
            },
            {
             "value": "no action",
             "probability": 0.9976826836344547
            },
            {
             "value": "advisory note",
             "probability": 0.001125364354354483
            }
           ]
          },
          "within_audit_window": {
           "value": false,
           "probability": 0.9985850023119155,
           "distribution": [
            {
             "value": false,
             "probability": 0.9985850023119155
            },
            {
             "value": true,
             "probability": 0.0014149976880845237
            }
           ]
          },
          "violations": {
           "value": 1,
           "probability": 0.9960174492965748,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007201187376277121
            },
            {
             "value": 1,
             "probability": 0.9960174492965748
            },
            {
             "value": 2,
             "probability": 0.0024693894033367405
            },
            {
             "value": 3,
             "probability": 0.0007930425624607996
            }
           ]
          }
         },
         "joint_probability": 0.9951209540146383,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": false,
        "correct_fields": 2,
        "joint_nll": 6.286520430556467,
        "field_brier": 0.6623770958182295
       }
      },
      "timing_ms": {
       "scoring": 34.23852217383683,
       "base_joint": 0.06576022133231163,
       "constrained_joint": 0.04711886867880821,
       "constrained_total": 34.285641042515635
      },
      "input_tokens": 653,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-063673af06a05b83-0033",
      "group": "bendsynth-063673af06a05b83",
      "provenance": "bendsynth-v1; family=bendsynth-063673af06a05b83; domain=museum photography permit compliance audit; parameters=audit_window=30; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "violations": 1,
       "within_audit_window": true,
       "decision": "advisory note"
      },
      "logits": [
       [
        -6.0846099853515625,
        1.0466740131378174,
        -6.171328067779541,
        -6.117942810058594
       ],
       [
        -5.238880157470703,
        1.0471175909042358
       ],
       [
        -5.797812461853027,
        1.0468116998672485,
        -5.7625579833984375
       ]
      ],
      "joint_logits": [
       -7.759798526763916,
       -7.676699638366699,
       -7.771890640258789,
       -8.138237953186035,
       -7.824805736541748,
       -8.201128005981445,
       -7.850710391998291,
       -7.298142433166504,
       -7.790945529937744,
       -7.455137252807617,
       1.0456970930099487,
       -7.231806755065918,
       -7.850100040435791,
       -7.738954067230225,
       -7.857397556304932,
       -8.258310317993164,
       -7.894835948944092,
       -8.315958023071289,
       -7.767210006713867,
       -7.678776264190674,
       -7.782538890838623,
       -8.197461128234863,
       -7.92239236831665,
       -8.246356964111328
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "violations": {
           "value": 1,
           "probability": 0.9976988743218907,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007978514974839774
            },
            {
             "value": 1,
             "probability": 0.9976988743218907
            },
            {
             "value": 2,
             "probability": 0.0007315787520893428
            },
            {
             "value": 3,
             "probability": 0.0007716954285359469
            }
           ]
          },
          "within_audit_window": {
           "value": true,
           "probability": 0.9981412625146414,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018587374853585685
            },
            {
             "value": true,
             "probability": 0.9981412625146414
            }
           ]
          },
          "decision": {
           "value": "advisory note",
           "probability": 0.9978361378486901,
           "distribution": [
            {
             "value": "escalated",
             "probability": 0.0010628618405096428
            },
            {
             "value": "advisory note",
             "probability": 0.9978361378486901
            },
            {
             "value": "no action",
             "probability": 0.0011010003108003455
            }
           ]
          }
         },
         "joint_probability": 0.9967245596805638,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0032808163164607385,
        "field_brier": 6.998818056214312e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "violations": {
           "value": 1,
           "probability": 0.9976988743218907,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0007978514974839774
            },
            {
             "value": 1,
             "probability": 0.9976988743218907
            },
            {
             "value": 2,
             "probability": 0.0007315787520893428
            },
            {
             "value": 3,
             "probability": 0.0007716954285359469
            }
           ]
          },
          "within_audit_window": {
           "value": true,
           "probability": 0.9981412625146414,
           "distribution": [
            {
             "value": false,
             "probability": 0.0018587374853585685
            },
            {
             "value": true,
             "probability": 0.9981412625146414
            }
           ]
          },
          "decision": {
           "value": "advisory note",
           "probability": 0.9978361378486901,
           "distribution": [
            {
             "value": "escalated",
             "probability": 0.0010628618405096428
            },
            {
             "value": "advisory note",
             "probability": 0.9978361378486901
            },
            {
             "value": "no action",
             "probability": 0.0011010003108003455
            }
           ]
          }
         },
         "joint_probability": 0.9967245596805638,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0032808163164607385,
        "field_brier": 6.998818056214312e-06
       }
      },
      "timing_ms": {
       "scoring": 34.7980260848999,
       "base_joint": 0.06365915760397911,
       "constrained_joint": 0.046819914132356644,
       "constrained_total": 34.84484599903226
      },
      "input_tokens": 641,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-063673af06a05b83-0034",
      "group": "bendsynth-063673af06a05b83",
      "provenance": "bendsynth-v1; family=bendsynth-063673af06a05b83; domain=museum photography permit compliance audit; parameters=audit_window=90; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "within_audit_window": true,
       "violations": 1,
       "decision": "advisory note"
      },
      "logits": [
       [
        -5.026594161987305,
        1.519203782081604
       ],
       [
        -5.525599479675293,
        1.5183600187301636,
        -5.755769729614258,
        -5.724077224731445
       ],
       [
        1.5184605121612549,
        -5.3872785568237305,
        -5.224947452545166
       ]
      ],
      "joint_logits": [
       -7.30706262588501,
       -7.388566970825195,
       -7.457925796508789,
       -7.354394435882568,
       -7.686041355133057,
       -7.704631328582764,
       -7.526216506958008,
       -7.562130451202393,
       -7.657732963562012,
       -7.4274187088012695,
       -7.518575191497803,
       -7.648080825805664,
       -7.094985008239746,
       -7.364753723144531,
       -7.330679893493652,
       1.517481803894043,
       -7.239716053009033,
       -6.354034423828125,
       -7.402584552764893,
       -7.5613298416137695,
       -7.5935378074646,
       -7.446946144104004,
       -7.481558799743652,
       -7.590590476989746
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_audit_window": {
           "value": true,
           "probability": 0.9985659213839124,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014340786160876263
            },
            {
             "value": true,
             "probability": 0.9985659213839124
            }
           ]
          },
          "violations": {
           "value": 1,
           "probability": 0.9977237199015098,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008706779665932459
            },
            {
             "value": 1,
             "probability": 0.9977237199015098
            },
            {
             "value": 2,
             "probability": 0.0006916653125038252
            },
            {
             "value": 3,
             "probability": 0.0007139368193931793
            }
           ]
          },
          "decision": {
           "value": "advisory note",
           "probability": 0.9978241034080885,
           "distribution": [
            {
             "value": "advisory note",
             "probability": 0.9978241034080885
            },
            {
             "value": "escalated",
             "probability": 0.000999837634295123
            },
            {
             "value": "no action",
             "probability": 0.0011760589576163723
            }
           ]
          }
         },
         "joint_probability": 0.9968479160294627,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0031570622512745216,
        "field_brier": 6.052705598191391e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "within_audit_window": {
           "value": true,
           "probability": 0.9985659213839124,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014340786160876263
            },
            {
             "value": true,
             "probability": 0.9985659213839124
            }
           ]
          },
          "violations": {
           "value": 1,
           "probability": 0.9977237199015098,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0008706779665932459
            },
            {
             "value": 1,
             "probability": 0.9977237199015098
            },
            {
             "value": 2,
             "probability": 0.0006916653125038252
            },
            {
             "value": 3,
             "probability": 0.0007139368193931793
            }
           ]
          },
          "decision": {
           "value": "advisory note",
           "probability": 0.9978241034080885,
           "distribution": [
            {
             "value": "advisory note",
             "probability": 0.9978241034080885
            },
            {
             "value": "escalated",
             "probability": 0.000999837634295123
            },
            {
             "value": "no action",
             "probability": 0.0011760589576163723
            }
           ]
          }
         },
         "joint_probability": 0.9968479160294627,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0031570622512745216,
        "field_brier": 6.052705598191391e-06
       }
      },
      "timing_ms": {
       "scoring": 35.53180815652013,
       "base_joint": 0.06603891961276531,
       "constrained_joint": 0.048318877816200256,
       "constrained_total": 35.58012703433633
      },
      "input_tokens": 629,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09f6b31df05e378f-0012",
      "group": "bendsynth-09f6b31df05e378f",
      "provenance": "bendsynth-v1; family=bendsynth-09f6b31df05e378f; domain=ambulance station shift cover escalation; parameters=escalation_threshold=2; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "critical_combo": false,
       "risk_factors": 2,
       "decision": "supervisor review"
      },
      "logits": [
       [
        0.5244815349578857,
        -6.137930870056152
       ],
       [
        -6.953191757202148,
        -6.877807140350342,
        0.5239602327346802,
        -6.8550028800964355
       ],
       [
        -6.467138767242432,
        -6.542689323425293,
        0.5239875316619873
       ]
      ],
      "joint_logits": [
       -8.633255004882812,
       -8.762121200561523,
       -8.993911743164062,
       -8.63161563873291,
       -8.711604118347168,
       -8.639493942260742,
       -8.518139839172363,
       -8.426280975341797,
       0.5233532786369324,
       -8.510038375854492,
       -8.688241958618164,
       -8.889772415161133,
       -8.607861518859863,
       -8.675743103027344,
       -8.849591255187988,
       -8.564817428588867,
       -8.672351837158203,
       -8.81539535522461,
       -8.422033309936523,
       -8.521454811096191,
       -8.585899353027344,
       -8.504136085510254,
       -8.56470012664795,
       -8.784835815429688
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "critical_combo": {
           "value": false,
           "probability": 0.998723571965615,
           "distribution": [
            {
             "value": false,
             "probability": 0.998723571965615
            },
            {
             "value": true,
             "probability": 0.001276428034385048
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9982029472650694,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005648497590133759
            },
            {
             "value": 1,
             "probability": 0.0006090768745239993
            },
            {
             "value": 2,
             "probability": 0.9982029472650694
            },
            {
             "value": 3,
             "probability": 0.0006231261013932827
            }
           ]
          },
          "decision": {
           "value": "supervisor review",
           "probability": 0.9982300648659905,
           "distribution": [
            {
             "value": "regional escalation",
             "probability": 0.0009183816890898326
            },
            {
             "value": "auto-fill roster",
             "probability": 0.0008515534449196286
            },
            {
             "value": "supervisor review",
             "probability": 0.9982300648659905
            }
           ]
          }
         },
         "joint_probability": 0.9975972510080698,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00240564022549014,
        "field_brier": 4.08916339621186e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "critical_combo": {
           "value": false,
           "probability": 0.998723571965615,
           "distribution": [
            {
             "value": false,
             "probability": 0.998723571965615
            },
            {
             "value": true,
             "probability": 0.001276428034385048
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9982029472650694,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005648497590133759
            },
            {
             "value": 1,
             "probability": 0.0006090768745239993
            },
            {
             "value": 2,
             "probability": 0.9982029472650694
            },
            {
             "value": 3,
             "probability": 0.0006231261013932827
            }
           ]
          },
          "decision": {
           "value": "supervisor review",
           "probability": 0.9982300648659905,
           "distribution": [
            {
             "value": "regional escalation",
             "probability": 0.0009183816890898326
            },
            {
             "value": "auto-fill roster",
             "probability": 0.0008515534449196286
            },
            {
             "value": "supervisor review",
             "probability": 0.9982300648659905
            }
           ]
          }
         },
         "joint_probability": 0.9975972510080698,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.00240564022549014,
        "field_brier": 4.08916339621186e-06
       }
      },
      "timing_ms": {
       "scoring": 38.05447090417147,
       "base_joint": 0.0656589400023222,
       "constrained_joint": 0.04742899909615517,
       "constrained_total": 38.10189990326762
      },
      "input_tokens": 733,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09f6b31df05e378f-0021",
      "group": "bendsynth-09f6b31df05e378f",
      "provenance": "bendsynth-v1; family=bendsynth-09f6b31df05e378f; domain=ambulance station shift cover escalation; parameters=escalation_threshold=3; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "critical_combo": true,
       "decision": "regional escalation",
       "risk_factors": 2
      },
      "logits": [
       [
        -6.778741836547852,
        -0.14963217079639435
       ],
       [
        -7.333619594573975,
        -0.14994847774505615,
        -7.18635368347168
       ],
       [
        -7.474043369293213,
        -7.5129265785217285,
        -0.15025906264781952,
        -7.481096267700195
       ]
      ],
      "joint_logits": [
       -9.325531959533691,
       -9.391278266906738,
       -9.441160202026367,
       -9.291852951049805,
       -9.104609489440918,
       -9.19975757598877,
       -9.190975189208984,
       -9.171232223510742,
       -9.242779731750488,
       -9.285173416137695,
       -9.343217849731445,
       -9.22787857055664,
       -9.422361373901367,
       -9.519795417785645,
       -9.484907150268555,
       -9.448512077331543,
       -9.321874618530273,
       -9.213272094726562,
       -0.1507650911808014,
       -9.258845329284668,
       -9.208264350891113,
       -9.255478858947754,
       -9.310300827026367,
       -9.260104179382324
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "critical_combo": {
           "value": true,
           "probability": 0.9986804049708681,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013195950291320238
            },
            {
             "value": true,
             "probability": 0.9986804049708681
            }
           ]
          },
          "decision": {
           "value": "regional escalation",
           "probability": 0.9983645207067172,
           "distribution": [
            {
             "value": "auto-fill roster",
             "probability": 0.0007576356555287594
            },
            {
             "value": "regional escalation",
             "probability": 0.9983645207067172
            },
            {
             "value": "supervisor review",
             "probability": 0.0008778436377539674
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9980546021257136,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006583778548633483
            },
            {
             "value": 1,
             "probability": 0.0006332692546735655
            },
            {
             "value": 2,
             "probability": 0.9980546021257136
            },
            {
             "value": 3,
             "probability": 0.0006537507647494731
            }
           ]
          }
         },
         "joint_probability": 0.9975496179674357,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024533891319830766,
        "field_brier": 4.1828433799425875e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "critical_combo": {
           "value": true,
           "probability": 0.9986804049708681,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013195950291320238
            },
            {
             "value": true,
             "probability": 0.9986804049708681
            }
           ]
          },
          "decision": {
           "value": "regional escalation",
           "probability": 0.9983645207067172,
           "distribution": [
            {
             "value": "auto-fill roster",
             "probability": 0.0007576356555287594
            },
            {
             "value": "regional escalation",
             "probability": 0.9983645207067172
            },
            {
             "value": "supervisor review",
             "probability": 0.0008778436377539674
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9980546021257136,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006583778548633483
            },
            {
             "value": 1,
             "probability": 0.0006332692546735655
            },
            {
             "value": 2,
             "probability": 0.9980546021257136
            },
            {
             "value": 3,
             "probability": 0.0006537507647494731
            }
           ]
          }
         },
         "joint_probability": 0.9975496179674357,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0024533891319830766,
        "field_brier": 4.1828433799425875e-06
       }
      },
      "timing_ms": {
       "scoring": 38.17258006893098,
       "base_joint": 0.06503891199827194,
       "constrained_joint": 0.04623900167644024,
       "constrained_total": 38.218819070607424
      },
      "input_tokens": 733,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-09f6b31df05e378f-0026",
      "group": "bendsynth-09f6b31df05e378f",
      "provenance": "bendsynth-v1; family=bendsynth-09f6b31df05e378f; domain=ambulance station shift cover escalation; parameters=escalation_threshold=5; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "risk_factors": 3,
       "decision": "regional escalation",
       "critical_combo": true
      },
      "logits": [
       [
        -7.896527290344238,
        -7.804825305938721,
        -7.8326616287231445,
        -0.38395434617996216
       ],
       [
        -0.3836592435836792,
        -7.539794921875,
        -7.72002649307251
       ],
       [
        -7.147673606872559,
        -0.3833825886249542
       ]
      ],
      "joint_logits": [
       -9.558646202087402,
       -9.680438041687012,
       -9.610247611999512,
       -9.677515983581543,
       -9.793194770812988,
       -9.837596893310547,
       -9.555920600891113,
       -9.481977462768555,
       -9.540022850036621,
       -9.54840087890625,
       -9.727226257324219,
       -9.756778717041016,
       -9.475855827331543,
       -9.311279296875,
       -9.67330551147461,
       -9.710009574890137,
       -9.823163986206055,
       -9.870203018188477,
       -9.434313774108887,
       -0.38443320989608765,
       -9.684060096740723,
       -9.529468536376953,
       -9.810078620910645,
       -9.784723281860352
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 3,
           "probability": 0.998275981611836,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005452325907107448
            },
            {
             "value": 1,
             "probability": 0.0005975957025596387
            },
            {
             "value": 2,
             "probability": 0.0005811900948936858
            },
            {
             "value": 3,
             "probability": 0.998275981611836
            }
           ]
          },
          "decision": {
           "value": "regional escalation",
           "probability": 0.9985705700927675,
           "distribution": [
            {
             "value": "regional escalation",
             "probability": 0.9985705700927675
            },
            {
             "value": "supervisor review",
             "probability": 0.0007789482255509317
            },
            {
             "value": "auto-fill roster",
             "probability": 0.0006504816816815224
            }
           ]
          },
          "critical_combo": {
           "value": true,
           "probability": 0.9988470653009762,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011529346990237334
            },
            {
             "value": true,
             "probability": 0.9988470653009762
            }
           ]
          }
         },
         "joint_probability": 0.9977980947089351,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002204333048970309,
        "field_brier": 3.232031329076686e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "risk_factors": {
           "value": 3,
           "probability": 0.998275981611836,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005452325907107448
            },
            {
             "value": 1,
             "probability": 0.0005975957025596387
            },
            {
             "value": 2,
             "probability": 0.0005811900948936858
            },
            {
             "value": 3,
             "probability": 0.998275981611836
            }
           ]
          },
          "decision": {
           "value": "regional escalation",
           "probability": 0.9985705700927675,
           "distribution": [
            {
             "value": "regional escalation",
             "probability": 0.9985705700927675
            },
            {
             "value": "supervisor review",
             "probability": 0.0007789482255509317
            },
            {
             "value": "auto-fill roster",
             "probability": 0.0006504816816815224
            }
           ]
          },
          "critical_combo": {
           "value": true,
           "probability": 0.9988470653009762,
           "distribution": [
            {
             "value": false,
             "probability": 0.0011529346990237334
            },
            {
             "value": true,
             "probability": 0.9988470653009762
            }
           ]
          }
         },
         "joint_probability": 0.9977980947089351,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.002204333048970309,
        "field_brier": 3.232031329076686e-06
       }
      },
      "timing_ms": {
       "scoring": 37.97617205418646,
       "base_joint": 0.07211905904114246,
       "constrained_joint": 0.04747905768454075,
       "constrained_total": 38.023651111871004
      },
      "input_tokens": 733,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03dc5c45493cf2a9-0002",
      "group": "bendsynth-03dc5c45493cf2a9",
      "provenance": "bendsynth-v1; family=bendsynth-03dc5c45493cf2a9; domain=concert ticket resale escalation review at a box office; parameters=cap_standard=29,cap_special=80; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "over_cap": true,
       "risk_factors": 2,
       "decision": "warn seller"
      },
      "logits": [
       [
        -5.754086494445801,
        0.7694123387336731
       ],
       [
        -6.886595726013184,
        -6.398382186889648,
        0.7687122821807861,
        -6.217228412628174
       ],
       [
        -6.151475429534912,
        -6.237046241760254,
        0.7689878344535828
       ]
      ],
      "joint_logits": [
       -8.57436752319336,
       -8.642925262451172,
       -8.693540573120117,
       -8.120262145996094,
       -8.236589431762695,
       -8.222274780273438,
       -8.347960472106934,
       -8.462517738342285,
       -8.054593086242676,
       -7.902954578399658,
       -7.987321853637695,
       -8.013723373413086,
       -8.608757972717285,
       -8.7139892578125,
       -8.861858367919922,
       -8.208366394042969,
       -8.287067413330078,
       -8.08122730255127,
       -8.344839096069336,
       -8.336431503295898,
       0.7681358456611633,
       -7.972008228302002,
       -8.085945129394531,
       -8.106201171875
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cap": {
           "value": true,
           "probability": 0.9985336307171487,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014663692828512766
            },
            {
             "value": true,
             "probability": 0.9985336307171487
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9978348196456232,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004724987166944068
            },
            {
             "value": 1,
             "probability": 0.0007698909095847923
            },
            {
             "value": 2,
             "probability": 0.9978348196456232
            },
            {
             "value": 3,
             "probability": 0.0009227907280976095
            }
           ]
          },
          "decision": {
           "value": "warn seller",
           "probability": 0.9981098111579692,
           "distribution": [
            {
             "value": "release ticket",
             "probability": 0.0009855061453157806
            },
            {
             "value": "reject listing",
             "probability": 0.0009046826967150582
            },
            {
             "value": "warn seller",
             "probability": 0.9981098111579692
            }
           ]
          }
         },
         "joint_probability": 0.9972597987090358,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027439625151004367,
        "field_brier": 5.339500164963253e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cap": {
           "value": true,
           "probability": 0.9985336307171487,
           "distribution": [
            {
             "value": false,
             "probability": 0.0014663692828512766
            },
            {
             "value": true,
             "probability": 0.9985336307171487
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9978348196456232,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0004724987166944068
            },
            {
             "value": 1,
             "probability": 0.0007698909095847923
            },
            {
             "value": 2,
             "probability": 0.9978348196456232
            },
            {
             "value": 3,
             "probability": 0.0009227907280976095
            }
           ]
          },
          "decision": {
           "value": "warn seller",
           "probability": 0.9981098111579692,
           "distribution": [
            {
             "value": "release ticket",
             "probability": 0.0009855061453157806
            },
            {
             "value": "reject listing",
             "probability": 0.0009046826967150582
            },
            {
             "value": "warn seller",
             "probability": 0.9981098111579692
            }
           ]
          }
         },
         "joint_probability": 0.9972597987090358,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0027439625151004367,
        "field_brier": 5.339500164963253e-06
       }
      },
      "timing_ms": {
       "scoring": 37.11895109154284,
       "base_joint": 0.06643892265856266,
       "constrained_joint": 0.0552500132471323,
       "constrained_total": 37.17420110478997
      },
      "input_tokens": 752,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03dc5c45493cf2a9-0032",
      "group": "bendsynth-03dc5c45493cf2a9",
      "provenance": "bendsynth-v1; family=bendsynth-03dc5c45493cf2a9; domain=concert ticket resale escalation review at a box office; parameters=cap_standard=12,cap_special=50; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "decision": "reject listing",
       "over_cap": true,
       "risk_factors": 2
      },
      "logits": [
       [
        -5.778471946716309,
        -5.485452651977539,
        0.884763240814209
       ],
       [
        -5.439758777618408,
        0.8859595060348511
       ],
       [
        -6.5206403732299805,
        -5.2638092041015625,
        0.8843262791633606,
        -6.406883716583252
       ]
      ],
      "joint_logits": [
       -8.257824897766113,
       -7.308557510375977,
       -8.384760856628418,
       -8.130620956420898,
       -8.292430877685547,
       -7.2554030418396,
       -7.79408073425293,
       -8.206743240356445,
       -8.015106201171875,
       -7.0573930740356445,
       -8.111759185791016,
       -7.886711597442627,
       -8.050020217895508,
       -6.9147233963012695,
       -7.403193473815918,
       -7.934822082519531,
       -8.948549270629883,
       -7.702796936035156,
       -8.135354995727539,
       -8.823780059814453,
       -8.602508544921875,
       -6.506443023681641,
       0.8835649490356445,
       -8.507164001464844
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "reject listing",
           "probability": 0.9970201086293768,
           "distribution": [
            {
             "value": "warn seller",
             "probability": 0.0012732027473514147
            },
            {
             "value": "release ticket",
             "probability": 0.001706688623271706
            },
            {
             "value": "reject listing",
             "probability": 0.9970201086293768
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9982135167158297,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017864832841702565
            },
            {
             "value": true,
             "probability": 0.9982135167158297
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9965845149671384,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006061470958173946
            },
            {
             "value": 1,
             "probability": 0.0021301627721293515
            },
            {
             "value": 2,
             "probability": 0.9965845149671384
            },
            {
             "value": 3,
             "probability": 0.0006791751649148687
            }
           ]
          }
         },
         "joint_probability": 0.9958260614321317,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004182673764709229,
        "field_brier": 1.2276151191498222e-05
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "decision": {
           "value": "reject listing",
           "probability": 0.9970201086293768,
           "distribution": [
            {
             "value": "warn seller",
             "probability": 0.0012732027473514147
            },
            {
             "value": "release ticket",
             "probability": 0.001706688623271706
            },
            {
             "value": "reject listing",
             "probability": 0.9970201086293768
            }
           ]
          },
          "over_cap": {
           "value": true,
           "probability": 0.9982135167158297,
           "distribution": [
            {
             "value": false,
             "probability": 0.0017864832841702565
            },
            {
             "value": true,
             "probability": 0.9982135167158297
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9965845149671384,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0006061470958173946
            },
            {
             "value": 1,
             "probability": 0.0021301627721293515
            },
            {
             "value": 2,
             "probability": 0.9965845149671384
            },
            {
             "value": 3,
             "probability": 0.0006791751649148687
            }
           ]
          }
         },
         "joint_probability": 0.9958260614321317,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.004182673764709229,
        "field_brier": 1.2276151191498222e-05
       }
      },
      "timing_ms": {
       "scoring": 38.11011090874672,
       "base_joint": 0.06531993858516216,
       "constrained_joint": 0.047359149903059006,
       "constrained_total": 38.15747005864978
      },
      "input_tokens": 812,
      "forward_passes": 1
     },
     {
      "id": "bendsynth-03dc5c45493cf2a9-0036",
      "group": "bendsynth-03dc5c45493cf2a9",
      "provenance": "bendsynth-v1; family=bendsynth-03dc5c45493cf2a9; domain=concert ticket resale escalation review at a box office; parameters=cap_standard=19,cap_special=49; labels from a checked Bend oracle; no teacher labels",
      "expected": {
       "over_cap": true,
       "risk_factors": 2,
       "decision": "reject listing"
      },
      "logits": [
       [
        -5.859313011169434,
        0.7490021586418152
       ],
       [
        -6.8282470703125,
        -6.430828094482422,
        0.7483994364738464,
        -6.547649383544922
       ],
       [
        -6.279453754425049,
        0.7482649087905884,
        -5.976719379425049
       ]
      ],
      "joint_logits": [
       -8.608134269714355,
       -9.077841758728027,
       -8.298768997192383,
       -8.270784378051758,
       -8.628389358520508,
       -7.977074146270752,
       -8.289490699768066,
       -8.368351936340332,
       -8.038363456726074,
       -8.336133003234863,
       -8.790651321411133,
       -8.01159381866455,
       -8.624557495117188,
       -9.053403854370117,
       -8.33271598815918,
       -8.272382736206055,
       -8.319035530090332,
       -8.00517749786377,
       -8.217474937438965,
       0.7476978898048401,
       -7.8081440925598145,
       -8.33456039428711,
       -8.740429878234863,
       -8.079005241394043
      ],
      "comparisons": {
       "base": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cap": {
           "value": true,
           "probability": 0.9986527142219062,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013472857780939334
            },
            {
             "value": true,
             "probability": 0.9986527142219062
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9980510597395115,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005112779644007512
            },
            {
             "value": 1,
             "probability": 0.0007607709277059349
            },
            {
             "value": 2,
             "probability": 0.9980510597395115
            },
            {
             "value": 3,
             "probability": 0.0006768913683817984
            }
           ]
          },
          "decision": {
           "value": "reject listing",
           "probability": 0.9979168558729878,
           "distribution": [
            {
             "value": "warn seller",
             "probability": 0.0008851053242596508
            },
            {
             "value": "reject listing",
             "probability": 0.9979168558729878
            },
            {
             "value": "release ticket",
             "probability": 0.0011980388027525368
            }
           ]
          }
         },
         "joint_probability": 0.997351076262923,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026524383435477714,
        "field_brier": 5.095094474138027e-06
       },
       "constrained": {
        "result": {
         "status": "ok",
         "decisions": {
          "over_cap": {
           "value": true,
           "probability": 0.9986527142219062,
           "distribution": [
            {
             "value": false,
             "probability": 0.0013472857780939334
            },
            {
             "value": true,
             "probability": 0.9986527142219062
            }
           ]
          },
          "risk_factors": {
           "value": 2,
           "probability": 0.9980510597395115,
           "distribution": [
            {
             "value": 0,
             "probability": 0.0005112779644007512
            },
            {
             "value": 1,
             "probability": 0.0007607709277059349
            },
            {
             "value": 2,
             "probability": 0.9980510597395115
            },
            {
             "value": 3,
             "probability": 0.0006768913683817984
            }
           ]
          },
          "decision": {
           "value": "reject listing",
           "probability": 0.9979168558729878,
           "distribution": [
            {
             "value": "warn seller",
             "probability": 0.0008851053242596508
            },
            {
             "value": "reject listing",
             "probability": 0.9979168558729878
            },
            {
             "value": "release ticket",
             "probability": 0.0011980388027525368
            }
           ]
          }
         },
         "joint_probability": 0.997351076262923,
         "valid_mass": 1.0,
         "log_valid_mass": 0.0,
         "total_assignments": 24,
         "valid_assignments": 24,
         "calibrated": false
        },
        "joint_correct": true,
        "correct_fields": 3,
        "joint_nll": 0.0026524383435477714,
        "field_brier": 5.095094474138027e-06
       }
      },
      "timing_ms": {
       "scoring": 37.01363201253116,
       "base_joint": 0.06483914330601692,
       "constrained_joint": 0.0473198015242815,
       "constrained_total": 37.06095181405544
      },
      "input_tokens": 752,
      "forward_passes": 1
     }
    ]
   }
  }
 },
 "order_stability": {
  "directory": "reports/order_joint-marker-stage-k-t-v1_cached",
  "low_evidence_variants": [
   "absent",
   "conflict",
   "quorum_shortfall",
   "repeated_source"
  ],
  "all": {
   "base_cases": 96,
   "action_accuracy": 0.8090277777777778,
   "order_winner_flip_rate": 0.03125,
   "order_total_variation_mean": 0.028297298617938556,
   "order_noul_shift_mean": 0.000702334253927374,
   "first_position_win_rate": 0.3368055555555556,
   "repeat_cases": 48,
   "repeat_winner_flip_rate": 0.0,
   "repeat_total_variation_mean": 0.0
  },
  "low_evidence": {
   "base_cases": 32,
   "action_accuracy": 0.4791666666666667,
   "order_winner_flip_rate": 0.03125,
   "order_total_variation_mean": 0.017094352584744562,
   "order_noul_shift_mean": 0.0016074531863695465,
   "first_position_win_rate": 0.34375,
   "repeat_cases": 16,
   "repeat_winner_flip_rate": 0.0,
   "repeat_total_variation_mean": 0.0
  },
  "resolved_evidence": {
   "base_cases": 64,
   "action_accuracy": 0.9739583333333334,
   "order_winner_flip_rate": 0.03125,
   "order_total_variation_mean": 0.03389877163453556,
   "order_noul_shift_mean": 0.00024977478770628765,
   "first_position_win_rate": 0.3333333333333333,
   "repeat_cases": 32,
   "repeat_winner_flip_rate": 0.0,
   "repeat_total_variation_mean": 0.0
  }
 },
 "cached_panel": {
  "complete_requests_correct": 992,
  "latency_ms_p50": 47.318708966486156,
  "latency_ms_p95": 80.32756601460278
 }
}