{
 "scope": "the reference API (v1.13.0) and joint-marker-stage-e-t-v1 on identical requests. Constraints are not sent to either model. cached = 1,000 development requests; blind = 500 test-split requests collected 2026-09-20; public = 99 official validation rows. One seed.",
 "cached": {
  "reference": "data/generated/reference_pilot_1000",
  "candidate": "reports/joint_marker_stage_e_t_v1_cached",
  "requests_sha256": "3abab7b41e3176e41dcceb0a7c6b01856b86d53157c4e3c3aac02035278f82a6",
  "requests": 1000,
  "fields": 2600,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 2600,
   "only_reference_correct": 9,
   "only_candidate_correct": 195,
   "both_wrong": 0,
   "mcnemar_exact_p": 1.1491893327778618e-46,
   "reference": {
    "accuracy": 0.925,
    "nll": 0.21902170358821046,
    "brier": 0.1191332119249998
   },
   "candidate": {
    "accuracy": 0.9965384615384615,
    "nll": 0.02358574458481544,
    "brier": 0.005493046041486454
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.059052059052059055,
     0.08451242829827915
    ],
    "nll": [
     -0.22236110357156633,
     -0.16982229596165213
    ],
    "brier": [
     -0.1303554096135234,
     -0.09770000001008583
    ]
   }
  },
  "groups": {
   "data/generated/judgment_v1/cases.jsonl:action": {
    "fields": 504,
    "only_reference_correct": 2,
    "only_candidate_correct": 102,
    "both_wrong": 0,
    "mcnemar_exact_p": 5.384961754264932e-28,
    "reference": {
     "accuracy": 0.7976190476190477,
     "nll": 0.5249258893736863,
     "brier": 0.30932152116807
    },
    "candidate": {
     "accuracy": 0.996031746031746,
     "nll": 0.0210578370805455,
     "brier": 0.0044894444638420095
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.16468253968253968,
      0.23412698412698413
     ],
     "nll": [
      -0.5948706414033379,
      -0.4174308019963265
     ],
     "brier": [
      -0.3554472214978163,
      -0.25564459648438076
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:evidence_quality": {
    "fields": 504,
    "only_reference_correct": 0,
    "only_candidate_correct": 27,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.4901161193847656e-08,
    "reference": {
     "accuracy": 0.9464285714285714,
     "nll": 0.14050475740192622,
     "brier": 0.07616837668815783
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.014660169300109155,
     "brier": 0.0003345715595372194
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.03373015873015873,
      0.07539682539682539
     ],
     "nll": [
      -0.16438874116947405,
      -0.09303071352356315
     ],
     "brier": [
      -0.10217195681350404,
      -0.053297044872440524
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:matches_requirement": {
    "fields": 504,
    "only_reference_correct": 0,
    "only_candidate_correct": 22,
    "both_wrong": 0,
    "mcnemar_exact_p": 4.76837158203125e-07,
    "reference": {
     "accuracy": 0.9563492063492064,
     "nll": 0.18589161439656215,
     "brier": 0.08657579365079365
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0110940448406785,
     "brier": 0.0007575509748263925
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.027777777777777776,
      0.06349206349206349
     ],
     "nll": [
      -0.20169542943071883,
      -0.15138690983232017
     ],
     "brier": [
      -0.10743243973811763,
      -0.0675046659377101
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:action": {
    "fields": 400,
    "only_reference_correct": 7,
    "only_candidate_correct": 28,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0005082604475319386,
    "reference": {
     "accuracy": 0.93,
     "nll": 0.28646668306359996,
     "brier": 0.14266510621365167
    },
    "candidate": {
     "accuracy": 0.9825,
     "nll": 0.07137598468332176,
     "brier": 0.028041129034753455
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.025,
      0.0825
     ],
     "nll": [
      -0.28682760310286914,
      -0.14656763180052837
     ],
     "brier": [
      -0.15308714959414974,
      -0.07770855982292431
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:department": {
    "fields": 400,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.0013886329684167317,
     "brier": 5.350000000000005e-05
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.014250866277672282,
     "brier": 0.00042255936592796044
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      0.012221706513776307,
      0.013496040117711447
     ],
     "brier": [
      0.0003232773908543254,
      0.0004150599381668005
     ]
    }
   },
   "data/semantic_v1/train.jsonl:action": {
    "fields": 96,
    "only_reference_correct": 0,
    "only_candidate_correct": 6,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.03125,
    "reference": {
     "accuracy": 0.9375,
     "nll": 0.07793953434301547,
     "brier": 0.05382708333333333
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.012029073756659505,
     "brier": 0.0002868543242750703
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.020833333333333332,
      0.11458333333333333
     ],
     "nll": [
      -0.11903032766227911,
      -0.021475208292545803
     ],
     "brier": [
      -0.0971106360306912,
      -0.01786290676574451
     ]
    }
   },
   "data/semantic_v1/train.jsonl:eligible": {
    "fields": 96,
    "only_reference_correct": 0,
    "only_candidate_correct": 4,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.125,
    "reference": {
     "accuracy": 0.9583333333333334,
     "nll": 0.10428509846405361,
     "brier": 0.04597500000000001
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.012057196642997656,
     "brier": 0.00028815238550322506
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.010416666666666666,
      0.08333333333333333
     ],
     "nll": [
      -0.13193128353339015,
      -0.05955837018651764
     ],
     "brier": [
      -0.07783632662832214,
      -0.01999005248041687
     ]
    }
   },
   "data/semantic_v1/train.jsonl:permission": {
    "fields": 96,
    "only_reference_correct": 0,
    "only_candidate_correct": 6,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.03125,
    "reference": {
     "accuracy": 0.9375,
     "nll": 0.08077415141964393,
     "brier": 0.05371666666666667
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.012152497522953528,
     "brier": 0.00029306183706107123
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.020833333333333332,
      0.11458333333333333
     ],
     "nll": [
      -0.12418274378925538,
      -0.02303005209586736
     ],
     "brier": [
      -0.09718873270133573,
      -0.01830890548952781
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 2600,
     "confidence_min": 0.35,
     "accuracy": 0.925
    },
    {
     "coverage": 0.9,
     "fields": 2340,
     "confidence_min": 0.73,
     "accuracy": 0.955982905982906
    },
    {
     "coverage": 0.75,
     "fields": 1950,
     "confidence_min": 0.89,
     "accuracy": 0.9697435897435898
    },
    {
     "coverage": 0.5,
     "fields": 1300,
     "confidence_min": 0.96,
     "accuracy": 0.9853846153846154
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 2600,
     "confidence_min": 0.4382959948907701,
     "accuracy": 0.9965384615384615
    },
    {
     "coverage": 0.9,
     "fields": 2340,
     "confidence_min": 0.981991834702081,
     "accuracy": 1.0
    },
    {
     "coverage": 0.75,
     "fields": 1950,
     "confidence_min": 0.9844696608975513,
     "accuracy": 1.0
    },
    {
     "coverage": 0.5,
     "fields": 1300,
     "confidence_min": 0.9869746021267632,
     "accuracy": 1.0
    }
   ]
  }
 },
 "blind": {
  "reference": "data/generated/reference_blind_500",
  "candidate": "reports/blind_joint-marker-stage-e-t-v1_cached",
  "requests_sha256": "4647dc3dd7edd87d97dc5e5a2f115eb1454a1006af2db5164b2eb3756f60afd7",
  "requests": 500,
  "fields": 1180,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1180,
   "only_reference_correct": 71,
   "only_candidate_correct": 117,
   "both_wrong": 50,
   "mcnemar_exact_p": 0.0009790560536514462,
   "reference": {
    "accuracy": 0.8584745762711864,
    "nll": 0.3308882778156683,
    "brier": 0.20319474576271188
   },
   "candidate": {
    "accuracy": 0.8974576271186441,
    "nll": 0.2403729395019762,
    "brier": 0.1449138425563558
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.012733446519524618,
     0.0658682634730539
    ],
    "nll": [
     -0.14380676678159254,
     -0.03519381540858258
    ],
    "brier": [
     -0.08881100164359779,
     -0.027269949261679197
    ]
   }
  },
  "groups": {
   "data/generated/judgment_v1/cases.jsonl:action": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 8,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0078125,
    "reference": {
     "accuracy": 0.8333333333333334,
     "nll": 0.5649138665795856,
     "brier": 0.29636666666666667
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.012296194969148461,
     "brier": 0.0002284230842867145
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0625,
      0.2916666666666667
     ],
     "nll": [
      -0.9380167151370755,
      -0.24505214251281046
     ],
     "brier": [
      -0.4853516023394051,
      -0.1460668133684144
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:evidence_quality": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 2,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.5,
    "reference": {
     "accuracy": 0.9583333333333334,
     "nll": 0.11543657120998631,
     "brier": 0.0624125
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.012839348880412568,
     "brier": 0.0002192260982907748
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.10416666666666667
     ],
     "nll": [
      -0.21262837820621625,
      -0.021973288307991015
     ],
     "brier": [
      -0.13747505724650158,
      -0.009465079146445963
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:matches_requirement": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 2,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.5,
    "reference": {
     "accuracy": 0.9583333333333334,
     "nll": 0.19715591766304155,
     "brier": 0.09675416666666668
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.009244324596915455,
     "brier": 0.0001792580725219774
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.10416666666666667
     ],
     "nll": [
      -0.28371700694295293,
      -0.11729532518109302
     ],
     "brier": [
      -0.1752984409690311,
      -0.03935345522032699
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:action": {
    "fields": 88,
    "only_reference_correct": 1,
    "only_candidate_correct": 15,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.000518798828125,
    "reference": {
     "accuracy": 0.8295454545454546,
     "nll": 0.4681900139759881,
     "brier": 0.258
    },
    "candidate": {
     "accuracy": 0.9886363636363636,
     "nll": 0.02687044006794181,
     "brier": 0.009297968939709859
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.07954545454545454,
      0.25
     ],
     "nll": [
      -0.6441841013092198,
      -0.2536991165923203
     ],
     "brier": [
      -0.36566547168487773,
      -0.13952086916016584
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:department": {
    "fields": 40,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.0002512583963375363,
     "brier": 5.000000000000005e-06
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.013502256933753476,
     "brier": 0.00036775977308007657
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      0.012480777834188467,
      0.014082832277140286
     ],
     "brier": [
      0.00032792534996939647,
      0.0004088276322285189
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:evidence_quality": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 4,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.125,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.19186169388221053,
     "brier": 0.11143333333333333
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.011853483819557456,
     "brier": 0.0001872663190297083
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.020833333333333332,
      0.16666666666666666
     ],
     "nll": [
      -0.33652415737855684,
      -0.06258272817261952
     ],
     "brier": [
      -0.20954632548166977,
      -0.035331203621425965
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:matches_requirement": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 3,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.25,
    "reference": {
     "accuracy": 0.9375,
     "nll": 0.2240429975360382,
     "brier": 0.110975
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.007921571495607032,
     "brier": 0.00012665316167295322
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.14583333333333334
     ],
     "nll": [
      -0.33846257026291804,
      -0.1199602593659491
     ],
     "brier": [
      -0.20409614942758444,
      -0.03758590298975098
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:action": {
    "fields": 40,
    "only_reference_correct": 0,
    "only_candidate_correct": 3,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.25,
    "reference": {
     "accuracy": 0.925,
     "nll": 0.3466242154785327,
     "brier": 0.17624
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.020232083716135156,
     "brier": 0.0007647905690260558
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.15
     ],
     "nll": [
      -0.5347076408734781,
      -0.1730456420346818
     ],
     "brier": [
      -0.29241156265275936,
      -0.08611447109003534
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:department": {
    "fields": 40,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.001005033585350145,
     "brier": 2.000000000000002e-05
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.013650005828798861,
     "brier": 0.0003726152686451732
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      0.011420371589797071,
      0.013701165581753647
     ],
     "brier": [
      0.0003147281762713001,
      0.0003912483050347418
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:action": {
    "fields": 72,
    "only_reference_correct": 13,
    "only_candidate_correct": 6,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.1670684814453125,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.2289145902005903,
     "brier": 0.1344972222222222
    },
    "candidate": {
     "accuracy": 0.8194444444444444,
     "nll": 0.5703879068067949,
     "brier": 0.2814514488353206
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.2222222222222222,
      0.013888888888888888
     ],
     "nll": [
      0.03821818303261392,
      0.6761329631675244
     ],
     "brier": [
      -0.009742680362246233,
      0.3145312290967965
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:evidence_quality": {
    "fields": 72,
    "only_reference_correct": 2,
    "only_candidate_correct": 3,
    "both_wrong": 2,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9305555555555556,
     "nll": 0.13080803955282563,
     "brier": 0.07639166666666666
    },
    "candidate": {
     "accuracy": 0.9444444444444444,
     "nll": 0.19661351745806402,
     "brier": 0.08188303337100332
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.041666666666666664,
      0.06944444444444445
     ],
     "nll": [
      -0.09767747492238132,
      0.29100823801001324
     ],
     "brier": [
      -0.06900678712898536,
      0.10268121242283122
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:matches_requirement": {
    "fields": 72,
    "only_reference_correct": 7,
    "only_candidate_correct": 10,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.629058837890625,
    "reference": {
     "accuracy": 0.8611111111111112,
     "nll": 0.36130463756224185,
     "brier": 0.21525
    },
    "candidate": {
     "accuracy": 0.9027777777777778,
     "nll": 0.35955110319439565,
     "brier": 0.16698954057909698
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.06944444444444445,
      0.1527777777777778
     ],
     "nll": [
      -0.25360250326417044,
      0.2768525949800911
     ],
     "brier": [
      -0.17953932295450192,
      0.09495819980708148
     ]
    }
   },
   "data/rlcd_synth_v3/majority_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 8,
    "only_candidate_correct": 15,
    "both_wrong": 18,
    "mcnemar_exact_p": 0.21003961563110352,
    "reference": {
     "accuracy": 0.5875,
     "nll": 0.7704734274880431,
     "brier": 0.53452
    },
    "candidate": {
     "accuracy": 0.675,
     "nll": 0.7365731944207703,
     "brier": 0.5248547526268317
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.025,
      0.2
     ],
     "nll": [
      -0.19990764652973794,
      0.13184832837434526
     ],
     "brier": [
      -0.12733212144406417,
      0.10946278756106327
     ]
    }
   },
   "data/rlcd_synth_v3/majority_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 11,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0009765625,
    "reference": {
     "accuracy": 0.8625,
     "nll": 0.30030802182472804,
     "brier": 0.17363
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.018820317362239675,
     "brier": 0.0006970743775889217
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0625,
      0.2125
     ],
     "nll": [
      -0.35511951245607326,
      -0.2109918657706074
     ],
     "brier": [
      -0.23631915480196714,
      -0.11390278270015161
     ]
    }
   },
   "data/rlcd_synth_v3/override_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 17,
    "only_candidate_correct": 12,
    "both_wrong": 16,
    "mcnemar_exact_p": 0.45825831964612007,
    "reference": {
     "accuracy": 0.65,
     "nll": 0.6607649163294025,
     "brier": 0.47980999999999996
    },
    "candidate": {
     "accuracy": 0.5875,
     "nll": 0.825213073085652,
     "brier": 0.5592589483329075
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.2,
      0.075
     ],
     "nll": [
      -0.027442936929660055,
      0.36085528459686195
     ],
     "brier": [
      -0.05470010443715649,
      0.2116200564635394
     ]
    }
   },
   "data/rlcd_synth_v3/override_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 7,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.015625,
    "reference": {
     "accuracy": 0.9125,
     "nll": 0.20750992356246,
     "brier": 0.1171775
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.01840454014373288,
     "brier": 0.0006669149880595984
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0375,
      0.15
     ],
     "nll": [
      -0.2652357768090621,
      -0.12365276333739539
     ],
     "brier": [
      -0.180694173517611,
      -0.06149384557304645
     ]
    }
   },
   "data/rlcd_synth_v3/veto_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 23,
    "only_candidate_correct": 13,
    "both_wrong": 14,
    "mcnemar_exact_p": 0.13249816396273673,
    "reference": {
     "accuracy": 0.6625,
     "nll": 0.6325119017462898,
     "brier": 0.4476975
    },
    "candidate": {
     "accuracy": 0.5375,
     "nll": 0.822962278299328,
     "brier": 0.5623672723068466
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.275,
      0.025
     ],
     "nll": [
      -0.0029790556353160172,
      0.3799469516347748
     ],
     "brier": [
      -0.025707222499420625,
      0.24504237224509895
     ]
    }
   },
   "data/rlcd_synth_v3/veto_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.13382107094010265,
     "brier": 0.045527500000000005
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.018366630046505766,
     "brier": 0.0006635137541949679
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      -0.142806858606292,
      -0.0914133384503451
     ],
     "brier": [
      -0.06169279419286513,
      -0.03023478916705455
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:action": {
    "fields": 12,
    "only_reference_correct": 0,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.14175288215384776,
     "brier": 0.09385
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.012260366274744023,
     "brier": 0.0002980775873391746
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.25
     ],
     "nll": [
      -0.2887926444157743,
      0.008674014896443063
     ],
     "brier": [
      -0.21138795225318163,
      0.0001743920710617731
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:eligible": {
    "fields": 12,
    "only_reference_correct": 0,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.1436268852546587,
     "brier": 0.07148333333333334
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.012193882492196231,
     "brier": 0.0002945367525270118
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.25
     ],
     "nll": [
      -0.2653197490715407,
      -0.031355768953637904
     ],
     "brier": [
      -0.1693525201152021,
      -0.003570592766045056
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:permission": {
    "fields": 12,
    "only_reference_correct": 0,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.12323621978999721,
     "brier": 0.07825
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.01235200464323972,
     "brier": 0.00030229320948221524
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.25
     ],
     "nll": [
      -0.27117664109611955,
      0.007173527719340631
     ],
     "brier": [
      -0.20208102017238294,
      0.0001488409230379443
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1180,
     "confidence_min": 0.47,
     "accuracy": 0.8584745762711864
    },
    {
     "coverage": 0.9,
     "fields": 1062,
     "confidence_min": 0.67,
     "accuracy": 0.8945386064030132
    },
    {
     "coverage": 0.75,
     "fields": 885,
     "confidence_min": 0.82,
     "accuracy": 0.9322033898305084
    },
    {
     "coverage": 0.5,
     "fields": 590,
     "confidence_min": 0.9299999999999999,
     "accuracy": 0.9677966101694915
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1180,
     "confidence_min": 0.4165017722444777,
     "accuracy": 0.8974576271186441
    },
    {
     "coverage": 0.9,
     "fields": 1062,
     "confidence_min": 0.5493712487392391,
     "accuracy": 0.931261770244821
    },
    {
     "coverage": 0.75,
     "fields": 885,
     "confidence_min": 0.9789871068749395,
     "accuracy": 0.9875706214689266
    },
    {
     "coverage": 0.5,
     "fields": 590,
     "confidence_min": 0.9830912872645099,
     "accuracy": 0.9864406779661017
    }
   ]
  },
  "latency_ms": {
   "reference_p50_p95": [
    109.39751204568893,
    191.90808408893645
   ],
   "marker_p50_p95": [
    40.631092502735555,
    63.31278895959258
   ]
  },
  "complete_requests_correct": {
   "reference": 363,
   "marker": 390
  },
  "reference_usage_as_reported": {
   "input_tokens": 427933,
   "output_tokens": 31093
  }
 },
 "public": {
  "reference": "data/generated/reference_public_99",
  "candidate": "reports/public_joint-marker-stage-e-t-v1_cached",
  "requests_sha256": "7dd21a875ecdcef675d862301fac14b697878d0ee70b92ca8b7c17af90d73245",
  "requests": 99,
  "fields": 99,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 99,
   "only_reference_correct": 59,
   "only_candidate_correct": 1,
   "both_wrong": 3,
   "mcnemar_exact_p": 1.0581813203458523e-16,
   "reference": {
    "accuracy": 0.9595959595959596,
    "nll": 0.17967170938188623,
    "brier": 0.08603636363636363
   },
   "candidate": {
    "accuracy": 0.37373737373737376,
    "nll": 1.3097377406520572,
    "brier": 0.7432957312587792
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     -0.6868686868686869,
     -0.48484848484848486
    ],
    "nll": [
     0.957250177715792,
     1.308172184554674
    ],
    "brier": [
     0.5558662587359178,
     0.7589565158552195
    ]
   }
  },
  "groups": {
   "data/public_validation_v1/hellaswag_calibration.jsonl:answer": {
    "fields": 33,
    "only_reference_correct": 22,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 4.76837158203125e-07,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.17844382589808147,
     "brier": 0.06521212121212121
    },
    "candidate": {
     "accuracy": 0.3333333333333333,
     "nll": 1.4990138021373298,
     "brier": 0.8024522627268081
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.8181818181818182,
      -0.5151515151515151
     ],
     "nll": [
      1.0712193381249833,
      1.6277206004191065
     ],
     "brier": [
      0.6014816871395603,
      0.8861135462006016
     ]
    }
   },
   "data/public_validation_v1/mmlu_calibration.jsonl:answer": {
    "fields": 33,
    "only_reference_correct": 24,
    "only_candidate_correct": 0,
    "both_wrong": 2,
    "mcnemar_exact_p": 1.1920928955078125e-07,
    "reference": {
     "accuracy": 0.9393939393939394,
     "nll": 0.2179869466603772,
     "brier": 0.11583030303030303
    },
    "candidate": {
     "accuracy": 0.21212121212121213,
     "nll": 1.713811838792962,
     "brier": 0.9110048168974905
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.8787878787878788,
      -0.5757575757575758
     ],
     "nll": [
      1.2150703551688484,
      1.7783408387569393
     ],
     "brier": [
      0.638041174985804,
      0.956763620251235
     ]
    }
   },
   "data/public_validation_v1/winogrande_calibration.jsonl:answer": {
    "fields": 33,
    "only_reference_correct": 13,
    "only_candidate_correct": 1,
    "both_wrong": 1,
    "mcnemar_exact_p": 0.0018310546875,
    "reference": {
     "accuracy": 0.9393939393939394,
     "nll": 0.14258435558720006,
     "brier": 0.07706666666666667
    },
    "candidate": {
     "accuracy": 0.5757575757575758,
     "nll": 0.7163875810258796,
     "brier": 0.5164301141520389
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5454545454545454,
      -0.18181818181818182
     ],
     "nll": [
      0.3363727258793026,
      0.8093626322573236
     ],
     "brier": [
      0.24932942423457205,
      0.6319019154932065
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 99,
     "confidence_min": 0.27999999999999997,
     "accuracy": 0.9595959595959596
    },
    {
     "coverage": 0.9,
     "fields": 89,
     "confidence_min": 0.67,
     "accuracy": 0.9775280898876404
    },
    {
     "coverage": 0.75,
     "fields": 74,
     "confidence_min": 0.8,
     "accuracy": 0.9864864864864865
    },
    {
     "coverage": 0.5,
     "fields": 50,
     "confidence_min": 0.95,
     "accuracy": 1.0
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 99,
     "confidence_min": 0.28873648856453915,
     "accuracy": 0.37373737373737376
    },
    {
     "coverage": 0.9,
     "fields": 89,
     "confidence_min": 0.35565563480399975,
     "accuracy": 0.4044943820224719
    },
    {
     "coverage": 0.75,
     "fields": 74,
     "confidence_min": 0.4256420875811859,
     "accuracy": 0.43243243243243246
    },
    {
     "coverage": 0.5,
     "fields": 50,
     "confidence_min": 0.5617986527679806,
     "accuracy": 0.48
    }
   ]
  },
  "latency_ms": {
   "reference_p50_p95": [
    115.80799403600395,
    167.67858597449958
   ],
   "marker_p50_p95": [
    24.072380969300866,
    37.29635989293456
   ]
  },
  "complete_requests_correct": {
   "reference": 95,
   "marker": 37
  },
  "reference_usage_as_reported": {
   "input_tokens": 43896,
   "output_tokens": 3993
  }
 },
 "blind_qwen_replay_plain": {
  "reference": "data/generated/reference_blind_500",
  "candidate": "reports/blind_joint-replay-plain-v1_cached",
  "requests_sha256": "4647dc3dd7edd87d97dc5e5a2f115eb1454a1006af2db5164b2eb3756f60afd7",
  "requests": 500,
  "fields": 1180,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1180,
   "only_reference_correct": 479,
   "only_candidate_correct": 63,
   "both_wrong": 104,
   "mcnemar_exact_p": 3.30854723350011e-80,
   "reference": {
    "accuracy": 0.8584745762711864,
    "nll": 0.3308882778156683,
    "brier": 0.20319474576271188
   },
   "candidate": {
    "accuracy": 0.5059322033898305,
    "nll": 1.582415293450241,
    "brier": 0.7445283073392328
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     -0.38879456706281834,
     -0.3153000845308538
    ],
    "nll": [
     1.1106961590956164,
     1.3993247691250281
    ],
    "brier": [
     0.4882445142594494,
     0.5954104207367525
    ]
   }
  },
  "groups": {
   "data/generated/judgment_v1/cases.jsonl:action": {
    "fields": 48,
    "only_reference_correct": 22,
    "only_candidate_correct": 1,
    "both_wrong": 7,
    "mcnemar_exact_p": 5.7220458984375e-06,
    "reference": {
     "accuracy": 0.8333333333333334,
     "nll": 0.5649138665795856,
     "brier": 0.29636666666666667
    },
    "candidate": {
     "accuracy": 0.3958333333333333,
     "nll": 2.7948271568678376,
     "brier": 1.0827861838721198
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5833333333333334,
      -0.2916666666666667
     ],
     "nll": [
      1.6333162638791594,
      2.861292659058148
     ],
     "brier": [
      0.5470207706448859,
      1.026607250263511
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:evidence_quality": {
    "fields": 48,
    "only_reference_correct": 32,
    "only_candidate_correct": 1,
    "both_wrong": 1,
    "mcnemar_exact_p": 7.916241884231567e-09,
    "reference": {
     "accuracy": 0.9583333333333334,
     "nll": 0.11543657120998631,
     "brier": 0.0624125
    },
    "candidate": {
     "accuracy": 0.3125,
     "nll": 1.7836390778314248,
     "brier": 0.9252109036357782
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.7916666666666666,
      -0.5
     ],
     "nll": [
      1.1960761403393911,
      2.219769288924037
     ],
     "brier": [
      0.6812347922343126,
      1.0452257076432485
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:matches_requirement": {
    "fields": 48,
    "only_reference_correct": 22,
    "only_candidate_correct": 0,
    "both_wrong": 2,
    "mcnemar_exact_p": 4.76837158203125e-07,
    "reference": {
     "accuracy": 0.9583333333333334,
     "nll": 0.19715591766304155,
     "brier": 0.09675416666666668
    },
    "candidate": {
     "accuracy": 0.5,
     "nll": 1.6648526599021292,
     "brier": 0.7291556220535904
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.6041666666666666,
      -0.3125
     ],
     "nll": [
      0.8202879209271545,
      2.1391777756621866
     ],
     "brier": [
      0.4154329281112408,
      0.8530130933644995
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:action": {
    "fields": 88,
    "only_reference_correct": 45,
    "only_candidate_correct": 0,
    "both_wrong": 15,
    "mcnemar_exact_p": 5.684341886080802e-14,
    "reference": {
     "accuracy": 0.8295454545454546,
     "nll": 0.4681900139759881,
     "brier": 0.258
    },
    "candidate": {
     "accuracy": 0.3181818181818182,
     "nll": 3.3223489398174664,
     "brier": 1.1401506575501086
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.6136363636363636,
      -0.4090909090909091
     ],
     "nll": [
      2.285415466123735,
      3.480114071227398
     ],
     "brier": [
      0.7280689047466342,
      1.029974761645465
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:department": {
    "fields": 40,
    "only_reference_correct": 8,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0078125,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.0002512583963375363,
     "brier": 5.000000000000005e-06
    },
    "candidate": {
     "accuracy": 0.8,
     "nll": 0.6298679969073155,
     "brier": 0.3600785298928498
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.325,
      -0.075
     ],
     "nll": [
      0.2908818519601747,
      1.013989533172574
     ],
     "brier": [
      0.15514493959516434,
      0.583349859711054
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:evidence_quality": {
    "fields": 48,
    "only_reference_correct": 20,
    "only_candidate_correct": 2,
    "both_wrong": 2,
    "mcnemar_exact_p": 0.00012111663818359375,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.19186169388221053,
     "brier": 0.11143333333333333
    },
    "candidate": {
     "accuracy": 0.5416666666666666,
     "nll": 1.598672529408326,
     "brier": 0.6936161600562261
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5208333333333334,
      -0.20833333333333334
     ],
     "nll": [
      0.842184074138198,
      2.024580713035338
     ],
     "brier": [
      0.3887827715744325,
      0.7778277071755518
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:matches_requirement": {
    "fields": 48,
    "only_reference_correct": 17,
    "only_candidate_correct": 0,
    "both_wrong": 3,
    "mcnemar_exact_p": 1.52587890625e-05,
    "reference": {
     "accuracy": 0.9375,
     "nll": 0.2240429975360382,
     "brier": 0.110975
    },
    "candidate": {
     "accuracy": 0.5833333333333334,
     "nll": 2.56514602465682,
     "brier": 0.778110015310586
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5,
      -0.22916666666666666
     ],
     "nll": [
      1.426802123563003,
      3.3466138830399426
     ],
     "brier": [
      0.428571404714426,
      0.9111477061613011
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:action": {
    "fields": 40,
    "only_reference_correct": 29,
    "only_candidate_correct": 0,
    "both_wrong": 3,
    "mcnemar_exact_p": 3.725290298461914e-09,
    "reference": {
     "accuracy": 0.925,
     "nll": 0.3466242154785327,
     "brier": 0.17624
    },
    "candidate": {
     "accuracy": 0.2,
     "nll": 2.2492143807491747,
     "brier": 1.0837678468187593
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.85,
      -0.575
     ],
     "nll": [
      1.4896732485542263,
      2.3193823985289965
     ],
     "brier": [
      0.7367448391967877,
      1.0633272424412241
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:department": {
    "fields": 40,
    "only_reference_correct": 9,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.00390625,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.001005033585350145,
     "brier": 2.000000000000002e-05
    },
    "candidate": {
     "accuracy": 0.775,
     "nll": 0.6540936474158364,
     "brier": 0.37174769528121393
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.375,
      -0.1
     ],
     "nll": [
      0.32853259094177484,
      1.016777094740983
     ],
     "brier": [
      0.17669556840000494,
      0.5939170277317779
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:action": {
    "fields": 72,
    "only_reference_correct": 34,
    "only_candidate_correct": 4,
    "both_wrong": 2,
    "mcnemar_exact_p": 6.038535502739251e-07,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.2289145902005903,
     "brier": 0.1344972222222222
    },
    "candidate": {
     "accuracy": 0.5,
     "nll": 2.050942505480204,
     "brier": 0.9180107372756271
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5416666666666666,
      -0.2777777777777778
     ],
     "nll": [
      1.3129047802216272,
      2.320608994801992
     ],
     "brier": [
      0.5626223559855421,
      1.0068331611311898
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:evidence_quality": {
    "fields": 72,
    "only_reference_correct": 39,
    "only_candidate_correct": 2,
    "both_wrong": 3,
    "mcnemar_exact_p": 7.839844329282641e-10,
    "reference": {
     "accuracy": 0.9305555555555556,
     "nll": 0.13080803955282563,
     "brier": 0.07639166666666666
    },
    "candidate": {
     "accuracy": 0.4166666666666667,
     "nll": 1.4482563364707053,
     "brier": 0.8110147526284233
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.6388888888888888,
      -0.3888888888888889
     ],
     "nll": [
      1.018371203008855,
      1.619291644555827
     ],
     "brier": [
      0.5854289841456803,
      0.8777002981101811
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:matches_requirement": {
    "fields": 72,
    "only_reference_correct": 25,
    "only_candidate_correct": 4,
    "both_wrong": 6,
    "mcnemar_exact_p": 0.00010371580719947815,
    "reference": {
     "accuracy": 0.8611111111111112,
     "nll": 0.36130463756224185,
     "brier": 0.21525
    },
    "candidate": {
     "accuracy": 0.5694444444444444,
     "nll": 0.729232612810025,
     "brier": 0.5275566612671914
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.4166666666666667,
      -0.1527777777777778
     ],
     "nll": [
      0.28183313025805234,
      0.46207860645810367
     ],
     "brier": [
      0.23566483184447534,
      0.3936725816267199
     ]
    }
   },
   "data/rlcd_synth_v3/majority_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 20,
    "only_candidate_correct": 17,
    "both_wrong": 16,
    "mcnemar_exact_p": 0.7428293587290682,
    "reference": {
     "accuracy": 0.5875,
     "nll": 0.7704734274880431,
     "brier": 0.53452
    },
    "candidate": {
     "accuracy": 0.55,
     "nll": 0.7290817664708136,
     "brier": 0.5303108662426146
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.1875,
      0.1
     ],
     "nll": [
      -0.2504292545455279,
      0.16806242515361997
     ],
     "brier": [
      -0.15018880508690513,
      0.1518650318581154
     ]
    }
   },
   "data/rlcd_synth_v3/majority_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 36,
    "only_candidate_correct": 6,
    "both_wrong": 5,
    "mcnemar_exact_p": 2.8288777684792876e-06,
    "reference": {
     "accuracy": 0.8625,
     "nll": 0.30030802182472804,
     "brier": 0.17363
    },
    "candidate": {
     "accuracy": 0.4875,
     "nll": 1.5314276681027923,
     "brier": 0.8650777764683608
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5125,
      -0.2375
     ],
     "nll": [
      0.8746273469024896,
      1.5881571259015712
     ],
     "brier": [
      0.5007476513346478,
      0.884618691678757
     ]
    }
   },
   "data/rlcd_synth_v3/override_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 24,
    "only_candidate_correct": 11,
    "both_wrong": 17,
    "mcnemar_exact_p": 0.04095959151163697,
    "reference": {
     "accuracy": 0.65,
     "nll": 0.6607649163294025,
     "brier": 0.47980999999999996
    },
    "candidate": {
     "accuracy": 0.4875,
     "nll": 0.7297866721225827,
     "brier": 0.5309625684155479
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.3,
      -0.025
     ],
     "nll": [
      -0.12024723054400874,
      0.26066473183869127
     ],
     "brier": [
      -0.08915557862552605,
      0.19295897724989416
     ]
    }
   },
   "data/rlcd_synth_v3/override_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 35,
    "only_candidate_correct": 1,
    "both_wrong": 6,
    "mcnemar_exact_p": 1.076841726899147e-09,
    "reference": {
     "accuracy": 0.9125,
     "nll": 0.20750992356246,
     "brier": 0.1171775
    },
    "candidate": {
     "accuracy": 0.4875,
     "nll": 2.226472358994251,
     "brier": 0.9660737348543851
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5375,
      -0.3125
     ],
     "nll": [
      1.5422447555228138,
      2.5125995095922393
     ],
     "brier": [
      0.6523933316240444,
      1.0475248885763517
     ]
    }
   },
   "data/rlcd_synth_v3/veto_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 25,
    "only_candidate_correct": 11,
    "both_wrong": 16,
    "mcnemar_exact_p": 0.028816719655878842,
    "reference": {
     "accuracy": 0.6625,
     "nll": 0.6325119017462898,
     "brier": 0.4476975
    },
    "candidate": {
     "accuracy": 0.4875,
     "nll": 0.7243433319815624,
     "brier": 0.5265073046443615
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.3125,
      -0.0375
     ],
     "nll": [
      -0.11737745750947318,
      0.2899217539932844
     ],
     "brier": [
      -0.07381537027096077,
      0.2249020043128925
     ]
    }
   },
   "data/rlcd_synth_v3/veto_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 26,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 2.9802322387695312e-08,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.13382107094010265,
     "brier": 0.045527500000000005
    },
    "candidate": {
     "accuracy": 0.675,
     "nll": 1.5973839385170598,
     "brier": 0.63771870787245
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.4375,
      -0.225
     ],
     "nll": [
      0.9912094765954956,
      2.0084827812577526
     ],
     "brier": [
      0.40304899265935645,
      0.8032371438089656
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:action": {
    "fields": 12,
    "only_reference_correct": 7,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0703125,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.14175288215384776,
     "brier": 0.09385
    },
    "candidate": {
     "accuracy": 0.4166666666666667,
     "nll": 1.1181741577676474,
     "brier": 0.7502379177509249
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.8333333333333334,
      -0.08333333333333333
     ],
     "nll": [
      0.364331160107078,
      1.7349034865214543
     ],
     "brier": [
      0.24325007667619689,
      1.0817299115257326
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:eligible": {
    "fields": 12,
    "only_reference_correct": 3,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.625,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.1436268852546587,
     "brier": 0.07148333333333334
    },
    "candidate": {
     "accuracy": 0.75,
     "nll": 0.5474884203376086,
     "brier": 0.33587792911329317
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5,
      0.16666666666666666
     ],
     "nll": [
      -0.04481062093258565,
      1.0341582790656452
     ],
     "brier": [
      -0.035322408865949693,
      0.6471018864576817
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:permission": {
    "fields": 12,
    "only_reference_correct": 1,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.12323621978999721,
     "brier": 0.07825
    },
    "candidate": {
     "accuracy": 0.9166666666666666,
     "nll": 0.5415855377737432,
     "brier": 0.2934011427580188
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.25,
      0.25
     ],
     "nll": [
      -0.04430928286655359,
      1.0588440564996515
     ],
     "brier": [
      -0.07564710681700151,
      0.5865231482709897
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1180,
     "confidence_min": 0.47,
     "accuracy": 0.8584745762711864
    },
    {
     "coverage": 0.9,
     "fields": 1062,
     "confidence_min": 0.67,
     "accuracy": 0.8945386064030132
    },
    {
     "coverage": 0.75,
     "fields": 885,
     "confidence_min": 0.82,
     "accuracy": 0.9322033898305084
    },
    {
     "coverage": 0.5,
     "fields": 590,
     "confidence_min": 0.9299999999999999,
     "accuracy": 0.9677966101694915
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1180,
     "confidence_min": 0.39673194151068436,
     "accuracy": 0.5059322033898305
    },
    {
     "coverage": 0.9,
     "fields": 1062,
     "confidence_min": 0.5191595869805448,
     "accuracy": 0.5169491525423728
    },
    {
     "coverage": 0.75,
     "fields": 885,
     "confidence_min": 0.6127992370435522,
     "accuracy": 0.5050847457627119
    },
    {
     "coverage": 0.5,
     "fields": 590,
     "confidence_min": 0.8003681600554057,
     "accuracy": 0.5508474576271186
    }
   ]
  }
 }
}