{
 "scope": "the reference API (v1.13.0) and joint-marker-stage-g-t-v1 on identical requests, constraints sent to neither. 3,639 paid reference API calls in total, none failed, no retries. unseen_v1 (inside blind) and order were used to diagnose stage F and are development sets for stage G; unseen2 was generated before stage G trained. One seed.",
 "public768": {
  "reference": "data/generated/reference_public_768",
  "candidate": "reports/public768_joint-marker-stage-g-t-v1_cached",
  "requests_sha256": "798d018d89c02c00e51c609463d1216cfcd777a20650cc8b6efbbe993518d27d",
  "requests": 768,
  "fields": 768,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 768,
   "only_reference_correct": 224,
   "only_candidate_correct": 29,
   "both_wrong": 38,
   "mcnemar_exact_p": 1.6568343860182494e-38,
   "reference": {
    "accuracy": 0.9127604166666666,
    "nll": 0.5297108014398391,
    "brier": 0.1502735482720726
   },
   "candidate": {
    "accuracy": 0.6588541666666666,
    "nll": 1.3155217936395442,
    "brier": 0.6015933939917243
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     -0.2903645833333333,
     -0.21875
    ],
    "nll": [
     0.553440039123717,
     0.9935327129801674
    ],
    "brier": [
     0.39294970433919,
     0.5118854794345016
    ]
   }
  },
  "groups": {
   "data/public_validation_v1/hellaswag_calibration.jsonl:answer": {
    "fields": 256,
    "only_reference_correct": 37,
    "only_candidate_correct": 12,
    "both_wrong": 7,
    "mcnemar_exact_p": 0.0004697573180081349,
    "reference": {
     "accuracy": 0.92578125,
     "nll": 0.3873098396840055,
     "brier": 0.15116439481621774
    },
    "candidate": {
     "accuracy": 0.828125,
     "nll": 0.6948386672309717,
     "brier": 0.3062793048549486
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.1484375,
      -0.046875
     ],
     "nll": [
      0.025542933115465405,
      0.560821436575822
     ],
     "brier": [
      0.07804036254250177,
      0.23682615430880177
     ]
    }
   },
   "data/public_validation_v1/mmlu_calibration.jsonl:answer": {
    "fields": 256,
    "only_reference_correct": 127,
    "only_candidate_correct": 5,
    "both_wrong": 24,
    "mcnemar_exact_p": 1.1820533154262882e-31,
    "reference": {
     "accuracy": 0.88671875,
     "nll": 0.8228434591672611,
     "brier": 0.1892578125
    },
    "candidate": {
     "accuracy": 0.41015625,
     "nll": 2.3108009352955388,
     "brier": 1.0258584375952036
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.5390625,
      -0.41015625
     ],
     "nll": [
      0.9275042528428731,
      1.9800661633108922
     ],
     "brier": [
      0.7238411990007586,
      0.9396904251471019
     ]
    }
   },
   "data/public_validation_v1/winogrande_calibration.jsonl:answer": {
    "fields": 256,
    "only_reference_correct": 60,
    "only_candidate_correct": 12,
    "both_wrong": 7,
    "mcnemar_exact_p": 8.055498526741105e-09,
    "reference": {
     "accuracy": 0.92578125,
     "nll": 0.37897910546825053,
     "brier": 0.1103984375
    },
    "candidate": {
     "accuracy": 0.73828125,
     "nll": 0.9409257783921223,
     "brier": 0.47264243952502055
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.24609375,
      -0.12890625
     ],
     "nll": [
      0.2091111277167614,
      0.8566901030348408
     ],
     "brier": [
      0.26671274939141126,
      0.45858451167516673
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 768,
     "confidence_min": 0.29,
     "accuracy": 0.9127604166666666
    },
    {
     "coverage": 0.9,
     "fields": 691,
     "confidence_min": 0.66,
     "accuracy": 0.9392185238784371
    },
    {
     "coverage": 0.75,
     "fields": 576,
     "confidence_min": 0.87,
     "accuracy": 0.9583333333333334
    },
    {
     "coverage": 0.5,
     "fields": 384,
     "confidence_min": 0.97,
     "accuracy": 0.9791666666666666
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 768,
     "confidence_min": 0.34623673902079793,
     "accuracy": 0.6588541666666666
    },
    {
     "coverage": 0.9,
     "fields": 691,
     "confidence_min": 0.7179564862002246,
     "accuracy": 0.683068017366136
    },
    {
     "coverage": 0.75,
     "fields": 576,
     "confidence_min": 0.9502823026118145,
     "accuracy": 0.7291666666666666
    },
    {
     "coverage": 0.5,
     "fields": 384,
     "confidence_min": 0.9793067898411559,
     "accuracy": 0.8046875
    }
   ]
  },
  "latency_ms_p50_p95": {
   "reference": [
    101.76942148245871,
    159.77542591281235
   ],
   "marker": [
    20.36565716844052,
    35.48129997216165
   ]
  }
 },
 "development": {
  "reference": "data/generated/reference_development_640",
  "candidate": "reports/development_joint-marker-stage-g-t-v1_cached",
  "requests_sha256": "d3389dbf1b9dade03c9ce23716a14b3ac2fb9760aa4e88be0ddc43e7d9ee2f97",
  "requests": 640,
  "fields": 1280,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1280,
   "only_reference_correct": 0,
   "only_candidate_correct": 276,
   "both_wrong": 2,
   "mcnemar_exact_p": 1.6472184286297693e-83,
   "reference": {
    "accuracy": 0.7828125,
    "nll": 0.5989872310599031,
    "brier": 0.32722859375
   },
   "candidate": {
    "accuracy": 0.9984375,
    "nll": 0.011883351324893378,
    "brier": 0.002202117916850232
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.19375,
     0.23671875
    ],
    "nll": [
     -0.675612662112753,
     -0.5000483592096641
    ],
    "brier": [
     -0.35593507615413805,
     -0.2937874270242461
    ]
   }
  },
  "groups": {
   "data/generated/reference_panels_v1/development.jsonl:action": {
    "fields": 640,
    "only_reference_correct": 0,
    "only_candidate_correct": 222,
    "both_wrong": 2,
    "mcnemar_exact_p": 2.967364920549937e-67,
    "reference": {
     "accuracy": 0.65,
     "nll": 0.9388350134625263,
     "brier": 0.5090984375
    },
    "candidate": {
     "accuracy": 0.996875,
     "nll": 0.01623468851971789,
     "brier": 0.004279858554965957
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.3109375,
      0.38125
     ],
     "nll": [
      -1.0814487964910184,
      -0.7654750237563618
     ],
     "brier": [
      -0.5558770816510643,
      -0.45111694764322374
     ]
    }
   },
   "data/generated/reference_panels_v1/development.jsonl:condition": {
    "fields": 640,
    "only_reference_correct": 0,
    "only_candidate_correct": 54,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.1102230246251565e-16,
    "reference": {
     "accuracy": 0.915625,
     "nll": 0.2591394486572801,
     "brier": 0.14535875
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.007532014130068865,
     "brier": 0.0001243772787345077
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0640625,
      0.10625
     ],
     "nll": [
      -0.28466011052426904,
      -0.22093255755660088
     ],
     "brier": [
      -0.17091432253089855,
      -0.12106522096061867
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1280,
     "confidence_min": 0.5,
     "accuracy": 0.7828125
    },
    {
     "coverage": 0.9,
     "fields": 1152,
     "confidence_min": 0.66,
     "accuracy": 0.8133680555555556
    },
    {
     "coverage": 0.75,
     "fields": 960,
     "confidence_min": 0.82,
     "accuracy": 0.8427083333333333
    },
    {
     "coverage": 0.5,
     "fields": 640,
     "confidence_min": 0.92,
     "accuracy": 0.8671875
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1280,
     "confidence_min": 0.5211373279846617,
     "accuracy": 0.9984375
    },
    {
     "coverage": 0.9,
     "fields": 1152,
     "confidence_min": 0.9897176741107855,
     "accuracy": 1.0
    },
    {
     "coverage": 0.75,
     "fields": 960,
     "confidence_min": 0.9922106994803181,
     "accuracy": 1.0
    },
    {
     "coverage": 0.5,
     "fields": 640,
     "confidence_min": 0.9930173432622086,
     "accuracy": 1.0
    }
   ]
  },
  "latency_ms_p50_p95": {
   "reference": [
    118.48623037803918,
    190.35738287493587
   ],
   "marker": [
    44.4509630324319,
    71.27445610240102
   ]
  }
 },
 "counterfactual": {
  "reference": "data/generated/reference_counterfactual_640",
  "candidate": "reports/counterfactual_joint-marker-stage-g-t-v1_cached",
  "requests_sha256": "768ef004a6a064db3d4ad6729bf7f02b15f3b6829221a6e5b546a29767af4ccc",
  "requests": 640,
  "fields": 1280,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1280,
   "only_reference_correct": 0,
   "only_candidate_correct": 297,
   "both_wrong": 1,
   "mcnemar_exact_p": 7.854549544476363e-90,
   "reference": {
    "accuracy": 0.7671875,
    "nll": 0.5850759392775601,
    "brier": 0.347788125
   },
   "candidate": {
    "accuracy": 0.99921875,
    "nll": 0.014128354749948788,
    "brier": 0.0031049937501169043
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.209375,
     0.2546875
    ],
    "nll": [
     -0.651730903839448,
     -0.5063478880612103
    ],
    "brier": [
     -0.37658643951977483,
     -0.3132887747021666
    ]
   }
  },
  "groups": {
   "data/generated/reference_panels_v1/counterfactual.jsonl:action": {
    "fields": 640,
    "only_reference_correct": 0,
    "only_candidate_correct": 234,
    "both_wrong": 1,
    "mcnemar_exact_p": 7.24454326306137e-71,
    "reference": {
     "accuracy": 0.6328125,
     "nll": 0.910007498678427,
     "brier": 0.54788625
    },
    "candidate": {
     "accuracy": 0.9984375,
     "nll": 0.02095535975066874,
     "brier": 0.006096805810533571
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.328125,
      0.403125
     ],
     "nll": [
      -1.0380150668530694,
      -0.7729225201365459
     ],
     "brier": [
      -0.5961886320178708,
      -0.4871012515783824
     ]
    }
   },
   "data/generated/reference_panels_v1/counterfactual.jsonl:condition": {
    "fields": 640,
    "only_reference_correct": 0,
    "only_candidate_correct": 63,
    "both_wrong": 0,
    "mcnemar_exact_p": 2.168404344971009e-19,
    "reference": {
     "accuracy": 0.9015625,
     "nll": 0.2601443798766932,
     "brier": 0.14769000000000002
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.007301349749228837,
     "brier": 0.00011318168970023774
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0765625,
      0.121875
     ],
     "nll": [
      -0.28657883059297623,
      -0.221840022582938
     ],
     "brier": [
      -0.17306925471647322,
      -0.12380466932705048
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1280,
     "confidence_min": 0.5,
     "accuracy": 0.7671875
    },
    {
     "coverage": 0.9,
     "fields": 1152,
     "confidence_min": 0.64,
     "accuracy": 0.7942708333333334
    },
    {
     "coverage": 0.75,
     "fields": 960,
     "confidence_min": 0.81,
     "accuracy": 0.8322916666666667
    },
    {
     "coverage": 0.5,
     "fields": 640,
     "confidence_min": 0.91,
     "accuracy": 0.8609375
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1280,
     "confidence_min": 0.525831301002928,
     "accuracy": 0.99921875
    },
    {
     "coverage": 0.9,
     "fields": 1152,
     "confidence_min": 0.9905432463203897,
     "accuracy": 0.9991319444444444
    },
    {
     "coverage": 0.75,
     "fields": 960,
     "confidence_min": 0.9922629555057465,
     "accuracy": 1.0
    },
    {
     "coverage": 0.5,
     "fields": 640,
     "confidence_min": 0.9930686243407834,
     "accuracy": 1.0
    }
   ]
  },
  "latency_ms_p50_p95": {
   "reference": [
    111.95720999967307,
    177.764164051041
   ],
   "marker": [
    46.47791641764343,
    70.93046908266842
   ]
  }
 },
 "retention": {
  "reference": "data/generated/reference_retention_512",
  "candidate": "reports/retention_joint-marker-stage-g-t-v1_cached",
  "requests_sha256": "011e90e2acc9d617401ee9402b7f8bdc988f6a91de14b866825cce1754d53b06",
  "requests": 512,
  "fields": 1024,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1024,
   "only_reference_correct": 0,
   "only_candidate_correct": 105,
   "both_wrong": 0,
   "mcnemar_exact_p": 4.930380657631324e-32,
   "reference": {
    "accuracy": 0.8974609375,
    "nll": 0.2866913663892488,
    "brier": 0.1680818359375
   },
   "candidate": {
    "accuracy": 1.0,
    "nll": 0.007438991063973252,
    "brier": 0.000513211546069468
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.080078125,
     0.1279296875
    ],
    "nll": [
     -0.31687737762507256,
     -0.24665259485691005
    ],
    "brier": [
     -0.19762517226395857,
     -0.14227210457635103
    ]
   }
  },
  "groups": {
   "data/random_lists_v1/equal_calibration.jsonl:action": {
    "fields": 512,
    "only_reference_correct": 0,
    "only_candidate_correct": 54,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.1102230246251565e-16,
    "reference": {
     "accuracy": 0.89453125,
     "nll": 0.30290819673278013,
     "brier": 0.18036875
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.008203214984697725,
     "brier": 0.0009364192186483898
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.08203125,
      0.1328125
     ],
     "nll": [
      -0.3373878246213467,
      -0.25650091973519223
     ],
     "brier": [
      -0.21191925027899475,
      -0.15085619655952365
     ]
    }
   },
   "data/random_lists_v1/equal_calibration.jsonl:condition": {
    "fields": 512,
    "only_reference_correct": 0,
    "only_candidate_correct": 51,
    "both_wrong": 0,
    "mcnemar_exact_p": 8.881784197001252e-16,
    "reference": {
     "accuracy": 0.900390625,
     "nll": 0.2704745360457175,
     "brier": 0.155794921875
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.006674767143248779,
     "brier": 9.000387349054625e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.076171875,
      0.126953125
     ],
     "nll": [
      -0.3008030007154012,
      -0.23235339646520345
     ],
     "brier": [
      -0.18593915974042105,
      -0.13027240323041958
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1024,
     "confidence_min": 0.5,
     "accuracy": 0.8974609375
    },
    {
     "coverage": 0.9,
     "fields": 922,
     "confidence_min": 0.65,
     "accuracy": 0.9219088937093276
    },
    {
     "coverage": 0.75,
     "fields": 768,
     "confidence_min": 0.78,
     "accuracy": 0.9557291666666666
    },
    {
     "coverage": 0.5,
     "fields": 512,
     "confidence_min": 0.88,
     "accuracy": 0.98046875
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1024,
     "confidence_min": 0.5367863355612645,
     "accuracy": 1.0
    },
    {
     "coverage": 0.9,
     "fields": 922,
     "confidence_min": 0.9922103558188807,
     "accuracy": 1.0
    },
    {
     "coverage": 0.75,
     "fields": 768,
     "confidence_min": 0.9929480578803399,
     "accuracy": 1.0
    },
    {
     "coverage": 0.5,
     "fields": 512,
     "confidence_min": 0.9933875534261347,
     "accuracy": 1.0
    }
   ]
  },
  "latency_ms_p50_p95": {
   "reference": [
    113.74656448606402,
    182.91272991336882
   ],
   "marker": [
    51.04239855427295,
    69.98322997242212
   ]
  }
 },
 "unseen2": {
  "reference": "data/generated/reference_unseen2_144",
  "candidate": "reports/unseen2_joint-marker-stage-g-t-v1_cached",
  "requests_sha256": "fc5a3cdacc4cbab287543c69d365664108b04ed4fcf1732e1ce3a63212e617cb",
  "requests": 144,
  "fields": 432,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 432,
   "only_reference_correct": 9,
   "only_candidate_correct": 51,
   "both_wrong": 0,
   "mcnemar_exact_p": 3.085035609438902e-08,
   "reference": {
    "accuracy": 0.8819444444444444,
    "nll": 0.35543278183580723,
    "brier": 0.20653177288220778
   },
   "candidate": {
    "accuracy": 0.9791666666666666,
    "nll": 0.05330499141858046,
    "brier": 0.03135905659684145
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.05092592592592592,
     0.14814814814814814
    ],
    "nll": [
     -0.41645212857743724,
     -0.19833588121152576
    ],
    "brier": [
     -0.2502908177238839,
     -0.10888886493759163
    ]
   }
  },
  "groups": {
   "data/generated/unseen_v2/test.jsonl:action": {
    "fields": 144,
    "only_reference_correct": 5,
    "only_candidate_correct": 23,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0009122341871261597,
    "reference": {
     "accuracy": 0.8402777777777778,
     "nll": 0.39054634438174823,
     "brier": 0.2395847222222222
    },
    "candidate": {
     "accuracy": 0.9652777777777778,
     "nll": 0.08215770149354144,
     "brier": 0.046235755326466385
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.05555555555555555,
      0.19444444444444445
     ],
     "nll": [
      -0.46135084258089193,
      -0.16560198322729563
     ],
     "brier": [
      -0.28956707095021633,
      -0.10391644180904683
     ]
    }
   },
   "data/generated/unseen_v2/test.jsonl:evidence_quality": {
    "fields": 144,
    "only_reference_correct": 2,
    "only_candidate_correct": 18,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0004024505615234375,
    "reference": {
     "accuracy": 0.875,
     "nll": 0.347194612209041,
     "brier": 0.19897309642440114
    },
    "candidate": {
     "accuracy": 0.9861111111111112,
     "nll": 0.038712679016252256,
     "brier": 0.022226009177383033
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.05555555555555555,
      0.1736111111111111
     ],
     "nll": [
      -0.4724873898288641,
      -0.16290941296418338
     ],
     "brier": [
      -0.2710991509990927,
      -0.0904979593205575
     ]
    }
   },
   "data/generated/unseen_v2/test.jsonl:matches_requirement": {
    "fields": 144,
    "only_reference_correct": 2,
    "only_candidate_correct": 10,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.03857421875,
    "reference": {
     "accuracy": 0.9305555555555556,
     "nll": 0.32855738891663255,
     "brier": 0.1810375
    },
    "candidate": {
     "accuracy": 0.9861111111111112,
     "nll": 0.03904459374594769,
     "brier": 0.025615405286674917
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.013888888888888888,
      0.10416666666666667
     ],
     "nll": [
      -0.35981615628111757,
      -0.2212910159040744
     ],
     "brier": [
      -0.21265418095059618,
      -0.10152054813309067
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 432,
     "confidence_min": 0.43,
     "accuracy": 0.8819444444444444
    },
    {
     "coverage": 0.9,
     "fields": 389,
     "confidence_min": 0.6699999999999999,
     "accuracy": 0.9023136246786633
    },
    {
     "coverage": 0.75,
     "fields": 324,
     "confidence_min": 0.8,
     "accuracy": 0.9135802469135802
    },
    {
     "coverage": 0.5,
     "fields": 216,
     "confidence_min": 0.93,
     "accuracy": 0.9398148148148148
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 432,
     "confidence_min": 0.5837434471130658,
     "accuracy": 0.9791666666666666
    },
    {
     "coverage": 0.9,
     "fields": 389,
     "confidence_min": 0.9959975037729137,
     "accuracy": 1.0
    },
    {
     "coverage": 0.75,
     "fields": 324,
     "confidence_min": 0.9964968042267776,
     "accuracy": 1.0
    },
    {
     "coverage": 0.5,
     "fields": 216,
     "confidence_min": 0.9968162794561308,
     "accuracy": 1.0
    }
   ]
  },
  "latency_ms_p50_p95": {
   "reference": [
    112.74057149421424,
    190.19817397929728
   ],
   "marker": [
    46.895837993361056,
    65.53984805941582
   ]
  }
 },
 "blind": {
  "reference": "data/generated/reference_blind_500",
  "candidate": "reports/blind_joint-marker-stage-g-t-v1_cached",
  "requests_sha256": "4647dc3dd7edd87d97dc5e5a2f115eb1454a1006af2db5164b2eb3756f60afd7",
  "requests": 500,
  "fields": 1180,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1180,
   "only_reference_correct": 18,
   "only_candidate_correct": 164,
   "both_wrong": 3,
   "mcnemar_exact_p": 1.1505290637140604e-30,
   "reference": {
    "accuracy": 0.8584745762711864,
    "nll": 0.3308882778156683,
    "brier": 0.20319474576271188
   },
   "candidate": {
    "accuracy": 0.9822033898305085,
    "nll": 0.07286408346985528,
    "brier": 0.029512386895811184
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.09991532599491956,
     0.1496196111580727
    ],
    "nll": [
     -0.3135314088948579,
     -0.19719032270707493
    ],
    "brier": [
     -0.20584814374964064,
     -0.1420894082823218
    ]
   }
  },
  "groups": {
   "data/generated/judgment_v1/cases.jsonl:action": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 8,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0078125,
    "reference": {
     "accuracy": 0.8333333333333334,
     "nll": 0.5649138665795856,
     "brier": 0.29636666666666667
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.00304133295758085,
     "brier": 1.3885397537184193e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0625,
      0.2916666666666667
     ],
     "nll": [
      -0.9468841202428852,
      -0.254274262187241
     ],
     "brier": [
      -0.48555690107356186,
      -0.14628216884309164
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:evidence_quality": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 2,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.5,
    "reference": {
     "accuracy": 0.9583333333333334,
     "nll": 0.11543657120998631,
     "brier": 0.0624125
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.003463273946747407,
     "brier": 1.5968651631998615e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.10416666666666667
     ],
     "nll": [
      -0.22187289524767848,
      -0.0314894020861342
     ],
     "brier": [
      -0.13767997587244205,
      -0.009671918716914366
     ]
    }
   },
   "data/generated/judgment_v1/cases.jsonl:matches_requirement": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 2,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.5,
    "reference": {
     "accuracy": 0.9583333333333334,
     "nll": 0.19715591766304155,
     "brier": 0.09675416666666668
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.002327940774657183,
     "brier": 1.0847693745455112e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.10416666666666667
     ],
     "nll": [
      -0.29063131295357464,
      -0.12433381805545514
     ],
     "brier": [
      -0.1754930132561355,
      -0.039506054057202845
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:action": {
    "fields": 88,
    "only_reference_correct": 1,
    "only_candidate_correct": 15,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.000518798828125,
    "reference": {
     "accuracy": 0.8295454545454546,
     "nll": 0.4681900139759881,
     "brier": 0.258
    },
    "candidate": {
     "accuracy": 0.9886363636363636,
     "nll": 0.06760372678019343,
     "brier": 0.022496240336061863
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.07954545454545454,
      0.25
     ],
     "nll": [
      -0.6316467413560318,
      -0.16662611259376714
     ],
     "brier": [
      -0.3617005937467957,
      -0.11609229441937496
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:department": {
    "fields": 40,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.0002512583963375363,
     "brier": 5.000000000000005e-06
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0051951855712445505,
     "brier": 5.391193418205039e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      0.0043702191544367055,
      0.005283349441267886
     ],
     "brier": [
      3.7404028062238856e-05,
      5.5775461981977996e-05
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:evidence_quality": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 4,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.125,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.19186169388221053,
     "brier": 0.11143333333333333
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.003542018203313983,
     "brier": 1.672503659543144e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.020833333333333332,
      0.16666666666666666
     ],
     "nll": [
      -0.3447344737930514,
      -0.07059262118977479
     ],
     "brier": [
      -0.20970833940738787,
      -0.035504276061903366
     ]
    }
   },
   "data/generated/reasoning_followup/validation.jsonl:matches_requirement": {
    "fields": 48,
    "only_reference_correct": 0,
    "only_candidate_correct": 3,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.25,
    "reference": {
     "accuracy": 0.9375,
     "nll": 0.2240429975360382,
     "brier": 0.110975
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.0023868366716566624,
     "brier": 1.145950109590191e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.14583333333333334
     ],
     "nll": [
      -0.34379845740580744,
      -0.12542572333665916
     ],
     "brier": [
      -0.2042099553471629,
      -0.03769704385952944
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:action": {
    "fields": 40,
    "only_reference_correct": 1,
    "only_candidate_correct": 3,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.625,
    "reference": {
     "accuracy": 0.925,
     "nll": 0.3466242154785327,
     "brier": 0.17624
    },
    "candidate": {
     "accuracy": 0.975,
     "nll": 0.1471803147653458,
     "brier": 0.04953085909051862
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.025,
      0.15
     ],
     "nll": [
      -0.5035927020054091,
      0.1463863444034371
     ],
     "brier": [
      -0.2687632846431255,
      0.007036847744555479
     ]
    }
   },
   "data/generated/support_v1/cases.jsonl:department": {
    "fields": 40,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.001005033585350145,
     "brier": 2.000000000000002e-05
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.005167152956086123,
     "brier": 5.3359203939697814e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      0.0030993987934132003,
      0.005009556702914175
     ],
     "brier": [
      1.213845786415514e-05,
      5.032422221075388e-05
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:action": {
    "fields": 72,
    "only_reference_correct": 8,
    "only_candidate_correct": 6,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.79052734375,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.2289145902005903,
     "brier": 0.1344972222222222
    },
    "candidate": {
     "accuracy": 0.8888888888888888,
     "nll": 0.45896538487405725,
     "brier": 0.1858988080957192
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.125,
      0.06944444444444445
     ],
     "nll": [
      -0.07980052578001325,
      0.6062529275634528
     ],
     "brier": [
      -0.0828173991512233,
      0.19908734625406133
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:evidence_quality": {
    "fields": 72,
    "only_reference_correct": 0,
    "only_candidate_correct": 3,
    "both_wrong": 2,
    "mcnemar_exact_p": 0.25,
    "reference": {
     "accuracy": 0.9305555555555556,
     "nll": 0.13080803955282563,
     "brier": 0.07639166666666666
    },
    "candidate": {
     "accuracy": 0.9722222222222222,
     "nll": 0.07899551207093208,
     "brier": 0.041862100518885136
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.09722222222222222
     ],
     "nll": [
      -0.12452543898810085,
      0.023728552215315675
     ],
     "brier": [
      -0.08240618252723565,
      0.009105436113433476
     ]
    }
   },
   "data/generated/unseen_v1/test.jsonl:matches_requirement": {
    "fields": 72,
    "only_reference_correct": 8,
    "only_candidate_correct": 10,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.8145294189453125,
    "reference": {
     "accuracy": 0.8611111111111112,
     "nll": 0.36130463756224185,
     "brier": 0.21525
    },
    "candidate": {
     "accuracy": 0.8888888888888888,
     "nll": 0.36447120432808894,
     "brier": 0.16691783296881754
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.08333333333333333,
      0.1388888888888889
     ],
     "nll": [
      -0.2655339585342734,
      0.3255847703359264
     ],
     "brier": [
      -0.17643347555496225,
      0.09538569431388748
     ]
    }
   },
   "data/rlcd_synth_v3/majority_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 33,
    "both_wrong": 0,
    "mcnemar_exact_p": 2.3283064365386963e-10,
    "reference": {
     "accuracy": 0.5875,
     "nll": 0.7704734274880431,
     "brier": 0.53452
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.02567110731848793,
     "brier": 0.0070972059052125756
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.3125,
      0.5125
     ],
     "nll": [
      -0.9368064040702123,
      -0.559389644410661
     ],
     "brier": [
      -0.6645703705023281,
      -0.393875917028412
     ]
    }
   },
   "data/rlcd_synth_v3/majority_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 11,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.0009765625,
    "reference": {
     "accuracy": 0.8625,
     "nll": 0.30030802182472804,
     "brier": 0.17363
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.00837064962686923,
     "brier": 0.00016339588548561248
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0625,
      0.2125
     ],
     "nll": [
      -0.36534555068099406,
      -0.22167441583185185
     ],
     "brier": [
      -0.23683572201480535,
      -0.11446105901142345
     ]
    }
   },
   "data/rlcd_synth_v3/override_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 28,
    "both_wrong": 0,
    "mcnemar_exact_p": 7.450580596923828e-09,
    "reference": {
     "accuracy": 0.65,
     "nll": 0.6607649163294025,
     "brier": 0.47980999999999996
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.008896574057313682,
     "brier": 0.00021578497296974482
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.25,
      0.45
     ],
     "nll": [
      -0.83432554295533,
      -0.4734155145430864
     ],
     "brier": [
      -0.6107666302523275,
      -0.3502991317863569
     ]
    }
   },
   "data/rlcd_synth_v3/override_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 7,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.015625,
    "reference": {
     "accuracy": 0.9125,
     "nll": 0.20750992356246,
     "brier": 0.1171775
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.007689291312432521,
     "brier": 0.00012503247598185172
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0375,
      0.15
     ],
     "nll": [
      -0.27586891936291874,
      -0.13451168699544475
     ],
     "brier": [
      -0.18123204532388565,
      -0.062038017219897545
     ]
    }
   },
   "data/rlcd_synth_v3/veto_test.jsonl:action": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 26,
    "both_wrong": 1,
    "mcnemar_exact_p": 2.9802322387695312e-08,
    "reference": {
     "accuracy": 0.6625,
     "nll": 0.6325119017462898,
     "brier": 0.4476975
    },
    "candidate": {
     "accuracy": 0.9875,
     "nll": 0.03978678313509351,
     "brier": 0.02271009551261601
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.225,
      0.425
     ],
     "nll": [
      -0.7870599823787245,
      -0.41232571154836106
     ],
     "brier": [
      -0.561940856083586,
      -0.2949004737008908
     ]
    }
   },
   "data/rlcd_synth_v3/veto_test.jsonl:condition": {
    "fields": 80,
    "only_reference_correct": 0,
    "only_candidate_correct": 0,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 1.0,
     "nll": 0.13382107094010265,
     "brier": 0.045527500000000005
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.008000201315569996,
     "brier": 0.0001586040500427679
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.0
     ],
     "nll": [
      -0.1529333501904499,
      -0.10176117792960948
     ],
     "brier": [
      -0.06218519237827321,
      -0.030726667555101596
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:action": {
    "fields": 12,
    "only_reference_correct": 0,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.14175288215384776,
     "brier": 0.09385
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.004769535027086412,
     "brier": 4.533490841912462e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.25
     ],
     "nll": [
      -0.29624214288187206,
      0.0013792912566064524
     ],
     "brier": [
      -0.21163897821915212,
      -8.547629783470279e-05
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:eligible": {
    "fields": 12,
    "only_reference_correct": 0,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.1436268852546587,
     "brier": 0.07148333333333334
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.004703261749102566,
     "brier": 4.406110105440493e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.25
     ],
     "nll": [
      -0.2728325015173538,
      -0.03881453744463873
     ],
     "brier": [
      -0.16960449438186745,
      -0.00382237441254087
     ]
    }
   },
   "data/semantic_v1/cases.jsonl:permission": {
    "fields": 12,
    "only_reference_correct": 0,
    "only_candidate_correct": 1,
    "both_wrong": 0,
    "mcnemar_exact_p": 1.0,
    "reference": {
     "accuracy": 0.9166666666666666,
     "nll": 0.12323621978999721,
     "brier": 0.07825
    },
    "candidate": {
     "accuracy": 1.0,
     "nll": 0.004856974103167526,
     "brier": 4.6998880358931594e-05
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.0,
      0.25
     ],
     "nll": [
      -0.2786519834036631,
      -0.00019356006163347207
     ],
     "brier": [
      -0.20233576101097772,
      -0.0001044162610324035
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1180,
     "confidence_min": 0.47,
     "accuracy": 0.8584745762711864
    },
    {
     "coverage": 0.9,
     "fields": 1062,
     "confidence_min": 0.67,
     "accuracy": 0.8945386064030132
    },
    {
     "coverage": 0.75,
     "fields": 885,
     "confidence_min": 0.82,
     "accuracy": 0.9322033898305084
    },
    {
     "coverage": 0.5,
     "fields": 590,
     "confidence_min": 0.9299999999999999,
     "accuracy": 0.9677966101694915
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1180,
     "confidence_min": 0.5083945849431952,
     "accuracy": 0.9822033898305085
    },
    {
     "coverage": 0.9,
     "fields": 1062,
     "confidence_min": 0.9908377091279952,
     "accuracy": 0.992467043314501
    },
    {
     "coverage": 0.75,
     "fields": 885,
     "confidence_min": 0.9927437704139631,
     "accuracy": 0.9932203389830508
    },
    {
     "coverage": 0.5,
     "fields": 590,
     "confidence_min": 0.9944064532102996,
     "accuracy": 0.9915254237288136
    }
   ]
  },
  "latency_ms_p50_p95": {
   "reference": [
    109.39751204568893,
    191.90808408893645
   ]
  }
 },
 "order": {
  "reference": "data/generated/reference_order_336",
  "candidate": "reports/order_joint-marker-stage-g-t-v1_cached",
  "requests_sha256": "17f9088d4fe2c7dd7461e1c04b14fca63cf8d6af90ff59bbd8c895c87743341e",
  "requests": 336,
  "fields": 1008,
  "bootstrap": "2000 resamples of whole requests, seed 419, 95% percentile interval",
  "all": {
   "fields": 1008,
   "only_reference_correct": 30,
   "only_candidate_correct": 61,
   "both_wrong": 0,
   "mcnemar_exact_p": 0.0015165131944572215,
   "reference": {
    "accuracy": 0.939484126984127,
    "nll": 0.19824015084544572,
    "brier": 0.10872287784855125
   },
   "candidate": {
    "accuracy": 0.9702380952380952,
    "nll": 0.11396306453193387,
    "brier": 0.0502421903405803
   },
   "candidate_minus_reference_ci95": {
    "accuracy": [
     0.008928571428571428,
     0.05257936507936508
    ],
    "nll": [
     -0.1450852189278436,
     -0.020063416648045583
    ],
    "brier": [
     -0.08772376039340626,
     -0.02746004435464497
    ]
   }
  },
  "groups": {
   "data/generated/order_v1/test.jsonl:action": {
    "fields": 336,
    "only_reference_correct": 10,
    "only_candidate_correct": 24,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.024306510109454393,
    "reference": {
     "accuracy": 0.9285714285714286,
     "nll": 0.17963740596887126,
     "brier": 0.09942440476190476
    },
    "candidate": {
     "accuracy": 0.9702380952380952,
     "nll": 0.14477996423331968,
     "brier": 0.055834106671631634
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      0.008928571428571428,
      0.07738095238095238
     ],
     "nll": [
      -0.1254934466745844,
      0.05863245291853901
     ],
     "brier": [
      -0.08627586653263705,
      -0.00127326588341813
     ]
    }
   },
   "data/generated/order_v1/test.jsonl:evidence_quality": {
    "fields": 336,
    "only_reference_correct": 3,
    "only_candidate_correct": 8,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.2265625,
    "reference": {
     "accuracy": 0.9761904761904762,
     "nll": 0.08507247933394088,
     "brier": 0.041618038307558516
    },
    "candidate": {
     "accuracy": 0.9910714285714286,
     "nll": 0.0302089001170281,
     "brier": 0.014035486750065572
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.005952380952380952,
      0.03571428571428571
     ],
     "nll": [
      -0.08370247407036456,
      -0.027815560893796797
     ],
     "brier": [
      -0.04661242929139306,
      -0.008873744030606458
     ]
    }
   },
   "data/generated/order_v1/test.jsonl:matches_requirement": {
    "fields": 336,
    "only_reference_correct": 17,
    "only_candidate_correct": 29,
    "both_wrong": 0,
    "mcnemar_exact_p": 0.10380535550410741,
    "reference": {
     "accuracy": 0.9136904761904762,
     "nll": 0.330010567233525,
     "brier": 0.18512619047619047
    },
    "candidate": {
     "accuracy": 0.9494047619047619,
     "nll": 0.16690032924545387,
     "brier": 0.08085697760004369
    },
    "candidate_minus_reference_ci95": {
     "accuracy": [
      -0.002976190476190476,
      0.0744047619047619
     ],
     "nll": [
      -0.25202439153034006,
      -0.06699983877260582
     ],
     "brier": [
      -0.1507070474837978,
      -0.05612011253916416
     ]
    }
   }
  },
  "selective": {
   "reference": [
    {
     "coverage": 1.0,
     "fields": 1008,
     "confidence_min": 0.44,
     "accuracy": 0.939484126984127
    },
    {
     "coverage": 0.9,
     "fields": 907,
     "confidence_min": 0.58,
     "accuracy": 0.9746416758544653
    },
    {
     "coverage": 0.75,
     "fields": 756,
     "confidence_min": 0.78,
     "accuracy": 0.9947089947089947
    },
    {
     "coverage": 0.5,
     "fields": 504,
     "confidence_min": 0.93,
     "accuracy": 1.0
    }
   ],
   "candidate": [
    {
     "coverage": 1.0,
     "fields": 1008,
     "confidence_min": 0.568634510582235,
     "accuracy": 0.9702380952380952
    },
    {
     "coverage": 0.9,
     "fields": 907,
     "confidence_min": 0.9914599523531759,
     "accuracy": 0.9878721058434399
    },
    {
     "coverage": 0.75,
     "fields": 756,
     "confidence_min": 0.9957343272586283,
     "accuracy": 1.0
    },
    {
     "coverage": 0.5,
     "fields": 504,
     "confidence_min": 0.9963233383598016,
     "accuracy": 1.0
    }
   ]
  },
  "latency_ms_p50_p95": {
   "reference": [
    108.69107861071825,
    178.54181304574013
   ],
   "marker": [
    45.87811848614365,
    60.5619209818542
   ]
  }
 },
 "order_stability": {
  "reference": {
   "directory": "data/generated/reference_order_336",
   "low_evidence_variants": [
    "absent",
    "conflict",
    "quorum_shortfall",
    "repeated_source"
   ],
   "all": {
    "base_cases": 96,
    "action_accuracy": 0.9375,
    "order_winner_flip_rate": 0.08333333333333333,
    "order_total_variation_mean": 0.04979166666666667,
    "order_noul_shift_mean": 0.0346875,
    "first_position_win_rate": 0.3229166666666667,
    "repeat_cases": 48,
    "repeat_winner_flip_rate": 0.020833333333333332,
    "repeat_total_variation_mean": 0.02125
   },
   "low_evidence": {
    "base_cases": 32,
    "action_accuracy": 0.9166666666666666,
    "order_winner_flip_rate": 0.15625,
    "order_total_variation_mean": 0.07875,
    "order_noul_shift_mean": 0.022187499999999995,
    "first_position_win_rate": 0.28125,
    "repeat_cases": 16,
    "repeat_winner_flip_rate": 0.0625,
    "repeat_total_variation_mean": 0.030000000000000002
   },
   "resolved_evidence": {
    "base_cases": 64,
    "action_accuracy": 0.9479166666666666,
    "order_winner_flip_rate": 0.046875,
    "order_total_variation_mean": 0.035312500000000004,
    "order_noul_shift_mean": 0.04093750000000001,
    "first_position_win_rate": 0.34375,
    "repeat_cases": 32,
    "repeat_winner_flip_rate": 0.0,
    "repeat_total_variation_mean": 0.016875
   }
  },
  "marker_g": {
   "directory": "reports/order_joint-marker-stage-g-t-v1_cached",
   "low_evidence_variants": [
    "absent",
    "conflict",
    "quorum_shortfall",
    "repeated_source"
   ],
   "all": {
    "base_cases": 96,
    "action_accuracy": 0.96875,
    "order_winner_flip_rate": 0.0,
    "order_total_variation_mean": 0.0060767388438868015,
    "order_noul_shift_mean": 0.0005605329533551726,
    "first_position_win_rate": 0.3333333333333333,
    "repeat_cases": 48,
    "repeat_winner_flip_rate": 0.0,
    "repeat_total_variation_mean": 0.0
   },
   "low_evidence": {
    "base_cases": 32,
    "action_accuracy": 0.96875,
    "order_winner_flip_rate": 0.0,
    "order_total_variation_mean": 0.004054465876397448,
    "order_noul_shift_mean": 0.00028963767245280083,
    "first_position_win_rate": 0.3333333333333333,
    "repeat_cases": 16,
    "repeat_winner_flip_rate": 0.0,
    "repeat_total_variation_mean": 0.0
   },
   "resolved_evidence": {
    "base_cases": 64,
    "action_accuracy": 0.96875,
    "order_winner_flip_rate": 0.0,
    "order_total_variation_mean": 0.007087875327631477,
    "order_noul_shift_mean": 0.0006959805938063585,
    "first_position_win_rate": 0.3333333333333333,
    "repeat_cases": 32,
    "repeat_winner_flip_rate": 0.0,
    "repeat_total_variation_mean": 0.0
   }
  },
  "marker_e": {
   "directory": "reports/order_joint-marker-stage-e-t-v1_cached",
   "low_evidence_variants": [
    "absent",
    "conflict",
    "quorum_shortfall",
    "repeated_source"
   ],
   "all": {
    "base_cases": 96,
    "action_accuracy": 0.8576388888888888,
    "order_winner_flip_rate": 0.03125,
    "order_total_variation_mean": 0.04774422795553457,
    "order_noul_shift_mean": 0.004267660763882562,
    "first_position_win_rate": 0.3298611111111111,
    "repeat_cases": 48,
    "repeat_winner_flip_rate": 0.0,
    "repeat_total_variation_mean": 0.0
   },
   "low_evidence": {
    "base_cases": 32,
    "action_accuracy": 0.7083333333333334,
    "order_winner_flip_rate": 0.0625,
    "order_total_variation_mean": 0.0641301473601443,
    "order_noul_shift_mean": 0.0045296199226371025,
    "first_position_win_rate": 0.3229166666666667,
    "repeat_cases": 16,
    "repeat_winner_flip_rate": 0.0,
    "repeat_total_variation_mean": 0.0
   },
   "resolved_evidence": {
    "base_cases": 64,
    "action_accuracy": 0.9322916666666666,
    "order_winner_flip_rate": 0.015625,
    "order_total_variation_mean": 0.03955126825322971,
    "order_noul_shift_mean": 0.004136681184505291,
    "first_position_win_rate": 0.3333333333333333,
    "repeat_cases": 32,
    "repeat_winner_flip_rate": 0.0,
    "repeat_total_variation_mean": 0.0
   }
  }
 }
}