{ "name": "containment judge measurement — semantic reference check on prose references (.docx / references[]), two sealed fresh draws", "sealed": true, "read_once_on": "2026-10-13", "protocols": [ "scripts/typesafe-probe/containment/v1-draw-2026-10-13/PROTOCOL.md", "scripts/typesafe-probe/containment/v2-draw-2026-10-13/PROTOCOL.md" ], "containment_version": "2026-08-26", "judge": { "vendor": "TypeSafe System One", "model": "jev-1.13.0", "question_version": "2026-10-13", "question_file": "scripts/typesafe-probe/containment/v1-draw-2026-10-13/questions.json" }, "threshold": { "flag_cites_work_below": 0.5, "lift_cites_work_at_or_above": 0.8, "pre_registered": { "flag": "v1 and v2", "lift": "v2 (0.8 chosen from the v1 sweep; v1 is burned for the lift figure)" } }, "truth": "title-independent: every row is a real Crossref journal-article record drawn with /works?sample=100; the DOI resolves to the record. Genuine arms keep the record's authors, year and DOI with the true, a model-translated, or a model-paraphrased title; the fabrication arm keeps them with a model-invented same-field title (<40% content-word overlap). References formatted by a deterministic five-style template (not the product formatter).", "rows_files": { "v1": "public/citation-integrity/validation-results-containment-v1.json", "v2": "public/citation-integrity/validation-results-containment-v2.json" }, "draws": { "v1": { "read_once_on": "2026-10-13", "purpose": "flag rate at 0.5 (pre-registered)", "rows": 1654, "errors": 0, "arms": { "genuine_clean": { "n": 1154, "flagged_below_0_5": { "flagged": 0, "n": 1154, "rate": 0, "wilson95": { "lo": 0, "hi": 0.003317897715887907 } }, "lift_eligible_at_or_above_0_8": { "flagged": 1153, "n": 1154, "rate": 0.9991334488734835, "wilson95": { "lo": 0.9951077302847173, "hi": 0.9998470200023684 } }, "min": 0.79, "max": 0.99 }, "genuine_translation": { "n": 150, "flagged_below_0_5": { "flagged": 0, "n": 150, "rate": 0, "wilson95": { "lo": 0, "hi": 0.02497113914571871 } }, "lift_eligible_at_or_above_0_8": { "flagged": 150, "n": 150, "rate": 1, "wilson95": { "lo": 0.9750288608542812, "hi": 1 } }, "min": 0.88, "max": 0.98 }, "genuine_paraphrase": { "n": 150, "flagged_below_0_5": { "flagged": 2, "n": 150, "rate": 0.013333333333333334, "wilson95": { "lo": 0.0036640523882232115, "hi": 0.047307856380276334 } }, "lift_eligible_at_or_above_0_8": { "flagged": 132, "n": 150, "rate": 0.88, "wilson95": { "lo": 0.8182903386460476, "hi": 0.9227315956032063 } }, "min": 0.35, "max": 0.97 }, "fab_real_authors": { "n": 200, "flagged_below_0_5": { "flagged": 196, "n": 200, "rate": 0.98, "wilson95": { "lo": 0.9497121344948062, "hi": 0.9921956998236058 } }, "lift_eligible_at_or_above_0_8": { "flagged": 0, "n": 200, "rate": 0, "wilson95": { "lo": 0, "hi": 0.018846005918320894 } }, "min": 0.02, "max": 0.61 } }, "per_style_clean_flags": { "vancouver": { "flagged": 0, "n": 231, "rate": 0, "wilson95": { "lo": 0, "hi": 0.016358260206028232 } }, "apa7": { "flagged": 0, "n": 231, "rate": 0, "wilson95": { "lo": 0, "hi": 0.016358260206028232 } }, "ieee": { "flagged": 0, "n": 231, "rate": 0, "wilson95": { "lo": 0, "hi": 0.016358260206028232 } }, "nature": { "flagged": 0, "n": 231, "rate": 0, "wilson95": { "lo": 0, "hi": 0.016358260206028232 } }, "chicago-ad": { "flagged": 0, "n": 230, "rate": 0, "wilson95": { "lo": 0, "hi": 0.01642821465470643 } } } }, "v2": { "read_once_on": "2026-10-13", "purpose": "lift rate at 0.8 (pre-registered) + flag confirmation", "rows": 1784, "errors": 0, "arms": { "genuine_clean": { "n": 1136, "flagged_below_0_5": { "flagged": 0, "n": 1136, "rate": 0, "wilson95": { "lo": 0, "hi": 0.0033702928547264814 } }, "lift_eligible_at_or_above_0_8": { "flagged": 1135, "n": 1136, "rate": 0.9991197183098591, "wilson95": { "lo": 0.9950304816193596, "hi": 0.9998445957598134 } }, "min": 0.77, "max": 0.99 }, "genuine_translation": { "n": 148, "flagged_below_0_5": { "flagged": 0, "n": 148, "rate": 0, "wilson95": { "lo": 0, "hi": 0.02530004952529478 } }, "lift_eligible_at_or_above_0_8": { "flagged": 147, "n": 148, "rate": 0.9932432432432432, "wilson95": { "lo": 0.9627220296906855, "hi": 0.9988062998316587 } }, "min": 0.75, "max": 0.98 }, "genuine_paraphrase": { "n": 200, "flagged_below_0_5": { "flagged": 0, "n": 200, "rate": 0, "wilson95": { "lo": 0, "hi": 0.018846005918320894 } }, "lift_eligible_at_or_above_0_8": { "flagged": 182, "n": 200, "rate": 0.91, "wilson95": { "lo": 0.8622332538615105, "hi": 0.9423130212854665 } }, "min": 0.64, "max": 0.98 }, "fab_real_authors": { "n": 300, "flagged_below_0_5": { "flagged": 300, "n": 300, "rate": 1, "wilson95": { "lo": 0.9873565700022643, "hi": 0.9999999999999999 } }, "lift_eligible_at_or_above_0_8": { "flagged": 0, "n": 300, "rate": 0, "wilson95": { "lo": 0, "hi": 0.012643429997735661 } }, "min": 0.02, "max": 0.41 } }, "per_style_clean_flags": { "vancouver": { "flagged": 0, "n": 228, "rate": 0, "wilson95": { "lo": 0, "hi": 0.016569933954907145 } }, "apa7": { "flagged": 0, "n": 227, "rate": 0, "wilson95": { "lo": 0, "hi": 0.016641714491668746 } }, "ieee": { "flagged": 0, "n": 227, "rate": 0, "wilson95": { "lo": 0, "hi": 0.016641714491668746 } }, "nature": { "flagged": 0, "n": 227, "rate": 0, "wilson95": { "lo": 0, "hi": 0.016641714491668746 } }, "chicago-ad": { "flagged": 0, "n": 227, "rate": 0, "wilson95": { "lo": 0, "hi": 0.016641714491668746 } } }, "disjoint_from": ["v1"] } }, "notes": [ "Both draws are burned for threshold tuning: their sweeps were read.", "A fabrication on real authors always corroborates by construction, so its lift-eligible share IS the wrong-lift rate.", "v2 worker report's summary line 'Total Jev calls made in this run: 0' is a counter defect; raw.jsonl holds 1,784 responses, one per row." ] }