{
  "name": "verifier measurement v7 — semantic title check on/off (sealed, two passes)",
  "sealed": true,
  "read_once_on": "2026-10-12",
  "protocol": "docs/VERIFIER_EVAL_v7_SEMANTIC_PROTOCOL.md",
  "verify_version": "2026-08-26",
  "corpus": {
    "file": "test/fixtures/verify/validation-set-v7.json",
    "seed": 20261012,
    "entries": 1921,
    "composition": {
      "clean": {
        "total": 1721
      },
      "fab:real-doi-fake-title": {
        "total": 40
      },
      "fab:real-doi-near-miss-title": {
        "total": 100
      },
      "fab:real-doi-topic-swap-title": {
        "total": 60
      }
    },
    "clean_target": 2000,
    "clean_drawn": 1721,
    "clean_shortfall_reason": "citing-work pool exhausted at 1,769 candidate references (pull-references --citing-n=1500); no row excluded after the draw",
    "clean_arm_identifier_types": ["doi"],
    "disjoint_from": [
      "v2",
      "v2-blind",
      "v3-blind",
      "author-axis-v1",
      "v4-blind",
      "v5",
      "v6",
      "v7-tuning"
    ]
  },
  "http": {
    "off_200": 1921,
    "on_200": 1921,
    "rows": 1921
  },
  "judge": {
    "vendor": "TypeSafe System One",
    "model": "jev-1.13.0",
    "question_version": "2026-10-12"
  },
  "threshold": {
    "same_work_below": 0.5,
    "polarity_flip_above": 0.5,
    "pre_registered": true
  },
  "default_path": {
    "clean": {
      "flagged": 36,
      "n": 1721,
      "rate": 0.020918070889018012,
      "wilson95": {
        "lo": 0.01514746719331609,
        "hi": 0.028822715344025307
      },
      "high_confidence": {
        "flagged": 13,
        "n": 1721,
        "rate": 0.007553747821034283,
        "wilson95": {
          "lo": 0.004419729014280922,
          "hi": 0.012881337935866849
        }
      }
    },
    "fab:real-doi-fake-title": {
      "flagged": 40,
      "n": 40,
      "rate": 1,
      "wilson95": {
        "lo": 0.9123754607496077,
        "hi": 1
      },
      "high_confidence": {
        "flagged": 34,
        "n": 40,
        "rate": 0.85,
        "wilson95": {
          "lo": 0.7092737396097997,
          "hi": 0.9293890829149256
        }
      }
    },
    "fab:real-doi-near-miss-title": {
      "flagged": 23,
      "n": 100,
      "rate": 0.23,
      "wilson95": {
        "lo": 0.15843154746876406,
        "hi": 0.32154564856827694
      },
      "high_confidence": {
        "flagged": 0,
        "n": 100,
        "rate": 0,
        "wilson95": {
          "lo": 0,
          "hi": 0.03699480747600191
        }
      }
    },
    "fab:real-doi-topic-swap-title": {
      "flagged": 0,
      "n": 60,
      "rate": 0,
      "wilson95": {
        "lo": 0,
        "hi": 0.06017393047793289
      },
      "high_confidence": {
        "flagged": 0,
        "n": 60,
        "rate": 0,
        "wilson95": {
          "lo": 0,
          "hi": 0.06017393047793289
        }
      }
    }
  },
  "end_to_end_on": {
    "clean": {
      "flagged": 44,
      "n": 1721,
      "rate": 0.02556653108657757,
      "wilson95": {
        "lo": 0.019099702229198656,
        "hi": 0.03414669442542748
      }
    },
    "fab:real-doi-fake-title": {
      "flagged": 40,
      "n": 40,
      "rate": 1,
      "wilson95": {
        "lo": 0.9123754607496077,
        "hi": 1
      }
    },
    "fab:real-doi-near-miss-title": {
      "flagged": 99,
      "n": 100,
      "rate": 0.99,
      "wilson95": {
        "lo": 0.9455124752390653,
        "hi": 0.9982326134344526
      }
    },
    "fab:real-doi-topic-swap-title": {
      "flagged": 58,
      "n": 60,
      "rate": 0.9666666666666667,
      "wilson95": {
        "lo": 0.8863601247107462,
        "hi": 0.9908108735098498
      }
    }
  },
  "opt_in_delta_on_string_matched": {
    "clean": {
      "flagged": 8,
      "n": 1685,
      "rate": 0.004747774480712166,
      "wilson95": {
        "lo": 0.002407684871580856,
        "hi": 0.009340959879945869
      },
      "judge_applied": 1685
    },
    "fab:real-doi-fake-title": {
      "flagged": 0,
      "n": 0,
      "rate": null,
      "wilson95": null,
      "judge_applied": 0
    },
    "fab:real-doi-near-miss-title": {
      "flagged": 76,
      "n": 77,
      "rate": 0.987012987012987,
      "wilson95": {
        "lo": 0.9300363315482386,
        "hi": 0.9977038418278451
      },
      "judge_applied": 77
    },
    "fab:real-doi-topic-swap-title": {
      "flagged": 58,
      "n": 60,
      "rate": 0.9666666666666667,
      "wilson95": {
        "lo": 0.8863601247107462,
        "hi": 0.9908108735098498
      },
      "judge_applied": 60
    }
  },
  "which_question_fired": {
    "same_work_only": 14,
    "polarity_only": 6,
    "both": 122
  },
  "clean_flagged_on": [
    {
      "id": "clean-ref-blind-10-1007-bf01417857",
      "same_work": 0.41,
      "polarity_flip": 0.07
    },
    {
      "id": "clean-ref-blind-10-2196-jmir-7087",
      "same_work": 0.91,
      "polarity_flip": 0.74
    },
    {
      "id": "clean-ref-blind-10-1177-1538574411419528",
      "same_work": 0.27,
      "polarity_flip": 0.07
    },
    {
      "id": "clean-ref-blind-10-1108-jadee-10-2012-0025",
      "same_work": 0.45,
      "polarity_flip": 0.07
    },
    {
      "id": "clean-ref-blind-10-1016-s1872-2067-22-64139-1",
      "same_work": 0.37,
      "polarity_flip": 0.37
    },
    {
      "id": "clean-ref-blind-10-1212-01-wnl-0000066683-34093-e2",
      "same_work": 0.33,
      "polarity_flip": 0.06
    },
    {
      "id": "clean-ref-blind-10-1097-sla-0b013e31825d355f",
      "same_work": 0.21,
      "polarity_flip": 0.45
    },
    {
      "id": "clean-ref-blind-10-1182-blood-2005-07-2972",
      "same_work": 0.4,
      "polarity_flip": 0.33
    }
  ],
  "fabrication_still_matched_on": [
    {
      "id": "fab-nearmiss-10-1002-jha2-602",
      "stratum": "fab:real-doi-near-miss-title",
      "same_work": 0.84,
      "polarity_flip": 0.18
    },
    {
      "id": "fab-topicswap-10-1021-acs-inorgchem-1c00745",
      "stratum": "fab:real-doi-topic-swap-title",
      "same_work": 0.6,
      "polarity_flip": 0.35
    },
    {
      "id": "fab-topicswap-10-1016-j-mtla-2018-08-012",
      "stratum": "fab:real-doi-topic-swap-title",
      "same_work": 0.64,
      "polarity_flip": 0.42
    }
  ],
  "sources": {
    "off": "scripts/typesafe-probe/v7/off-rows.json",
    "on": "scripts/typesafe-probe/v7/on-rows.json",
    "report": "scripts/typesafe-probe/v7/report.md"
  }
}