187 lines
11 KiB
Python
187 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""WS5 (г) verification — deterministic defect-class checkers.
|
||
|
||
For each checker: (a) POSITIVES — the known empirical defects (exp15 §7.9 / §2 traps) must be
|
||
caught; (b) FALSE-POSITIVE rate — firing on the FULL 25-chapter clean ru final. A checker that
|
||
flags a notable share of clean text does not land.
|
||
|
||
Corpus: rerun/records.json {source, final} per chunk (25 chapters). The prod FP measurement
|
||
(after build) runs through `tmctl export` (invariant #8); the rerun store is schema v7 vs HEAD
|
||
v8 (read-only cannot migrate), and exportNormalize is a cosmetic width-fold that does not alter
|
||
the words these checkers key on, so records.json final is a faithful $0 pre-build FP corpus.
|
||
|
||
$0: deterministic, no LLM, no network. Reuses eval/exp15/q4a_traps rule_b1/rule_b2 machinery.
|
||
Writes /home/ubuntu/books/gu-zhenren/design11/ws5_checkers_verify.json (OUT of git).
|
||
"""
|
||
import json, os, sys, re
|
||
|
||
sys.path.insert(0, "/home/ubuntu/projects/textmachine/eval/exp15")
|
||
import q4a_traps as Q # rule_b1, rule_b2, sentence_with, LOCATOR
|
||
|
||
RECORDS = "/home/ubuntu/books/gu-zhenren/rerun/records.json"
|
||
OUT = "/home/ubuntu/books/gu-zhenren/design11/ws5_checkers_verify.json"
|
||
|
||
# ---------- DC-1: 时辰 (double-hour) unit checker ----------
|
||
_CN_NUM = {"一":1,"二":2,"两":2,"三":3,"四":4,"五":5,"六":6,"七":7,"八":8,"九":9,"十":10}
|
||
def cn_num(s):
|
||
if s.isdigit():
|
||
return int(s)
|
||
return _CN_NUM.get(s)
|
||
_SHICHEN = re.compile(r"([0-9一二两三四五六七八九十])\s*个?\s*时辰")
|
||
_RU_HOURS = re.compile(r"(\d+|один|два|двух|три|трёх|трех|четыре|пять|шесть)\s+час")
|
||
def shichen_checker(src, tgt):
|
||
m = _SHICHEN.search(src)
|
||
if not m:
|
||
return None
|
||
n = cn_num(m.group(1))
|
||
if n is None:
|
||
return None
|
||
expected_hours = n * 2 # 1 时辰 = 2 modern hours
|
||
# find an hours rendering in target; flag ONLY on an explicit mismatch (count rendered as hours).
|
||
# A paraphrase with no explicit hours count is a valid rendering, NOT a defect (refinement: the
|
||
# "no hours found" branch false-flagged ch3/chunk0 at 1.8% — dropped).
|
||
hm = _RU_HOURS.search(tgt)
|
||
if not hm:
|
||
return {"flag": False}
|
||
tok = hm.group(1)
|
||
ru_num = {"один":1,"два":2,"двух":2,"три":3,"трёх":3,"трех":3,"четыре":4,"пять":5,"шесть":6}.get(tok)
|
||
if tok.isdigit():
|
||
ru_num = int(tok)
|
||
if ru_num == n and ru_num != expected_hours:
|
||
return {"flag": True, "reason": f"{n}个时辰 rendered as {ru_num} часов (count) instead of ~{expected_hours}"}
|
||
return {"flag": False}
|
||
|
||
# ---------- DC-2: number-scale magnitude checker (千万 / 数十万 + b1/b2 fractions) ----------
|
||
_MAG = [("千万", 10_000_000, r"(десят\w* миллион|10\s*000\s*000|10000000)"),
|
||
("数十万", 100_000, r"(сотн\w* тысяч|нескольк\w* сот\w* тысяч|[1-9]00\s*000)")]
|
||
def magnitude_checker(src, tgt):
|
||
flags = []
|
||
for token, value, ok_re in _MAG:
|
||
if token in src:
|
||
if not re.search(ok_re, tgt, re.I):
|
||
# positive iff a WRONG smaller magnitude is rendered
|
||
if token == "千万" and re.search(r"тысяч", tgt) and not re.search(r"миллион", tgt):
|
||
flags.append(f"{token}=10M rendered as 'тысячи' (10000x under)")
|
||
elif token == "数十万" and re.search(r"десятк\w* тысяч", tgt):
|
||
flags.append(f"{token}~several×100k rendered as 'десятки тысяч' (10x under)")
|
||
# fraction/percent via q4a rules on the located sentence
|
||
for cls, rule in (("b1", Q.rule_b1), ("b2", Q.rule_b2)):
|
||
loc = Q.LOCATOR.get(cls)
|
||
if loc:
|
||
sent = Q.sentence_with(tgt, loc)
|
||
if sent:
|
||
v = rule(sent)
|
||
if v == "fail":
|
||
flags.append(f"{cls}: numeric magnitude/fraction inverted in '{sent[:40]}...'")
|
||
return {"flag": bool(flags), "reasons": flags} if flags else {"flag": False}
|
||
|
||
# ---------- DC-6: register-lexicon negative-list ----------
|
||
NEG_LIST = {"терем", "терема", "тереме", "теремом", "терему", "теремах"} # сказочно-русский регистр в сянься
|
||
def register_checker(tgt):
|
||
low = tgt.lower()
|
||
hits = sorted({w for w in NEG_LIST if re.search(r"\b" + re.escape(w) + r"\b", low)})
|
||
return {"flag": bool(hits), "hits": hits} if hits else {"flag": False}
|
||
|
||
# ---------- DC-5: verse / allusion detector (source-side, structural) ----------
|
||
# Poetic parallel lines: >=2 short clauses split by CJK comma with near-equal Han length, or
|
||
# known couplet markers; flag when target renders them as flat prose (no line breaks / all one sentence).
|
||
# A loose "short clauses" heuristic false-fires on 73.7% of prose (measured). Landable version
|
||
# requires BOTH structural regularity (>=3 clauses of EQUAL Han length 4-7) AND a poetic-register
|
||
# marker (classical imagery lexeme), so ordinary comma-listing prose does not trip it.
|
||
_VERSE_LEX = set("落日青丝暮雪朝明月清风山河湖海霜霞烟雨云天地花酒诗词赋")
|
||
def verse_checker(src, strict=True):
|
||
for line in src.split("\n"):
|
||
clauses = [c.strip() for c in re.split(r"[，、；]", line) if c.strip()]
|
||
if len(clauses) < 3:
|
||
continue
|
||
hlens = [sum(1 for ch in c if "一" <= ch <= "鿿") for c in clauses]
|
||
if not strict:
|
||
if len([h for h in hlens if 3 <= h <= 8]) >= 2 and max(hlens) <= 8:
|
||
return {"flag": True, "reason": f"loose: {len(clauses)} short clauses"}
|
||
continue
|
||
# strict: >=3 clauses of the SAME length in [4,7] (regular meter) AND a poetic lexeme present
|
||
from collections import Counter
|
||
common_len, cnt = Counter([h for h in hlens if 4 <= h <= 7]).most_common(1)[0] if any(4 <= h <= 7 for h in hlens) else (0, 0)
|
||
regular = cnt >= 3
|
||
lex = any(ch in _VERSE_LEX for ch in line)
|
||
if regular and lex:
|
||
return {"flag": True, "reason": f"strict verse: {cnt} clauses len={common_len} + poetic lexeme"}
|
||
return {"flag": False}
|
||
|
||
# ---------- DC-3: gender-enforce (hidden character before reveal) ----------
|
||
# 白凝冰 = hidden; before until_ch the target must NOT use female surface forms for this character.
|
||
HIDDEN_NAME_RU = re.compile(r"Бай\s+Нинбин|Бай\s+Нин\s*Бин|白凝冰")
|
||
_FEMALE_FORM = re.compile(r"\b(она|её|ей|ею|неё)\b|\b\w+(ла|лась)\b")
|
||
def gender_hidden_checker(src, tgt):
|
||
if "白凝冰" not in src and not HIDDEN_NAME_RU.search(tgt):
|
||
return None
|
||
# flag if a female surface form co-occurs near the hidden character (pre-reveal window)
|
||
if HIDDEN_NAME_RU.search(tgt):
|
||
window = tgt
|
||
if re.search(r"\b(она|её|ей|неё)\b", window):
|
||
return {"flag": True, "reason": "female pronoun for gender:hidden character before reveal"}
|
||
return {"flag": False}
|
||
|
||
def main():
|
||
recs = json.load(open(RECORDS))
|
||
n_chunks = len(recs)
|
||
|
||
# ---- (a) POSITIVES: synthetic trap fixtures from exp15 §7.9 / §2 (checker must catch) ----
|
||
positives = {
|
||
"DC1_shichen": shichen_checker("他闭关了三个时辰。", "Он затворился на три часа."), # want flag: 三时辰=6h, rendered "три часа"
|
||
"DC2_qianwan": magnitude_checker("有千万条蛊虫。", "Там были тысячи гу-червей."), # want flag: 千万=10M as "тысячи"
|
||
"DC2_shushiwan": magnitude_checker("聚集了数十万人。", "Собрались десятки тысяч человек."), # want flag: 数十万 as "десятки тысяч"
|
||
"DC6_terem": register_checker("Он вошёл в высокий терем."), # want flag: терем
|
||
# DC5: the loose line-profile detector CATCHES the trap, but at 74% FP (measured below) it is
|
||
# NOT landable as a gate; the strict version drops FP to 42% but LOSES this positive =>
|
||
# deterministic verse detection is unviable -> DC5 is a pack-curated locus + owner decision.
|
||
"DC5_verse": verse_checker("青山落日，长河渐隐，朝如青丝暮成雪。", strict=False), # caught (loose only)
|
||
"DC3_gender": gender_hidden_checker("白凝冰走了过来。", "Бай Нинбин подошла, её взгляд был холоден."), # want flag: female pre-reveal
|
||
}
|
||
positives_caught = {k: (v is not None and v.get("flag", False)) for k, v in positives.items()}
|
||
|
||
# ---- (b) FALSE-POSITIVE rate on the 25-ch clean final ----
|
||
fp = {c: 0 for c in ("DC1_shichen", "DC2_magnitude", "DC6_register", "DC5_verse_strict", "DC5_verse_loose", "DC3_gender")}
|
||
fp_detail = {c: [] for c in fp}
|
||
for r in recs:
|
||
src = r.get("source", "") or ""
|
||
fin = r.get("final", "") or ""
|
||
ch, ci = r.get("chapter"), r.get("chunk_idx")
|
||
for name, fn, arg in (
|
||
("DC1_shichen", shichen_checker, (src, fin)),
|
||
("DC2_magnitude", magnitude_checker, (src, fin)),
|
||
("DC6_register", register_checker, (fin,)),
|
||
("DC5_verse_strict", lambda s: verse_checker(s, strict=True), (src,)),
|
||
("DC5_verse_loose", lambda s: verse_checker(s, strict=False), (src,)),
|
||
("DC3_gender", gender_hidden_checker, (src, fin)),
|
||
):
|
||
res = fn(*arg)
|
||
if res is not None and res.get("flag"):
|
||
fp[name] += 1
|
||
if len(fp_detail[name]) < 5:
|
||
fp_detail[name].append({"ch": ch, "chunk": ci, "why": res})
|
||
|
||
report = {
|
||
"n_chunks": n_chunks,
|
||
"positives_caught": positives_caught,
|
||
"positives_all_caught": all(positives_caught.values()),
|
||
"fp_fire_counts_on_25ch_final": fp,
|
||
"fp_fire_rate_pct": {k: round(100 * v / n_chunks, 1) for k, v in fp.items()},
|
||
"fp_detail_sample": fp_detail,
|
||
"note": ("FP fires are inspected: DC5 verse and DC6 register are the FP-prone target-side "
|
||
"checkers; DC1/DC2/DC3 are src<->target conversions with structurally low FP. "
|
||
"A checker flagging a notable share of clean text is refined before landing."),
|
||
}
|
||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||
json.dump(report, open(OUT, "w"), ensure_ascii=False, indent=2)
|
||
print(json.dumps({k: report[k] for k in ("n_chunks", "positives_caught", "positives_all_caught",
|
||
"fp_fire_counts_on_25ch_final", "fp_fire_rate_pct")}, ensure_ascii=False, indent=2))
|
||
print("\nFP detail sample:")
|
||
for k, v in report["fp_detail_sample"].items():
|
||
if v:
|
||
print(f" {k}: {v[:2]}")
|
||
print(f"\nVERDICT: positives {'ALL CAUGHT' if report['positives_all_caught'] else 'MISSED SOME'}; "
|
||
f"FP rates on 25-ch final: " + ", ".join(f"{k}={report['fp_fire_rate_pct'][k]}%" for k in fp))
|
||
|
||
if __name__ == "__main__":
|
||
main()
|