247 lines
11 KiB
Python
247 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""$0-агрегация проб консилиума по пре-регистрации research/24 §0.
|
||
|
||
Формы: P0 (топ §C2-3) · P1 (прод coldrun-a) · S1 (ds1) · SC (majority ds1..3) ·
|
||
C3 (majority ds1,glm1,mis1) · C3conf (взвешенный уверенностью) · E1 (улики-первым).
|
||
Метрики: точность против голда · парные сравнения · кластерная связность ·
|
||
B2 (AUC уверенности, устойчивость, сравнимость шкал, канал отказа) · деньги вторым путём.
|
||
"""
|
||
import collections
|
||
import json
|
||
import re
|
||
import unicodedata
|
||
from pathlib import Path
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
RAW = Path.home() / "books" / "gu-zhenren" / "bank-arbitration"
|
||
GOLD = [json.loads(l) for l in open(HERE / "gold" / "gold.jsonl", encoding="utf-8")]
|
||
NODST = "⟦TM-NO-DST⟧"
|
||
WORD2NUM = {"точно": 95, "скорее да": 70, "не уверен": 40, "не знаю": 10}
|
||
FIELD = re.compile(r"\t| {2,}|\s*\|\s*")
|
||
|
||
|
||
def ntgt(s: str) -> str:
|
||
s = unicodedata.normalize("NFKC", s or "").casefold().replace("ё", "е")
|
||
for ch in "«»\"'":
|
||
s = s.replace(ch, "")
|
||
return " ".join(s.split())
|
||
|
||
|
||
def eq(a: str, b: str) -> bool:
|
||
return ntgt(a) == ntgt(b) and ntgt(a) != ""
|
||
|
||
|
||
def parse_pass(tag: str) -> dict:
|
||
"""src -> (dst, conf|None). Терпимый к «третье поле съедено в dst»."""
|
||
out = {}
|
||
cost = 0.0
|
||
for f in sorted(RAW.glob(f"{tag}-b*.json")):
|
||
rec = json.load(open(f, encoding="utf-8"))
|
||
cost += rec["cost_usd"]
|
||
for ln in rec["content"].split("\n"):
|
||
ln = ln.strip()
|
||
if not ln or ln.startswith("#"):
|
||
continue
|
||
parts = [p.strip() for p in FIELD.split(ln) if p.strip()]
|
||
if len(parts) < 2:
|
||
continue
|
||
src = parts[0]
|
||
conf = None
|
||
rest = parts[1:]
|
||
last = rest[-1].casefold() if rest else ""
|
||
if len(rest) >= 2:
|
||
if re.fullmatch(r"\d{1,3}", rest[-1]):
|
||
conf = int(rest[-1])
|
||
rest = rest[:-1]
|
||
elif last in WORD2NUM:
|
||
conf = WORD2NUM[last]
|
||
rest = rest[:-1]
|
||
dst = " ".join(rest)
|
||
if src not in out:
|
||
out[src] = (dst, conf)
|
||
return {"map": out, "cost": cost}
|
||
|
||
|
||
def majority(votes: list[str], fallback: str) -> str:
|
||
cnt = collections.Counter(ntgt(v) for v in votes if v and v != NODST)
|
||
if not cnt:
|
||
return fallback
|
||
top, n = cnt.most_common(1)[0]
|
||
if n >= 2:
|
||
for v in votes: # вернуть сырую форму первого голоса за победителя
|
||
if ntgt(v) == top:
|
||
return v
|
||
return fallback
|
||
|
||
|
||
def conf_weighted(pairs: list[tuple[str, int | None]], fallback: str) -> str:
|
||
w = collections.defaultdict(float)
|
||
raw = {}
|
||
for dst, conf in pairs:
|
||
if not dst or dst == NODST:
|
||
continue
|
||
w[ntgt(dst)] += (conf if conf is not None else 50)
|
||
raw.setdefault(ntgt(dst), dst)
|
||
if not w:
|
||
return fallback
|
||
return raw[max(w, key=lambda k: (w[k], k))]
|
||
|
||
|
||
def auc(scores_pos, scores_neg):
|
||
"""Ранговая AUC (Mann-Whitney), плюс bootstrap 90% CI."""
|
||
import random
|
||
def one(p, n):
|
||
if not p or not n:
|
||
return None
|
||
wins = ties = 0
|
||
for a in p:
|
||
for b in n:
|
||
if a > b:
|
||
wins += 1
|
||
elif a == b:
|
||
ties += 1
|
||
return (wins + ties / 2) / (len(p) * len(n))
|
||
point = one(scores_pos, scores_neg)
|
||
if point is None:
|
||
return None, (None, None)
|
||
rng = random.Random(24)
|
||
bs = []
|
||
for _ in range(2000):
|
||
p = [rng.choice(scores_pos) for _ in scores_pos]
|
||
n = [rng.choice(scores_neg) for _ in scores_neg]
|
||
v = one(p, n)
|
||
if v is not None:
|
||
bs.append(v)
|
||
bs.sort()
|
||
return point, (bs[int(0.05 * len(bs))], bs[int(0.95 * len(bs))])
|
||
|
||
|
||
def main():
|
||
inb = [r for r in GOLD if r["in_bank"]]
|
||
gold_by_src = {r["bank_src"]: r for r in inb}
|
||
|
||
passes = {t: parse_pass(t) for t in ["ds1", "ds2", "ds3", "dsw", "glm1", "mis1"]}
|
||
for t, p in passes.items():
|
||
print(f"pass {t}: {len(p['map'])} строк, ${p['cost']:.6f}")
|
||
total = sum(p["cost"] for p in passes.values())
|
||
print(f"деньги проб B (второй путь, из сырья): ${total:.6f}")
|
||
|
||
# формы
|
||
forms = {}
|
||
for src, g in gold_by_src.items():
|
||
variants = g.get("variants") or []
|
||
p0 = variants[0][0] if variants else ""
|
||
ds = [passes[t]["map"].get(src, ("", None)) for t in ("ds1", "ds2", "ds3")]
|
||
glm = passes["glm1"]["map"].get(src, ("", None))
|
||
mis = passes["mis1"]["map"].get(src, ("", None))
|
||
s1 = ds[0][0]
|
||
sc = majority([d[0] for d in ds], fallback=s1)
|
||
c3votes = [ds[0][0], glm[0], mis[0]]
|
||
c3 = majority(c3votes, fallback=(p0 if any(eq(v, p0) for v in c3votes) else s1))
|
||
c3conf = conf_weighted([ds[0], glm, mis], fallback=s1)
|
||
contested = (g.get("spread", 0) >= 2 and len(variants) >= 2 and variants[0][1] - variants[1][1] <= 1) \
|
||
or (variants and variants[0][1] == 1)
|
||
e1 = c3 if contested else p0
|
||
forms[src] = dict(P0=p0, P1=g.get("bank_dst", ""), S1=s1, SC=sc, C3=c3, C3conf=c3conf, E1=e1,
|
||
contested=contested)
|
||
|
||
names = ["P0", "P1", "S1", "SC", "C3", "C3conf", "E1"]
|
||
print("\n=== Точность против голда (45) ===")
|
||
accs = {}
|
||
for n in names:
|
||
ok = [s for s, f in forms.items() if eq(f[n], gold_by_src[s]["gold_dst"])]
|
||
accs[n] = set(ok)
|
||
print(f" {n}: {len(ok)}/45")
|
||
print(f" спорных термов (E1 зовёт совет): {sum(1 for f in forms.values() if f['contested'])}/45")
|
||
|
||
print("\n=== Парные сравнения (A-прав-B-неправ / B-прав-A-неправ) ===")
|
||
for a, b in [("S1", "P0"), ("SC", "P0"), ("C3", "P0"), ("C3conf", "P0"), ("E1", "P0"),
|
||
("C3", "SC"), ("C3", "S1"), ("SC", "S1"), ("C3conf", "C3"), ("E1", "C3"), ("S1", "P1"), ("C3", "P1")]:
|
||
x = len(accs[a] - accs[b]); y = len(accs[b] - accs[a])
|
||
print(f" {a} vs {b}: +{x} / -{y} (net {x - y:+d})")
|
||
|
||
print("\n=== Единицы: где C3 разошёлся с голдом, а S1/SC были правы (и наоборот) ===")
|
||
for s in sorted((accs["S1"] | accs["SC"]) - accs["C3"]):
|
||
f = forms[s]
|
||
print(f" C3 потерял {s}: gold={gold_by_src[s]['gold_dst']!r} S1={f['S1']!r} C3={f['C3']!r}")
|
||
for s in sorted(accs["C3"] - (accs["S1"] | accs["SC"])):
|
||
f = forms[s]
|
||
print(f" C3 добыл {s}: gold={gold_by_src[s]['gold_dst']!r} S1={f['S1']!r} C3={f['C3']!r}")
|
||
|
||
# кластеры
|
||
FAMS = {
|
||
"古月-семья": [s for s in forms if s.startswith("古月")],
|
||
"转-серия": [s for s in forms if s.endswith("转") and len(s) == 2],
|
||
"等-серия": [s for s in forms if s.endswith("等") and len(s) == 2],
|
||
}
|
||
NUMERALS = {"первый", "второй", "третий", "четвертый", "пятый", "шестой", "седьмой", "восьмой", "девятый"}
|
||
|
||
def kind_of(fam: str, v: str) -> str:
|
||
"""Конвенция строки: 古月 — слитно/раздельно ГДЕ БЫ ни стояла фамилия; серии — не-числительная голова."""
|
||
s = ntgt(v)
|
||
if not s:
|
||
return "<пусто>"
|
||
if fam == "古月-семья":
|
||
if "гу юэ" in s:
|
||
return "раздельно"
|
||
if "гуюэ" in s:
|
||
return "слитно"
|
||
return "иное:" + s
|
||
words = [w for w in s.replace("«", " ").replace("»", " ").split() if w not in NUMERALS]
|
||
# конвенция серии = ГОЛОВА (родовое слово) — первое не-числительное слово («уровень цзя» и
|
||
# «уровень и» делят конвенцию «уровень»; модификатор серии различен по построению)
|
||
return words[0] if words else s
|
||
|
||
print("\n=== Кластерная связность (одна конвенция на семью?) ===")
|
||
for fam, members in FAMS.items():
|
||
if len(members) < 2:
|
||
continue
|
||
print(f" {fam} ({len(members)}):")
|
||
for n in ["P1", "S1", "SC", "C3", "C3conf", "E1"]:
|
||
kinds = {kind_of(fam, forms[m][n]) for m in members}
|
||
coherent = len(kinds) == 1
|
||
print(f" {n}: {'КОНСИСТЕНТНА (' + next(iter(kinds)) + ')' if coherent else 'ХИМЕРА ' + str(sorted(kinds))}")
|
||
for m in members:
|
||
print(f" {m}: gold={gold_by_src[m]['gold_dst']!r} S1={forms[m]['S1']!r} glm={passes['glm1']['map'].get(m,('',))[0]!r} mis={passes['mis1']['map'].get(m,('',))[0]!r}")
|
||
|
||
# B2: уверенность
|
||
print("\n=== B2: уверенность ===")
|
||
for t in ["ds1", "glm1", "mis1", "dsw"]:
|
||
pos, neg = [], []
|
||
for s, g in gold_by_src.items():
|
||
dst, conf = passes[t]["map"].get(s, ("", None))
|
||
if conf is None or not dst or dst == NODST:
|
||
continue
|
||
(pos if eq(dst, g["gold_dst"]) else neg).append(conf)
|
||
a, (lo, hi) = auc(pos, neg)
|
||
mean = (sum(pos) + sum(neg)) / max(1, len(pos) + len(neg))
|
||
print(f" {t}: AUC={a if a is None else round(a,3)} CI90=({lo and round(lo,3)},{hi and round(hi,3)}) "
|
||
f"n_prav={len(pos)} n_neprav={len(neg)} средняя conf={round(mean,1)}")
|
||
# устойчивость ds1..ds3
|
||
diffs, flips = [], 0
|
||
n_common = 0
|
||
for s in gold_by_src:
|
||
recs = [passes[t]["map"].get(s) for t in ("ds1", "ds2", "ds3")]
|
||
if any(r is None for r in recs):
|
||
continue
|
||
n_common += 1
|
||
cs = [r[1] for r in recs if r[1] is not None]
|
||
if len(cs) >= 2:
|
||
diffs.append(max(cs) - min(cs))
|
||
if len({ntgt(r[0]) for r in recs}) > 1:
|
||
flips += 1
|
||
diffs.sort()
|
||
med = diffs[len(diffs) // 2] if diffs else None
|
||
print(f" устойчивость ds1..ds3: медиана размаха conf={med} п., флипов ответа {flips}/{n_common}")
|
||
# отказы
|
||
for t in passes:
|
||
nod = sum(1 for d, _ in passes[t]["map"].values() if d == NODST)
|
||
print(f" {t}: ⟦TM-NO-DST⟧ = {nod}")
|
||
|
||
(HERE / "forms_dump.json").write_text(
|
||
json.dumps({s: forms[s] for s in sorted(forms)}, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print("\nформы записаны в forms_dump.json")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|