#!/usr/bin/env python3 """Скоринг арма Q4b по пре-регу §1.4–1.5 (`docs/experiments/19-editor-contract-q4b.md`). Все метрики детерминированные, $0. Судейского рига нет и он вне скоупа. """ from __future__ import annotations import json import re import sys import statistics as st from pathlib import Path REPO = Path("/home/ubuntu/projects/textmachine") RAW = Path.home() / "books" / "gu-zhenren" / "bank-arbitration" HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) sys.path.insert(0, str(REPO / "eval" / "bank_arbitration")) import editor_wire_probe as P # noqa: E402 from inject_probe import pick_windows # noqa: E402 from offblock import offblock_cells, norm # noqa: E402 WINS = pick_windows() REPS = (1, 2, 3) def out_text(tag: str) -> str | None: """Финальный текст арма: для диффа — ПРИМЕНЁННЫЙ, для full-regen — content.""" ap = RAW / f"{tag}.applied.json" if ap.exists(): return json.load(open(ap, encoding="utf-8"))["text"] f = RAW / f"{tag}.json" if not f.exists(): return None return json.load(open(f, encoding="utf-8"))["content"] def arm_tags(arm: str, reps=REPS) -> dict[int, list[str]]: """{окно: [теги реплик]}, существующие. A1/A5 пробы 18 = реплика r1 армов F1/F5.""" legacy = {"F1": "edp-A1-w{k}", "F5": "edp-A5-w{k}"} out = {} for k in range(6): tags = [] if arm in legacy and (RAW / (legacy[arm].format(k=k) + ".json")).exists(): tags.append(legacy[arm].format(k=k)) for r in reps: t = f"ec-{arm}-w{k}-r{r}" if (RAW / f"{t}.json").exists(): tags.append(t) if tags: out[k] = tags return out # --- 1. формат-комплаенс ------------------------------------------------------------------- def compliance(): print("=== 1. ФОРМАТ-КОМПЛАЕНС (стоп-гейт Q4b: ≥0.90) ===") for arm in ("D1", "D2"): ops = ap = nc = mal = 0 rej = dict(notfound=0, ambiguous=0, overlap=0, empty=0) n = 0 for k, tags in arm_tags(arm).items(): for t in tags: f = RAW / f"{t}.applied.json" if not f.exists(): continue d = json.load(open(f, encoding="utf-8")) n += 1 ops += d["ops_total"]; ap += d["applied"] nc += d["no_change"]; mal += d["malformed"] for key in rej: rej[key] += d[f"rejected_{key}"] c = ap / ops if ops else 1.0 print(f" {arm}: вызовов {n} · операций {ops} · применено {ap} · " f"КОМПЛАЕНС {c:.3f} {'✓' if c >= 0.90 else '✗'}") print(f" reject: не найден {rej['notfound']} · неоднозначен {rej['ambiguous']} · " f"пересечение {rej['overlap']} · пустой {rej['empty']} | " f"NO_CHANGE {nc} · малформед {mal}") print(f" правок на вызов: медиана {ops/n:.1f}" if n else "") # --- 2/3. канон в блоке и вне блока -------------------------------------------------------- def canon_inblock(): print("\n=== 2. КАНОН В БЛОКЕ (21 клетка; знаменатель пробы 18) ===") base = {} for k, (_, buf, terms) in enumerate(WINS): d = norm(P.draft_of(k)) base[k] = [t for t in terms if t != "元海"] dn = sum(1 for k in base for t in base[k] if P.hits(norm(P.draft_of(k)), t)[0]) print(f" черновик: {dn}/21") for arm in ("D1", "F1"): per = [] for k, tags in arm_tags(arm).items(): pass rows = {} for k in range(6): for i, t in enumerate(arm_tags(arm).get(k, [])): o = out_text(t) if o is None: continue rows.setdefault(i, [0, 0]) n = base[k] rows[i][0] += sum(1 for x in n if P.hits(norm(o), x)[0]) rows[i][1] += len(n) vals = [f"{v[0]}/{v[1]}" for v in rows.values()] nums = [v[0] for v in rows.values()] print(f" {arm}: розыгрыши {' · '.join(vals)} медиана {st.median(nums):.0f}/21" if nums else f" {arm}: нет данных") def canon_offblock(): print("\n=== 3. КАНОН ВНЕ БЛОКА (41 клетка; §0.1) ===") cells = offblock_cells() dn = sum(1 for k, s, d, rx in cells if re.search(rx, norm(P.draft_of(k)))) print(f" черновик: {dn}/{len(cells)}") for arm in ("D1", "F1", "D2", "F5"): rows = {} for k, s, dst, rx in cells: for i, t in enumerate(arm_tags(arm).get(k, [])): o = out_text(t) if o is None: continue rows.setdefault(i, [0, 0]) rows[i][1] += 1 rows[i][0] += bool(re.search(rx, norm(o))) nums = [v[0] for v in rows.values()] if nums: print(f" {arm}: розыгрыши {' · '.join(f'{v[0]}/{v[1]}' for v in rows.values())}" f" медиана {st.median(nums):.0f}") # --- 4. фикс-рейт посадок ------------------------------------------------------------------ def plants(): print("\n=== 4. ФИКС-РЕЙТ 12 ПОСАДОК (пре-рег: дифф НЕ ХУЖЕ full-regen) ===") for arm in ("D2", "F5"): rows = {} for k in range(6): for i, t in enumerate(arm_tags(arm).get(k, [])): o = out_text(t) if o is None: continue n = norm(o) rows.setdefault(i, {"k1": [0, 0], "k2": [0, 0]}) for cls, old, new, bad_rx, fix_rx in P.DEFECTS[k]: rows[i][cls][1] += 1 rows[i][cls][0] += (not re.search(bad_rx, n)) if rows: for cls in ("k1", "k2"): v = [f"{r[cls][0]}/{r[cls][1]}" for r in rows.values()] print(f" {arm} {cls}: розыгрыши {' · '.join(v)}") # --- 5. атом-omission ---------------------------------------------------------------------- NUMWORDS = ("два", "две", "три", "четыр", "пят", "шест", "сем", "восем", "девят", "десят", "двадцат", "тридцат", "сорок", "пятьдесят", "сто", "тысяч", "пятьсот", "полов") NEG = ("не ", "ни ", "нет", "без ") def omission(): print("\n=== 5. АТОМ-OMISSION (числа + полярность; пре-рег: у диффа должно быть 0) ===") for arm in ("D1", "F1", "D2", "F5"): rows = {} for k in range(6): draft = P.draft_of(k) if arm in ("D1", "F1") else P.planted_draft(k, P.draft_of(k))[0] dnums = set(re.findall(r"\d{2,}", draft)) | {w for w in NUMWORDS if w in draft.lower()} dneg = sum(draft.lower().count(x) for x in NEG) for i, t in enumerate(arm_tags(arm).get(k, [])): o = out_text(t) if o is None: continue onums = set(re.findall(r"\d{2,}", o)) | {w for w in NUMWORDS if w in o.lower()} oneg = sum(o.lower().count(x) for x in NEG) rows.setdefault(i, [0, 0, 0]) rows[i][0] += len(dnums - onums) # исчезнувшие числовые атомы rows[i][1] += max(0, dneg - oneg) # просевшая полярность rows[i][2] += 1 if rows: v = [f"чисел-{r[0]}/полярн-{r[1]}" for r in rows.values()] print(f" {arm}: розыгрыши {' · '.join(v)}") # --- 6. внесённые дефекты + 7. деньги ------------------------------------------------------ def guards_and_money(): print("\n=== 6. ВНЕСЁННЫЕ ДЕФЕКТЫ (форма regression_guard: обвал длины >40%) ===") for arm in ("D1", "F1", "D2", "F5"): drops = tot = 0 for k in range(6): draft = P.draft_of(k) if arm in ("D1", "F1") else P.planted_draft(k, P.draft_of(k))[0] dl = len(re.sub(r"\s", "", draft)) for t in arm_tags(arm).get(k, []): o = out_text(t) if o is None: continue tot += 1 if len(re.sub(r"\s", "", o)) < dl * 0.60: drops += 1 if tot: print(f" {arm}: обвалов длины {drops}/{tot}") print("\n=== 7. ДЕНЬГИ И ВЫХОДНЫЕ ТОКЕНЫ (COGS-ось) ===") for arm in ("D1", "D2", "F1", "F5"): outs, cost, n = [], 0.0, 0 for k in range(6): for t in arm_tags(arm).get(k, []): f = RAW / f"{t}.json" if not f.exists(): continue r = json.load(open(f, encoding="utf-8")) outs.append(r["completion_tokens"]); cost += r["cost_usd"]; n += 1 if n: print(f" {arm}: вызовов {n} · выход медиана {st.median(outs):.0f} ток. " f"(мин {min(outs)} макс {max(outs)}) · ${cost:.6f} · ${cost/n:.6f}/вызов") tot = sum(json.load(open(f, encoding="utf-8"))["cost_usd"] for f in RAW.glob("ec-*.json") if not f.name.endswith(".applied.json")) print(f" ИТОГО проба Q4b: ${tot:.6f} (потолок $0.40)") if __name__ == "__main__": compliance(); canon_inblock(); canon_offblock(); plants(); omission(); guards_and_money()