217 lines
9.7 KiB
Python
217 lines
9.7 KiB
Python
#!/usr/bin/env python3
|
||
"""Скоринг арма Q4b по пре-регу §1.4–1.5 (`docs/experiments/19-editor-contract-q4b.md`).
|
||
Все метрики детерминированные, $0. Судейского рига нет и он вне скоупа.
|
||
"""
|
||
from __future__ import annotations
|
||
import json
|
||
import re
|
||
import sys
|
||
import statistics as st
|
||
from pathlib import Path
|
||
|
||
REPO = Path("/home/ubuntu/projects/textmachine")
|
||
RAW = Path.home() / "books" / "gu-zhenren" / "bank-arbitration"
|
||
HERE = Path(__file__).resolve().parent
|
||
sys.path.insert(0, str(HERE))
|
||
sys.path.insert(0, str(REPO / "eval" / "bank_arbitration"))
|
||
|
||
import editor_wire_probe as P # noqa: E402
|
||
from inject_probe import pick_windows # noqa: E402
|
||
from offblock import offblock_cells, norm # noqa: E402
|
||
|
||
WINS = pick_windows()
|
||
REPS = (1, 2, 3)
|
||
|
||
|
||
def out_text(tag: str) -> str | None:
|
||
"""Финальный текст арма: для диффа — ПРИМЕНЁННЫЙ, для full-regen — content."""
|
||
ap = RAW / f"{tag}.applied.json"
|
||
if ap.exists():
|
||
return json.load(open(ap, encoding="utf-8"))["text"]
|
||
f = RAW / f"{tag}.json"
|
||
if not f.exists():
|
||
return None
|
||
return json.load(open(f, encoding="utf-8"))["content"]
|
||
|
||
|
||
def arm_tags(arm: str, reps=REPS) -> dict[int, list[str]]:
|
||
"""{окно: [теги реплик]}, существующие. A1/A5 пробы 18 = реплика r1 армов F1/F5."""
|
||
legacy = {"F1": "edp-A1-w{k}", "F5": "edp-A5-w{k}"}
|
||
out = {}
|
||
for k in range(6):
|
||
tags = []
|
||
if arm in legacy and (RAW / (legacy[arm].format(k=k) + ".json")).exists():
|
||
tags.append(legacy[arm].format(k=k))
|
||
for r in reps:
|
||
t = f"ec-{arm}-w{k}-r{r}"
|
||
if (RAW / f"{t}.json").exists():
|
||
tags.append(t)
|
||
if tags:
|
||
out[k] = tags
|
||
return out
|
||
|
||
|
||
# --- 1. формат-комплаенс -------------------------------------------------------------------
|
||
def compliance():
|
||
print("=== 1. ФОРМАТ-КОМПЛАЕНС (стоп-гейт Q4b: ≥0.90) ===")
|
||
for arm in ("D1", "D2"):
|
||
ops = ap = nc = mal = 0
|
||
rej = dict(notfound=0, ambiguous=0, overlap=0, empty=0)
|
||
n = 0
|
||
for k, tags in arm_tags(arm).items():
|
||
for t in tags:
|
||
f = RAW / f"{t}.applied.json"
|
||
if not f.exists():
|
||
continue
|
||
d = json.load(open(f, encoding="utf-8"))
|
||
n += 1
|
||
ops += d["ops_total"]; ap += d["applied"]
|
||
nc += d["no_change"]; mal += d["malformed"]
|
||
for key in rej:
|
||
rej[key] += d[f"rejected_{key}"]
|
||
c = ap / ops if ops else 1.0
|
||
print(f" {arm}: вызовов {n} · операций {ops} · применено {ap} · "
|
||
f"КОМПЛАЕНС {c:.3f} {'✓' if c >= 0.90 else '✗'}")
|
||
print(f" reject: не найден {rej['notfound']} · неоднозначен {rej['ambiguous']} · "
|
||
f"пересечение {rej['overlap']} · пустой {rej['empty']} | "
|
||
f"NO_CHANGE {nc} · малформед {mal}")
|
||
print(f" правок на вызов: медиана {ops/n:.1f}" if n else "")
|
||
|
||
|
||
# --- 2/3. канон в блоке и вне блока --------------------------------------------------------
|
||
def canon_inblock():
|
||
print("\n=== 2. КАНОН В БЛОКЕ (21 клетка; знаменатель пробы 18) ===")
|
||
base = {}
|
||
for k, (_, buf, terms) in enumerate(WINS):
|
||
d = norm(P.draft_of(k))
|
||
base[k] = [t for t in terms if t != "元海"]
|
||
dn = sum(1 for k in base for t in base[k] if P.hits(norm(P.draft_of(k)), t)[0])
|
||
print(f" черновик: {dn}/21")
|
||
for arm in ("D1", "F1"):
|
||
per = []
|
||
for k, tags in arm_tags(arm).items():
|
||
pass
|
||
rows = {}
|
||
for k in range(6):
|
||
for i, t in enumerate(arm_tags(arm).get(k, [])):
|
||
o = out_text(t)
|
||
if o is None:
|
||
continue
|
||
rows.setdefault(i, [0, 0])
|
||
n = base[k]
|
||
rows[i][0] += sum(1 for x in n if P.hits(norm(o), x)[0])
|
||
rows[i][1] += len(n)
|
||
vals = [f"{v[0]}/{v[1]}" for v in rows.values()]
|
||
nums = [v[0] for v in rows.values()]
|
||
print(f" {arm}: розыгрыши {' · '.join(vals)} медиана {st.median(nums):.0f}/21" if nums else f" {arm}: нет данных")
|
||
|
||
|
||
def canon_offblock():
|
||
print("\n=== 3. КАНОН ВНЕ БЛОКА (41 клетка; §0.1) ===")
|
||
cells = offblock_cells()
|
||
dn = sum(1 for k, s, d, rx in cells
|
||
if re.search(rx, norm(P.draft_of(k))))
|
||
print(f" черновик: {dn}/{len(cells)}")
|
||
for arm in ("D1", "F1", "D2", "F5"):
|
||
rows = {}
|
||
for k, s, dst, rx in cells:
|
||
for i, t in enumerate(arm_tags(arm).get(k, [])):
|
||
o = out_text(t)
|
||
if o is None:
|
||
continue
|
||
rows.setdefault(i, [0, 0])
|
||
rows[i][1] += 1
|
||
rows[i][0] += bool(re.search(rx, norm(o)))
|
||
nums = [v[0] for v in rows.values()]
|
||
if nums:
|
||
print(f" {arm}: розыгрыши {' · '.join(f'{v[0]}/{v[1]}' for v in rows.values())}"
|
||
f" медиана {st.median(nums):.0f}")
|
||
|
||
|
||
# --- 4. фикс-рейт посадок ------------------------------------------------------------------
|
||
def plants():
|
||
print("\n=== 4. ФИКС-РЕЙТ 12 ПОСАДОК (пре-рег: дифф НЕ ХУЖЕ full-regen) ===")
|
||
for arm in ("D2", "F5"):
|
||
rows = {}
|
||
for k in range(6):
|
||
for i, t in enumerate(arm_tags(arm).get(k, [])):
|
||
o = out_text(t)
|
||
if o is None:
|
||
continue
|
||
n = norm(o)
|
||
rows.setdefault(i, {"k1": [0, 0], "k2": [0, 0]})
|
||
for cls, old, new, bad_rx, fix_rx in P.DEFECTS[k]:
|
||
rows[i][cls][1] += 1
|
||
rows[i][cls][0] += (not re.search(bad_rx, n))
|
||
if rows:
|
||
for cls in ("k1", "k2"):
|
||
v = [f"{r[cls][0]}/{r[cls][1]}" for r in rows.values()]
|
||
print(f" {arm} {cls}: розыгрыши {' · '.join(v)}")
|
||
|
||
|
||
# --- 5. атом-omission ----------------------------------------------------------------------
|
||
NUMWORDS = ("два", "две", "три", "четыр", "пят", "шест", "сем", "восем", "девят", "десят",
|
||
"двадцат", "тридцат", "сорок", "пятьдесят", "сто", "тысяч", "пятьсот", "полов")
|
||
NEG = ("не ", "ни ", "нет", "без ")
|
||
|
||
|
||
def omission():
|
||
print("\n=== 5. АТОМ-OMISSION (числа + полярность; пре-рег: у диффа должно быть 0) ===")
|
||
for arm in ("D1", "F1", "D2", "F5"):
|
||
rows = {}
|
||
for k in range(6):
|
||
draft = P.draft_of(k) if arm in ("D1", "F1") else P.planted_draft(k, P.draft_of(k))[0]
|
||
dnums = set(re.findall(r"\d{2,}", draft)) | {w for w in NUMWORDS if w in draft.lower()}
|
||
dneg = sum(draft.lower().count(x) for x in NEG)
|
||
for i, t in enumerate(arm_tags(arm).get(k, [])):
|
||
o = out_text(t)
|
||
if o is None:
|
||
continue
|
||
onums = set(re.findall(r"\d{2,}", o)) | {w for w in NUMWORDS if w in o.lower()}
|
||
oneg = sum(o.lower().count(x) for x in NEG)
|
||
rows.setdefault(i, [0, 0, 0])
|
||
rows[i][0] += len(dnums - onums) # исчезнувшие числовые атомы
|
||
rows[i][1] += max(0, dneg - oneg) # просевшая полярность
|
||
rows[i][2] += 1
|
||
if rows:
|
||
v = [f"чисел-{r[0]}/полярн-{r[1]}" for r in rows.values()]
|
||
print(f" {arm}: розыгрыши {' · '.join(v)}")
|
||
|
||
|
||
# --- 6. внесённые дефекты + 7. деньги ------------------------------------------------------
|
||
def guards_and_money():
|
||
print("\n=== 6. ВНЕСЁННЫЕ ДЕФЕКТЫ (форма regression_guard: обвал длины >40%) ===")
|
||
for arm in ("D1", "F1", "D2", "F5"):
|
||
drops = tot = 0
|
||
for k in range(6):
|
||
draft = P.draft_of(k) if arm in ("D1", "F1") else P.planted_draft(k, P.draft_of(k))[0]
|
||
dl = len(re.sub(r"\s", "", draft))
|
||
for t in arm_tags(arm).get(k, []):
|
||
o = out_text(t)
|
||
if o is None:
|
||
continue
|
||
tot += 1
|
||
if len(re.sub(r"\s", "", o)) < dl * 0.60:
|
||
drops += 1
|
||
if tot:
|
||
print(f" {arm}: обвалов длины {drops}/{tot}")
|
||
print("\n=== 7. ДЕНЬГИ И ВЫХОДНЫЕ ТОКЕНЫ (COGS-ось) ===")
|
||
for arm in ("D1", "D2", "F1", "F5"):
|
||
outs, cost, n = [], 0.0, 0
|
||
for k in range(6):
|
||
for t in arm_tags(arm).get(k, []):
|
||
f = RAW / f"{t}.json"
|
||
if not f.exists():
|
||
continue
|
||
r = json.load(open(f, encoding="utf-8"))
|
||
outs.append(r["completion_tokens"]); cost += r["cost_usd"]; n += 1
|
||
if n:
|
||
print(f" {arm}: вызовов {n} · выход медиана {st.median(outs):.0f} ток. "
|
||
f"(мин {min(outs)} макс {max(outs)}) · ${cost:.6f} · ${cost/n:.6f}/вызов")
|
||
tot = sum(json.load(open(f, encoding="utf-8"))["cost_usd"] for f in RAW.glob("ec-*.json")
|
||
if not f.name.endswith(".applied.json"))
|
||
print(f" ИТОГО проба Q4b: ${tot:.6f} (потолок $0.40)")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
compliance(); canon_inblock(); canon_offblock(); plants(); omission(); guards_and_money()
|