134 lines
5.9 KiB
Python
134 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
||
"""B1: голд-набор арбитража банка (промт BANK_ARBITRATION §4 B1, $0).
|
||
|
||
Ядро голда = сид v2 (~/books/gu-zhenren/guzhenren-seed-v2.yaml): 53 записи status:approved
|
||
с подписанным владельцем dst. Джойн против банка coldrun-a (BANK-FULL.tsv + bank-stop
|
||
таблица с KWIC и вариантами черновиков) — по нормализованной поверхности src + алиасам.
|
||
|
||
Выход: gold/gold.jsonl — по строке на голд-терм с провенансом:
|
||
src, gold_dst (подпись владельца), gold_alt (decl-формы и алиасы dst — НЕ считаются
|
||
верным ответом арбитража, лежат для справки), type, seed_note,
|
||
in_bank (нашёлся ли в BANK-FULL), bank_dst (что сконсолидировал терминолог coldrun-a),
|
||
bank_origin/freq/spread, variants [(dst, chunks)] в ПОРЯДКЕ ранжирования §C2-3
|
||
(bankStopRows рендерит best-ranked first — это и есть бесплатный базис P0),
|
||
contexts (KWIC из bank-stop), provenance.
|
||
|
||
Смещение голда named честно в отчёте: сид = термы, до которых дошли руки владельца.
|
||
Читает всё READ-ONLY. Не вызывает моделей. $0.
|
||
"""
|
||
import json
|
||
import os
|
||
import sys
|
||
import unicodedata
|
||
|
||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bank_autonomy"))
|
||
import parse_common # noqa: E402
|
||
|
||
import yaml # noqa: E402
|
||
|
||
SEED = os.path.expanduser("~/books/gu-zhenren/guzhenren-seed-v2.yaml")
|
||
BANKFULL = os.path.join(parse_common.SCRATCH, "BANK-FULL.tsv")
|
||
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gold", "gold.jsonl")
|
||
|
||
|
||
def norm_src(s: str) -> str:
|
||
"""Зеркало text.NormalizeSourceKey для нашего среза: NFKC + lower.
|
||
trad2simp/kana-fold не нужны — сид и банк оба simplified-zh (проверяется джойном:
|
||
непарные строки печатаются, глазами)."""
|
||
return unicodedata.normalize("NFKC", s or "").strip().lower()
|
||
|
||
|
||
def load_seed():
|
||
doc = yaml.safe_load(open(SEED, encoding="utf-8"))
|
||
out = []
|
||
for t in doc.get("terms", []):
|
||
if t.get("status") != "approved" or not t.get("dst"):
|
||
continue
|
||
aliases = [a["alias"] for a in t.get("aliases", []) if a.get("alias")]
|
||
decl = (t.get("decl") or {}).get("forms") or []
|
||
out.append({
|
||
"src": t["src"], "gold_dst": t["dst"], "type": t.get("type", "term"),
|
||
"aliases": aliases, "gold_alt": decl, "seed_note": t.get("note", ""),
|
||
"since_ch": t.get("since_ch", 0),
|
||
})
|
||
return out
|
||
|
||
|
||
def load_bankfull():
|
||
"""BANK-FULL.tsv: src, сведённый_dst, origin, type, freq, spread, варианты, n_ctx."""
|
||
rows = {}
|
||
with open(BANKFULL, encoding="utf-8") as f:
|
||
header = f.readline()
|
||
assert header.startswith("src\t"), header
|
||
for ln in f:
|
||
p = ln.rstrip("\n").split("\t")
|
||
if len(p) < 8:
|
||
continue
|
||
variants = []
|
||
for part in p[6].split(" | "):
|
||
part = part.strip()
|
||
if not part:
|
||
continue
|
||
if "×" in part:
|
||
d, _, n = part.rpartition("×")
|
||
variants.append((d.strip(), int(n)))
|
||
else:
|
||
variants.append((part, 1))
|
||
rows[norm_src(p[0])] = {
|
||
"src": p[0], "bank_dst": p[1] if p[1] != "—" else "",
|
||
"origin": p[2], "type": p[3], "freq": int(p[4]), "spread": int(p[5]),
|
||
"variants": variants,
|
||
}
|
||
return rows
|
||
|
||
|
||
def main():
|
||
seed = load_seed()
|
||
bank = load_bankfull()
|
||
stop = parse_common.parse_bankstop() # src -> {..., ctx: [...]}
|
||
stop_by_norm = {norm_src(k): v for k, v in stop.items()}
|
||
|
||
gold, matched, unmatched = [], 0, []
|
||
for t in seed:
|
||
surfaces = [t["src"]] + t["aliases"]
|
||
hit = None
|
||
hit_surface = None
|
||
for s in surfaces:
|
||
if norm_src(s) in bank:
|
||
hit = bank[norm_src(s)]
|
||
hit_surface = s
|
||
break
|
||
rec = {
|
||
"src": t["src"], "gold_dst": t["gold_dst"], "type": t["type"],
|
||
"gold_alt": t["gold_alt"], "aliases": t["aliases"], "seed_note": t["seed_note"],
|
||
"in_bank": hit is not None,
|
||
"provenance": f"seed v2 approved (владелец) src={t['src']}; bank=coldrun-a BANK-FULL"
|
||
+ (f" via surface {hit_surface}" if hit_surface and hit_surface != t["src"] else ""),
|
||
}
|
||
if hit:
|
||
matched += 1
|
||
st = stop_by_norm.get(norm_src(hit["src"]), {})
|
||
rec.update({
|
||
"bank_src": hit["src"], "bank_dst": hit["bank_dst"], "bank_origin": hit["origin"],
|
||
"bank_type": hit["type"], "freq": hit["freq"], "spread": hit["spread"],
|
||
"variants": hit["variants"], "contexts": st.get("ctx", []),
|
||
"evidence": st.get("evidence", []),
|
||
})
|
||
else:
|
||
unmatched.append(t["src"])
|
||
gold.append(rec)
|
||
|
||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||
with open(OUT, "w", encoding="utf-8") as f:
|
||
for r in gold:
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
|
||
n_var2 = sum(1 for r in gold if r["in_bank"] and len(r.get("variants", [])) >= 2)
|
||
print(f"seed approved: {len(seed)} | matched in BANK-FULL: {matched} | not in bank: {len(unmatched)}")
|
||
print(f"of matched: spread>=2 (арбитражные, есть из чего выбирать): {n_var2}")
|
||
print("not in bank:", ", ".join(unmatched))
|
||
print(f"wrote {OUT}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|