173 lines
8.3 KiB
Python
173 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
||
"""WS3 (г) verification, part 2 — emit a seed-delta (§C2-7) from the persisted exp16
|
||
tables (A3 candidates + banknote dst), which exp16 itself never emitted (only sidecars),
|
||
then DRY-LINT it against a faithful Python replication of the memseed.loadGlossarySeed
|
||
fail-louds (the (д) after-build spec runs the real Go loadGlossarySeed for byte parity).
|
||
|
||
Emission rules (research/20 §C2-7, memseed.go:30-57 seedTerm schema, miner NEVER writes approved):
|
||
- non-seed A3 candidate WITH a banknote dst -> status: draft, dst=<majority banknote dst>
|
||
- non-seed A3 candidate WITHOUT a dst -> status: auto (no dst, inert in automaton)
|
||
- since_ch = first chapter of occurrence (auto, by source); until_ch left 0 (owner only)
|
||
|
||
$0: deterministic, no LLM, no network. Reuses eval/exp16 miner + banknote artifacts.
|
||
Writes seed-delta YAML + lint report to /home/ubuntu/books/gu-zhenren/design11/ (OUT of git).
|
||
"""
|
||
import json, os, sys, glob, re, collections
|
||
import yaml
|
||
|
||
EVAL = "/home/ubuntu/projects/textmachine/eval"
|
||
sys.path.insert(0, f"{EVAL}/exp15")
|
||
sys.path.insert(0, f"{EVAL}/exp16")
|
||
import exp16_common as X
|
||
import arms as A
|
||
import mem_select as MS # normalize_source_key, significant_len (byte-verified port of memnorm.go)
|
||
|
||
A3 = "/home/ubuntu/books/gu-zhenren/exp16/A3_candidates_full.json"
|
||
BANK = "/home/ubuntu/books/gu-zhenren/exp16/coldstart/banknote/*.banknote.json"
|
||
OUT_YAML = "/home/ubuntu/books/gu-zhenren/design11/ws3_seed_delta.yaml"
|
||
OUT_LINT = "/home/ubuntu/books/gu-zhenren/design11/ws3_seeddelta_lint.json"
|
||
|
||
MIN_KEY_HAN, MIN_KEY_PHON = 2, 3
|
||
def min_key_len(nk):
|
||
return MIN_KEY_HAN if any('㐀' <= c <= '鿿' or '豈' <= c <= '' for c in nk) else MIN_KEY_PHON
|
||
|
||
# ---------- faithful replication of memseed.loadGlossarySeed fail-louds ----------
|
||
def lint_seed(terms):
|
||
problems = []
|
||
seen = {}
|
||
for i, t in enumerate(terms):
|
||
src = (t.get("src") or "").strip()
|
||
sense = (t.get("sense") or "").strip()
|
||
dst = (t.get("dst") or "").strip()
|
||
if not src:
|
||
problems.append(f"term {i}: src is required"); continue
|
||
status = t.get("status") or "approved"
|
||
if status not in ("auto", "draft", "approved"):
|
||
problems.append(f"term {src!r}: status must be auto|draft|approved, got {status!r}"); continue
|
||
if status in ("approved", "draft") and not dst:
|
||
problems.append(f"term {src!r}: a {status} term must have a non-empty dst (silently inert)"); continue
|
||
key = (src, sense, t.get("since_ch", 0), t.get("until_ch", 0))
|
||
if key in seen:
|
||
problems.append(f"term {src!r}: duplicate (src,sense,window)"); continue
|
||
seen[key] = i
|
||
# spoiler-window overlap same src different dst (D16.1) + shared-key collision (approved only)
|
||
def win_overlap(s1, u1, s2, u2):
|
||
s1 = s1 or 1; s2 = s2 or 1; u1 = u1 or (1 << 62); u2 = u2 or (1 << 62)
|
||
return max(s1, s2) <= min(u1, u2)
|
||
for i in range(len(terms)):
|
||
for j in range(i):
|
||
a, b = terms[i], terms[j]
|
||
if a.get("src") != b.get("src"):
|
||
continue
|
||
if (a.get("dst") or "") == (b.get("dst") or ""):
|
||
continue
|
||
if win_overlap(a.get("since_ch", 0), a.get("until_ch", 0), b.get("since_ch", 0), b.get("until_ch", 0)):
|
||
problems.append(f"term {a['src']!r}: overlapping spoiler windows with different dst")
|
||
return problems
|
||
|
||
def is_fragment(src, gt_norm, particles="的了在是和就也都不心面前后上下今少多大小与之其"):
|
||
if len(src) >= 2 and MS.normalize_source_key(src[:-1]) in gt_norm and src[-1] in particles:
|
||
return True
|
||
return False
|
||
|
||
def main():
|
||
chunks = X.load_chunks()
|
||
gt = X.load_gt()
|
||
gt_norm = set()
|
||
for e in gt:
|
||
for s in e.norm_surfaces:
|
||
gt_norm.add(s)
|
||
|
||
a3 = json.load(open(A3))
|
||
# banknote dst: majority per src; strip trailing [type] annotations the model added
|
||
bn = collections.defaultdict(collections.Counter)
|
||
for f in glob.glob(BANK):
|
||
d = json.load(open(f))
|
||
# chapter from filename e.g. 4.0.banknote.json -> chapter 4
|
||
ch = int(os.path.basename(f).split(".")[0])
|
||
for e in d.get("entries", []):
|
||
dst = re.sub(r"\[[^\]]*\]", "", e["dst"]).strip()
|
||
bn[MS.normalize_source_key(e["src"])][(dst, e.get("type", "term"))] += 1
|
||
bn_first_ch = {}
|
||
for f in glob.glob(BANK):
|
||
ch = int(os.path.basename(f).split(".")[0])
|
||
d = json.load(open(f))
|
||
for e in d.get("entries", []):
|
||
k = MS.normalize_source_key(e["src"])
|
||
bn_first_ch[k] = min(bn_first_ch.get(k, 1 << 30), ch)
|
||
|
||
# candidate chapters helper (first chapter of occurrence)
|
||
def first_chapter(src_norm):
|
||
for c in chunks:
|
||
if src_norm in c.nsource:
|
||
return c.chapter
|
||
return 0
|
||
|
||
delta = []
|
||
seen_src = set()
|
||
n_draft = n_auto = 0
|
||
# (1) non-seed candidates WITH a banknote dst -> draft
|
||
for cand in a3:
|
||
srcn = MS.normalize_source_key(cand["src"])
|
||
if srcn in gt_norm or srcn in seen_src:
|
||
continue
|
||
if is_fragment(cand["src"], gt_norm):
|
||
continue
|
||
typ = (cand["types"] or ["term"])[0]
|
||
if srcn in bn:
|
||
(dst, bt) = bn[srcn].most_common(1)[0][0], bn[srcn].most_common(1)[0]
|
||
dstv = bn[srcn].most_common(1)[0][0][0]
|
||
if not dstv:
|
||
continue
|
||
delta.append({"src": cand["src"], "dst": dstv, "type": typ, "status": "draft",
|
||
"since_ch": bn_first_ch.get(srcn) or first_chapter(srcn),
|
||
"note": "mined draft (banknote dst); ⟨проверить⟩ — canon proposal, owner signs"})
|
||
seen_src.add(srcn); n_draft += 1
|
||
# (2) top non-seed WHICH-only candidates WITHOUT dst -> auto (inert), a bounded sample.
|
||
# Apply the miner's operating point (top_k=90 rank window) + single-key ban + fragment filter
|
||
# so the owner review-sheet is clean (raw 13618-dump would carry particle/fragment noise).
|
||
a3_top = a3[: A.FROZEN["top_k"] * 3] # generous rank window; only pattern-typed or freq-anchored survive filters
|
||
for cand in a3_top:
|
||
if n_auto >= 40:
|
||
break
|
||
srcn = MS.normalize_source_key(cand["src"])
|
||
if srcn in gt_norm or srcn in seen_src:
|
||
continue
|
||
if is_fragment(cand["src"], gt_norm):
|
||
continue
|
||
if MS.significant_len(srcn) < min_key_len(srcn):
|
||
continue # single-key ban (A3): a too-short key never fires / is not an entity
|
||
if cand["freq"] < X.FREQ_FLOOR and not cand["from_pattern"]:
|
||
continue
|
||
typ = (cand["types"] or ["term"])[0]
|
||
if typ not in ("name", "place", "title", "term", "nickname"):
|
||
typ = "term"
|
||
delta.append({"src": cand["src"], "dst": "", "type": typ, "status": "auto",
|
||
"since_ch": first_chapter(srcn),
|
||
"note": "mined WHICH-only auto-candidate (no dst; inert until owner assigns)"})
|
||
seen_src.add(srcn); n_auto += 1
|
||
|
||
seed_doc = {"terms": delta}
|
||
os.makedirs(os.path.dirname(OUT_YAML), exist_ok=True)
|
||
with open(OUT_YAML, "w") as fh:
|
||
yaml.safe_dump(seed_doc, fh, allow_unicode=True, sort_keys=False)
|
||
|
||
# round-trip: re-parse the emitted YAML and lint it (as loadGlossarySeed would)
|
||
reparsed = yaml.safe_load(open(OUT_YAML))["terms"]
|
||
problems = lint_seed(reparsed)
|
||
|
||
report = {
|
||
"n_delta_terms": len(delta), "n_draft_with_banknote_dst": n_draft, "n_auto_no_dst": n_auto,
|
||
"lint_problems": problems, "lint_pass": len(problems) == 0,
|
||
"sample_draft": [t for t in delta if t["status"] == "draft"][:8],
|
||
"sample_auto": [t for t in delta if t["status"] == "auto"][:8],
|
||
}
|
||
json.dump(report, open(OUT_LINT, "w"), ensure_ascii=False, indent=2)
|
||
print(json.dumps({k: report[k] for k in ("n_delta_terms", "n_draft_with_banknote_dst", "n_auto_no_dst", "lint_pass", "lint_problems")}, ensure_ascii=False, indent=2))
|
||
print("\nsample draft (mined dst):", [(t["src"], t["dst"], t["type"]) for t in report["sample_draft"]])
|
||
print("sample auto (which-only):", [(t["src"], t["type"]) for t in report["sample_auto"]])
|
||
print(f"\nVERDICT: seed-delta of {len(delta)} terms ({n_draft} draft+dst / {n_auto} auto) "
|
||
f"{'PASSES' if report['lint_pass'] else 'FAILS'} memseed fail-loud dry-lint.")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|