heaven/textmachine
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix stale trad2simp data paths in eval after the pack-15 move to internal/text

Browse source
This commit is contained in:
Claude (backend session) 2026-07-25 01:20:42 +03:00
parent 4bfc097966
commit e241b9572c
2 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
eval/exp15/mem_select.py
View file

@ -33,7 +33,7 @@ import regex # \p{Han} etc. — exact Unicode script tables, matching Go's unic
import yaml import yaml
BACKEND = Path("/home/ubuntu/projects/textmachine/backend") BACKEND = Path("/home/ubuntu/projects/textmachine/backend")
TRAD2SIMP_FILE = BACKEND / "internal/pipeline/data/trad2simp.txt" TRAD2SIMP_FILE = BACKEND / "internal/text/data/trad2simp.txt" # moved from internal/pipeline/data in pack-15 (byte-identical)
MIN_KEY_LEN_HAN = 2 MIN_KEY_LEN_HAN = 2
MIN_KEY_LEN_PHONETIC = 3 MIN_KEY_LEN_PHONETIC = 3

6
eval/pilot/memory_eval.py
View file

@ -90,7 +90,7 @@ ROOT = Path(__file__).resolve().parent
SAMPLES = ROOT.parent / "data" / "samples" SAMPLES = ROOT.parent / "data" / "samples"
OUTDIR = ROOT.parent / "data" / "pilot" OUTDIR = ROOT.parent / "data" / "pilot"
OUTDIR.mkdir(parents=True, exist_ok=True) OUTDIR.mkdir(parents=True, exist_ok=True)
TRAD_TABLE = ROOT / "trad2simp.txt" # vendored from backend/internal/pipeline/data/trad2simp.txt TRAD_TABLE = ROOT / "trad2simp.txt" # vendored from backend/internal/text/data/trad2simp.txt
# --- normalization (faithful mirror of memnorm.go) -------------------------------- # --- normalization (faithful mirror of memnorm.go) --------------------------------
@ -104,7 +104,7 @@ def _load_trad2simp(path: Path) -> dict[str, str]:
m: dict[str, str] = {} m: dict[str, str] = {}
if not path.exists(): if not path.exists():
raise SystemExit(f"memory_eval: vendored trad2simp table missing: {path}\n" raise SystemExit(f"memory_eval: vendored trad2simp table missing: {path}\n"
f" cp backend/internal/pipeline/data/trad2simp.txt {path}") f" cp backend/internal/text/data/trad2simp.txt {path}")
for i, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): for i, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
s = line.strip() s = line.strip()
if not s or s.startswith("#"): if not s or s.startswith("#"):
@ -765,7 +765,7 @@ def self_test() -> int:
# İ→lower Go-parity (parity-review #1): norm_src must fold İ to 'i' (Python .lower() → 'i̇') # İ→lower Go-parity (parity-review #1): norm_src must fold İ to 'i' (Python .lower() → 'i̇')
check(norm_src("İ") == "i" and dm.normalize_target("İ") == "i", "İ: not folded to Go's lowercase 'i'") check(norm_src("İ") == "i" and dm.normalize_target("İ") == "i", "İ: not folded to Go's lowercase 'i'")
# trad2simp byte-parity with the Go embed (package-2 Task 1d): the vendored table must stay # trad2simp byte-parity with the Go embed (package-2 Task 1d): the vendored table must stay
# byte-identical to backend/internal/pipeline/data/trad2simp.txt (memoryNormVersion hashes its # byte-identical to backend/internal/text/data/trad2simp.txt (memoryNormVersion hashes its
# bytes) — a drift silently diverges the normalizers. Skipped if the Go source isn't checked out. # bytes) — a drift silently diverges the normalizers. Skipped if the Go source isn't checked out.
go_embed = ROOT.parent.parent / "backend" / "internal" / "pipeline" / "data" / "trad2simp.txt" go_embed = ROOT.parent.parent / "backend" / "internal" / "pipeline" / "data" / "trad2simp.txt"
if go_embed.exists(): if go_embed.exists():