#!/usr/bin/env python3 """WS4 (г) verification — banknote-v1 channel parser. (1) Re-parse the 14 persisted raw model outputs with the frozen banknote-v1 parser and reproduce: 95 total lines, distinct_src 65, parse_fail 0.0%, truncated 0. (2) Synthetic truncation: a block whose LAST line is cut mid-field parses tolerantly (banknote_truncated) under truncated_generation=True and as bad (parse_fail) under False. (3) fresh<->resume equivalence: the tm-banknote-v1 DERIVED-checkpoint id is a content-addressed hash of (reqHash, stripped) with the exact commitSanitizedExport formula (NULL-byte separators), so a fresh run and a resume re-derive the IDENTICAL id => the OK-path final_hash re-point is resume-stable by construction. (4) Parser invariants: SEP not a note-word; src must contain Han; tolerant field split (tab|>=2sp|pipe). $0: deterministic, no LLM, no network. Reuses eval/exp16/banknote.py + persisted raw outputs. Writes /home/ubuntu/books/gu-zhenren/design11/ws4_banknote_verify.json (OUT of git). """ import json, os, sys, glob, hashlib sys.path.insert(0, "/home/ubuntu/projects/textmachine/eval/exp16") import banknote as BN RAW = "/home/ubuntu/books/gu-zhenren/exp16/coldstart/banknote/*.raw.txt" OUT = "/home/ubuntu/books/gu-zhenren/design11/ws4_banknote_verify.json" def derived_hash(req_hash, stripped, channel="tm-banknote-v1"): """Exact commitSanitizedExport precedent (stagerun.go:230-232): NULL-byte separators + version INSIDE the hash; id prefix = channel + ':'. This is the WS4 tm-banknote-v1 derived-checkpoint id.""" payload = (channel + "\x00" + req_hash + "\x00" + stripped).encode("utf-8") return channel + ":" + hashlib.sha256(payload).hexdigest() def main(): result = {} # ---- (1) re-parse the 14 persisted raw outputs ---- total_lines = 0 parse_fails = 0 truncated = 0 distinct_src = set() per_file = {} for f in sorted(glob.glob(RAW)): raw = open(f).read() clean, block = BN.split_banknote(raw) entries, flags = BN.parse_banknote(block, truncated_generation=False) # prod: stop-only gate => False total_lines += flags["n_banknote_lines"] parse_fails += 1 if flags["banknote_parse_fail"] else 0 truncated += 1 if flags["banknote_truncated"] else 0 for e in entries: distinct_src.add(e["src"]) # invariant: clean must NOT contain the SEP assert BN.SEP not in clean, f"{f}: SEP leaked into clean text" per_file[os.path.basename(f)] = {"lines": flags["n_banknote_lines"], "parse_fail": flags["banknote_parse_fail"]} result["reparse_14_files"] = { "total_banknote_lines": total_lines, "distinct_src": len(distinct_src), "files_with_parse_fail": parse_fails, "files_truncated": truncated, "expected": {"total_lines": 95, "distinct_src": 65, "parse_fail": 0, "truncated": 0}, "match": (total_lines == 95 and len(distinct_src) == 65 and parse_fails == 0 and truncated == 0), } # ---- (2) synthetic truncation ---- good_block = "蛊\tгу\tterm\n方源\tФан Юань\tname" trunc_block = good_block + "\n希望蛊" # last line cut before the dst/type field e_t, f_t = BN.parse_banknote(trunc_block, truncated_generation=True) e_f, f_f = BN.parse_banknote(trunc_block, truncated_generation=False) result["synthetic_truncation"] = { "truncated_gen_True": {"n_lines": f_t["n_banknote_lines"], "parse_fail": f_t["banknote_parse_fail"], "truncated": f_t["banknote_truncated"]}, "truncated_gen_False": {"n_lines": f_f["n_banknote_lines"], "parse_fail": f_f["banknote_parse_fail"], "truncated": f_f["banknote_truncated"]}, "verdict": ("tolerant under True (truncated flag, 2 good lines kept), bad under False (parse_fail)" if (f_t["banknote_truncated"] and not f_t["banknote_parse_fail"] and f_t["n_banknote_lines"] == 2 and f_f["banknote_parse_fail"]) else "UNEXPECTED"), } # ---- (3) fresh<->resume derived-hash equivalence ---- req_hash = "a1b2c3deadbeef" * 4 # a sample attempt request_hash stripped = "Чистовик перевода без блока сноски." h1 = derived_hash(req_hash, stripped) # fresh run h2 = derived_hash(req_hash, stripped) # resume re-derive h3 = derived_hash(req_hash, stripped + "x") # different stripped -> different id result["fresh_resume_hash"] = { "id_fresh": h1[:40] + "...", "id_resume": h1 == h2, "stable": h1 == h2, "collision_free_on_content_change": h1 != h3, "formula": "sha256('tm-banknote-v1\\x00'+reqHash+'\\x00'+stripped), id-prefix 'tm-banknote-v1:'", "verdict": "resume re-derives IDENTICAL id (content-addressed) => OK-path final_hash re-point resume-stable" if (h1 == h2 and h1 != h3) else "UNEXPECTED", } # ---- (4) parser invariants ---- inv = {} # SEP is not a reserved note-word inv["SEP_not_note_word"] = all(w not in BN.SEP for w in ("Примечание", "Сноска", "Комментарий", "Note")) # non-Han src -> bad _, f_nh = BN.parse_banknote("hello\tworld\tterm") inv["non_han_src_is_bad"] = f_nh["banknote_parse_fail"] # SEP-less output -> whole thing is clean, empty block clean_ns, block_ns = BN.split_banknote("Просто перевод без блока.") inv["no_sep_all_clean"] = (block_ns == "" and clean_ns == "Просто перевод без блока.") # tolerant field split: >=2 spaces and pipe both parse e_sp, f_sp = BN.parse_banknote("蛊 гу term") e_pipe, f_pipe = BN.parse_banknote("蛊 | гу | term") inv["tolerant_split_spaces"] = (not f_sp["banknote_parse_fail"] and f_sp["n_banknote_lines"] == 1) inv["tolerant_split_pipe"] = (not f_pipe["banknote_parse_fail"] and f_pipe["n_banknote_lines"] == 1) result["parser_invariants"] = inv result["parser_invariants_all_pass"] = all(inv.values()) os.makedirs(os.path.dirname(OUT), exist_ok=True) json.dump(result, open(OUT, "w"), ensure_ascii=False, indent=2) print(json.dumps(result, ensure_ascii=False, indent=2)) r1 = result["reparse_14_files"] print(f"\nVERDICT: reparse {r1['total_banknote_lines']} lines / {r1['distinct_src']} distinct_src, " f"parse_fail={r1['files_with_parse_fail']} truncated={r1['files_truncated']} " f"({'MATCH' if r1['match'] else 'MISMATCH'} golden 95/65/0/0); " f"truncation {result['synthetic_truncation']['verdict'][:30]}...; " f"hash {'STABLE' if result['fresh_resume_hash']['stable'] else 'UNSTABLE'}; " f"invariants {'ALL PASS' if result['parser_invariants_all_pass'] else 'FAIL'}.") if __name__ == "__main__": main()