package pipeline import ( "crypto/sha256" "encoding/hex" "fmt" "regexp" "strings" "unicode" "textmachine/backend/internal/config" "textmachine/backend/internal/store" ) // banknote.go: the banknote-v1 in-band footnote channel (WS4, план §4 / research/20 §B3 / exp16 // banknote.py — RATIFIED D39.10 as the dst-delivery channel). The translator, AFTER the translation, // MAY emit a versioned separator + tab-delimited term lines for NEW terms only (not in the injected // glossary). This file is the pure, deterministic PARSER + SLICER — the wire-invariant core; the // integration seams (slice BEFORE classifyOutput, derived checkpoint, snapshot fold, telemetry — // план §4(а) 12 points) consume these functions. A faithful Go↔Python port (banknote.py is the // reference): same separator, same tolerant field split, same Han-in-src rule, same truncation // tolerance. Pin (ws4_banknote_verify.py): re-parse of 95 saved lines = 0 parse_fail / 0 truncated. // bankSeparator marks the start of the banknote block. Chosen to (a) not occur in natural prose and // (b) NOT be a "Примечание/Сноска/Комментарий" trailing-note word the sanitizer reserves (§B3-1), so // the two channels never collide. Changing it is a wire/parser change → a loud --resnapshot (§4в). const bankSeparator = "⟦TM-BANK-v1⟧" // bankMaxLines is the per-chunk banknote line budget (§B3-5, feeds the max_tokens sizing at // integration point 5). The parser does not enforce it (a model over-emitting is tolerated and // merely over-counted); it bounds the prompt-side budget. const bankMaxLines = 12 // bankParserVersion versions the split/parse algorithm — the verdict-axis component folded into // banknoteSnap{enabled, parser_version} (§4в/§4а point 6), mirroring sanitizerSnap: a parser change // re-resolves the stripped draft, so it must be a loud --resnapshot even without a prompt edit. const bankParserVersion = "banknote-v1" // bankDerivedNS is the derived-checkpoint id NAMESPACE (§4б EXACT formula) — the "tm--v1" prefix // convention the sibling commitSanitizedExport uses ("tm-sanitized-v1"), kept DISTINCT from the parser // version above exactly as sanitizerSnap.Version (sanitizer-v6) is distinct from the "tm-sanitized-v1" // namespace: it embeds the channel version so a version change yields a fresh (never colliding) id. const bankDerivedNS = "tm-banknote-v1" // bankTypeOK is the accepted type set; anything else falls back to "term" (a benign default). var bankTypeOK = map[string]bool{"name": true, "place": true, "title": true, "term": true, "nickname": true} // bankFieldSplit tolerates a tab, a run of ≥2 spaces, or a pipe (optionally whitespace-padded) as the // field delimiter — the exact tolerance of banknote.py so a model that emits spaces instead of a real // TAB still parses. (Go \s is ASCII-whitespace; banknote lines are tab/space-delimited, so the parity // with Python's unicode \s around the pipe is exact on the term corpus.) var bankFieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`) // bankEntry is one parsed candidate line (evidence for the bank-mining stop (§C); NOT written to the store until owner // signoff). Dst is the model's proposed translation — the direct dst delivery the co-occurrence // miner could not extract (蛊→гу, non-seed 龙公→Лун Гун). type bankEntry struct { Src string Dst string Type string } // bankFlags is the per-chunk telemetry surfaced loud at integration point 10. type bankFlags struct { NLines int // accepted (well-formed) banknote lines ParseFail bool // any malformed residual line (non-Han src / <2 fields) that was NOT a tolerated truncation Truncated bool // the LAST line was cut by generation length (tolerated, not a parse fail) } // splitBanknote slices the banknote block off the raw model output BEFORE any gate/editor (integration // point 1). Returns (clean_translation, raw_block). No separator → the whole (right-trimmed) output is // the clean translation and the block is "". Faithful to banknote.py: clean = output[:idx].rstrip(), // block = output[idx+len(SEP):].strip("\n"). func splitBanknote(output string) (clean, block string) { idx := strings.Index(output, bankSeparator) if idx < 0 { return strings.TrimRightFunc(output, unicode.IsSpace), "" } clean = strings.TrimRightFunc(output[:idx], unicode.IsSpace) block = strings.Trim(output[idx+len(bankSeparator):], "\n") return clean, block } // hasBankSrcHan reports whether src contains a CJK ideograph in the exact [㐀-鿿] range (U+3400–U+9FFF) // banknote.py checks — a zh→ru channel line whose src has no Han is malformed. Kept as the exact // Python range (NOT unicode.Han, which is wider) for byte-faithful parity. func hasBankSrcHan(src string) bool { for _, r := range src { if r >= 0x3400 && r <= 0x9FFF { return true } } return false } // parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry). // Tolerant of a TRUNCATED final line when truncatedGeneration is set (§B3-5): a short last line under // truncation is flagged banknote_truncated, not counted as a parse fail. Any other malformed line // (fewer than 2 fields, or a src with no Han) sets banknote_parse_fail. Deterministic, no time/rand. func parseBanknote(block string, truncatedGeneration bool) ([]bankEntry, bankFlags) { var entries []bankEntry var flags bankFlags if strings.TrimSpace(block) == "" { return entries, flags } var lines []string for _, ln := range strings.Split(block, "\n") { if strings.TrimSpace(ln) != "" { lines = append(lines, ln) } } bad := 0 for i, ln := range lines { raw := bankFieldSplit.Split(strings.TrimSpace(ln), -1) var parts []string for _, p := range raw { if p = strings.TrimSpace(p); p != "" { parts = append(parts, p) } } if len(parts) < 2 { // A short LAST line under a truncated generation is a tolerated cut, not a failure. if i == len(lines)-1 && truncatedGeneration { flags.Truncated = true continue } bad++ continue } src, dst := parts[0], parts[1] typ := "term" if len(parts) >= 3 { typ = strings.ToLower(parts[2]) } if !bankTypeOK[typ] { typ = "term" } if !hasBankSrcHan(src) { bad++ continue } entries = append(entries, bankEntry{Src: src, Dst: dst, Type: typ}) } flags.NLines = len(entries) flags.ParseFail = bad > 0 return entries, flags } // --- integration seam (план §4(а) 12 points) ---------------------------------------------------- // bankTokenBudget is the extra max_tokens the translator draft reserves for the ≤12-line footnote block // (integration point 5): the translation comes FIRST and the block LAST, so a length cut hits the block // (tolerated), not the translation. ~12 tokens per line. Added ONLY when the channel is enabled. const bankTokenBudget = bankMaxLines * 12 // applyBanknote slices the banknote block off a translator draft (integration points 1–5, 8). It runs // ONLY on the translator role with the channel enabled — the editor never emits banknotes, and a normal // draft has no separator (the slice is then a no-op that returns the raw text unchanged, so a // banknote-OFF run is byte-identical). When a ⟦TM-BANK-v1⟧ block IS present it returns the CLEANED // translation (what classify/coverage/echo/editor/export all see — points 2/3/4) plus that same cleaned // text as the `stripped` export to commit as a derived checkpoint (point 8), and the per-chunk telemetry // (point 10). Candidates are PARSED/accepted only under finish=="stop" (the finish=stop-only gate §4б — // a truncated block is not trusted); the slice itself runs regardless, so the length/echo classify never // sees the footnote's Han src column. Deterministic (a pure function of the raw text + gate state). func (r *Runner) applyBanknote(role, rawText, finish string) (clean, stripped string, flags bankFlags) { if role != roleTranslator || !r.Pipeline.Gates.Banknote.Enabled { return rawText, "", bankFlags{} } cleanText, block := splitBanknote(rawText) if block == "" { return rawText, "", bankFlags{} // no separator → no strip, no derived checkpoint (byte-identical path) } if finish == "stop" { // finish=stop-only gate: accept candidates + count telemetry only for a complete generation. // truncatedGeneration=false is unreachable-otherwise in prod (a truncated gen is finish≠stop). _, flags = parseBanknote(block, false) } return cleanText, cleanText, flags } // bankDerivedHash is the CONTENT-ADDRESSED id of a banknote-stripped export checkpoint (§4б EXACT // formula, mirroring commitSanitizedExport's namespacing): sha256("tm-banknote-v1\x00"+reqHash+"\x00"+ // stripped), prefixed "tm-banknote-v1:" so it can never collide with a real hex attempt hash and a // resume re-derives the identical id for free. The namespace embeds the channel version → a version // change yields a new id. func bankDerivedHash(reqHash, stripped string) string { sum := sha256.Sum256([]byte(bankDerivedNS + "\x00" + reqHash + "\x00" + stripped)) return bankDerivedNS + ":" + hex.EncodeToString(sum[:]) } // commitBanknoteExport persists the banknote-stripped CLEAN draft as a $0 derived checkpoint and returns // its request_hash (→ chunk_status.final_hash), so the OK-path final_hash→checkpoint export contract // (and the editor's resume read) yields the CLEANED draft instead of the raw one carrying the footnote // (integration point 8). Mirrors commitSanitizedExport: cost 0, escalation 0, idempotent, written BEFORE // chunk_status references it. `att` is the terminal (possibly escalated) OK attempt whose block was // sliced; att.bankStripped is the cleaned text. func (r *Runner) commitBanknoteExport(st config.Stage, ch Chunk, job *store.Job, att stageAttempt) (string, error) { derivedHash := bankDerivedHash(att.reqHash, att.bankStripped) if err := r.Store.PutDerivedCheckpoint(store.Checkpoint{ RequestHash: derivedHash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: att.attempt, Stage: st.Name, Role: st.Role, ModelRequested: att.modelActual, ModelActual: att.modelActual, ResponseText: att.bankStripped, UsageJSON: "{}", FinishReason: "banknote_export", }); err != nil { return "", fmt.Errorf("pipeline: commit banknote export ch%d/chunk%d/%s: %w", ch.Chapter, ch.ChunkIdx, st.Name, err) } return derivedHash, nil }