207 lines
10 KiB
Go
207 lines
10 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"regexp"
|
||
"strings"
|
||
"unicode"
|
||
|
||
"textmachine/backend/internal/config"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// banknote.go: the banknote-v1 in-band footnote channel (WS4, план §4 / research/20 §B3 / exp16
|
||
// banknote.py — RATIFIED D39.10 as the dst-delivery channel). The translator, AFTER the translation,
|
||
// MAY emit a versioned separator + tab-delimited term lines for NEW terms only (not in the injected
|
||
// glossary). This file is the pure, deterministic PARSER + SLICER — the wire-invariant core; the
|
||
// integration seams (slice BEFORE classifyOutput, derived checkpoint, snapshot fold, telemetry —
|
||
// план §4(а) 12 points) consume these functions. A faithful Go↔Python port (banknote.py is the
|
||
// reference): same separator, same tolerant field split, same Han-in-src rule, same truncation
|
||
// tolerance. Pin (ws4_banknote_verify.py): re-parse of 95 saved lines = 0 parse_fail / 0 truncated.
|
||
|
||
// bankSeparator marks the start of the banknote block. Chosen to (a) not occur in natural prose and
|
||
// (b) NOT be a "Примечание/Сноска/Комментарий" trailing-note word the sanitizer reserves (§B3-1), so
|
||
// the two channels never collide. Changing it is a wire/parser change → a loud --resnapshot (§4в).
|
||
const bankSeparator = "⟦TM-BANK-v1⟧"
|
||
|
||
// bankMaxLines is the per-chunk banknote line budget (§B3-5, feeds the max_tokens sizing at
|
||
// integration point 5). The parser does not enforce it (a model over-emitting is tolerated and
|
||
// merely over-counted); it bounds the prompt-side budget.
|
||
const bankMaxLines = 12
|
||
|
||
// bankParserVersion versions the split/parse algorithm — the verdict-axis component folded into
|
||
// banknoteSnap{enabled, parser_version} (§4в/§4а point 6), mirroring sanitizerSnap: a parser change
|
||
// re-resolves the stripped draft, so it must be a loud --resnapshot even without a prompt edit.
|
||
const bankParserVersion = "banknote-v1"
|
||
|
||
// bankDerivedNS is the derived-checkpoint id NAMESPACE (§4б EXACT formula) — the "tm-<name>-v1" prefix
|
||
// convention the sibling commitSanitizedExport uses ("tm-sanitized-v1"), kept DISTINCT from the parser
|
||
// version above exactly as sanitizerSnap.Version (sanitizer-v6) is distinct from the "tm-sanitized-v1"
|
||
// namespace: it embeds the channel version so a version change yields a fresh (never colliding) id.
|
||
const bankDerivedNS = "tm-banknote-v1"
|
||
|
||
// bankTypeOK is the accepted type set; anything else falls back to "term" (a benign default).
|
||
var bankTypeOK = map[string]bool{"name": true, "place": true, "title": true, "term": true, "nickname": true}
|
||
|
||
// bankFieldSplit tolerates a tab, a run of ≥2 spaces, or a pipe (optionally whitespace-padded) as the
|
||
// field delimiter — the exact tolerance of banknote.py so a model that emits spaces instead of a real
|
||
// TAB still parses. (Go \s is ASCII-whitespace; banknote lines are tab/space-delimited, so the parity
|
||
// with Python's unicode \s around the pipe is exact on the term corpus.)
|
||
var bankFieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`)
|
||
|
||
// bankEntry is one parsed candidate line (evidence for W1.5 §C; NOT written to the store until owner
|
||
// signoff). Dst is the model's proposed translation — the direct dst delivery the co-occurrence
|
||
// miner could not extract (蛊→гу, non-seed 龙公→Лун Гун).
|
||
type bankEntry struct {
|
||
Src string
|
||
Dst string
|
||
Type string
|
||
}
|
||
|
||
// bankFlags is the per-chunk telemetry surfaced loud at integration point 10.
|
||
type bankFlags struct {
|
||
NLines int // accepted (well-formed) banknote lines
|
||
ParseFail bool // any malformed residual line (non-Han src / <2 fields) that was NOT a tolerated truncation
|
||
Truncated bool // the LAST line was cut by generation length (tolerated, not a parse fail)
|
||
}
|
||
|
||
// splitBanknote slices the banknote block off the raw model output BEFORE any gate/editor (integration
|
||
// point 1). Returns (clean_translation, raw_block). No separator → the whole (right-trimmed) output is
|
||
// the clean translation and the block is "". Faithful to banknote.py: clean = output[:idx].rstrip(),
|
||
// block = output[idx+len(SEP):].strip("\n").
|
||
func splitBanknote(output string) (clean, block string) {
|
||
idx := strings.Index(output, bankSeparator)
|
||
if idx < 0 {
|
||
return strings.TrimRightFunc(output, unicode.IsSpace), ""
|
||
}
|
||
clean = strings.TrimRightFunc(output[:idx], unicode.IsSpace)
|
||
block = strings.Trim(output[idx+len(bankSeparator):], "\n")
|
||
return clean, block
|
||
}
|
||
|
||
// hasBankSrcHan reports whether src contains a CJK ideograph in the exact [㐀-鿿] range (U+3400–U+9FFF)
|
||
// banknote.py checks — a zh→ru channel line whose src has no Han is malformed. Kept as the exact
|
||
// Python range (NOT unicode.Han, which is wider) for byte-faithful parity.
|
||
func hasBankSrcHan(src string) bool {
|
||
for _, r := range src {
|
||
if r >= 0x3400 && r <= 0x9FFF {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry).
|
||
// Tolerant of a TRUNCATED final line when truncatedGeneration is set (§B3-5): a short last line under
|
||
// truncation is flagged banknote_truncated, not counted as a parse fail. Any other malformed line
|
||
// (fewer than 2 fields, or a src with no Han) sets banknote_parse_fail. Deterministic, no time/rand.
|
||
func parseBanknote(block string, truncatedGeneration bool) ([]bankEntry, bankFlags) {
|
||
var entries []bankEntry
|
||
var flags bankFlags
|
||
if strings.TrimSpace(block) == "" {
|
||
return entries, flags
|
||
}
|
||
var lines []string
|
||
for _, ln := range strings.Split(block, "\n") {
|
||
if strings.TrimSpace(ln) != "" {
|
||
lines = append(lines, ln)
|
||
}
|
||
}
|
||
bad := 0
|
||
for i, ln := range lines {
|
||
raw := bankFieldSplit.Split(strings.TrimSpace(ln), -1)
|
||
var parts []string
|
||
for _, p := range raw {
|
||
if p = strings.TrimSpace(p); p != "" {
|
||
parts = append(parts, p)
|
||
}
|
||
}
|
||
if len(parts) < 2 {
|
||
// A short LAST line under a truncated generation is a tolerated cut, not a failure.
|
||
if i == len(lines)-1 && truncatedGeneration {
|
||
flags.Truncated = true
|
||
continue
|
||
}
|
||
bad++
|
||
continue
|
||
}
|
||
src, dst := parts[0], parts[1]
|
||
typ := "term"
|
||
if len(parts) >= 3 {
|
||
typ = strings.ToLower(parts[2])
|
||
}
|
||
if !bankTypeOK[typ] {
|
||
typ = "term"
|
||
}
|
||
if !hasBankSrcHan(src) {
|
||
bad++
|
||
continue
|
||
}
|
||
entries = append(entries, bankEntry{Src: src, Dst: dst, Type: typ})
|
||
}
|
||
flags.NLines = len(entries)
|
||
flags.ParseFail = bad > 0
|
||
return entries, flags
|
||
}
|
||
|
||
// --- integration seam (план §4(а) 12 points) ----------------------------------------------------
|
||
|
||
// bankTokenBudget is the extra max_tokens the translator draft reserves for the ≤12-line footnote block
|
||
// (integration point 5): the translation comes FIRST and the block LAST, so a length cut hits the block
|
||
// (tolerated), not the translation. ~12 tokens per line. Added ONLY when the channel is enabled.
|
||
const bankTokenBudget = bankMaxLines * 12
|
||
|
||
// applyBanknote slices the banknote block off a translator draft (integration points 1–5, 8). It runs
|
||
// ONLY on the translator role with the channel enabled — the editor never emits banknotes, and a normal
|
||
// draft has no separator (the slice is then a no-op that returns the raw text unchanged, so a
|
||
// banknote-OFF run is byte-identical). When a ⟦TM-BANK-v1⟧ block IS present it returns the CLEANED
|
||
// translation (what classify/coverage/echo/editor/export all see — points 2/3/4) plus that same cleaned
|
||
// text as the `stripped` export to commit as a derived checkpoint (point 8), and the per-chunk telemetry
|
||
// (point 10). Candidates are PARSED/accepted only under finish=="stop" (the finish=stop-only gate §4б —
|
||
// a truncated block is not trusted); the slice itself runs regardless, so the length/echo classify never
|
||
// sees the footnote's Han src column. Deterministic (a pure function of the raw text + gate state).
|
||
func (r *Runner) applyBanknote(role, rawText, finish string) (clean, stripped string, flags bankFlags) {
|
||
if role != roleTranslator || !r.Pipeline.Gates.Banknote.Enabled {
|
||
return rawText, "", bankFlags{}
|
||
}
|
||
cleanText, block := splitBanknote(rawText)
|
||
if block == "" {
|
||
return rawText, "", bankFlags{} // no separator → no strip, no derived checkpoint (byte-identical path)
|
||
}
|
||
if finish == "stop" {
|
||
// finish=stop-only gate: accept candidates + count telemetry only for a complete generation.
|
||
// truncatedGeneration=false is unreachable-otherwise in prod (a truncated gen is finish≠stop).
|
||
_, flags = parseBanknote(block, false)
|
||
}
|
||
return cleanText, cleanText, flags
|
||
}
|
||
|
||
// bankDerivedHash is the CONTENT-ADDRESSED id of a banknote-stripped export checkpoint (§4б EXACT
|
||
// formula, mirroring commitSanitizedExport's namespacing): sha256("tm-banknote-v1\x00"+reqHash+"\x00"+
|
||
// stripped), prefixed "tm-banknote-v1:" so it can never collide with a real hex attempt hash and a
|
||
// resume re-derives the identical id for free. The namespace embeds the channel version → a version
|
||
// change yields a new id.
|
||
func bankDerivedHash(reqHash, stripped string) string {
|
||
sum := sha256.Sum256([]byte(bankDerivedNS + "\x00" + reqHash + "\x00" + stripped))
|
||
return bankDerivedNS + ":" + hex.EncodeToString(sum[:])
|
||
}
|
||
|
||
// commitBanknoteExport persists the banknote-stripped CLEAN draft as a $0 derived checkpoint and returns
|
||
// its request_hash (→ chunk_status.final_hash), so the OK-path final_hash→checkpoint export contract
|
||
// (and the editor's resume read) yields the CLEANED draft instead of the raw one carrying the footnote
|
||
// (integration point 8). Mirrors commitSanitizedExport: cost 0, escalation 0, idempotent, written BEFORE
|
||
// chunk_status references it. `att` is the terminal (possibly escalated) OK attempt whose block was
|
||
// sliced; att.bankStripped is the cleaned text.
|
||
func (r *Runner) commitBanknoteExport(st config.Stage, ch Chunk, job *store.Job, att stageAttempt) (string, error) {
|
||
derivedHash := bankDerivedHash(att.reqHash, att.bankStripped)
|
||
if err := r.Store.PutDerivedCheckpoint(store.Checkpoint{
|
||
RequestHash: derivedHash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: att.attempt,
|
||
Stage: st.Name, Role: st.Role, ModelRequested: att.modelActual, ModelActual: att.modelActual,
|
||
ResponseText: att.bankStripped, UsageJSON: "{}", FinishReason: "banknote_export",
|
||
}); err != nil {
|
||
return "", fmt.Errorf("pipeline: commit banknote export ch%d/chunk%d/%s: %w", ch.Chapter, ch.ChunkIdx, st.Name, err)
|
||
}
|
||
return derivedHash, nil
|
||
}
|