textmachine/backend/internal/pipeline/banknote.go

package pipeline

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"regexp"
	"sort"
	"strings"
	"sync"
	"unicode"

	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/config"
	"textmachine/backend/internal/store"
	"textmachine/backend/internal/terminology"
	"textmachine/backend/internal/text"
)

// banknote.go: the banknote-v1 in-band footnote channel (WS4, plan §4 / research/20 §B3 / exp16
// banknote.py — RATIFIED D39.10 as the dst-delivery channel). The translator, AFTER the translation,
// MAY emit a versioned separator + tab-delimited term lines for NEW terms only (not in the injected
// glossary). This file is the pure, deterministic PARSER + SLICER — the wire-invariant core; the
// integration seams (slice BEFORE classifyOutput, derived checkpoint, snapshot fold, telemetry —
// plan §4(а) 12 points) consume these functions. A faithful Go↔Python port (banknote.py is the
// reference): same separator, same tolerant field split, same truncation tolerance. The src rule
// DIVERGES deliberately (v2, backlog 19): attested-in-source instead of the reference's Han range. Pin (ws4_banknote_verify.py): re-parse of 95 saved lines = 0 parse_fail / 0 truncated.

// bankSeparator marks the start of the banknote block. Chosen to (a) not occur in natural prose and
// (b) NOT be a "Примечание/Сноска/Комментарий" trailing-note word the sanitizer reserves (§B3-1), so
// the two channels never collide. Changing it is a wire/parser change → a loud --resnapshot (§4в).
const bankSeparator = "⟦TM-BANK-v1⟧"

// bankMaxLines is the per-chunk banknote line budget (§B3-5, feeds the max_tokens sizing at
// integration point 5). The parser does not enforce it (a model over-emitting is tolerated and
// merely over-counted); it bounds the prompt-side budget.
//
// RAISED 12 → 20 (pack-20, on the mini-run measurement D39.41 re-read): 2 of the probe's 9 blocks came
// back at exactly 12 lines — a cap a fifth of the blocks press against is not a budget, it is a silent
// truncation of the WHAT channel on precisely the densest chapters, which are the ones with the most new
// terms to declare. Raising it costs only reserved max_tokens (billing is by ACTUAL usage), and the
// reservation is now derived rather than guessed — see bankTokenBudget.
const bankMaxLines = 20

// The channel has TWO versioned algorithms, and only one of them can change a paid artifact. Splitting
// them (cold-run session) is what makes the fold honest in both directions:
//
//   - bankSliceVersion governs splitBanknote — where the block is cut off the raw answer. Its output IS
//     the stripped draft: a checkpoint, the editor's input, the export. FOLDED into banknoteSnap; a change
//     re-resolves paid text, so it must be a loud --resnapshot.
//   - bankParseVersion governs parseBanknote — which candidate lines are accepted. Its output is EVIDENCE
//     for the owner's signature, never a checkpoint and never a wire byte; every run recomputes it from
//     the stored answers (bankObservedForBook). NOT folded, and LOGGED with the run at the bank-mining stop
//     (mining.go, "draft-side proposals folded") — the same contract gates.terminology and gates.voice
//     carry, and for the same reason. The logging is what makes staying out of the snapshot honest rather
//     than merely cheap: without a consumer the tag is a comment, and two maps produced by different rules
//     are indistinguishable after the fact.
//
// Folding the parse rule (as a single parser_version did) would re-bill a whole draft wave for a change
// that cannot alter one byte of what was bought — which is a standing incentive to leave the rule wrong.
const (
	bankSliceVersion = "banknote-slice-v1"
	// v2 (backlog 19 / D39.53 default): the src rule stopped being "contains a Han ideograph" (a language
	// FAMILY hardcoded in the engine) and became "occurs in the book"; v2.1 adds the column-order recovery
	// that same invariant makes possible; v2.2 (backlog 19 fix-pack) makes the reading DELIMITER-AWARE — a
	// tab/pipe line is positional, only a space-run line re-joins a split rendering. Bumping this is not
	// decoration: the tag is what attributes a stored signature map to the rule that produced it, and a rule
	// change under an unchanged tag is exactly the drift the split into two versions exists to prevent.
	bankParseVersion = "banknote-parse-v2.2-src-attested+column-order+delimiter-aware"
)

// bankDerivedNS is the derived-checkpoint id NAMESPACE (§4б EXACT formula) — the "tm-<name>-v1" prefix
// convention the sibling commitSanitizedExport uses ("tm-sanitized-v1"), kept DISTINCT from the parser
// version above exactly as sanitizerSnap.Version (sanitizer-v6) is distinct from the "tm-sanitized-v1"
// namespace: it embeds the channel version so a version change yields a fresh (never colliding) id.
const bankDerivedNS = "tm-banknote-v1"

// bankTypeOK is the accepted type set; anything else falls back to "term" (a benign default).
var bankTypeOK = map[string]bool{"name": true, "place": true, "title": true, "term": true, "nickname": true}

// bankFieldSplit tolerates a tab, a run of ≥2 spaces, or a pipe (optionally whitespace-padded) as the
// field delimiter — the exact tolerance of banknote.py so a model that emits spaces instead of a real
// TAB still parses. (Go \s is ASCII-whitespace; banknote lines are tab/space-delimited, so the parity
// with Python's unicode \s around the pipe is exact on the term corpus.)
var bankFieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`)

// bankColumnSplit is the UNAMBIGUOUS half of that tolerance: a tab or a pipe is a delimiter the model chose,
// never something that can occur inside a rendering. A line carrying one has declared its own columns.
var bankColumnSplit = regexp.MustCompile(`\t|\s*\|\s*`)

// bankSpaceRun collapses the internal whitespace of a column read positionally, so «Фан␣␣Юань» arrives as
// one clean rendering instead of carrying the model's stray spacing into the bank.
var bankSpaceRun = regexp.MustCompile(` {2,}`)

// bankColumns splits one banknote line into (src + rendering fields, type). See parseBanknote for why the
// delimiter decides which of the two readings applies.
func bankColumns(line string) ([]string, string) {
	typ := "term"
	if cols := nonEmptyFields(bankColumnSplit, line); len(cols) >= 2 {
		if len(cols) >= 3 {
			if t := strings.ToLower(cols[2]); bankTypeOK[t] {
				typ = t
			}
		}
		// BOTH columns are collapsed, not just the second: the order-recovery below may swap them, and the
		// column that becomes the rendering must arrive clean either way.
		return []string{collapseRuns(cols[0]), collapseRuns(cols[1])}, typ
	}
	fields := nonEmptyFields(bankFieldSplit, line)
	if len(fields) >= 3 {
		if last := strings.ToLower(fields[len(fields)-1]); bankTypeOK[last] {
			typ = last
			fields = fields[:len(fields)-1]
		}
	}
	return fields, typ
}

// collapseRuns folds a run of spaces inside a positionally-read column into one.
func collapseRuns(s string) string {
	return strings.TrimSpace(bankSpaceRun.ReplaceAllString(s, " "))
}

// nonEmptyFields splits a line on re with empty and whitespace-only fields dropped.
func nonEmptyFields(re *regexp.Regexp, line string) []string {
	var out []string
	for _, p := range re.Split(strings.TrimSpace(line), -1) {
		if p = strings.TrimSpace(p); p != "" {
			out = append(out, p)
		}
	}
	return out
}

// bankEntry is one parsed candidate line (evidence for the bank-mining stop (§C); NOT written to the store until owner
// signoff). Dst is the model's proposed translation — the direct dst delivery the co-occurrence
// miner could not extract (蛊→гу, non-seed 龙公→Лун Гун).
type bankEntry struct {
	Src  string
	Dst  string
	Type string
}

// bankFlags is the per-chunk telemetry surfaced loud at integration point 10.
type bankFlags struct {
	NLines    int  // accepted (well-formed) banknote lines
	ParseFail bool // any malformed residual line (src not in the book / <2 fields) that was NOT a tolerated truncation
	// Truncated marks a block the run did NOT complete: the separator was there, so the model opened the
	// channel, but the generation ended on a non-stop finish and the stop-only gate refuses it. Read off the
	// finish reason, not from the parser (which never sees such a block). ⚠ It cannot see a cut that removed
	// the separator itself — on the reference that is the majority of length-cut chunks — so it is a floor.
	Truncated bool
}

// splitBanknote slices the banknote block off the raw model output BEFORE any gate/editor (integration
// point 1). Returns (clean_translation, raw_block). No separator → the whole (right-trimmed) output is
// the clean translation and the block is "". Faithful to banknote.py: clean = output[:idx].rstrip(),
// block = output[idx+len(SEP):].strip("\n").
func splitBanknote(output string) (clean, block string, malformedSep bool) {
	idx := strings.Index(output, bankSeparator)
	if idx < 0 {
		// S12 (§B3-1, the case that provision was written for and the code never covered): the model TRIED
		// to open the channel and got the marker wrong — «Примечание:», a stray space inside the brackets, a
		// half-typed token. The exact-match parser silently treated the whole thing as prose, so the table
		// travelled into the editor and the export while banknote_parse_fail stayed 0.
		//
		// The recovery keys on the engine's own distinctive fragment, which cannot occur in a translation:
		// the line carrying it is service output by construction, so slicing there is safe, and the run is
		// told LOUDLY (parse_fail) that the channel was malformed rather than absent.
		if at := malformedSeparatorAt(output); at >= 0 {
			return strings.TrimRightFunc(output[:at], unicode.IsSpace), "", true
		}
		return strings.TrimRightFunc(output, unicode.IsSpace), "", false
	}
	clean = strings.TrimRightFunc(output[:idx], unicode.IsSpace)
	block = strings.Trim(output[idx+len(bankSeparator):], "\n")
	return clean, block, false
}

// bankMarkerFragment is the part of the separator distinctive enough to recognise a MISSPELLED one. It is
// the channel's own version token, so a translation cannot contain it by accident.
const bankMarkerFragment = "TM-BANK"

// malformedSeparatorAt returns the byte offset of the LINE that carries a broken separator, or -1. It is
// only ever consulted when the exact separator is absent.
func malformedSeparatorAt(output string) int {
	i := strings.Index(output, bankMarkerFragment)
	if i < 0 {
		return -1
	}
	if nl := strings.LastIndexByte(output[:i], '\n'); nl >= 0 {
		return nl
	}
	return 0
}

// bankSourceIndex answers the parser's src question — "is this surface actually in the book?" — for one
// run. It replaces the Han-range test the Python reference used (D39.53 default, backlog 19): that rule
// was a language FAMILY hardcoded in the engine, so on any non-ideographic source EVERY line was rejected
// with parse_fail and the channel that carries 5/6 of the bank's candidates was dead. "Occurs in the
// source" is pair-blind by construction and strictly stronger where the old rule worked at all: it also
// rejects a surface the model invented, which the Han test accepted as long as it looked Chinese.
//
// The book side is the CONCATENATION of the chunk sources under the same normalization the miner uses
// (NormalizeSourceKey), which is what makes the per-chunk fast path a pure optimisation rather than a
// second rule: a chunk's normalized text is a substring of the joined text, so the two can never disagree.
// The "\n" join is the boundary guard — a term cannot straddle two chunks and count as attested.
type bankSourceIndex struct {
	book string
	mu   sync.Mutex
	memo map[string]bool
}

// newBankSourceIndex builds the index once per run, in the composite root, so no call path can forget it.
func newBankSourceIndex(chunks []chunk.Chunk) *bankSourceIndex {
	parts := make([]string, 0, len(chunks))
	for _, ch := range chunks {
		parts = append(parts, text.NormalizeSourceKey(ch.Text))
	}
	return &bankSourceIndex{book: strings.Join(parts, "\n"), memo: map[string]bool{}}
}

// attested reports whether a NORMALIZED key occurs in the book. Memoized because the wave workers call it
// per banknote line and the book side is megabytes on a real book; the memo is a cache of a pure function,
// so it changes no verdict — only the cost of repeating one.
func (x *bankSourceIndex) attested(key string) bool {
	x.mu.Lock()
	defer x.mu.Unlock()
	if v, ok := x.memo[key]; ok {
		return v
	}
	v := strings.Contains(x.book, key)
	x.memo[key] = v
	return v
}

// bankSrcAttested builds the parser's src predicate for one parse. The chunk that produced the block is
// the fast path (a banknote declares the terms of ITS fragment, so nearly every legitimate line hits it);
// the book-wide index is the actual rule and the fallback for a term attested elsewhere — and for the
// re-fold path, which reads stored answers and passes no chunk text.
func (r *Runner) bankSrcAttested(chunkSource string) func(string) bool {
	nchunk := text.NormalizeSourceKey(chunkSource)
	return func(src string) bool {
		key := text.NormalizeSourceKey(strings.TrimSpace(src))
		if key == "" {
			return false
		}
		if nchunk != "" && strings.Contains(nchunk, key) {
			return true
		}
		if r.bankSrc == nil {
			return false
		}
		return r.bankSrc.attested(key)
	}
}

// parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry). A
// malformed line (fewer than 2 fields, or a src NOT attested in the source) sets banknote_parse_fail.
// Deterministic, no time/rand: srcAttested is a pure function of the book text (bankSourceIndex).
//
// It carried a `truncatedGeneration` tolerance for a short LAST line (§B3-5, faithful to banknote.py). The
// ratified finish=stop-only gate means this parser is never handed a truncated block at all, so the only
// caller pinned the argument to false and the branch could not execute in production while its comment said
// it could. The truncation FACT is now read where it actually exists — off the finish reason, before the
// gate refuses the block (applyBanknoteWithEntries) — and the unreachable branch is gone rather than left
// as a tolerance nobody can reach.
func parseBanknote(block string, srcAttested func(string) bool) ([]bankEntry, bankFlags) {
	var entries []bankEntry
	var flags bankFlags
	if strings.TrimSpace(block) == "" {
		return entries, flags
	}
	var lines []string
	for _, ln := range strings.Split(block, "\n") {
		if strings.TrimSpace(ln) != "" {
			lines = append(lines, ln)
		}
	}
	bad := 0
	for _, ln := range lines {
		// THE DELIMITER CARRIES INFORMATION, and reading the columns without it is what made the first cut of
		// this fix a regression on the frozen reference (−6 lines over 56 real blocks).
		//
		// A line the model delimited with a TAB or a PIPE has told us where its columns are, so it is read
		// POSITIONALLY: src, rendering, type — exactly as it was before the fix-pack. A third column outside
		// the closed type vocabulary («clan», «proverb», «onomatopoeia» — 1.25% of the reference) is then a
		// benign unknown type, as it always was; guessing it into the rendering corrupts a term the owner is
		// about to sign, and on a REVERSED line it made the src unfindable and dropped the line entirely.
		//
		// Only a line with NO tab and NO pipe is ambiguous, because the ≥2-space tolerance exists for models
		// that substitute spaces for tabs — and that same tolerance is what splits «Фан␣␣Юань» into two
		// fields. There, and only there, is the type identified by the closed vocabulary on the LAST field
		// and everything between re-joined, which is the discipline terminology.ParseReply already applies.
		fields, typ := bankColumns(ln)
		if len(fields) < 2 {
			bad++
			continue
		}
		// The src rule: a bank line must name something the BOOK contains. A surface no chunk of the source
		// carries is either a hallucination or a mis-split line — not evidence about this book, and letting
		// it through would put an invented term on the owner's sign map.
		//
		// The same invariant IDENTIFIES the source column, so a model that emits the pair in the other order
		// is recovered instead of discarded (measured: one chunk emitted all 12 of its lines reversed,
		// including a term the owner had signed by hand). Order is decided, never guessed: the declared
		// order wins whenever it is attested, the reversal is taken only when it is the ONLY reading the
		// book supports, and a line neither reading supports is still a parse fail. The src is one field on
		// either reading — a source surface has no spaces to be split on — so the rendering is whatever
		// remains, which is what makes the recovery survive a split rendering instead of failing on it.
		src, dst := fields[0], strings.Join(fields[1:], " ")
		revSrc, revDst := fields[len(fields)-1], strings.Join(fields[:len(fields)-1], " ")
		switch {
		case srcAttested == nil:
			bad++
			continue
		case srcAttested(src): // declared order — including the ambiguous case where both sides are attested
		case srcAttested(revSrc):
			src, dst = revSrc, revDst
		default:
			bad++
			continue
		}
		entries = append(entries, bankEntry{Src: src, Dst: dst, Type: typ})
	}
	flags.NLines = len(entries)
	flags.ParseFail = bad > 0
	return entries, flags
}

// --- integration seam (plan §4(а) 12 points) ----------------------------------------------------

// bankTokenBudget is the extra max_tokens the translator draft reserves for the footnote block
// (integration point 5): the translation comes FIRST and the block LAST, so a length cut hits the block
// (tolerated), not the translation. Added ONLY when the channel is enabled.
//
// DERIVED, not guessed (pack-20, closing the D39.41 acceptance finding). The old value was
// `bankMaxLines * 12` — a round number nobody had checked. Measured with the engine's OWN estimator over
// the probe's real blocks it was 142.6/147.2 est-tokens against a budget of 144: the reservation was at
// best exactly the need and at worst already negative, which is how a "generous" budget silently truncates
// the channel. It is now computed from a WORST-CASE line under the same EstimateTokens the rest of the
// max_tokens arithmetic uses (CJK ≈ 1 token/char, other ≈ 1/3), so a change to either the line format or
// the estimator moves it automatically instead of leaving a stale literal behind.
//
// Over-reserving is the safe direction and nearly free: max_tokens is a CEILING and a reservation, while
// billing is by ACTUAL usage — an unused ceiling costs nothing but headroom in the spend estimate.
var bankTokenBudget = bankMaxLines*bankWorstLineTokens + EstimateTokens(bankSeparator+"\n")

// bankWorstLineTokens is one banknote line's worst-case cost under EstimateTokens: a long source surface
// (all CJK, 1 token each), a long target rendering (Cyrillic, ≈1/3), and the type column with its tabs.
// The literal below IS the arithmetic — it is measured, not asserted (see TestBankTokenBudgetIsDerived).
var bankWorstLineTokens = EstimateTokens(strings.Repeat("蛊", 8) + "\t" + strings.Repeat("я", 40) + "\tnickname\n")

// applyBanknoteWithEntries slices the banknote block off a translator draft (integration points 1–5, 8)
// and returns the PARSED entries — the WHAT itself. It runs ONLY on the translator role with the channel
// enabled — the editor never emits banknotes, and a normal draft has no separator (the slice is then a
// no-op that returns the raw text unchanged, so a banknote-OFF run is byte-identical). When a ⟦TM-BANK-v1⟧
// block IS present it returns the CLEANED translation (what classify/coverage/echo/editor/export all see —
// points 2/3/4) plus that same cleaned text as the `stripped` export to commit as a derived checkpoint
// (point 8), and the per-chunk telemetry (point 10). Candidates are PARSED/accepted only under
// finish=="stop" (the finish=stop-only gate §4б — a truncated block is not trusted); the slice itself runs
// regardless, so the length/echo classify never sees the footnote's Han src column. Deterministic (a pure
// function of the raw text + gate state).
//
// Until the mini-run of 25.07 the entries were dropped into `_` here (D39.36): the model proposed a
// rendering for every new term, the parser accepted it, and the code kept only the counters. The owner
// then met the bank-mining stop holding bare source terms and had to invent the Russian himself — the
// opposite of the ratified design ("the dst is the owner's to attach via the banknote at sign time",
// mining.go:20). Returning them costs nothing: the parse already ran.
//
// They stay OUT of the bank. A proposal is evidence for the SIGNATURE, never a canon: it reaches the
// owner's sign map as `status: auto` and becomes a term only when the owner signs it. That boundary is
// what keeps "the model suggested it" from silently becoming "the book uses it".
func (r *Runner) applyBanknoteWithEntries(role, rawText, finish, chunkSource string) (clean, stripped string, flags bankFlags, entries []bankEntry) {
	if role != roleTranslator {
		return rawText, "", bankFlags{}, nil // the editor never emits banknotes
	}
	// THE SLICE IS UNCONDITIONAL (pack-20, D39.42 п.4). It used to be gated on Gates.Banknote.Enabled, and
	// that gate is what made the literal reading of "move the block into translator.md" unsafe: the key is
	// in no shipping config, so a prompt that started asking for the block would have had its raw
	// ⟦TM-BANK-v1⟧ table travel into the draft, the editor and the EXPORT with nothing to catch it (the
	// output sanitizer is final-stage-only). Making the slice a property of the TEXT rather than of a
	// config key removes that failure mode entirely — and it is byte-identical for every existing run,
	// because a draft with no separator takes the same early return it always did.
	//
	// The gate itself stays (owner: «булев остаётся»): it governs the max_tokens reservation and the
	// snapshot fold — the two things that ARE wire axes.
	cleanText, block, malformedSep := splitBanknote(rawText)
	if malformedSep {
		// A broken separator: the table is cut off the draft anyway (that line is service output), and the
		// run is told loudly. No candidates are taken from it — a block we could not even open is not
		// evidence.
		return cleanText, cleanText, bankFlags{ParseFail: true}, nil
	}
	if block == "" {
		return rawText, "", bankFlags{}, nil // no separator → no strip, no derived checkpoint (byte-identical path)
	}
	if finish != "stop" {
		// The finish=stop-only gate (§4б, ratified) refuses the block, and that refusal is the whole content
		// of banknote_truncated: a block that IS there under a finish the run did not complete is a block the
		// bank never got. (Length is the usual cause and the one the column was named for; a refusal or a
		// filter cut lands here too, and for the owner it means the same thing — the WHAT of this chunk is
		// missing, and the counters must not read as if it were clean.)
		//
		// The flag used to be derived INSIDE the parser from a truncatedGeneration argument the only caller
		// pinned to false, so the column could not be 1 in production and the comment beside it said the
		// opposite. Reading it off the finish reason needs no parse of untrusted text, no schema change, and
		// leaves the gate exactly where it was ratified: nothing is accepted, the fact is recorded.
		return cleanText, cleanText, bankFlags{Truncated: true}, nil
	}
	entries, flags = parseBanknote(block, r.bankSrcAttested(chunkSource))
	return cleanText, cleanText, flags, entries
}

// bankProposal is one durable WHAT proposal as it travels from the draft to the signature map: the
// normalized source key (the miner's candidate space, so the join needs no fuzzy matching), the source
// surface as the model wrote it, the proposed rendering and the type.
type bankProposal struct {
	SrcKey string `json:"k"`
	Src    string `json:"s"`
	Dst    string `json:"d"`
	Type   string `json:"t"`
}

// bankProposalsOf keys a chunk's parsed entries. Deterministic: the parser's line order is preserved.
func bankProposalsOf(entries []bankEntry) []bankProposal {
	out := make([]bankProposal, 0, len(entries))
	for _, e := range entries {
		key := text.NormalizeSourceKey(e.Src)
		if key == "" || strings.TrimSpace(e.Dst) == "" {
			continue // a proposal with no key or no rendering carries nothing to sign
		}
		out = append(out, bankProposal{SrcKey: key, Src: e.Src, Dst: strings.TrimSpace(e.Dst), Type: e.Type})
	}
	return out
}

// bankProposalsJSON serializes them for retrieval_state.banknote_detail. Empty → "" (the column stays
// empty on every channel-off chunk, so a banknote-free book's row bytes are unchanged).
func bankProposalsJSON(entries []bankEntry) string {
	out := bankProposalsOf(entries)
	if len(out) == 0 {
		return ""
	}
	b, err := json.Marshal(out)
	if err != nil {
		return ""
	}
	return string(b)
}

// bankFold accumulates the draft side of the bank over every sampling of the book that exists. It is the
// one place a proposal becomes evidence, so the three disciplines apply here: the enclosure trim, the
// answer-language screen, and per-CHUNK counting.
type bankFold struct {
	target      *unicode.RangeTable // nil → the language screen is inert
	votes       map[string]*bankAgg
	seen        map[bankVote]bool
	OffLanguage int
}

type bankAgg struct {
	src   string
	typ   string
	byDst map[string]*terminology.Proposal
}

// bankVote is one chunk's vote for one rendering, so a count means "how many chunks proposed this" and
// not "how many stored rows mention it" — they diverge the moment a chunk is re-drafted.
type bankVote struct {
	chapter, chunk int
	key, dst       string
}

func newBankFold(target *unicode.RangeTable) *bankFold {
	return &bankFold{target: target, votes: map[string]*bankAgg{}, seen: map[bankVote]bool{}}
}

// add folds one chunk's parsed proposals; src/dst arrive as the model wrote them.
func (f *bankFold) add(chapter, chunk int, props []bankProposal) {
	for _, p := range props {
		// Key and displayed surface are trimmed together: a decorated surface beside a bare key would be
		// one thing described two ways, and no matcher will ever see the decoration in the source.
		key, src := text.TrimEnclosure(p.SrcKey), text.TrimEnclosure(p.Src)
		if key == "" || strings.TrimSpace(p.Dst) == "" {
			continue
		}
		if terminology.OffLanguage(p.Dst, f.target) {
			f.OffLanguage++
			continue
		}
		v := bankVote{chapter, chunk, key, p.Dst}
		if f.seen[v] {
			continue
		}
		f.seen[v] = true
		a := f.votes[key]
		if a == nil {
			a = &bankAgg{src: src, typ: p.Type, byDst: map[string]*terminology.Proposal{}}
			f.votes[key] = a
		}
		if cur := a.byDst[p.Dst]; cur != nil {
			cur.Chunks++
			continue
		}
		a.byDst[p.Dst] = &terminology.Proposal{Dst: p.Dst, Type: p.Type, Chunks: 1}
	}
}

// observed materializes the fold: key-ordered, each key's renderings most-proposed-first with ties broken
// on the rendering itself — never on map order.
func (f *bankFold) observed() []terminology.Observed {
	keys := make([]string, 0, len(f.votes))
	for k := range f.votes {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	out := make([]terminology.Observed, 0, len(keys))
	for _, key := range keys {
		a := f.votes[key]
		list := make([]terminology.Proposal, 0, len(a.byDst))
		for _, p := range a.byDst {
			list = append(list, *p)
		}
		sort.Slice(list, func(i, j int) bool {
			if list[i].Chunks != list[j].Chunks {
				return list[i].Chunks > list[j].Chunks
			}
			return list[i].Dst < list[j].Dst
		})
		out = append(out, terminology.Observed{Key: key, Src: a.src, Type: a.typ, Proposals: list})
	}
	return out
}

// bankObservedByKey folds the durable per-chunk banknote rows into the draft-side view of each source
// surface. Both consumers read it: the terminologist's merge (whose candidates the signature-map join
// then reads, proposalsFromCandidates) — which also needs the SURFACE, so a proposal for a term the miner
// never found is not lost.
func bankObservedByKey(states []store.RetrievalState, target *unicode.RangeTable) []terminology.Observed {
	f := newBankFold(target)
	addRetrievalStates(f, states)
	return f.observed()
}

// bankObservedForBook is the draft side over EVERY sampling of the book that exists: the per-chunk
// telemetry rows plus every stored translator answer, including the ones a later run superseded. Both
// sources are folded into one accumulator, and votes are counted per chunk, so a chunk present in both
// (the normal case) counts once and the union only ever grows when new drafts were actually bought.
//
// Superseded attempts are included deliberately: a draft rejected for a defect in its TEXT does not make
// the terms it declared untrue, and nothing here enters the bank without a signature anyway.
func (r *Runner) bankObservedForBook() ([]terminology.Observed, int, error) {
	f := newBankFold(r.targetScript)
	states, err := r.Store.RetrievalStatesForBook(r.Book.BookID)
	if err != nil {
		return nil, 0, err
	}
	addRetrievalStates(f, states)
	// Only answers that carry the channel's own marker can contribute, and a book's answer history is
	// megabytes: the marker fragment (which a malformed separator still contains) bounds the read.
	answers, err := r.Store.RoleResponsesForBook(r.Book.BookID, roleTranslator, bankMarkerFragment)
	if err != nil {
		return nil, 0, err
	}
	for _, a := range answers {
		// The same slice/parse the live path runs, so the trust rule (a complete generation only) and the
		// derived $0 rows are handled by one definition rather than by a second copy here. No chunk text is
		// passed: this path has only the stored answer, so the src rule resolves against the whole book —
		// the same verdict the live path reaches, since the chunk check is a subset of it.
		_, _, _, entries := r.applyBanknoteWithEntries(roleTranslator, a.ResponseText, a.FinishReason, "")
		f.add(a.Chapter, a.ChunkIdx, bankProposalsOf(entries))
	}
	return f.observed(), f.OffLanguage, nil
}

func addRetrievalStates(f *bankFold, states []store.RetrievalState) {
	for _, rs := range states {
		if rs.BanknoteDetail == "" {
			continue
		}
		var props []bankProposal
		if json.Unmarshal([]byte(rs.BanknoteDetail), &props) != nil {
			continue // a corrupt detail blob is observability, never a reason to fail a run
		}
		f.add(rs.Chapter, rs.ChunkIdx, props)
	}
}

// bankDerivedHash is the CONTENT-ADDRESSED id of a banknote-stripped export checkpoint (§4б EXACT
// formula, mirroring commitSanitizedExport's namespacing): sha256("tm-banknote-v1\x00"+reqHash+"\x00"+
// stripped), prefixed "tm-banknote-v1:" so it can never collide with a real hex attempt hash and a
// resume re-derives the identical id for free. The namespace embeds the channel version → a version
// change yields a new id.
func bankDerivedHash(reqHash, stripped string) string {
	sum := sha256.Sum256([]byte(bankDerivedNS + "\x00" + reqHash + "\x00" + stripped))
	return bankDerivedNS + ":" + hex.EncodeToString(sum[:])
}

// commitBanknoteExport persists the banknote-stripped CLEAN draft as a $0 derived checkpoint and returns
// its request_hash (→ chunk_status.final_hash), so the OK-path final_hash→checkpoint export contract
// (and the editor's resume read) yields the CLEANED draft instead of the raw one carrying the footnote
// (integration point 8). Mirrors commitSanitizedExport: cost 0, escalation 0, idempotent, written BEFORE
// chunk_status references it. `att` is the terminal (possibly escalated) OK attempt whose block was
// sliced; att.bankStripped is the cleaned text.
func (r *Runner) commitBanknoteExport(st config.Stage, ch chunk.Chunk, job *store.Job, att stageAttempt) (string, error) {
	derivedHash := bankDerivedHash(att.reqHash, att.bankStripped)
	if err := r.Store.PutDerivedCheckpoint(store.Checkpoint{
		RequestHash: derivedHash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: att.attempt,
		Stage: st.Name, Role: st.Role, ModelRequested: att.modelActual, ModelActual: att.modelActual,
		ResponseText: att.bankStripped, UsageJSON: "{}", FinishReason: "banknote_export",
	}); err != nil {
		return "", fmt.Errorf("pipeline: commit banknote export ch%d/chunk%d/%s: %w", ch.Chapter, ch.ChunkIdx, st.Name, err)
	}
	return derivedHash, nil
}