407 lines
20 KiB
Go
407 lines
20 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"regexp"
|
||
"sort"
|
||
"strings"
|
||
"unicode"
|
||
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/config"
|
||
"textmachine/backend/internal/miner"
|
||
"textmachine/backend/internal/store"
|
||
"textmachine/backend/internal/terminology"
|
||
"textmachine/backend/internal/text"
|
||
)
|
||
|
||
// banknote.go: the banknote-v1 in-band footnote channel (WS4, plan §4 / research/20 §B3 / exp16
|
||
// banknote.py — RATIFIED D39.10 as the dst-delivery channel). The translator, AFTER the translation,
|
||
// MAY emit a versioned separator + tab-delimited term lines for NEW terms only (not in the injected
|
||
// glossary). This file is the pure, deterministic PARSER + SLICER — the wire-invariant core; the
|
||
// integration seams (slice BEFORE classifyOutput, derived checkpoint, snapshot fold, telemetry —
|
||
// plan §4(а) 12 points) consume these functions. A faithful Go↔Python port (banknote.py is the
|
||
// reference): same separator, same tolerant field split, same Han-in-src rule, same truncation
|
||
// tolerance. Pin (ws4_banknote_verify.py): re-parse of 95 saved lines = 0 parse_fail / 0 truncated.
|
||
|
||
// bankSeparator marks the start of the banknote block. Chosen to (a) not occur in natural prose and
|
||
// (b) NOT be a "Примечание/Сноска/Комментарий" trailing-note word the sanitizer reserves (§B3-1), so
|
||
// the two channels never collide. Changing it is a wire/parser change → a loud --resnapshot (§4в).
|
||
const bankSeparator = "⟦TM-BANK-v1⟧"
|
||
|
||
// bankMaxLines is the per-chunk banknote line budget (§B3-5, feeds the max_tokens sizing at
|
||
// integration point 5). The parser does not enforce it (a model over-emitting is tolerated and
|
||
// merely over-counted); it bounds the prompt-side budget.
|
||
//
|
||
// RAISED 12 → 20 (pack-20, on the mini-run measurement D39.41 re-read): 2 of the probe's 9 blocks came
|
||
// back at exactly 12 lines — a cap a fifth of the blocks press against is not a budget, it is a silent
|
||
// truncation of the WHAT channel on precisely the densest chapters, which are the ones with the most new
|
||
// terms to declare. Raising it costs only reserved max_tokens (billing is by ACTUAL usage), and the
|
||
// reservation is now derived rather than guessed — see bankTokenBudget.
|
||
const bankMaxLines = 20
|
||
|
||
// bankParserVersion versions the split/parse algorithm — the verdict-axis component folded into
|
||
// banknoteSnap{enabled, parser_version} (§4в/§4а point 6), mirroring sanitizerSnap: a parser change
|
||
// re-resolves the stripped draft, so it must be a loud --resnapshot even without a prompt edit.
|
||
const bankParserVersion = "banknote-v1"
|
||
|
||
// bankDerivedNS is the derived-checkpoint id NAMESPACE (§4б EXACT formula) — the "tm-<name>-v1" prefix
|
||
// convention the sibling commitSanitizedExport uses ("tm-sanitized-v1"), kept DISTINCT from the parser
|
||
// version above exactly as sanitizerSnap.Version (sanitizer-v6) is distinct from the "tm-sanitized-v1"
|
||
// namespace: it embeds the channel version so a version change yields a fresh (never colliding) id.
|
||
const bankDerivedNS = "tm-banknote-v1"
|
||
|
||
// bankTypeOK is the accepted type set; anything else falls back to "term" (a benign default).
|
||
var bankTypeOK = map[string]bool{"name": true, "place": true, "title": true, "term": true, "nickname": true}
|
||
|
||
// bankFieldSplit tolerates a tab, a run of ≥2 spaces, or a pipe (optionally whitespace-padded) as the
|
||
// field delimiter — the exact tolerance of banknote.py so a model that emits spaces instead of a real
|
||
// TAB still parses. (Go \s is ASCII-whitespace; banknote lines are tab/space-delimited, so the parity
|
||
// with Python's unicode \s around the pipe is exact on the term corpus.)
|
||
var bankFieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`)
|
||
|
||
// bankEntry is one parsed candidate line (evidence for the bank-mining stop (§C); NOT written to the store until owner
|
||
// signoff). Dst is the model's proposed translation — the direct dst delivery the co-occurrence
|
||
// miner could not extract (蛊→гу, non-seed 龙公→Лун Гун).
|
||
type bankEntry struct {
|
||
Src string
|
||
Dst string
|
||
Type string
|
||
}
|
||
|
||
// bankFlags is the per-chunk telemetry surfaced loud at integration point 10.
|
||
type bankFlags struct {
|
||
NLines int // accepted (well-formed) banknote lines
|
||
ParseFail bool // any malformed residual line (non-Han src / <2 fields) that was NOT a tolerated truncation
|
||
Truncated bool // the LAST line was cut by generation length (tolerated, not a parse fail)
|
||
}
|
||
|
||
// splitBanknote slices the banknote block off the raw model output BEFORE any gate/editor (integration
|
||
// point 1). Returns (clean_translation, raw_block). No separator → the whole (right-trimmed) output is
|
||
// the clean translation and the block is "". Faithful to banknote.py: clean = output[:idx].rstrip(),
|
||
// block = output[idx+len(SEP):].strip("\n").
|
||
func splitBanknote(output string) (clean, block string, malformedSep bool) {
|
||
idx := strings.Index(output, bankSeparator)
|
||
if idx < 0 {
|
||
// S12 (§B3-1, the case that provision was written for and the code never covered): the model TRIED
|
||
// to open the channel and got the marker wrong — «Примечание:», a stray space inside the brackets, a
|
||
// half-typed token. The exact-match parser silently treated the whole thing as prose, so the table
|
||
// travelled into the editor and the export while banknote_parse_fail stayed 0.
|
||
//
|
||
// The recovery keys on the engine's own distinctive fragment, which cannot occur in a translation:
|
||
// the line carrying it is service output by construction, so slicing there is safe, and the run is
|
||
// told LOUDLY (parse_fail) that the channel was malformed rather than absent.
|
||
if at := malformedSeparatorAt(output); at >= 0 {
|
||
return strings.TrimRightFunc(output[:at], unicode.IsSpace), "", true
|
||
}
|
||
return strings.TrimRightFunc(output, unicode.IsSpace), "", false
|
||
}
|
||
clean = strings.TrimRightFunc(output[:idx], unicode.IsSpace)
|
||
block = strings.Trim(output[idx+len(bankSeparator):], "\n")
|
||
return clean, block, false
|
||
}
|
||
|
||
// bankMarkerFragment is the part of the separator distinctive enough to recognise a MISSPELLED one. It is
|
||
// the channel's own version token, so a translation cannot contain it by accident.
|
||
const bankMarkerFragment = "TM-BANK"
|
||
|
||
// malformedSeparatorAt returns the byte offset of the LINE that carries a broken separator, or -1. It is
|
||
// only ever consulted when the exact separator is absent.
|
||
func malformedSeparatorAt(output string) int {
|
||
i := strings.Index(output, bankMarkerFragment)
|
||
if i < 0 {
|
||
return -1
|
||
}
|
||
if nl := strings.LastIndexByte(output[:i], '\n'); nl >= 0 {
|
||
return nl
|
||
}
|
||
return 0
|
||
}
|
||
|
||
// hasBankSrcHan reports whether src contains a CJK ideograph in the exact [㐀-鿿] range (U+3400–U+9FFF)
|
||
// banknote.py checks — a zh→ru channel line whose src has no Han is malformed. Kept as the exact
|
||
// Python range (NOT unicode.Han, which is wider) for byte-faithful parity.
|
||
func hasBankSrcHan(src string) bool {
|
||
for _, r := range src {
|
||
if r >= 0x3400 && r <= 0x9FFF {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry).
|
||
// Tolerant of a TRUNCATED final line when truncatedGeneration is set (§B3-5): a short last line under
|
||
// truncation is flagged banknote_truncated, not counted as a parse fail. Any other malformed line
|
||
// (fewer than 2 fields, or a src with no Han) sets banknote_parse_fail. Deterministic, no time/rand.
|
||
func parseBanknote(block string, truncatedGeneration bool) ([]bankEntry, bankFlags) {
|
||
var entries []bankEntry
|
||
var flags bankFlags
|
||
if strings.TrimSpace(block) == "" {
|
||
return entries, flags
|
||
}
|
||
var lines []string
|
||
for _, ln := range strings.Split(block, "\n") {
|
||
if strings.TrimSpace(ln) != "" {
|
||
lines = append(lines, ln)
|
||
}
|
||
}
|
||
bad := 0
|
||
for i, ln := range lines {
|
||
raw := bankFieldSplit.Split(strings.TrimSpace(ln), -1)
|
||
var parts []string
|
||
for _, p := range raw {
|
||
if p = strings.TrimSpace(p); p != "" {
|
||
parts = append(parts, p)
|
||
}
|
||
}
|
||
if len(parts) < 2 {
|
||
// A short LAST line under a truncated generation is a tolerated cut, not a failure.
|
||
if i == len(lines)-1 && truncatedGeneration {
|
||
flags.Truncated = true
|
||
continue
|
||
}
|
||
bad++
|
||
continue
|
||
}
|
||
src, dst := parts[0], parts[1]
|
||
typ := "term"
|
||
if len(parts) >= 3 {
|
||
typ = strings.ToLower(parts[2])
|
||
}
|
||
if !bankTypeOK[typ] {
|
||
typ = "term"
|
||
}
|
||
if !hasBankSrcHan(src) {
|
||
bad++
|
||
continue
|
||
}
|
||
entries = append(entries, bankEntry{Src: src, Dst: dst, Type: typ})
|
||
}
|
||
flags.NLines = len(entries)
|
||
flags.ParseFail = bad > 0
|
||
return entries, flags
|
||
}
|
||
|
||
// --- integration seam (plan §4(а) 12 points) ----------------------------------------------------
|
||
|
||
// bankTokenBudget is the extra max_tokens the translator draft reserves for the footnote block
|
||
// (integration point 5): the translation comes FIRST and the block LAST, so a length cut hits the block
|
||
// (tolerated), not the translation. Added ONLY when the channel is enabled.
|
||
//
|
||
// DERIVED, not guessed (pack-20, closing the D39.41 acceptance finding). The old value was
|
||
// `bankMaxLines * 12` — a round number nobody had checked. Measured with the engine's OWN estimator over
|
||
// the probe's real blocks it was 142.6/147.2 est-tokens against a budget of 144: the reservation was at
|
||
// best exactly the need and at worst already negative, which is how a "generous" budget silently truncates
|
||
// the channel. It is now computed from a WORST-CASE line under the same EstimateTokens the rest of the
|
||
// max_tokens arithmetic uses (CJK ≈ 1 token/char, other ≈ 1/3), so a change to either the line format or
|
||
// the estimator moves it automatically instead of leaving a stale literal behind.
|
||
//
|
||
// Over-reserving is the safe direction and nearly free: max_tokens is a CEILING and a reservation, while
|
||
// billing is by ACTUAL usage — an unused ceiling costs nothing but headroom in the spend estimate.
|
||
var bankTokenBudget = bankMaxLines*bankWorstLineTokens + EstimateTokens(bankSeparator+"\n")
|
||
|
||
// bankWorstLineTokens is one banknote line's worst-case cost under EstimateTokens: a long source surface
|
||
// (all CJK, 1 token each), a long target rendering (Cyrillic, ≈1/3), and the type column with its tabs.
|
||
// The literal below IS the arithmetic — it is measured, not asserted (see TestBankTokenBudgetIsDerived).
|
||
var bankWorstLineTokens = EstimateTokens(strings.Repeat("蛊", 8) + "\t" + strings.Repeat("я", 40) + "\tnickname\n")
|
||
|
||
// applyBanknote slices the banknote block off a translator draft (integration points 1–5, 8). It runs
|
||
// ONLY on the translator role with the channel enabled — the editor never emits banknotes, and a normal
|
||
// draft has no separator (the slice is then a no-op that returns the raw text unchanged, so a
|
||
// banknote-OFF run is byte-identical). When a ⟦TM-BANK-v1⟧ block IS present it returns the CLEANED
|
||
// translation (what classify/coverage/echo/editor/export all see — points 2/3/4) plus that same cleaned
|
||
// text as the `stripped` export to commit as a derived checkpoint (point 8), and the per-chunk telemetry
|
||
// (point 10). Candidates are PARSED/accepted only under finish=="stop" (the finish=stop-only gate §4б —
|
||
// a truncated block is not trusted); the slice itself runs regardless, so the length/echo classify never
|
||
// sees the footnote's Han src column. Deterministic (a pure function of the raw text + gate state).
|
||
func (r *Runner) applyBanknote(role, rawText, finish string) (clean, stripped string, flags bankFlags) {
|
||
clean, stripped, flags, _ = r.applyBanknoteWithEntries(role, rawText, finish)
|
||
return clean, stripped, flags
|
||
}
|
||
|
||
// applyBanknoteWithEntries is applyBanknote plus the PARSED entries — the WHAT itself.
|
||
//
|
||
// Until the mini-run of 25.07 the entries were dropped into `_` here (D39.36): the model proposed a
|
||
// rendering for every new term, the parser accepted it, and the code kept only the counters. The owner
|
||
// then met the bank-mining stop holding bare source terms and had to invent the Russian himself — the
|
||
// opposite of the ratified design ("the dst is the owner's to attach via the banknote at sign time",
|
||
// mining.go:20). Returning them costs nothing: the parse already ran.
|
||
//
|
||
// They stay OUT of the bank. A proposal is evidence for the SIGNATURE, never a canon: it reaches the
|
||
// owner's sign map as `status: auto` and becomes a term only when the owner signs it. That boundary is
|
||
// what keeps "the model suggested it" from silently becoming "the book uses it".
|
||
func (r *Runner) applyBanknoteWithEntries(role, rawText, finish string) (clean, stripped string, flags bankFlags, entries []bankEntry) {
|
||
if role != roleTranslator {
|
||
return rawText, "", bankFlags{}, nil // the editor never emits banknotes
|
||
}
|
||
// THE SLICE IS UNCONDITIONAL (pack-20, D39.42 п.4). It used to be gated on Gates.Banknote.Enabled, and
|
||
// that gate is what made the literal reading of "move the block into translator.md" unsafe: the key is
|
||
// in no shipping config, so a prompt that started asking for the block would have had its raw
|
||
// ⟦TM-BANK-v1⟧ table travel into the draft, the editor and the EXPORT with nothing to catch it (the
|
||
// output sanitizer is final-stage-only). Making the slice a property of the TEXT rather than of a
|
||
// config key removes that failure mode entirely — and it is byte-identical for every existing run,
|
||
// because a draft with no separator takes the same early return it always did.
|
||
//
|
||
// The gate itself stays (owner: «булев остаётся»): it governs the max_tokens reservation and the
|
||
// snapshot fold — the two things that ARE wire axes.
|
||
cleanText, block, malformedSep := splitBanknote(rawText)
|
||
if malformedSep {
|
||
// A broken separator: the table is cut off the draft anyway (that line is service output), and the
|
||
// run is told loudly. No candidates are taken from it — a block we could not even open is not
|
||
// evidence.
|
||
return cleanText, cleanText, bankFlags{ParseFail: true}, nil
|
||
}
|
||
if block == "" {
|
||
return rawText, "", bankFlags{}, nil // no separator → no strip, no derived checkpoint (byte-identical path)
|
||
}
|
||
if finish == "stop" {
|
||
// finish=stop-only gate: accept candidates + count telemetry only for a complete generation.
|
||
// truncatedGeneration=false is unreachable-otherwise in prod (a truncated gen is finish≠stop).
|
||
entries, flags = parseBanknote(block, false)
|
||
}
|
||
return cleanText, cleanText, flags, entries
|
||
}
|
||
|
||
// bankProposal is one durable WHAT proposal as it travels from the draft to the signature map: the
|
||
// normalized source key (the miner's candidate space, so the join needs no fuzzy matching), the source
|
||
// surface as the model wrote it, the proposed rendering and the type.
|
||
type bankProposal struct {
|
||
SrcKey string `json:"k"`
|
||
Src string `json:"s"`
|
||
Dst string `json:"d"`
|
||
Type string `json:"t"`
|
||
}
|
||
|
||
// bankProposalsJSON serializes a chunk's parsed entries for retrieval_state.banknote_detail. Empty
|
||
// entries → "" (the column stays empty on every channel-off chunk, so a banknote-free book's row bytes
|
||
// are unchanged). Deterministic: the parser's line order is preserved, nothing is sorted by map order.
|
||
func bankProposalsJSON(entries []bankEntry) string {
|
||
if len(entries) == 0 {
|
||
return ""
|
||
}
|
||
out := make([]bankProposal, 0, len(entries))
|
||
for _, e := range entries {
|
||
key := text.NormalizeSourceKey(e.Src)
|
||
if key == "" || strings.TrimSpace(e.Dst) == "" {
|
||
continue // a proposal with no key or no rendering carries nothing to sign
|
||
}
|
||
out = append(out, bankProposal{SrcKey: key, Src: e.Src, Dst: strings.TrimSpace(e.Dst), Type: e.Type})
|
||
}
|
||
if len(out) == 0 {
|
||
return ""
|
||
}
|
||
b, err := json.Marshal(out)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
return string(b)
|
||
}
|
||
|
||
// bankProposalsByKey folds every chunk's stored proposals into src-key → proposals, counting how many
|
||
// chunks proposed each rendering. The COUNT is the signal the mini-run surfaced: the same term came back
|
||
// with three different renderings across chunks (学堂家老 → «старейшина школы» / «старейшина-наставник» /
|
||
// «учитель-старейшина»), which is precisely the drift a canon exists to close — so the owner sees the
|
||
// alternatives and their weight, not one arbitrary winner.
|
||
func bankProposalsByKey(states []store.RetrievalState) map[string][]miner.DstProposal {
|
||
obs := bankObservedByKey(states)
|
||
out := make(map[string][]miner.DstProposal, len(obs))
|
||
for _, o := range obs {
|
||
list := make([]miner.DstProposal, 0, len(o.Proposals))
|
||
for _, p := range o.Proposals {
|
||
list = append(list, miner.DstProposal{Dst: p.Dst, Type: p.Type, Chunks: p.Chunks})
|
||
}
|
||
out[o.Key] = list
|
||
}
|
||
return out
|
||
}
|
||
|
||
// bankObservedByKey is the ONE fold of the durable per-chunk banknote rows into the draft-side view of
|
||
// each source surface: its key, the surface as the model wrote it, and every rendering with the number of
|
||
// chunks that proposed it. Both consumers read it — the signature-map join (through bankProposalsByKey)
|
||
// and the terminologist's merge, which additionally needs the SURFACE, so that a proposal for a term the
|
||
// miner never found can appear in the reverse section instead of being silently dropped on the floor.
|
||
//
|
||
// Deterministic throughout: the output slice is key-ordered and each key's renderings are ordered
|
||
// most-proposed-first with ties broken on the rendering itself — never on map order.
|
||
func bankObservedByKey(states []store.RetrievalState) []terminology.Observed {
|
||
type agg struct {
|
||
src string
|
||
typ string
|
||
byDst map[string]*terminology.Proposal
|
||
}
|
||
votes := map[string]*agg{}
|
||
for _, rs := range states {
|
||
if rs.BanknoteDetail == "" {
|
||
continue
|
||
}
|
||
var props []bankProposal
|
||
if json.Unmarshal([]byte(rs.BanknoteDetail), &props) != nil {
|
||
continue // a corrupt detail blob is observability, never a reason to fail a run
|
||
}
|
||
for _, p := range props {
|
||
a := votes[p.SrcKey]
|
||
if a == nil {
|
||
a = &agg{src: p.Src, typ: p.Type, byDst: map[string]*terminology.Proposal{}}
|
||
votes[p.SrcKey] = a
|
||
}
|
||
if cur := a.byDst[p.Dst]; cur != nil {
|
||
cur.Chunks++
|
||
continue
|
||
}
|
||
a.byDst[p.Dst] = &terminology.Proposal{Dst: p.Dst, Type: p.Type, Chunks: 1}
|
||
}
|
||
}
|
||
keys := make([]string, 0, len(votes))
|
||
for k := range votes {
|
||
keys = append(keys, k)
|
||
}
|
||
sort.Strings(keys)
|
||
out := make([]terminology.Observed, 0, len(keys))
|
||
for _, key := range keys {
|
||
a := votes[key]
|
||
list := make([]terminology.Proposal, 0, len(a.byDst))
|
||
for _, p := range a.byDst {
|
||
list = append(list, *p)
|
||
}
|
||
sort.Slice(list, func(i, j int) bool {
|
||
if list[i].Chunks != list[j].Chunks {
|
||
return list[i].Chunks > list[j].Chunks
|
||
}
|
||
return list[i].Dst < list[j].Dst
|
||
})
|
||
out = append(out, terminology.Observed{Key: key, Src: a.src, Type: a.typ, Proposals: list})
|
||
}
|
||
return out
|
||
}
|
||
|
||
// bankDerivedHash is the CONTENT-ADDRESSED id of a banknote-stripped export checkpoint (§4б EXACT
|
||
// formula, mirroring commitSanitizedExport's namespacing): sha256("tm-banknote-v1\x00"+reqHash+"\x00"+
|
||
// stripped), prefixed "tm-banknote-v1:" so it can never collide with a real hex attempt hash and a
|
||
// resume re-derives the identical id for free. The namespace embeds the channel version → a version
|
||
// change yields a new id.
|
||
func bankDerivedHash(reqHash, stripped string) string {
|
||
sum := sha256.Sum256([]byte(bankDerivedNS + "\x00" + reqHash + "\x00" + stripped))
|
||
return bankDerivedNS + ":" + hex.EncodeToString(sum[:])
|
||
}
|
||
|
||
// commitBanknoteExport persists the banknote-stripped CLEAN draft as a $0 derived checkpoint and returns
|
||
// its request_hash (→ chunk_status.final_hash), so the OK-path final_hash→checkpoint export contract
|
||
// (and the editor's resume read) yields the CLEANED draft instead of the raw one carrying the footnote
|
||
// (integration point 8). Mirrors commitSanitizedExport: cost 0, escalation 0, idempotent, written BEFORE
|
||
// chunk_status references it. `att` is the terminal (possibly escalated) OK attempt whose block was
|
||
// sliced; att.bankStripped is the cleaned text.
|
||
func (r *Runner) commitBanknoteExport(st config.Stage, ch chunk.Chunk, job *store.Job, att stageAttempt) (string, error) {
|
||
derivedHash := bankDerivedHash(att.reqHash, att.bankStripped)
|
||
if err := r.Store.PutDerivedCheckpoint(store.Checkpoint{
|
||
RequestHash: derivedHash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: att.attempt,
|
||
Stage: st.Name, Role: st.Role, ModelRequested: att.modelActual, ModelActual: att.modelActual,
|
||
ResponseText: att.bankStripped, UsageJSON: "{}", FinishReason: "banknote_export",
|
||
}); err != nil {
|
||
return "", fmt.Errorf("pipeline: commit banknote export ch%d/chunk%d/%s: %w", ch.Chapter, ch.ChunkIdx, st.Name, err)
|
||
}
|
||
return derivedHash, nil
|
||
}
|