From bccf2d8842b21263c626608f4ac7a091783eddca Mon Sep 17 00:00:00 2001 From: "Claude (backend session)" Date: Sat, 8 Aug 2026 21:03:31 +0300 Subject: [PATCH] Land the bank cluster fixpack: family co-batching as pair data, arbitration observability, banknote parser rejoin, auto-bank drop diff, confidence field, after adversarial acceptance and a fix round --- backend/cmd/tmctl/render.go | 52 +- backend/cmd/tmctl/render_test.go | 31 ++ backend/docs/pipeline.puml | 12 +- backend/internal/lang/bankdata.go | 250 ++++++++++ .../lang/bankdata/family-morphology.txt | 37 ++ backend/internal/lang/bankdata_test.go | 106 +++++ backend/internal/membank/memory.go | 10 + backend/internal/miner/miner_emit.go | 39 +- backend/internal/pipeline/bankfixpack_test.go | 383 +++++++++++++++ backend/internal/pipeline/banknote.go | 155 ++++-- backend/internal/pipeline/banknote_test.go | 139 +++++- backend/internal/pipeline/chunkrun.go | 70 +++ backend/internal/pipeline/mining.go | 219 ++++++++- .../internal/pipeline/miningstop_join_test.go | 53 ++- .../internal/pipeline/minirun_fixes_test.go | 8 +- backend/internal/pipeline/runner.go | 12 + backend/internal/pipeline/runner_test.go | 10 +- backend/internal/pipeline/terminologist.go | 146 +++++- backend/internal/pipeline/waverun.go | 2 + .../internal/terminology/arbitration_test.go | 215 +++++++++ backend/internal/terminology/classify.go | 30 +- backend/internal/terminology/family_test.go | 348 ++++++++++++++ backend/internal/terminology/script_test.go | 6 +- backend/internal/terminology/series.go | 445 +++++++++++++++++- backend/internal/terminology/terminology.go | 319 +++++++++++-- .../internal/terminology/terminology_test.go | 18 +- backend/prompts/zh-ru/terminologist.md | 19 +- 27 files changed, 2934 insertions(+), 200 deletions(-) create mode 100644 backend/internal/lang/bankdata.go create mode 100644 backend/internal/lang/bankdata/family-morphology.txt create mode 100644 backend/internal/lang/bankdata_test.go create mode 100644 backend/internal/pipeline/bankfixpack_test.go create mode 100644 backend/internal/terminology/arbitration_test.go create mode 100644 backend/internal/terminology/family_test.go diff --git a/backend/cmd/tmctl/render.go b/backend/cmd/tmctl/render.go index 3c18117d..894e3f20 100644 --- a/backend/cmd/tmctl/render.go +++ b/backend/cmd/tmctl/render.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "sort" + "strconv" "strings" "unicode/utf8" @@ -103,14 +104,36 @@ func renderBankStopRows(w io.Writer, rows []pipeline.BankStopRow) { if len(rows) == 0 { return } - shown := rows + // THE ONLY use the confidence is allowed to have (D39.102): the capped stdout view is the REVIEW surface, + // so it is ordered least-sure-first — an ordinal inside one model's reply and nothing more. It is not a + // weight, not a threshold, not comparable between models, and it never decides which rendering wins; the + // full sidecar keeps the source-key order, because that is the reference document. + shown := append([]pipeline.BankStopRow(nil), rows...) + sort.SliceStable(shown, func(i, j int) bool { return reviewRank(shown[i]) < reviewRank(shown[j]) }) if len(shown) > bankStopStdoutCap { shown = shown[:bankStopStdoutCap] } fmt.Fprintln(w, "") - fmt.Fprintln(w, " src proposed dst origin freq spread") + fmt.Fprintln(w, " (least-confident first — the role's own ordering of what to read)") + fmt.Fprintln(w, " src proposed dst origin freq spread conv conf") for _, r := range shown { - fmt.Fprintf(w, " %-20s %-25s %-9s %5d %6d\n", trunc(r.Src, 20), trunc(dashIfEmpty(r.Dst), 25), r.Origin, r.Freq, r.Spread) + fmt.Fprintf(w, " %-20s %-25s %-9s %5d %6d %5d %5s\n", trunc(r.Src, 20), trunc(dashIfEmpty(r.Dst), 25), + r.Origin, r.Freq, r.Spread, r.Conventions, confCell(r.Conf)) + // WHY this row has this dst — the question the artifacts could not answer before the fix-pack. The + // signals are the ranking factors that fired for the winning variant; "invented" says the rendering is + // the role's own, not one the drafts proposed (legitimate — it saw the whole book — and worth a look). + if len(r.Signals) > 0 || r.Invented { + why := strings.Join(r.Signals, ", ") + if r.Invented { + why = strings.TrimPrefix(why+" · invented (no draft proposed it)", " · ") + } + fmt.Fprintf(w, " why: %s\n", trunc(why, 90)) + } + if len(r.Contradicts) > 0 { + // A contradiction against this run's OWN consolidations: the compound dropped the rendering the + // same reply gave its part. Loudest line of the row — it is a canon breaking inside one call. + fmt.Fprintf(w, " ⚠ contradicts: %s\n", trunc(strings.Join(r.Contradicts, "; "), 80)) + } if len(r.Variants) > 1 { // The disagreement is the reason to sign: one term coming back several ways is exactly what a // canon exists to close, and hiding it behind one winner throws that reason away. @@ -126,6 +149,29 @@ func renderBankStopRows(w io.Writer, rows []pipeline.BankStopRow) { fmt.Fprintln(w, "") } +// reviewRank orders the review surface: rows the role was least sure of first, rows it said nothing about +// after them (silence is not a low score), and rows it never consolidated last. +func reviewRank(r pipeline.BankStopRow) int { + switch { + case r.Dst == "": + return 1000 + case r.Conf < 0: + return 200 + default: + return r.Conf + } +} + +// confCell renders the role's stated confidence, or "—" when the reply carried none (a negative). Printing +// a bare 0 for both would hide the single most important row on the sheet — the one the role itself said it +// was least sure of — behind the rows it never mentioned. +func confCell(conf int) string { + if conf < 0 { + return "—" + } + return strconv.Itoa(conf) +} + // trunc shortens s to n runes with an ellipsis (rune-safe, so a CJK surface is never split mid-character). func trunc(s string, n int) string { rs := []rune(s) diff --git a/backend/cmd/tmctl/render_test.go b/backend/cmd/tmctl/render_test.go index e8cbbbab..02ba529c 100644 --- a/backend/cmd/tmctl/render_test.go +++ b/backend/cmd/tmctl/render_test.go @@ -290,3 +290,34 @@ func TestRenderSignatureStopShowsTheTable(t *testing.T) { t.Fatalf("a term with no proposed dst must show a dash:\n%s", b2.String()) } } + +// TestRenderSignatureStopOrdersTheReviewLeastSureFirst pins the ONE use the role's stated confidence is +// allowed to have (D39.102): it orders the review list, an ordinal inside one model's reply, and nothing +// else. Without it the number is printed and never used — a column that claims to answer «what do I read +// first» and does not. The full sidecar keeps source-key order; only this capped view is the review surface. +func TestRenderSignatureStopOrdersTheReviewLeastSureFirst(t *testing.T) { + rows := []pipeline.BankStopRow{ + {Src: "уверенный", Dst: "д", Origin: "mined", Freq: 9, Conf: 95}, + {Src: "неконсолидированный", Origin: "mined", Freq: 9, Conf: -1}, + {Src: "неуверенный", Dst: "д", Origin: "mined", Freq: 9, Conf: 20}, + {Src: "безмолвный", Dst: "д", Origin: "mined", Freq: 9, Conf: -1}, + } + var b bytes.Buffer + renderSignatureStop(&b, &pipeline.WaveSignatureStop{Terms: len(rows), SignaturePath: "p", Rows: rows}) + out := b.String() + order := []string{"неуверенный", "уверенный", "безмолвный", "неконсолидированный"} + at := -1 + for _, name := range order { + i := strings.Index(out, name) + if i < 0 { + t.Fatalf("row %q missing:\n%s", name, out) + } + if i < at { + t.Fatalf("review order broken at %q — want least-confident first, then silence, then unconsolidated:\n%s", name, out) + } + at = i + } + if !strings.Contains(out, "least-confident first") { + t.Fatalf("the banner must say what its order means, or a sorted table reads as arbitrary:\n%s", out) + } +} diff --git a/backend/docs/pipeline.puml b/backend/docs/pipeline.puml index f185fae4..fbb791f2 100644 --- a/backend/docs/pipeline.puml +++ b/backend/docs/pipeline.puml @@ -47,8 +47,16 @@ partition "ВОЛНА ДРАФТА [OK] (∥ workers, по draft-чанкам)" if (Майнинг вкл. И дельта непуста? [OK]) then (да) :МАЙНИНГ-СТОП: MineBank по нормализованному источнику - против контраста (сид и reject-set исключены) → - signature map → прогон ОСТАНОВЛЕН (exit 3); + против контраста (сид и reject-set исключены); + :ТЕРМИНОЛОГ [OK]: Merge(WHICH∪WHAT) → KWIC → ранг §C2-3 + (голос по ФОЛДУ целевой формы, сырые формы в показе) → + батчи ЮНИТАМИ: серия (равнодлинная) + СЕМЬЯ + (общий корень: имя-префикс либо реалия-суффикс, данные скрипта) + и containment-пара — юнит целиком в ОДИН вызов, кап членов; + :$0-экраны: канон-конфликты · КОНФЛИКТЫ СВОИХ ЖЕ + консолидаций · label-mismatch → стоп-таблица + (сигналы топ-варианта · invented · уверенность · конвенций); + :signature map → прогон ОСТАНОВЛЕН (exit 3); :Владелец: каждый терм → promote в mined_delta (approved+dst) ИЛИ decline в mined_rejects; :Ре-ран: драфт $0-резюм · пустая дельта → авто-континью; diff --git a/backend/internal/lang/bankdata.go b/backend/internal/lang/bankdata.go new file mode 100644 index 00000000..d0134ae8 --- /dev/null +++ b/backend/internal/lang/bankdata.go @@ -0,0 +1,250 @@ +package lang + +import ( + "crypto/sha256" + "embed" + "encoding/hex" + "fmt" + "sort" + "strconv" + "strings" + "sync" +) + +// bankdata.go: the BANK-ASSEMBLY data plane — language data read ONLY when the terminologist assembles its +// batches, and by nothing that a wave touches. +// +// WHY A PLANE OF ITS OWN, and why it is NOT in data/ beside script-series.txt. Both existing data planes +// fold into the WAVE snapshot: the embedded one through EmbeddedVersion, the pack one through +// Pack.Version(). That is right for bytes that ride the wire or resolve a verdict — editing them must be a +// loud --resnapshot. Family morphology does neither: it decides only WHICH candidates the terminologist +// shows in ONE call. Folding it into the wave would re-bill a whole book's draft for a knob that cannot +// change a single byte of what that draft bought — and it would contradict the ratified stance that the +// terminology axis stays OFF the snapshot (config.TerminologyGate, terminologyVersion, +// TestTerminologyIsNotSnapshotFolded). +// +// It is not unversioned, though: an edit here changes a batch's CONTENT, which is inside the bank-role +// RequestHash, so the affected batches re-pay by mechanism (cents), and BankDataVersion() is logged with +// the run so a signature map stays attributable to the data that produced it. + +//go:embed bankdata/family-morphology.txt +var bankFS embed.FS + +// bankDataAlgoVersion tags the bank plane's hashing RECIPE (like embedAlgoVersion tags the embedded one). +// Bump it only when the framing or the file set changes; a DATA edit already moves the hash via the bytes. +const bankDataAlgoVersion = "bankdata-v1" + +// bankDataFiles are the plane's files, in a FIXED order (the order is folded into the hash). +var bankDataFiles = []string{"bankdata/family-morphology.txt"} + +// FamilyAffix is one engine type's family rule: which side of the surface carries the shared root morpheme +// and how many runes of it must be shared before two surfaces count as one family. +type FamilyAffix struct { + Suffix bool // the root is the TRAILING morpheme (head-final realia); false → the LEADING one (names) + MinRunes int // ≤0 → this type forms no families +} + +// FamilyMorph is a source language's resolved family-channel data. The zero value is INERT — a language +// written in no dense script, or in a dense script with no rows, co-batches exactly as it did before this +// channel existed. +type FamilyMorph struct { + Affix map[string]FamilyAffix // engine type (name|place|title|term) → its rule + MinMembers int // smallest set that counts as a family + MaxMembers int // largest batch unit a family merge may produce (0 = unbounded) + ContainmentRunes int // shortest candidate that may anchor the compositional channel +} + +// Enabled reports whether the language declares enough to form a family at all. +func (f FamilyMorph) Enabled() bool { + return f.MinMembers > 0 && (len(f.Affix) > 0 || f.ContainmentRunes > 0) +} + +// scriptFamily is one script's authored rows, before a language merges its dense scripts. +type scriptFamily struct { + affix map[string]FamilyAffix + minMembers, maxMembers, containmentRunes int +} + +var ( + familyOnce sync.Once + familyByScript map[string]*scriptFamily + bankDataOnce sync.Once + bankDataVal string +) + +func loadFamilyMorphology() { + familyOnce.Do(func() { + m, err := parseFamilyMorphology(mustBankData(bankDataFiles[0])) + if err != nil { + panic(fmt.Sprintf("lang: embedded %s is corrupt: %v", bankDataFiles[0], err)) + } + familyByScript = m + }) +} + +// BankDataVersion is the content hash of the bank-assembly plane (recipe tag + each file's path and bytes, +// same framing as EmbeddedVersion). It is deliberately NOT in the wave snapshot — see the file comment — +// and is logged with the run instead, so "which data produced this signature map" stays answerable. +func BankDataVersion() string { + bankDataOnce.Do(func() { + h := sha256.New() + h.Write([]byte(bankDataAlgoVersion)) + for _, name := range bankDataFiles { + h.Write([]byte("\x00" + name + "\x00")) + h.Write(mustBankData(name)) + } + bankDataVal = bankDataAlgoVersion + "-" + hex.EncodeToString(h.Sum(nil))[:12] + }) + return bankDataVal +} + +// FamilyMorphology resolves the family-channel data for a SOURCE language by unioning the rows of its +// DENSE scripts (the same gate the series channel uses: one rune ≈ one morpheme is what makes a shared +// affix a shared MORPHEME rather than a coincidence). Merge rules, all on the conservative side: +// - two dense scripts declaring the same type with DIFFERENT sides disagree about the language's +// morphology, so that type forms no families rather than one guessed direction; +// - otherwise the STRICTER rule wins (the longer required root), the strictest min_members, and the +// SMALLEST max_members, so adding a script never loosens what another script asked for. +func FamilyMorphology(lng string) FamilyMorph { + loadLangScripts() + loadFamilyMorphology() + var out FamilyMorph + names := make([]string, 0, 4) + for name := range langScriptBy[strings.ToLower(strings.TrimSpace(lng))] { + if cjkScriptNames[name] { + names = append(names, name) + } + } + sort.Strings(names) // deterministic merge order (the rules are order-free, the iteration must be too) + conflict := map[string]bool{} + for _, name := range names { + sf := familyByScript[name] + if sf == nil { + continue + } + if out.Affix == nil { + out.Affix = map[string]FamilyAffix{} + } + for typ, a := range sf.affix { + cur, had := out.Affix[typ] + switch { + case !had: + out.Affix[typ] = a + case cur.Suffix != a.Suffix: + conflict[typ] = true + case a.MinRunes > cur.MinRunes: + out.Affix[typ] = a + } + } + if sf.minMembers > out.MinMembers { + out.MinMembers = sf.minMembers + } + if sf.containmentRunes > out.ContainmentRunes { + out.ContainmentRunes = sf.containmentRunes + } + if sf.maxMembers > 0 && (out.MaxMembers == 0 || sf.maxMembers < out.MaxMembers) { + out.MaxMembers = sf.maxMembers + } + } + for typ := range conflict { + delete(out.Affix, typ) + } + return out +} + +// parseFamilyMorphology reads the plane's rows into script → rules. Pure, so a test can drive it with a +// synthetic table and prove the channel is DATA. Fail-loud on an unknown key / side / non-positive integer: +// a malformed row is a corrupt asset, and a silently-inert family channel is exactly the drift this +// codebase refuses (the Palladius typo precedent). +func parseFamilyMorphology(b []byte) (map[string]*scriptFamily, error) { + out := map[string]*scriptFamily{} + at := func(script string) *scriptFamily { + if out[script] == nil { + out[script] = &scriptFamily{affix: map[string]FamilyAffix{}} + } + return out[script] + } + posInt := func(i int, key, raw string) (int, error) { + n, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil || n <= 0 { + return 0, fmt.Errorf("line %d: %s must be a positive integer (%q)", i+1, key, raw) + } + return n, nil + } + line := 0 + for i, raw := range strings.Split(string(b), "\n") { + t := strings.TrimSpace(strings.TrimRight(raw, "\r")) + if t == "" || strings.HasPrefix(t, "#") { + continue + } + line = i + f := strings.Split(t, "\t") + for j := range f { + f[j] = strings.TrimSpace(f[j]) + } + if len(f) < 3 || f[1] == "" { + return nil, fmt.Errorf("line %d: want `keyscript…`, got %q", i+1, t) + } + // The SCRIPT column is checked against the range registry. A typo parses fine and then simply never + // matches a language's declared scripts — the channel goes silently inert for the pair that asked for + // it, which is the exact silent-empty-table class the pack loader refuses everywhere else. + if _, known := scriptRanges[f[1]]; !known { + return nil, fmt.Errorf("line %d: unknown script %q — it resolves to no Unicode range, so every row naming it would be silently ignored (add it to scriptRanges, or fix the spelling)", i+1, f[1]) + } + switch f[0] { + case "family_affix": + if len(f) != 5 || f[2] == "" { + return nil, fmt.Errorf("line %d: family_affix wants `family_affixscripttypeprefix|suffixmin_root_runes`, got %q", i+1, t) + } + n, err := posInt(i, "min_root_runes", f[4]) + if err != nil { + return nil, err + } + switch f[3] { + case "prefix": + at(f[1]).affix[f[2]] = FamilyAffix{MinRunes: n} + case "suffix": + at(f[1]).affix[f[2]] = FamilyAffix{Suffix: true, MinRunes: n} + default: + return nil, fmt.Errorf("line %d: side must be prefix|suffix, got %q", i+1, f[3]) + } + case "family_min_members", "family_max_members", "family_containment_runes": + if len(f) != 3 { + return nil, fmt.Errorf("line %d: %s wants `%sscriptn`, got %q", i+1, f[0], f[0], t) + } + n, err := posInt(i, f[0], f[2]) + if err != nil { + return nil, err + } + switch f[0] { + case "family_min_members": + at(f[1]).minMembers = n + case "family_max_members": + at(f[1]).maxMembers = n + default: + at(f[1]).containmentRunes = n + } + default: + return nil, fmt.Errorf("line %d: unknown key %q (want family_affix|family_min_members|family_max_members|family_containment_runes)", i+1, f[0]) + } + } + // A HALF table is a corrupt one, for the same reason: family_affix with no family_min_members leaves + // Enabled() false, so the rows are read, parsed, and then quietly do nothing. + for name, sf := range out { + switch { + case sf.minMembers == 0: + return nil, fmt.Errorf("script %q declares family rules but no family_min_members — the channel would load and stay inert (line %d region)", name, line+1) + case len(sf.affix) == 0 && sf.containmentRunes == 0: + return nil, fmt.Errorf("script %q declares family_min_members but neither family_affix nor family_containment_runes — nothing can form a family", name) + } + } + return out, nil +} + +func mustBankData(name string) []byte { + b, err := bankFS.ReadFile(name) + if err != nil { + panic(fmt.Sprintf("lang: missing bank-data asset %q: %v", name, err)) + } + return b +} diff --git a/backend/internal/lang/bankdata/family-morphology.txt b/backend/internal/lang/bankdata/family-morphology.txt new file mode 100644 index 00000000..7d60643b --- /dev/null +++ b/backend/internal/lang/bankdata/family-morphology.txt @@ -0,0 +1,37 @@ +# FAMILY morphology per writing SCRIPT (bank fix-pack §G1, research/24 §B4): which candidates of a dense +# script belong to ONE term FAMILY and must therefore travel in ONE terminologist call. Measured: the same +# model rendered the 古月-family one way in batch 0 and another way in batch 2 — a chimera that correlates +# with the batch boundary perfectly, on a family DetectSeries cannot co-batch (its members differ in length). +# +# A family is anchored on a shared ROOT MORPHEME, and where that root sits is ASYMMETRIC by class: a NAME +# carries its clan/surname morpheme in FRONT (古月方源, 古月正), while a realia term carries its generic head +# LAST (head-final writing: *-海空窍, *-寨). The asymmetry is DATA, not a Go branch, so a pair whose names +# are head-final (or whose script is head-initial) declares its own rows and needs no engine edit. +# +# Rows (TAB-separated), all keyed by SCRIPT name (the names lang/script.go resolves to Unicode ranges): +# family_affixscripttypeprefix|suffixmin_root_runes +# one rule per engine type (name|place|title|term). A type with no row never forms a family. +# family_min_membersscriptn — below n a "family" is a coincidence, not a convention. +# family_max_membersscriptn — the blob backstop: a merge that would push a unit past n +# is refused and COUNTED. Measured on the coldrun-a bank +# distillation (150 surfaces): 24 refuses nothing and the +# largest real unit is 20 — the 古月 clan with the surfaces +# built on it; 16 splits that unit, and a split family is +# the very chimera this channel exists to prevent. +# family_containment_runesscriptn — the compositional channel: a candidate that is itself a +# substring of another candidate (元海 ⊂ 元海空窍) joins its +# unit. n bounds the anchor so a single generic morpheme +# cannot sweep half the bank into one call. +# +# A script with NO rows here leaves the family channel INERT (the batcher then behaves exactly as before +# this file existed), so a pair that has not been measured never silently changes what it buys. +family_affix han name prefix 2 +# `nickname` is a person-surface the BANKNOTE channel accepts from a draft, so a candidate can carry it and +# it needs a rule of its own — otherwise a whole legitimate type forms no families, silently. +family_affix han nickname prefix 2 +family_affix han place suffix 2 +family_affix han title suffix 2 +family_affix han term suffix 2 +family_min_members han 2 +family_max_members han 24 +family_containment_runes han 2 diff --git a/backend/internal/lang/bankdata_test.go b/backend/internal/lang/bankdata_test.go new file mode 100644 index 00000000..4d2bd801 --- /dev/null +++ b/backend/internal/lang/bankdata_test.go @@ -0,0 +1,106 @@ +package lang + +import ( + "strings" + "testing" +) + +// TestBankPlaneIsOutsideTheWaveFold is the load-bearing property of the whole plane, and the reason it is a +// separate embed rather than another row in data/script-series.txt: editing family morphology must NOT move +// the wave snapshot. EmbeddedVersion folds into every book's snapshot, so a row added there would re-bill a +// whole draft wave for a knob that only decides which candidates share a terminologist CALL — the axis +// config.TerminologyGate is deliberately kept off the snapshot for. +func TestBankPlaneIsOutsideTheWaveFold(t *testing.T) { + for _, f := range embeddedDataFiles() { + if strings.Contains(f.name, "family") { + t.Fatalf("the bank plane's %s reached the WAVE-folded embed set — editing it would re-snapshot every book", f.name) + } + } + // Both versions exist and are distinct namespaces, so a log line naming one can never be read as the other. + if v := BankDataVersion(); !strings.HasPrefix(v, bankDataAlgoVersion+"-") { + t.Fatalf("BankDataVersion must be tagged with its own recipe, got %q", v) + } + if strings.HasPrefix(BankDataVersion(), embedAlgoVersion) { + t.Fatal("the two data planes must not share a version namespace") + } +} + +// TestFamilyMorphologyIsData drives the parser with a synthetic table: a script's family rules — including +// the name/realia asymmetry and the head side — are DATA, so a pair that is not in the repo declares its own +// and needs no Go edit. The script here is a REGISTERED one that ships no family rows of its own — the first +// version of this test declared «devanagari», which resolves to no Unicode range and was therefore dropped +// in silence, so the flagship generality test proved nothing at all. +func TestFamilyMorphologyIsData(t *testing.T) { + got, err := parseFamilyMorphology([]byte(strings.Join([]string{ + "# a source whose NAMES are head-final and whose realia lead with the root", + "family_affix\thangul\tname\tsuffix\t3", + "family_affix\thangul\tterm\tprefix\t2", + "family_min_members\thangul\t4", + "family_containment_runes\thangul\t3", + }, "\n"))) + if err != nil { + t.Fatal(err) + } + sf := got["hangul"] + if sf == nil { + t.Fatalf("the declared script must be present: %v", got) + } + if a := sf.affix["name"]; !a.Suffix || a.MinRunes != 3 { + t.Fatalf("a head-final name rule is one data row, got %+v", a) + } + if a := sf.affix["term"]; a.Suffix || a.MinRunes != 2 { + t.Fatalf("the asymmetry is per-type data, got %+v", a) + } + if sf.minMembers != 4 || sf.containmentRunes != 3 || sf.maxMembers != 0 { + t.Fatalf("bounds come from the rows, absent ones stay unset: %+v", sf) + } +} + +// TestFamilyMorphologyRefusesCorruptRows: a malformed row is a corrupt asset, and the failure mode it would +// otherwise produce — a channel that parses fine and silently forms nothing — is the exact drift the pack +// loader already refuses for the Palladius tables. +func TestFamilyMorphologyRefusesCorruptRows(t *testing.T) { + for _, bad := range []string{ + "family_affix\than\tname\tmiddle\t2", // no such side + "family_affix\than\tname\tprefix\t0", // a zero-rune root matches everything + "family_affix\than\tname\tprefix", // truncated row + "family_minmembers\than\t2", // typo'd key + "family_min_members\than\tlots", // non-integer + // A typo'd SCRIPT resolves to no range, so every row naming it is silently ignored and the pair that + // asked for the channel gets an inert one. + "family_affix\thann\tname\tprefix\t2\nfamily_min_members\thann\t2", + // A HALF table: rules with no quorum, or a quorum with nothing to form a family from. Both parse and + // then do nothing, which is the failure this pack declared intolerable everywhere else. + "family_affix\than\tname\tprefix\t2", + "family_min_members\than\t2", + } { + if _, err := parseFamilyMorphology([]byte(bad)); err == nil { + t.Fatalf("a corrupt row must fail loud: %q", bad) + } + } +} + +// TestFamilyMorphologyForZhIsLive pins the shipped table through the resolver a book actually calls: zh is +// written in a dense script that declares rules, an alphabetic source is inert, and the ja case shows the +// union — kana declare nothing, so the language inherits han's rules rather than losing them. +func TestFamilyMorphologyForZhIsLive(t *testing.T) { + zh := FamilyMorphology("zh") + if !zh.Enabled() { + t.Fatal("zh is the measured pair and must have a live family channel") + } + if a := zh.Affix["name"]; a.Suffix || a.MinRunes < 2 { + t.Fatalf("a han name family leads with the clan morpheme: %+v", a) + } + if a := zh.Affix["term"]; !a.Suffix || a.MinRunes < 2 { + t.Fatalf("a han realia family ends with the generic head: %+v", a) + } + if zh.MinMembers < 2 || zh.MaxMembers <= 0 || zh.ContainmentRunes < 2 { + t.Fatalf("the bounds must all be declared: %+v", zh) + } + if en := FamilyMorphology("en"); en.Enabled() { + t.Fatalf("an alphabetic source forms no morpheme families: %+v", en) + } + if ja := FamilyMorphology("ja"); !ja.Enabled() || ja.Affix["term"] != zh.Affix["term"] { + t.Fatalf("a language written in han inherits han's rules through the union: %+v", ja) + } +} diff --git a/backend/internal/membank/memory.go b/backend/internal/membank/memory.go index c32a4d1d..ae204ffd 100644 --- a/backend/internal/membank/memory.go +++ b/backend/internal/membank/memory.go @@ -202,6 +202,16 @@ type PickedEntry struct { // no real path changes (pack-16 tail). func (p PickedEntry) valid() bool { return p.entry != nil } +// Src is the bank row's SOURCE surface — the only thing about a record a caller outside the bank can NAME. +// It exists because n_evicted counted budget drops without ever saying WHICH rows the model did not get to +// see, and a bare count is not something an operator can act on. Empty for a zero value. +func (p PickedEntry) Src() string { + if p.entry == nil { + return "" + } + return p.entry.src +} + // TrustGateEvent records a longest-match suppression the DISPOSITION-gate REFUSED (D39 layer 4): // a longer but LOWER-trust key (draft/ambiguous, e.g. draft 四代族长) that would have deleted a nested // HIGHER-trust key (approved/confirmed, e.g. approved 族长). The draft is editor-excluded (the block diff --git a/backend/internal/miner/miner_emit.go b/backend/internal/miner/miner_emit.go index 181e513a..8c34c5ae 100644 --- a/backend/internal/miner/miner_emit.go +++ b/backend/internal/miner/miner_emit.go @@ -240,7 +240,14 @@ func DeltaYAML(mined []Term, proposals map[string][]DstProposal) (string, error) // The join key is the miner's own normalized surface, so a proposal written with different // orthography still lands on its term (the same normalization the seed-surface exclusion uses). if props := proposals[text.NormalizeSourceKey(m.Src)]; len(props) > 0 { - st.Dst = props[0].Dst + // The dst comes from a proposal made ON THIS SURFACE. A rendering that reached the term through an + // ALIAS of its cluster is evidence — it is listed in the note with the surface that proposed it — + // but the miner's clustering is itself unverified (measured: a surname cluster holding a clan + // title), and promoting an alias's word to the term's own rendering is the clustering DECISION, + // which belongs to the owner's signature and to nothing else. + if props[0].Via == "" { + st.Dst = props[0].Dst + } note = appendProposalNote(note, props) } // TWO-MODE EMISSION (§C2-7, ratified and until pack-20 unbuilt): a term that arrives with a @@ -272,19 +279,40 @@ func appendProposalNote(note string, props []DstProposal) string { b.WriteString(note) b.WriteString("; ") } - fmt.Fprintf(&b, "dst PROPOSED by the translator (banknote, %d chunk(s)) — NOT approved, sign or replace it", props[0].Chunks) + if props[0].Via != "" { + // Nothing was proposed on this surface itself: every rendering came through an alias, so the row + // carries no dst and the note says who proposed what for which surface. + b.WriteString("dst PROPOSED only through cluster alias(es) — NOT attached, sign or replace it: ") + for i, p := range props { + if i > 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "%q ×%d%s", p.Dst, p.Chunks, viaNote(p.Via)) + } + return b.String() + } + fmt.Fprintf(&b, "dst PROPOSED by the translator (banknote, %d chunk(s)) — NOT approved, sign or replace it", + props[0].Chunks) if len(props) > 1 { b.WriteString("; other proposals: ") for i, p := range props[1:] { if i > 0 { b.WriteString(", ") } - fmt.Fprintf(&b, "%q ×%d", p.Dst, p.Chunks) + fmt.Fprintf(&b, "%q ×%d%s", p.Dst, p.Chunks, viaNote(p.Via)) } } return b.String() } +// viaNote renders the alias provenance of a proposal, or "" for one made on the term's own surface. +func viaNote(via string) string { + if via == "" { + return "" + } + return " [proposed for " + via + "]" +} + // appendConsolidatedNote records that the dst on this row is the TERMINOLOGIST's consolidation, not a // per-chunk guess and not a signature. The distinction is the whole trust story of the auto mode: a // `draft` row reaches the wire with an unverified marker, so the sign map must say, in the artifact @@ -374,4 +402,9 @@ type DstProposal struct { Dst string Type string Chunks int + // Via names the surface that actually proposed this rendering when it is not the term's own — an ALIAS + // the miner clustered onto it. It travels into the note because the alternative is a sign map that + // presents a rendering proposed FOR ANOTHER SURFACE as a rendering of this one, and the clustering that + // joined them is itself unverified (the polygon's surname cluster holding a clan title). + Via string } diff --git a/backend/internal/pipeline/bankfixpack_test.go b/backend/internal/pipeline/bankfixpack_test.go new file mode 100644 index 00000000..3a2daf86 --- /dev/null +++ b/backend/internal/pipeline/bankfixpack_test.go @@ -0,0 +1,383 @@ +package pipeline + +import ( + "bytes" + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "textmachine/backend/internal/config" + "textmachine/backend/internal/miner" + "textmachine/backend/internal/terminology" + "textmachine/backend/internal/text" +) + +// bankfixpack_test.go: the fix-pack's pipeline-level seams — the arbitration record on the stop table (§G3), +// the signature map's alias join (§G3), the auto-bank rewrite diff (row 130) and the eviction list. + +// TestBankStopTableCarriesTheArbitrationRecord pins §A7's finding closed: «why does this term have THIS +// dst» has to be answerable from the artifacts. The row now carries the winning variant's ranking signals, +// whether the rendering is one the drafts ever proposed, how many CONVENTIONS they actually offered (as +// against raw forms), and the role's own confidence. +func TestBankStopTableCarriesTheArbitrationRecord(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isTerminologyBody(body) { + // A rendering NO draft proposed, with a confidence field — both the invented flag and the + // confidence column have to survive the parser. + return "方源\tГуюэ Фан Юань\t40", "stop" + } + // The drafts disagree on the CASE only: one convention written two ways. + return "Фан Юань пришёл." + "\n" + bankSeparator + + "\n方源\tФан Юань\tname\n方源\tфан юань\tname", "stop" + }) + defer srv.Close() + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})) + defer r.Close() + stop := runToSignatureStop(t, r) + + var row *BankStopRow + for i := range stop.Rows { + if stop.Rows[i].Src == "方源" { + row = &stop.Rows[i] + } + } + if row == nil { + t.Fatalf("方源 missing from the table: %+v", stop.Rows) + } + if row.Conf != 40 { + t.Fatalf("the role's stated confidence must reach the review table, got %d", row.Conf) + } + if !row.Invented { + t.Fatalf("a rendering no draft proposed must say so: %+v", *row) + } + if row.Conventions != 1 { + t.Fatalf("two spellings of one rendering are ONE convention, got %d (%v)", row.Conventions, row.Variants) + } + if row.Spread < 2 { + t.Fatalf("the owner must still see that the drafts wrote it two ways, got spread=%d", row.Spread) + } + raw, err := os.ReadFile(stop.TablePath) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"conventions=", "confidence=40", "INVENTED"} { + if !strings.Contains(string(raw), want) { + t.Fatalf("the sidecar must carry %q:\n%s", want, raw) + } + } +} + +// TestBankStopRowCarriesTheWinningVariantsSignals pins the OTHER half of §A7's answer: not only WHAT was +// chosen but WHY. The signals are the §C2-3 factors that fired for the variant the ranking put first, and +// they were already computed and thrown away before this pack — so the failure mode is a row that looks +// complete and explains nothing. +func TestBankStopRowCarriesTheWinningVariantsSignals(t *testing.T) { + c := terminology.Candidate{Key: "元海", Src: "元海", Type: "term", Variants: []terminology.Variant{ + {Dst: "море истинной ци", Chunks: 5, Forms: 1}, + {Dst: "первозданное море", Chunks: 1, Forms: 1}, + }} + terminology.ScoreVariants(&c, terminology.ScoreOpts{}) + if len(c.Variants[0].Signals) == 0 { + t.Fatalf("test premise broken: the winner must have fired at least one factor: %+v", c.Variants) + } + rows := bankStopRows([]terminology.Candidate{c}, map[string]string{"元海": "море истинной ци"}, terminologyResult{}) + if len(rows[0].Signals) == 0 { + t.Fatalf("the winning variant's audit trail must reach the row: %+v", rows[0]) + } + if strings.Join(rows[0].Signals, ",") != strings.Join(c.Variants[0].Signals, ",") { + t.Fatalf("the row must carry the TOP variant's signals, got %v want %v", rows[0].Signals, c.Variants[0].Signals) + } + if !strings.Contains(renderBankStopTable(rows), "why: ") { + t.Fatalf("and the sidecar must print them:\n%s", renderBankStopTable(rows)) + } + // A row with no consolidation carries no confidence rather than a zero one: 0 means «the role said it was + // not sure at all», which is the first row to review, not the same as silence. + if rows[0].Conf >= 0 { + t.Fatalf("an absent confidence must stay absent, got %d", rows[0].Conf) + } +} + +// TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply guards the seam between the two «this batch bought +// nothing» signals. A batch the budget never reached has an empty reply text for a completely different +// reason, and warning about an EMPTY COMPLETION there would cry wolf on every budget-bounded run — the +// budget already said what happened, one line earlier. +func TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply(t *testing.T) { + var logBuf bytes.Buffer + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isTerminologyBody(body) { + return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop" + } + return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop" + }) + defer srv.Close() + // batch_runes tiny → several batches; a budget sized under one batch → the pass stops at the first. + probe := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, batchRunes: 400})) + _ = runToSignatureStop(t, probe) + if probe.lastTerminology.Batches < 2 { + t.Fatalf("the fixture must actually batch, got %d", probe.lastTerminology.Batches) + } + perBatch := probe.lastTerminology.EstimateUSD / float64(probe.lastTerminology.Batches) + probe.Close() + + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{ + terminology: true, batchRunes: 400, budgetUSD: perBatch * 1.5, + })) + defer r.Close() + r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) + _ = runToSignatureStop(t, r) + out := logBuf.String() + if !strings.Contains(out, "budget would be exceeded") { + t.Fatalf("the fixture must actually hit the budget:\n%s", out) + } + if strings.Contains(out, "EMPTY completion") { + t.Fatalf("a batch the budget never reached is not a paid batch that came back empty:\n%s", out) + } +} + +// TestSelfContradictionIsReportedAtTheStop is §G2 end to end: the run consolidates a part and then renders +// the compound without it, and both the log and the row say so. The canon check cannot see this — the book +// has no signed rows at all — which is precisely why 18 of the 149 contradictions on the live bank were +// invisible. +func TestSelfContradictionIsReportedAtTheStop(t *testing.T) { + var logBuf bytes.Buffer + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isTerminologyBody(body) { + // 花家 → «клан Хуа», and the compound built on it drops «Хуа» entirely. + return "花家\tклан Хуа\n花家的人\tлюди клана", "stop" + } + return "Фан Юань пришёл." + "\n" + bankSeparator + + "\n花家\tклан Хуа\tname\n花家的人\tлюди клана\tterm", "stop" + }) + defer srv.Close() + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})) + defer r.Close() + r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) + _ = runToSignatureStop(t, r) + + if r.lastTerminology.SelfConflicts == 0 { + t.Fatalf("the compound contradicts the part the same reply consolidated: %+v", r.lastTerminology) + } + if !strings.Contains(logBuf.String(), "contradict each other") { + t.Fatalf("a run breaking its own canon must say so:\n%s", logBuf.String()) + } + var flagged bool + for _, row := range r.lastBankStopRows { + if row.Src == "花家的人" && len(row.Contradicts) > 0 { + flagged = true + } + } + if !flagged { + t.Fatalf("the contradicting row must be marked in the review table: %+v", r.lastBankStopRows) + } +} + +// TestSignatureMapJoinsAliasProposals is the acceptance finding of §G3: a rendering that reached the +// terminologist through an ALIAS of a cluster showed up in the stop table (Merge routes it to the owner) +// and was ABSENT from the owner's row in the signature map, which joined on the alias's own key. The map +// and the table then described the same bank differently, for exactly the terms the miner clustered. +func TestSignatureMapJoinsAliasProposals(t *testing.T) { + mined := []terminology.Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子"}}} + observed := []terminology.Observed{ + {Key: "方小子", Src: "方小子", Type: "name", Proposals: []terminology.Proposal{{Dst: "малыш Фан", Chunks: 3}}}, + } + cands := terminology.Merge(mined, observed, text.NormalizeTargetForm) + props := proposalsFromCandidates(cands) + if got := props["方源"]; len(got) == 0 || got[0].Dst != "малыш Фан" { + t.Fatalf("an alias-routed proposal must land on the OWNER's key, got %+v", props) + } + if props["方源"][0].Via != "方小子" { + t.Fatalf("and it must say which surface actually proposed it, got %q", props["方源"][0].Via) + } + yaml, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, props) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(yaml, "малыш Фан") { + t.Fatalf("the signature map must carry the rendering the table showed:\n%s", yaml) + } + if !strings.Contains(yaml, "proposed for 方小子") { + t.Fatalf("and it must not present an alias's guess as the term's own:\n%s", yaml) + } + // AND it must not become the term's dst. Merge routes an alias's rendering to the cluster owner so the + // ranking sees all the evidence; promoting it to the owner's rendering is the clustering DECISION, which + // belongs to a signature. Here NOTHING was proposed on 方源 itself, so the row carries no dst at all. + if strings.Contains(yaml, "dst: малыш Фан") { + t.Fatalf("an alias's rendering became the term's dst — it reaches the wire with no model and no gate:\n%s", yaml) + } + // With a DIRECT proposal present, that one is the dst and the alias stays evidence in the note. + observed = append(observed, terminology.Observed{ + Key: "方源", Src: "方源", Type: "name", Proposals: []terminology.Proposal{{Dst: "Фан Юань", Chunks: 1}}, + }) + direct := proposalsFromCandidates(terminology.Merge(mined, observed, text.NormalizeTargetForm)) + yaml2, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, direct) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(yaml2, "dst: Фан Юань") { + t.Fatalf("a rendering proposed on the term's own surface must be its dst:\n%s", yaml2) + } + if !strings.Contains(yaml2, "малыш Фан") { + t.Fatalf("and the alias proposal must still be listed as evidence:\n%s", yaml2) + } +} + +// TestAutoBankRewriteNamesDroppedTerms is the $0 minimum of backlog row 130. The auto-bank file is rewritten +// WHOLE from a top-N-capped delta, so a term that was in the bank for twenty chapters can vanish from it +// mid-run with nothing in the artifacts saying so. The pack does not change that behaviour — accumulating is +// the owner's STOP decision, because it moves memory_version — it makes the loss visible. +func TestAutoBankRewriteNamesDroppedTerms(t *testing.T) { + var logBuf bytes.Buffer + r := &Runner{ + Log: slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})), + Book: &config.Book{BookID: "test-book", ProjectDB: filepath.Join(t.TempDir(), "project.db")}, + } + ctx := context.Background() + + first := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "青茅山", Type: "place", Freq: 12}} + if err := r.writeAutoBank(ctx, first, nil, nil); err != nil { + t.Fatal(err) + } + if logBuf.Len() != 0 { + t.Fatalf("the first write has nothing to diff against:\n%s", logBuf.String()) + } + // The next run's delta no longer holds 青茅山 — the rank cap cut it, and it silently leaves the bank. + second := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "花家", Type: "name", Freq: 9}} + if err := r.writeAutoBank(ctx, second, nil, nil); err != nil { + t.Fatal(err) + } + out := logBuf.String() + if !strings.Contains(out, "青茅山") { + t.Fatalf("a term dropped by the rewrite must be NAMED:\n%s", out) + } + if strings.Contains(out, "方源") { + t.Fatalf("a term that is still there is not a loss:\n%s", out) + } + // The boundary is intact: the file is the new delta, not a merge of both runs. + raw, err := os.ReadFile(r.autoBankPath()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "青茅山") { + t.Fatalf("the pack must REPORT the drop, not accumulate the file (that moves memory_version):\n%s", raw) + } + + // And the owner's OWN two verbs are not a loss. Promoting a term into the mined-delta (it becomes a seed + // surface) or declining it in mined-rejects removes it from the next delta ON PURPOSE — warning about + // that fires a false alarm on the normal signature cycle and advises the owner to do what he just did. + logBuf.Reset() + third := []miner.Term{{Src: "方源", Type: "name", Freq: 40}} + handled := map[string]bool{text.NormalizeSourceKey("花家"): true} + if err := r.writeAutoBank(ctx, third, nil, handled); err != nil { + t.Fatal(err) + } + if strings.Contains(logBuf.String(), "花家") { + t.Fatalf("a term the owner promoted or declined is not a silent loss:\n%s", logBuf.String()) + } +} + +// TestAutoBankDiffSurvivesTheEngineOwnRows is the row-130 warning through the path production takes: two +// real auto-mode runs, a term that leaves the delta between them, and the warning that has to name it. +// +// It has to be end-to-end, because the defect lives in the CALL SITE, not in the helper. From the second +// auto-mode run the stored glossary also holds the engine's own unsigned rows — seedGlossary re-seeds the +// auto-bank at start (seeding.go:83-93) — so passing the raw seed makes every term the engine ever proposed +// read as «the owner decided about this». The diff then empties and the warning is structurally dead in +// production while every unit test stays green. Risk 2, the self-exclusion trap, arriving at the one place +// in this file that did not filter for it. +func TestAutoBankDiffSurvivesTheEngineOwnRows(t *testing.T) { + var logBuf bytes.Buffer + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isTerminologyBody(body) { + return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop" + } + return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop" + }) + defer srv.Close() + bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}) + + // Run 1: the auto mode mines 花家 among others, writes the auto-bank and seeds it into the store. + r1 := newRunner(t, bookPath) + if _, err := r1.TranslateBook(context.Background()); err != nil { + t.Fatal(err) + } + var had bool + for _, src := range r1.autoBankSurfaces() { + if src == "花家" { + had = true + } + } + if !had { + t.Fatalf("test premise broken: the first run must bank 花家, got %v", r1.autoBankSurfaces()) + } + r1.Close() + + // The book changes and 花家 stops occurring, so it leaves the delta — standing in for the rank cap doing + // the same thing mid-book, which is the case backlog row 130 is about. + writeFile(t, filepath.Join(filepath.Dir(bookPath), "source.txt"), + strings.Repeat("方源来到青茅山。方源很强。青茅山很高。", 6)) + + r2 := newRunner(t, bookPath) + defer r2.Close() + r2.Resnapshot = true // the source moved, and this fixture is about the bank diff, not the snapshot gate + r2.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) + if _, err := r2.TranslateBook(context.Background()); err != nil { + t.Fatal(err) + } + // The store now holds the engine's own unsigned rows — the state that used to swallow the warning. + seed, err := r2.Store.GlossaryForBook("test-book") + if err != nil { + t.Fatal(err) + } + engineRows := 0 + for _, e := range seed { + if e.Source == "mined" && e.Status != "approved" { + engineRows++ + } + } + if engineRows == 0 { + t.Fatal("test premise broken: the store must hold the engine's own unsigned rows by the second run") + } + out := logBuf.String() + if !strings.Contains(out, "NOT in the one this run just wrote") || !strings.Contains(out, "花家") { + t.Fatalf("the row-130 warning must NAME the term the rewrite dropped, engine rows in the seed and all:\n%s", out) + } +} + +// TestEvictedBankRowsAreNamed: n_evicted counted the rows the injection budget dropped and never said which, +// so «the model had no canon for this term» was an unactionable integer. The names go to the log rather than +// to a new column — a per-row detail column is a schema migration, and this repository has already paid for +// a non-idempotent one (backlog row 49а). +func TestEvictedBankRowsAreNamed(t *testing.T) { + var logBuf bytes.Buffer + rec := &reqRec{} + srv := newJSONProvider(rec, draftEdit) + defer srv.Close() + // A budget of one token cannot fit even the first glossary line, so every matched row is evicted. + bookPath := setupProjectOpts(t, srv.URL, projectOpts{ + source: suzukiSource, glossarySeed: suzukiSeed, glossaryTokenBudget: 1, + }) + r := newRunner(t, bookPath) + defer r.Close() + r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) + if _, err := r.TranslateBook(context.Background()); err != nil { + t.Fatal(err) + } + rs, err := r.Store.GetRetrievalState("test-book", 1, 0) + if err != nil || rs == nil { + t.Fatalf("retrieval_state: %v %v", rs, err) + } + if rs.NEvicted == 0 { + t.Fatal("test premise broken: a one-token budget must evict the matched row") + } + out := logBuf.String() + if !strings.Contains(out, "鈴木") { + t.Fatalf("the evicted rows must be NAMED, not counted:\n%s", out) + } +} diff --git a/backend/internal/pipeline/banknote.go b/backend/internal/pipeline/banknote.go index 36f59fb3..f18c843b 100644 --- a/backend/internal/pipeline/banknote.go +++ b/backend/internal/pipeline/banknote.go @@ -51,8 +51,11 @@ const bankMaxLines = 20 // re-resolves paid text, so it must be a loud --resnapshot. // - bankParseVersion governs parseBanknote — which candidate lines are accepted. Its output is EVIDENCE // for the owner's signature, never a checkpoint and never a wire byte; every run recomputes it from -// the stored answers (bankObservedForBook). NOT folded, logged with the run — the same contract -// gates.terminology and gates.voice already carry, and for the same reason. +// the stored answers (bankObservedForBook). NOT folded, and LOGGED with the run at the bank-mining stop +// (mining.go, "draft-side proposals folded") — the same contract gates.terminology and gates.voice +// carry, and for the same reason. The logging is what makes staying out of the snapshot honest rather +// than merely cheap: without a consumer the tag is a comment, and two maps produced by different rules +// are indistinguishable after the fact. // // Folding the parse rule (as a single parser_version did) would re-bill a whole draft wave for a change // that cannot alter one byte of what was bought — which is a standing incentive to leave the rule wrong. @@ -60,8 +63,11 @@ const ( bankSliceVersion = "banknote-slice-v1" // v2 (backlog 19 / D39.53 default): the src rule stopped being "contains a Han ideograph" (a language // FAMILY hardcoded in the engine) and became "occurs in the book"; v2.1 adds the column-order recovery - // that same invariant makes possible. - bankParseVersion = "banknote-parse-v2.1-src-attested+column-order" + // that same invariant makes possible; v2.2 (backlog 19 fix-pack) makes the reading DELIMITER-AWARE — a + // tab/pipe line is positional, only a space-run line re-joins a split rendering. Bumping this is not + // decoration: the tag is what attributes a stored signature map to the rule that produced it, and a rule + // change under an unchanged tag is exactly the drift the split into two versions exists to prevent. + bankParseVersion = "banknote-parse-v2.2-src-attested+column-order+delimiter-aware" ) // bankDerivedNS is the derived-checkpoint id NAMESPACE (§4б EXACT formula) — the "tm--v1" prefix @@ -79,6 +85,54 @@ var bankTypeOK = map[string]bool{"name": true, "place": true, "title": true, "te // with Python's unicode \s around the pipe is exact on the term corpus.) var bankFieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`) +// bankColumnSplit is the UNAMBIGUOUS half of that tolerance: a tab or a pipe is a delimiter the model chose, +// never something that can occur inside a rendering. A line carrying one has declared its own columns. +var bankColumnSplit = regexp.MustCompile(`\t|\s*\|\s*`) + +// bankSpaceRun collapses the internal whitespace of a column read positionally, so «Фан␣␣Юань» arrives as +// one clean rendering instead of carrying the model's stray spacing into the bank. +var bankSpaceRun = regexp.MustCompile(` {2,}`) + +// bankColumns splits one banknote line into (src + rendering fields, type). See parseBanknote for why the +// delimiter decides which of the two readings applies. +func bankColumns(line string) ([]string, string) { + typ := "term" + if cols := nonEmptyFields(bankColumnSplit, line); len(cols) >= 2 { + if len(cols) >= 3 { + if t := strings.ToLower(cols[2]); bankTypeOK[t] { + typ = t + } + } + // BOTH columns are collapsed, not just the second: the order-recovery below may swap them, and the + // column that becomes the rendering must arrive clean either way. + return []string{collapseRuns(cols[0]), collapseRuns(cols[1])}, typ + } + fields := nonEmptyFields(bankFieldSplit, line) + if len(fields) >= 3 { + if last := strings.ToLower(fields[len(fields)-1]); bankTypeOK[last] { + typ = last + fields = fields[:len(fields)-1] + } + } + return fields, typ +} + +// collapseRuns folds a run of spaces inside a positionally-read column into one. +func collapseRuns(s string) string { + return strings.TrimSpace(bankSpaceRun.ReplaceAllString(s, " ")) +} + +// nonEmptyFields splits a line on re with empty and whitespace-only fields dropped. +func nonEmptyFields(re *regexp.Regexp, line string) []string { + var out []string + for _, p := range re.Split(strings.TrimSpace(line), -1) { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + // bankEntry is one parsed candidate line (evidence for the bank-mining stop (§C); NOT written to the store until owner // signoff). Dst is the model's proposed translation — the direct dst delivery the co-occurrence // miner could not extract (蛊→гу, non-seed 龙公→Лун Гун). @@ -92,7 +146,11 @@ type bankEntry struct { type bankFlags struct { NLines int // accepted (well-formed) banknote lines ParseFail bool // any malformed residual line (src not in the book / <2 fields) that was NOT a tolerated truncation - Truncated bool // the LAST line was cut by generation length (tolerated, not a parse fail) + // Truncated marks a block the run did NOT complete: the separator was there, so the model opened the + // channel, but the generation ended on a non-stop finish and the stop-only gate refuses it. Read off the + // finish reason, not from the parser (which never sees such a block). ⚠ It cannot see a cut that removed + // the separator itself — on the reference that is the majority of length-cut chunks — so it is a floor. + Truncated bool } // splitBanknote slices the banknote block off the raw model output BEFORE any gate/editor (integration @@ -198,12 +256,17 @@ func (r *Runner) bankSrcAttested(chunkSource string) func(string) bool { } } -// parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry). -// Tolerant of a TRUNCATED final line when truncatedGeneration is set (§B3-5): a short last line under -// truncation is flagged banknote_truncated, not counted as a parse fail. Any other malformed line -// (fewer than 2 fields, or a src NOT attested in the source) sets banknote_parse_fail. Deterministic, no -// time/rand: srcAttested is a pure function of the book text (bankSourceIndex). -func parseBanknote(block string, truncatedGeneration bool, srcAttested func(string) bool) ([]bankEntry, bankFlags) { +// parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry). A +// malformed line (fewer than 2 fields, or a src NOT attested in the source) sets banknote_parse_fail. +// Deterministic, no time/rand: srcAttested is a pure function of the book text (bankSourceIndex). +// +// It carried a `truncatedGeneration` tolerance for a short LAST line (§B3-5, faithful to banknote.py). The +// ratified finish=stop-only gate means this parser is never handed a truncated block at all, so the only +// caller pinned the argument to false and the branch could not execute in production while its comment said +// it could. The truncation FACT is now read where it actually exists — off the finish reason, before the +// gate refuses the block (applyBanknoteWithEntries) — and the unreachable branch is gone rather than left +// as a tolerance nobody can reach. +func parseBanknote(block string, srcAttested func(string) bool) ([]bankEntry, bankFlags) { var entries []bankEntry var flags bankFlags if strings.TrimSpace(block) == "" { @@ -216,31 +279,25 @@ func parseBanknote(block string, truncatedGeneration bool, srcAttested func(stri } } bad := 0 - for i, ln := range lines { - raw := bankFieldSplit.Split(strings.TrimSpace(ln), -1) - var parts []string - for _, p := range raw { - if p = strings.TrimSpace(p); p != "" { - parts = append(parts, p) - } - } - if len(parts) < 2 { - // A short LAST line under a truncated generation is a tolerated cut, not a failure. - if i == len(lines)-1 && truncatedGeneration { - flags.Truncated = true - continue - } + for _, ln := range lines { + // THE DELIMITER CARRIES INFORMATION, and reading the columns without it is what made the first cut of + // this fix a regression on the frozen reference (−6 lines over 56 real blocks). + // + // A line the model delimited with a TAB or a PIPE has told us where its columns are, so it is read + // POSITIONALLY: src, rendering, type — exactly as it was before the fix-pack. A third column outside + // the closed type vocabulary («clan», «proverb», «onomatopoeia» — 1.25% of the reference) is then a + // benign unknown type, as it always was; guessing it into the rendering corrupts a term the owner is + // about to sign, and on a REVERSED line it made the src unfindable and dropped the line entirely. + // + // Only a line with NO tab and NO pipe is ambiguous, because the ≥2-space tolerance exists for models + // that substitute spaces for tabs — and that same tolerance is what splits «Фан␣␣Юань» into two + // fields. There, and only there, is the type identified by the closed vocabulary on the LAST field + // and everything between re-joined, which is the discipline terminology.ParseReply already applies. + fields, typ := bankColumns(ln) + if len(fields) < 2 { bad++ continue } - src, dst := parts[0], parts[1] - typ := "term" - if len(parts) >= 3 { - typ = strings.ToLower(parts[2]) - } - if !bankTypeOK[typ] { - typ = "term" - } // The src rule: a bank line must name something the BOOK contains. A surface no chunk of the source // carries is either a hallucination or a mis-split line — not evidence about this book, and letting // it through would put an invented term on the owner's sign map. @@ -249,14 +306,18 @@ func parseBanknote(block string, truncatedGeneration bool, srcAttested func(stri // is recovered instead of discarded (measured: one chunk emitted all 12 of its lines reversed, // including a term the owner had signed by hand). Order is decided, never guessed: the declared // order wins whenever it is attested, the reversal is taken only when it is the ONLY reading the - // book supports, and a line neither reading supports is still a parse fail. + // book supports, and a line neither reading supports is still a parse fail. The src is one field on + // either reading — a source surface has no spaces to be split on — so the rendering is whatever + // remains, which is what makes the recovery survive a split rendering instead of failing on it. + src, dst := fields[0], strings.Join(fields[1:], " ") + revSrc, revDst := fields[len(fields)-1], strings.Join(fields[:len(fields)-1], " ") switch { case srcAttested == nil: bad++ continue case srcAttested(src): // declared order — including the ambiguous case where both sides are attested - case srcAttested(dst): - src, dst = dst, src + case srcAttested(revSrc): + src, dst = revSrc, revDst default: bad++ continue @@ -335,11 +396,20 @@ func (r *Runner) applyBanknoteWithEntries(role, rawText, finish, chunkSource str if block == "" { return rawText, "", bankFlags{}, nil // no separator → no strip, no derived checkpoint (byte-identical path) } - if finish == "stop" { - // finish=stop-only gate: accept candidates + count telemetry only for a complete generation. - // truncatedGeneration=false is unreachable-otherwise in prod (a truncated gen is finish≠stop). - entries, flags = parseBanknote(block, false, r.bankSrcAttested(chunkSource)) + if finish != "stop" { + // The finish=stop-only gate (§4б, ratified) refuses the block, and that refusal is the whole content + // of banknote_truncated: a block that IS there under a finish the run did not complete is a block the + // bank never got. (Length is the usual cause and the one the column was named for; a refusal or a + // filter cut lands here too, and for the owner it means the same thing — the WHAT of this chunk is + // missing, and the counters must not read as if it were clean.) + // + // The flag used to be derived INSIDE the parser from a truncatedGeneration argument the only caller + // pinned to false, so the column could not be 1 in production and the comment beside it said the + // opposite. Reading it off the finish reason needs no parse of untrusted text, no schema change, and + // leaves the gate exactly where it was ratified: nothing is accepted, the fact is recorded. + return cleanText, cleanText, bankFlags{Truncated: true}, nil } + entries, flags = parseBanknote(block, r.bankSrcAttested(chunkSource)) return cleanText, cleanText, flags, entries } @@ -465,8 +535,9 @@ func (f *bankFold) observed() []terminology.Observed { } // bankObservedByKey folds the durable per-chunk banknote rows into the draft-side view of each source -// surface. Both consumers read it: the signature-map join (proposalsFromObserved) and the terminologist's -// merge, which also needs the SURFACE so a proposal for a term the miner never found is not lost. +// surface. Both consumers read it: the terminologist's merge (whose candidates the signature-map join +// then reads, proposalsFromCandidates) — which also needs the SURFACE, so a proposal for a term the miner +// never found is not lost. func bankObservedByKey(states []store.RetrievalState, target *unicode.RangeTable) []terminology.Observed { f := newBankFold(target) addRetrievalStates(f, states) diff --git a/backend/internal/pipeline/banknote_test.go b/backend/internal/pipeline/banknote_test.go index 900a171e..0408427c 100644 --- a/backend/internal/pipeline/banknote_test.go +++ b/backend/internal/pipeline/banknote_test.go @@ -40,7 +40,7 @@ func TestSplitBanknoteSlicesBlock(t *testing.T) { if strings.Contains(clean, bankSeparator) || strings.Contains(clean, "方源") { t.Fatalf("the banknote block leaked into the clean translation: %q", clean) } - ents, flags := parseBanknote(block, false, attestedIn(srcAll)) + ents, flags := parseBanknote(block, attestedIn(srcAll)) if len(ents) != 2 || flags.ParseFail || flags.Truncated || flags.NLines != 2 { t.Fatalf("parse = %+v flags=%+v, want 2 clean entries", ents, flags) } @@ -56,7 +56,7 @@ func TestParseBanknoteTolerantFieldSplit(t *testing.T) { // tab, ≥2 spaces, and pipe are all accepted delimiters (a model that emits spaces instead of a // real TAB still parses). type falls back to "term" when absent or unknown. block := "方源\tФан Юань\tname\n蛊师 гу-мастер title\n青茅山 | гора Цинмао | place\n古月\tГу Юэ" - ents, flags := parseBanknote(block, false, attestedIn(srcAll)) + ents, flags := parseBanknote(block, attestedIn(srcAll)) if flags.ParseFail { t.Fatalf("tolerant split should not fail: %+v", flags) } @@ -82,7 +82,7 @@ func TestParseBanknoteSrcMustBeAttested(t *testing.T) { "San Michon\tСан-Мишон\tplace", // multi-word latin src "リン\tРин\tname", // kana src }, "\n") - ents, flags := parseBanknote(block, false, attestedIn(src)) + ents, flags := parseBanknote(block, attestedIn(src)) if flags.ParseFail { t.Fatalf("every src here is in the source; none may be a parse fail: %+v", flags) } @@ -91,7 +91,7 @@ func TestParseBanknoteSrcMustBeAttested(t *testing.T) { } // The other half of the rule: a plausible, well-formed, Han-bearing line the book never contains. invented := "天魔宗\tСекта Небесного Демона\tplace" - ents, flags = parseBanknote(invented, false, attestedIn(src)) + ents, flags = parseBanknote(invented, attestedIn(src)) if !flags.ParseFail || len(ents) != 0 { t.Fatalf("an invented src must be a parse fail and yield nothing, got %+v %+v", ents, flags) } @@ -111,7 +111,7 @@ func TestParseBanknoteRecoversColumnOrder(t *testing.T) { "花海\tМоре цветов\tplace", // declared order "выдумка\tтоже выдумка\tterm", // neither side attested → still a parse fail }, "\n") - ents, flags := parseBanknote(block, false, attestedIn(src)) + ents, flags := parseBanknote(block, attestedIn(src)) if !flags.ParseFail { t.Fatalf("the unattested line must still fail: %+v", flags) } @@ -127,7 +127,7 @@ func TestParseBanknoteRecoversColumnOrder(t *testing.T) { // Ambiguity is resolved by the DECLARED order, never by a guess: when the book attests both fields // (a source that quotes the target language, or a same-script pair), the model's order stands. both := "San Michon\tСан-Мишон\tplace" - ents, flags = parseBanknote(both, false, attestedIn(src+" Сан-Мишон")) + ents, flags = parseBanknote(both, attestedIn(src+" Сан-Мишон")) if flags.ParseFail || len(ents) != 1 || ents[0].Src != "San Michon" { t.Fatalf("both-attested must keep the declared order, got %+v %+v", ents, flags) } @@ -157,22 +157,21 @@ func TestBankSourceIndexParity(t *testing.T) { } } -func TestParseBanknoteTruncationTolerated(t *testing.T) { - // The LAST line cut by generation length is tolerated (banknote_truncated), not a parse fail — - // but ONLY under truncated_generation; the same short line otherwise IS a parse fail. +// TestParseBanknoteShortLastLineIsAParseFail: the parser's truncation TOLERANCE is gone with the fix-pack, +// and this pins what replaced it. A short last line reaches this parser only under finish=stop — a complete +// generation that simply emitted a broken line — and that is a parse fail, not a tolerated cut. The +// truncation FACT now lives where it is actually known, one level up, off the finish reason +// (TestBanknoteTruncatedIsReachable). +func TestParseBanknoteShortLastLineIsAParseFail(t *testing.T) { block := "方源\tФан Юань\tname\n蛊" // last line has no dst - entsT, flagsT := parseBanknote(block, true, attestedIn(srcAll)) - if !flagsT.Truncated || flagsT.ParseFail || len(entsT) != 1 { - t.Fatalf("truncated=true: want 1 entry + truncated flag + no parse_fail, got %+v %+v", entsT, flagsT) - } - entsF, flagsF := parseBanknote(block, false, attestedIn(srcAll)) - if flagsF.Truncated || !flagsF.ParseFail || len(entsF) != 1 { - t.Fatalf("truncated=false: the short last line must be a parse_fail, got %+v %+v", entsF, flagsF) + ents, flags := parseBanknote(block, attestedIn(srcAll)) + if flags.Truncated || !flags.ParseFail || len(ents) != 1 { + t.Fatalf("a complete generation's short line is a parse_fail, got %+v %+v", ents, flags) } } func TestParseBanknoteEmptyBlock(t *testing.T) { - ents, flags := parseBanknote("", false, attestedIn(srcAll)) + ents, flags := parseBanknote("", attestedIn(srcAll)) if len(ents) != 0 || flags.ParseFail || flags.Truncated || flags.NLines != 0 { t.Fatalf("empty block must yield no entries and clean flags, got %+v %+v", ents, flags) } @@ -277,3 +276,109 @@ func TestBankTokenBudgetIsDerived(t *testing.T) { t.Fatalf("the line cap was raised on the mini-run measurement (2 of 9 blocks pressed against 12), got %d", bankMaxLines) } } + +// TestParseBanknoteRejoinsASplitRendering is backlog row 129. The field splitter tolerates a run of ≥2 +// spaces because models substitute spaces for tabs — and that same tolerance splits INSIDE a rendering the +// moment a model writes «Фан␣␣Юань». Taking parts[1] alone banked the half-name and pushed the tail into +// the type column, where it failed the closed vocabulary and silently became "term": a mangled surface on +// the owner's sign map with parse_fail still reading 0. The terminologist's parser was fixed for exactly +// this; the banknote's was not. +func TestParseBanknoteRejoinsASplitRendering(t *testing.T) { + attested := attestedIn(srcAll) + // BOTH shapes: with the type column present, and as a bare two-column line. + entries, flags := parseBanknote("方源\tФан Юань\tname\n蛊师\tмастер гу", attested) + if flags.ParseFail { + t.Fatalf("neither line is malformed: %+v", flags) + } + if len(entries) != 2 { + t.Fatalf("want both lines, got %+v", entries) + } + if entries[0].Dst != "Фан Юань" || entries[0].Type != "name" { + t.Fatalf("a split rendering must be re-joined and the type column still read: %+v", entries[0]) + } + if entries[1].Dst != "мастер гу" || entries[1].Type != "term" { + t.Fatalf("with no type column the whole tail is the rendering: %+v", entries[1]) + } + // THE SPACE-RUN PATH is the only one where the re-join can be observed at all: on a TAB line the + // rendering is one column, so join(fields[1:]) and fields[1] are the same string and the re-join could be + // reverted with every test still green. Here they differ, and «Фан» alone is what used to be banked. + spaced, sflags := parseBanknote("方源 Фан Юань name", attested) + if sflags.ParseFail || len(spaced) != 1 || spaced[0].Dst != "Фан Юань" || spaced[0].Type != "name" { + t.Fatalf("a space-delimited line must re-join its rendering: %+v %+v", spaced, sflags) + } + if two, _ := parseBanknote("蛊师 мастер гу", attested); len(two) != 1 || two[0].Dst != "мастер гу" { + t.Fatalf("and so must a space-delimited line with no type column: %+v", two) + } + // The recognised shapes are unchanged: a real three-field line is still src/dst/type. + plain, _ := parseBanknote("青茅山\tгора Цинмао\tplace", attested) + if len(plain) != 1 || plain[0].Dst != "гора Цинмао" || plain[0].Type != "place" { + t.Fatalf("the ordinary three-field line must be untouched: %+v", plain) + } + // A TAB-delimited third column outside the type vocabulary is a benign unknown TYPE, exactly as it was + // before the fix-pack — NOT a piece of the rendering. Guessing it into the rendering is what the first cut + // of this fix did, and it corrupted four terms and dropped two on the frozen coldrun-a reference. + odd, _ := parseBanknote("古月\tклан Гуюэ\tсемья", attested) + if len(odd) != 1 || odd[0].Dst != "клан Гуюэ" || odd[0].Type != "term" { + t.Fatalf("an unknown TYPE column must be discarded, not glued onto the rendering: %+v", odd) + } + // And the column-order recovery still works, including over a rendering the model spaced out. + rev, _ := parseBanknote("Фан Юань\t方源\tname", attested) + if len(rev) != 1 || rev[0].Src != "方源" || rev[0].Dst != "Фан Юань" { + t.Fatalf("a reversed line must still be recovered: %+v", rev) + } +} + +// TestParseBanknoteReadsDeclaredColumnsPositionally is the pin the frozen reference bought. A tab or a pipe +// is a delimiter the model CHOSE — it cannot occur inside a rendering — so such a line has declared its own +// columns and is read by position, as it was before the fix-pack. Only a line delimited by a run of spaces +// is ambiguous, and only there may the type be guessed from the closed vocabulary. +// +// Measured cost of getting this wrong: replaying 56 real banknote blocks of ~/books/gu-zhenren/coldrun-a +// through the first cut of the fix gave −6 lines against HEAD and +0 — four renderings corrupted by a glued +// type word (clan/event/proverb/onomatopoeia — 1.25% of the reference carries a type outside the +// vocabulary) and two lines dropped outright, with banknote_parse_fail unchanged in every one of them. +func TestParseBanknoteReadsDeclaredColumnsPositionally(t *testing.T) { + attested := attestedIn(srcAll) + for _, tc := range []struct{ line, src, dst, typ string }{ + // An unknown third column: discarded as an unknown type, never glued (the four corrupted terms). + {"古月\tГу Юэ\tclan", "古月", "Гу Юэ", "term"}, + {"方源|Фан Юань|proverb", "方源", "Фан Юань", "term"}, + // The same line REVERSED: the src is still found by attestation, and the unknown type still discarded. + // The first cut looked for the src in the last field, found «clan», and dropped the line (the two lost). + {"Гу Юэ\t古月\tclan", "古月", "Гу Юэ", "term"}, + // A known type column is read as a type, in both orders. + {"青茅山\tгора Цинмао\tplace", "青茅山", "гора Цинмао", "place"}, + // And a rendering the model spaced out inside a REAL column arrives whole and clean. + {"方源\tФан Юань\tname", "方源", "Фан Юань", "name"}, + } { + got, flags := parseBanknote(tc.line, attested) + if len(got) != 1 { + t.Fatalf("%q: want one entry, got %+v (parse_fail=%v)", tc.line, got, flags.ParseFail) + } + if got[0].Src != tc.src || got[0].Dst != tc.dst || got[0].Type != tc.typ { + t.Fatalf("%q: got %+v, want src=%q dst=%q type=%q", tc.line, got[0], tc.src, tc.dst, tc.typ) + } + } +} + +// TestBanknoteTruncatedIsReachable closes the hygiene finding beside row 129: banknote_truncated was +// derived inside the parser from an argument the only production caller pinned to false, so the column was +// structurally 0 while its comment claimed otherwise. The refusal itself is the signal — a block present +// under a non-stop finish was cut by generation length — and reading it off the finish reason changes +// nothing about the ratified stop-only gate: still no candidates, still no telemetry lines. +func TestBanknoteTruncatedIsReachable(t *testing.T) { + r := &Runner{bankSrc: newBankSourceIndex([]chunk.Chunk{{Text: srcAll}})} + raw := "Незаконченный перевод\n" + bankSeparator + "\n方源\tФан" + _, _, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "length", srcAll) + if !flags.Truncated { + t.Fatalf("a block under a non-stop finish IS the truncation the column was meant to record: %+v", flags) + } + if len(entries) != 0 || flags.NLines != 0 { + t.Fatalf("the stop-only gate still accepts nothing: %+v / %+v", entries, flags) + } + // A complete generation is unaffected. + _, _, ok, got := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", srcAll) + if ok.Truncated || len(got) != 1 { + t.Fatalf("a complete generation is not truncated and its block is parsed: %+v / %+v", ok, got) + } +} diff --git a/backend/internal/pipeline/chunkrun.go b/backend/internal/pipeline/chunkrun.go index 4766bf57..23219ba3 100644 --- a/backend/internal/pipeline/chunkrun.go +++ b/backend/internal/pipeline/chunkrun.go @@ -1,9 +1,12 @@ package pipeline import ( + "context" "encoding/json" + "fmt" "maps" "slices" + "sort" "strings" "textmachine/backend/internal/checks" @@ -130,6 +133,7 @@ func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel memban } rs.NSpoilerBlocked = len(sel.Rejected) rs.NEvicted = len(sel.Evicted) + r.recordEvicted(sel.Evicted) // Loud record of the disposition-gated suppressor (D39 layer 4): a longer lower-trust key refused // from eating a nested higher-trust one — the term-drift code root, now visible instead of a // silent drop (research/13 §7). Detail carries the refused suppressor→protected pairs for a human. @@ -164,6 +168,72 @@ func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel memban return r.Store.UpsertRetrievalState(rs) } +// recordEvicted accumulates WHICH bank rows the injection budget dropped, book-wide, so the wave can name +// them once instead of leaving n_evicted as a number nobody can act on. +// +// The list is kept in memory and reported to the LOG rather than stored beside the counter, deliberately: a +// per-row detail column would be a schema migration, and this repository has already paid for a +// non-idempotent one (backlog row 49а). The count keeps its column; the names ride the plane an operator +// reads when deciding whether to raise gates.glossary token budget or prune the seed. +func (r *Runner) recordEvicted(evicted []membank.PickedEntry) { + if len(evicted) == 0 { + return + } + r.evictMu.Lock() + defer r.evictMu.Unlock() + if r.evictedRows == nil { + r.evictedRows = map[string]int{} + } + for _, p := range evicted { + if src := p.Src(); src != "" { + r.evictedRows[src]++ + } + } +} + +// evictedNameCap bounds the names one warning carries: the point is to name the rows that lose most often, +// not to reprint the bank. +const evictedNameCap = 20 + +// reportEvicted drains the accumulator and names the rows the budget dropped most often. Called once per +// wave, so a book that never overflows its injection budget stays silent. +// +// ⚠ Scope, stated: only the DRAFT wave feeds it, because n_evicted is a draft-wave column +// (persistRetrievalState) — the edit wave runs its own Select whose evictions this pack does not record. +func (r *Runner) reportEvicted(ctx context.Context, wave string) { + r.evictMu.Lock() + rows := r.evictedRows + r.evictedRows = nil + r.evictMu.Unlock() + if len(rows) == 0 { + return + } + type row struct { + src string + n int + } + list := make([]row, 0, len(rows)) + for src, n := range rows { + list = append(list, row{src, n}) + } + sort.Slice(list, func(i, j int) bool { + if list[i].n != list[j].n { + return list[i].n > list[j].n + } + return list[i].src < list[j].src + }) + named := make([]string, 0, evictedNameCap) + for _, e := range list { + if len(named) == evictedNameCap { + break + } + named = append(named, fmt.Sprintf("%s ×%d", e.src, e.n)) + } + r.Log.WarnContext(ctx, "memory: the injection token budget DROPPED bank rows before the model saw them — these terms had no canon on the wire for those units", + "book", r.Book.BookID, "wave", wave, "rows", len(list), "shown", len(named), + "budget_tokens", r.Pipeline.Context.GlossaryTokenBudget, "terms", strings.Join(named, ", ")) +} + // boolToStoreInt maps a bool telemetry flag to the store's 0/1 integer column form. func boolToStoreInt(b bool) int { if b { diff --git a/backend/internal/pipeline/mining.go b/backend/internal/pipeline/mining.go index 1a8819e5..fcb394cb 100644 --- a/backend/internal/pipeline/mining.go +++ b/backend/internal/pipeline/mining.go @@ -86,10 +86,17 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr // exists. They arrive as EVIDENCE — nothing proposed enters the bank without a signature. A read failure // degrades to the WHICH-only map rather than blocking the stop: the map is what the owner needs, the // dst is a bonus. + // The parse rule is LOGGED with the run, which is the whole contract that lets it stay out of the + // snapshot: it cannot change a paid byte, but it decides which candidate lines became evidence, so a + // signature map has to be attributable to the rule that produced it. Until the fix-pack the constant was + // declared "logged with the run" and read by nothing — the tag was a comment, not a mechanism, and two + // artifacts produced by different rules were indistinguishable. observed, offLanguage, oerr := r.bankObservedForBook() if oerr != nil { r.Log.WarnContext(ctx, "bank-mining: could not read the banknote proposals; the signature map falls back to WHICH-only (bare terms)", "err", oerr) } + r.Log.InfoContext(ctx, "bank-mining: draft-side proposals folded", "book", r.Book.BookID, + "surfaces", len(observed), "parse_version", bankParseVersion, "slice_version", bankSliceVersion) if offLanguage > 0 { r.Log.WarnContext(ctx, "bank-mining: draft-side proposals were written in another script and are NOT offered for signature", "book", r.Book.BookID, "dropped", offLanguage, "target_script", r.Pipeline.Gates.Terminology.TargetScript) @@ -97,8 +104,12 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr // The TERMINOLOGIST (pack-20, D39.42): merge both channels, gather each candidate's source contexts, // rank the renderings the drafts produced (§C2-3), and — when the gate is on — consolidate the whole - // bank in a handful of batched calls. With the gate off this is $0 assembly whose only consumer is the - // stop's own table, and the emitted delta is byte-identical to before. + // bank in a handful of batched calls. With the gate off this is $0 assembly, but NOT a no-op for the + // artifact: since the fix-pack the delta's dst comes from these ranked, target-form-folded candidates + // rather than from the raw draft-side order, so a term whose drafts disagreed can carry a different + // proposal than it did before — see proposalsFromCandidates. That is bank CONTENT, so it moves + // memory_version and the edit-wave snapshot with it (bank-only → $0 re-pin for every unit the term does + // not occur in). The draft wave is untouched: these rows are Source:"mined" and base-excluded. tchunks := make([]terminology.Chunk, len(minerChunks)) for i, c := range minerChunks { tchunks[i] = terminology.Chunk{Chapter: c.Chapter, ChunkIdx: c.ChunkIdx, NSource: c.NSource} @@ -146,7 +157,7 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr mined = attachConsolidatedDst(mined, consolidated) // Non-empty delta → write the owner signature map (the mined seed-delta YAML) and STOP before the edit wave. - proposals := proposalsFromObserved(observed) + proposals := proposalsFromCandidates(cands) withDst := 0 for _, m := range mined { if m.Dst != "" || len(proposals[text.NormalizeSourceKey(m.Src)]) > 0 { @@ -163,7 +174,7 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr // The RICH table (D39.36's «стоп с таблицей»: src · dst · frequency · variant spread · evidence). It is // written as a sidecar on EVERY run, signed or not, because it is also the auto mode's record of what // the book decided on its own — and it is capped on stdout, never in the file (emitRankCap is 200). - rows := bankStopRows(cands, consolidated) + rows := bankStopRows(cands, consolidated, tres) if werr := os.WriteFile(r.bankStopTablePath(), []byte(renderBankStopTable(rows)), 0o644); werr != nil { r.Log.WarnContext(ctx, "bank-mining: could not write the stop table sidecar (the signature map is unaffected)", "err", werr) } @@ -180,7 +191,7 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr // re-seeding through the ordinary seedGlossary path (rather than a second, private write) is what // keeps ONE definition of what the bank is: every guard the seed path owns — the collision checks, // the reject filter, the fail-louds — applies to the engine's rows exactly as to the owner's. - if err := r.writeAutoBank(mined, proposals); err != nil { + if err := r.writeAutoBank(ctx, mined, proposals, ownerHandled(unsignedEngineSurfaces(seed), rejects)); err != nil { return false, err } if err := r.seedGlossary(ctx); err != nil { @@ -251,34 +262,98 @@ type BankStopRow struct { Variants []string // "rendering ×N", best-ranked first Contexts []string // source KWIC Evidence []string + // The §G3 arbitration record: until this pack, «why does this term have THIS dst» was unanswerable from + // the artifacts (research/24 §A7). Every field below is read off work the ranking already did. + // + // Conventions is how many genuinely different DECISIONS the drafts made, once renderings differing only + // in target form are folded (Spread counts the raw forms). Signals are the §C2-3 factors that fired for + // the TOP-ranked variant — the winner's audit trail, printed for the row rather than per variant, + // because the row is what the owner signs. Invented says the consolidated rendering is NOT one the + // drafts proposed: legitimate (the role sees the whole book, the drafts saw fragments) and exactly the + // class to read first. Conf is the role's own stated confidence, which orders the review list and + // nothing else (D39.102) — NEGATIVE when the reply carried none, because «the role said it was 0% sure» + // is the most important row on the sheet and «the role said nothing» is not a row at all. + // Contradicts names THIS RUN's other consolidations the rendering breaks (§G2). + Conventions int + Signals []string + Invented bool + Conf int + Contradicts []string } // bankStopRows projects the merged candidates into the operator table, best-ranked variants first. // Deterministic: cands is key-ordered and nothing here iterates a map for output. -func bankStopRows(cands []terminology.Candidate, consolidated map[string]string) []BankStopRow { +func bankStopRows(cands []terminology.Candidate, consolidated map[string]string, tres terminologyResult) []BankStopRow { out := make([]BankStopRow, 0, len(cands)) for _, c := range cands { + dst := consolidated[c.Key] row := BankStopRow{ - Src: c.Src, Dst: consolidated[c.Key], Origin: string(c.Origin), Type: c.Type, - Freq: c.Freq, Spread: c.Spread(), Contexts: c.KWIC, Evidence: c.Evidence, + Src: c.Src, Dst: dst, Origin: string(c.Origin), Type: c.Type, + Freq: c.Freq, Spread: c.Spread(), Conventions: c.Conventions(), + Contexts: c.KWIC, Evidence: c.Evidence, + Conf: confOrAbsent(tres.Conf, c.Key), Contradicts: tres.Contradictions[c.Src], } - for _, v := range c.Variants { - row.Variants = append(row.Variants, fmt.Sprintf("%s ×%d", v.Dst, v.Chunks)) + for i, v := range c.Variants { + label := fmt.Sprintf("%s ×%d", v.Dst, v.Chunks) + if v.Via != "" { + label += " (proposed for " + v.Via + ")" + } + row.Variants = append(row.Variants, label) + if i == 0 { + row.Signals = v.Signals + } } + row.Invented = dst != "" && !proposedByDrafts(dst, c.Variants) out = append(out, row) } return out } +// confOrAbsent reads the role's stated confidence for a key, or -1 when the reply carried none. A plain +// zero would merge the two, and they are opposites: one is the first row to review, the other is silence. +func confOrAbsent(conf map[string]int, key string) int { + if v, ok := conf[key]; ok { + return v + } + return -1 +} + +// proposedByDrafts reports whether the consolidated rendering is one the drafts actually produced, compared +// under the same target-form fold the vote is counted with — so a case or ё difference is not reported as +// an invention. +func proposedByDrafts(dst string, vs []terminology.Variant) bool { + want := text.NormalizeTargetForm(dst) + for _, v := range vs { + if text.NormalizeTargetForm(v.Dst) == want { + return true + } + } + return false +} + // renderBankStopTable serializes the FULL table for the sidecar. One block per term, the same shape the // stdout banner prints — so the capped view and the file cannot describe the bank differently. func renderBankStopTable(rows []BankStopRow) string { var b strings.Builder fmt.Fprintf(&b, "BANK VERIFICATION TABLE — %d term(s)\n", len(rows)) - b.WriteString("src · proposed dst · origin · type · freq · variant spread · evidence · source contexts\n\n") + b.WriteString("src · proposed dst · origin · type · freq · variant spread · conventions · confidence ·\n") + b.WriteString("why (the ranking factors that won) · contradictions · drafts · evidence · source contexts\n\n") for _, r := range rows { fmt.Fprintf(&b, "%s\t%s\n", r.Src, dashIfEmpty(r.Dst)) - fmt.Fprintf(&b, " origin=%s type=%s freq=%d spread=%d\n", r.Origin, dashIfEmpty(r.Type), r.Freq, r.Spread) + fmt.Fprintf(&b, " origin=%s type=%s freq=%d spread=%d conventions=%d", r.Origin, dashIfEmpty(r.Type), r.Freq, r.Spread, r.Conventions) + if r.Conf >= 0 { + fmt.Fprintf(&b, " confidence=%d", r.Conf) + } + if r.Invented { + b.WriteString(" INVENTED(no draft proposed it)") + } + b.WriteString("\n") + if len(r.Signals) > 0 { + fmt.Fprintf(&b, " why: %s\n", strings.Join(r.Signals, ", ")) + } + if len(r.Contradicts) > 0 { + fmt.Fprintf(&b, " CONTRADICTS this run's own: %s\n", strings.Join(r.Contradicts, "; ")) + } if len(r.Variants) > 0 { fmt.Fprintf(&b, " drafts: %s\n", strings.Join(r.Variants, " | ")) } @@ -348,17 +423,40 @@ func reverseSectionTerms(cands []terminology.Candidate, seed []store.GlossaryEnt return out, eligible } -// proposalsFromObserved re-shapes the folded draft-side view into the signature-map join's input. One -// definition of the fold (bankObservedByKey) feeds both consumers, so the map the owner signs and the -// table the terminologist read can never disagree about what a chunk proposed. -func proposalsFromObserved(obs []terminology.Observed) map[string][]miner.DstProposal { - out := make(map[string][]miner.DstProposal, len(obs)) - for _, o := range obs { - list := make([]miner.DstProposal, 0, len(o.Proposals)) - for _, p := range o.Proposals { - list = append(list, miner.DstProposal{Dst: p.Dst, Type: p.Type, Chunks: p.Chunks}) +// proposalsFromCandidates re-shapes the MERGED candidates into the signature-map join's input. +// +// It reads the candidates rather than the raw draft-side view, and that is the fix (§G3, acceptance +// finding): a proposal that arrived under an ALIAS of a cluster is routed to the cluster's owner by +// Merge — the stop table therefore showed it — while the signature map joined on the alias's own key and +// the owner's row never mentioned it. The map the owner signs then disagreed with the table he was reading +// it against, for exactly the terms the miner clustered. One source for both removes the divergence +// structurally instead of keeping two joins in step by hand. +// +// The list arrives §C2-3-ranked and target-form folded, so the note's alternatives are the ones the table +// shows, in the order it shows them. +func proposalsFromCandidates(cands []terminology.Candidate) map[string][]miner.DstProposal { + out := make(map[string][]miner.DstProposal, len(cands)) + for _, c := range cands { + if len(c.Variants) == 0 { + continue } - out[o.Key] = list + // DIRECT proposals first, alias-routed ones after — and DeltaYAML takes the term's dst from a DIRECT + // one only. Merge routes an alias's rendering to the cluster owner so the ranking sees all of the + // entity's evidence; letting that rendering become the OWNER's dst is a different act entirely. It + // would put «Малыш Фан» on 方源 with no model involved, on the $0 path, with the terminology gate OFF — + // the exact harm Variant.Via was introduced to prevent, arriving through the fix that introduced Via. + list := make([]miner.DstProposal, 0, len(c.Variants)) + for _, v := range c.Variants { + if v.Via == "" { + list = append(list, miner.DstProposal{Dst: v.Dst, Type: c.Type, Chunks: v.Chunks}) + } + } + for _, v := range c.Variants { + if v.Via != "" { + list = append(list, miner.DstProposal{Dst: v.Dst, Type: c.Type, Chunks: v.Chunks, Via: v.Via}) + } + } + out[c.Key] = list } return out } @@ -448,7 +546,19 @@ func unsignedEngineSurfaces(rows []store.GlossaryEntry) []store.GlossaryEntry { // writeAutoBank persists the unsigned rows the auto mode decided to carry forward, as the same seed-YAML // schema everything else in this pipeline speaks (so `tmctl seed-lint` reads it, and a row can be moved // into the owner's delta by copy-paste). Deterministic: the mined list is already sorted by src. -func (r *Runner) writeAutoBank(mined []miner.Term, proposals map[string][]miner.DstProposal) error { +// +// It DIFFS the file it is about to replace, and that is the $0 minimum of backlog row 130. The file is +// rewritten WHOLE from this run's delta, and the delta is capped at the miner's top-200 (emitRankCap, +// applied BEFORE the emission filters, so seed and declined terms do not free their slots). The two +// mechanisms were ratified separately and their INTERACTION never was: as a book grows, a term that was in +// the bank for twenty chapters silently vanishes from it mid-run, with nothing in the artifacts saying so. +// Naming the losers costs nothing and moves no bytes. +// +// The BOUNDARY is deliberate and is the owner's STOP: this reports, it does not accumulate. Merging the old +// file into the new one would change what the bank CONTAINS, which moves memory_version and re-prices the +// edit wave — a decision, not a hygiene fix. +func (r *Runner) writeAutoBank(ctx context.Context, mined []miner.Term, proposals map[string][]miner.DstProposal, signed map[string]bool) error { + before := r.autoBankSurfaces() body, err := miner.DeltaYAML(mined, proposals) if err != nil { return fmt.Errorf("pipeline: marshal auto-bank: %w", err) @@ -456,9 +566,72 @@ func (r *Runner) writeAutoBank(mined []miner.Term, proposals map[string][]miner. if err := os.WriteFile(r.autoBankPath(), []byte(body), 0o644); err != nil { return fmt.Errorf("pipeline: write auto-bank %s: %w", r.autoBankPath(), err) } + if len(before) == 0 { + return nil + } + now := make(map[string]bool, len(mined)) + for _, m := range mined { + now[text.NormalizeSourceKey(m.Src)] = true + } + var gone []string + for _, src := range before { + key := text.NormalizeSourceKey(src) + if now[key] || signed[key] { + // signed[] is the owner's two verbs — PROMOTED into the mined-delta or DECLINED in mined-rejects. + // Both remove the term from this run's delta on purpose, and reporting them as a silent loss would + // fire a false alarm on the normal signature cycle — advising the owner to do what he just did. + continue + } + gone = append(gone, src) + } + if len(gone) > 0 { + sort.Strings(gone) + r.Log.WarnContext(ctx, "bank-mining: terms that were in the auto-bank are NOT in the one this run just wrote — the file is rewritten whole from a top-N-capped delta, so a term the book still uses can drop out of the bank mid-run; if one of these matters, promote it into the owner's mined-delta file (it is then a seed surface and cannot be cut again)", + "book", r.Book.BookID, "dropped", len(gone), "kept", len(mined), "rank_cap", miner.EmitRankCap(), + "terms", strings.Join(gone, ", "), "auto_bank", r.autoBankPath()) + } return nil } +// ownerHandled is the surface set the OWNER has already decided about: every signed seed surface (a promoted +// term is one) plus every declined one. A term leaving the delta through either door is not a loss. +// +// ⚠ The caller MUST pass it through unsignedEngineSurfaces — risk 2, the self-exclusion trap this file +// documents twice and works around in two other places. From the SECOND auto-mode run the stored glossary +// also holds the engine's OWN unsigned rows (seedGlossary re-seeds the auto-bank at start), so a raw seed +// makes every term the engine ever proposed look owner-decided: the diff empties, and the row-130 warning — +// whose entire purpose is to name terms the rewrite dropped — can never fire again in production. +func ownerHandled(seed []store.GlossaryEntry, rejects map[string]bool) map[string]bool { + out := make(map[string]bool, len(seed)+len(rejects)) + for _, e := range seed { + out[text.NormalizeSourceKey(e.Src)] = true + for _, a := range e.Aliases { + out[text.NormalizeSourceKey(a.Alias)] = true + } + } + for k := range rejects { + out[k] = true + } + return out +} + +// autoBankSurfaces reads the src surfaces of the auto-bank file as it stands BEFORE this run rewrites it, +// in file order. Absent or unreadable → nil: the diff is observability, and failing a paid run because the +// PREVIOUS artifact cannot be parsed would be the tail wagging the dog. +func (r *Runner) autoBankSurfaces() []string { + entries, err := membank.LoadGlossarySeed(r.autoBankPath()) + if err != nil { + return nil + } + out := make([]string, 0, len(entries)) + for _, e := range entries { + if e.Src != "" { + out = append(out, e.Src) + } + } + return out +} + // minedRejectFile is the owner's mined-term reject list (Book.MinedRejects, R1-FL-B): the src surfaces the // owner reviewed and DECLINED. It is a PROPOSAL filter only — rejects never enter the bank content, so this // file is deliberately NOT folded into the snapshot (a reject affects the next mining PROPOSAL, not any diff --git a/backend/internal/pipeline/miningstop_join_test.go b/backend/internal/pipeline/miningstop_join_test.go index 55c98e57..0f439992 100644 --- a/backend/internal/pipeline/miningstop_join_test.go +++ b/backend/internal/pipeline/miningstop_join_test.go @@ -576,29 +576,40 @@ func TestTerminologistSpendIsInTheRunTotal(t *testing.T) { } } -// TestTerminologistEmptyReplyIsLoud: a paid batch that comes back unreadable (empty completion, prose, -// truncation) would otherwise be indistinguishable from «the role declined these terms» — which is a -// DECISION in §C2-7 — and the run would exit 0 having bought nothing. +// TestTerminologistEmptyReplyIsLoud: a paid batch that comes back unreadable would otherwise be +// indistinguishable from «the role declined these terms» — which is a DECISION in §C2-7 — and the run would +// exit 0 having bought nothing. +// +// TWO branches, and until the fix-pack only the first was covered while the test's NAME claimed the second +// (§G3): prose is a non-empty completion the parser rejects line by line, whereas an EMPTY completion never +// reached the parser at all and took a silent `continue`. The fixture below is the honest one. func TestTerminologistEmptyReplyIsLoud(t *testing.T) { - var logBuf bytes.Buffer - rec := &reqRec{} - srv := newJSONProvider(rec, func(body string) (string, string) { - if isTerminologyBody(body) { - return "Извините, я не понял задание.", "stop" // prose: no line the parser can use - } - return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop" - }) - defer srv.Close() - bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}) + for _, tc := range []struct{ name, reply, want string }{ + {"prose", "Извините, я не понял задание.", "returned nothing the parser could use"}, + {"empty completion", "", "came back with an EMPTY completion"}, + } { + t.Run(tc.name, func(t *testing.T) { + var logBuf bytes.Buffer + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isTerminologyBody(body) { + return tc.reply, "stop" + } + return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop" + }) + defer srv.Close() + bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}) - r := newRunner(t, bookPath) - defer r.Close() - r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) - if _, err := r.TranslateBook(context.Background()); err != nil { - t.Fatal(err) - } - if !strings.Contains(logBuf.String(), "returned nothing the parser could use") { - t.Fatalf("a paid batch that bought no terminology must say so:\n%s", logBuf.String()) + r := newRunner(t, bookPath) + defer r.Close() + r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) + if _, err := r.TranslateBook(context.Background()); err != nil { + t.Fatal(err) + } + if !strings.Contains(logBuf.String(), tc.want) { + t.Fatalf("a paid batch that bought no terminology must say so (%s):\n%s", tc.name, logBuf.String()) + } + }) } } diff --git a/backend/internal/pipeline/minirun_fixes_test.go b/backend/internal/pipeline/minirun_fixes_test.go index 6d5451a1..c9db6fe9 100644 --- a/backend/internal/pipeline/minirun_fixes_test.go +++ b/backend/internal/pipeline/minirun_fixes_test.go @@ -10,6 +10,8 @@ import ( "textmachine/backend/internal/obs" "textmachine/backend/internal/store" + "textmachine/backend/internal/terminology" + "textmachine/backend/internal/text" ) // minirun_fixes_test.go: the two seams the 25.07 mini-run proved were producing a fact and losing it. @@ -165,7 +167,11 @@ func TestBanknoteProposalsArePersistedAndJoined(t *testing.T) { if len(props) != 2 { t.Fatalf("want 2 stored proposals, got %d: %s", len(props), rs.BanknoteDetail) } - byKey := proposalsFromObserved(bankObservedByKey([]store.RetrievalState{*rs}, nil)) + // The signature-map join reads the MERGED candidates (fix-pack §G3), so the assertion follows the same + // path production takes: a proposal must land on the miner's normalized surface after the merge, not + // only in the raw draft-side view. + observed := bankObservedByKey([]store.RetrievalState{*rs}, nil) + byKey := proposalsFromCandidates(terminology.Merge(nil, observed, text.NormalizeTargetForm)) if got := byKey[props[0].SrcKey]; len(got) == 0 || got[0].Dst != props[0].Dst { t.Fatalf("the fold must key proposals by the miner's normalized surface, got %+v", byKey) } diff --git a/backend/internal/pipeline/runner.go b/backend/internal/pipeline/runner.go index c25e404c..6f35544d 100644 --- a/backend/internal/pipeline/runner.go +++ b/backend/internal/pipeline/runner.go @@ -19,6 +19,7 @@ import ( "textmachine/backend/internal/llm" "textmachine/backend/internal/membank" "textmachine/backend/internal/store" + "textmachine/backend/internal/terminology" ) // runner.go: runner setup — Runner and its wiring (config stack, store in the right @@ -78,10 +79,21 @@ type Runner struct { // .target_script (config validates the name). nil when the book declares none — the answer-language // screen is then inert, which loadTargetScript says out loud if any channel could have used it. targetScript *unicode.RangeTable + // familyParams is the §G1 family channel resolved from the source's declared morphology, once, at load + // time (loadFamilyParams). The zero value is a disabled channel — the batcher then co-batches series + // only, exactly as before the channel existed. Like the rest of the terminology axis it is NOT + // snapshot-folded: it changes which candidates share a CALL, never a wave byte. + familyParams terminology.FamilyParams // rateGuards is the per-model wave-concurrency guard set (WS1 §1б), built once in the precompute pass and // read-only in the waves — a transport axis, never snapshot-folded. nil until buildRateGuards. rateGuards map[string]*rateGuard + // evictedRows accumulates WHICH bank rows the injection budget dropped (src → how many units), so the + // wave can NAME them once rather than leave n_evicted as an unactionable count. Written from N draft + // workers, hence the mutex; drained by reportEvicted. + evictMu sync.Mutex + evictedRows map[string]int + // bankSrc is the banknote parser's "is this surface in the book?" index (backlog 19), built once from // the chunk manifest in the precompute pass. nil outside a book run → the parser then judges a line by // the chunk text it was handed, and accepts nothing without one. diff --git a/backend/internal/pipeline/runner_test.go b/backend/internal/pipeline/runner_test.go index 72767cfd..59670e58 100644 --- a/backend/internal/pipeline/runner_test.go +++ b/backend/internal/pipeline/runner_test.go @@ -117,6 +117,9 @@ type projectOpts struct { // rebillConsentUSD writes book.yaml's `rebill_consent_usd` override (D20.2-Q2). 0 = omit the key, so // the book takes the ratified default threshold and every pre-existing fixture is byte-unchanged. rebillConsentUSD float64 + // glossaryTokenBudget overrides context.glossary_token_budget. 0 keeps the fixture's historical 800, so + // every pre-existing project is byte-identical; a tiny value is how the EVICTION path is reachable at all. + glossaryTokenBudget int } func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string { @@ -130,6 +133,9 @@ func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string { if o.bookUSD == 0 { o.bookUSD = 1.0 } + if o.glossaryTokenBudget == 0 { + o.glossaryTokenBudget = 800 + } dir := t.TempDir() writeFile(t, filepath.Join(dir, "prompts", "translator.md"), @@ -163,12 +169,12 @@ core: C1 version: 1 defaults: { max_output_ratio: 2.0, min_max_tokens: %d } retries: { regenerate_before_escalate: %d } -context: { glossary_injection: selective, glossary_token_budget: 800 } +context: { glossary_injection: selective, glossary_token_budget: %d } waves: { workers: %d } stages: - { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" } - { name: edit, role: editor, model: fake-model, prompt_override: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" } -%s`, o.minMaxTokens, o.regenerate, o.waveWorkers, gatesBlock)) +%s`, o.minMaxTokens, o.regenerate, o.glossaryTokenBudget, o.waveWorkers, gatesBlock)) sourceName := "source.txt" if len(o.epub) > 0 { diff --git a/backend/internal/pipeline/terminologist.go b/backend/internal/pipeline/terminologist.go index 7fc35b8e..9022e51f 100644 --- a/backend/internal/pipeline/terminologist.go +++ b/backend/internal/pipeline/terminologist.go @@ -57,7 +57,7 @@ const terminologyStageName = "terminology" // terminologyVersion versions the ASSEMBLY algorithm (merge → KWIC → §C2-3 ranking → batching → parse). It // is deliberately NOT snapshot-folded — see config.TerminologyGate — but it is logged with the run so a // signature map can be attributed to the algorithm that produced it. -const terminologyVersion = "terminology-v2-merge+kwic+c2-3+series" +const terminologyVersion = "terminology-v3-merge+kwic+c2-3+series+families+formfold" // Engine defaults for the block sizing. They bound ONE call's input; the whole book is covered by batching. const ( @@ -83,14 +83,29 @@ type terminologyResult struct { // OBSERVABILITY, never a gate: the rows stay unverified either way, and this is what tells the owner // which of them to look at first. CanonConflicts int - CostUSD float64 // what THIS run's RENDER phase paid - CumUSD float64 // what the render calls cost in total (a replayed checkpoint is $0 now, not then) - Fresh bool // at least one call actually reached the provider this run - EstimateUSD float64 // the pre-call projection, logged before any money moves + // SelfConflicts counts consolidations that contradict ANOTHER consolidation of the same run (§G2) — the + // majority class on a live bank, and the one nothing looked at before this pack. Contradictions carries + // the same finding per key, so the stop table can mark the rows instead of printing a bare total. + SelfConflicts int + Contradictions map[string][]string + // Conf is the role's own stated confidence per key. It sorts the review list «least sure first» and does + // nothing else — never a weight, a threshold or a cross-model comparison (D39.102). + Conf map[string]int + CostUSD float64 // what THIS run's RENDER phase paid + CumUSD float64 // what the render calls cost in total (a replayed checkpoint is $0 now, not then) + Fresh bool // at least one call actually reached the provider this run + EstimateUSD float64 // the pre-call projection, logged before any money moves // Reclassified is how many candidate types the §2 classifier phase actually changed; ClassifyCostUSD is // what that phase paid this run. Both zero when classify_types is off. Reclassified int ClassifyCostUSD float64 + // Families is how many family GROUPS §G1 detected; FamiliesRefused how many of their merges the member + // cap turned down, and FamiliesHeld how many a series held part of back. Both of the latter mean the same + // thing to the owner — a family met split across two calls — and both are zero when the source declares + // no family data. + Families int + FamiliesRefused int + FamiliesHeld int } // loadTerminologyTemplate loads the pair's terminologist prompt when the gate is on. Mirrors @@ -104,9 +119,42 @@ func (r *Runner) loadTerminologyTemplate() error { return err } r.terminologyTemplate = tpl + if err := r.loadFamilyParams(); err != nil { + return err + } return r.loadClassifierTemplate() } +// loadFamilyParams resolves the §G1 family channel from the SOURCE language's declared morphology, once, at +// LOAD time — before any billing, like every other gate precondition. A source with no family data yields a +// disabled channel and the batcher behaves exactly as it did before the channel existed. +// +// It is also where a data typo dies: the file names engine TYPES, and lang cannot check them against the +// engine's closed vocabulary without depending on this layer. A rule for a type nothing emits would parse +// fine and leave the channel quietly half-off — the same silent-empty-table class the pack loader refuses. +func (r *Runner) loadFamilyParams() error { + fm := lang.FamilyMorphology(r.Book.SourceLang) + if !fm.Enabled() { + r.familyParams = terminology.FamilyParams{} + return nil + } + sEnabled, headFinal := lang.SeriesMorphology(r.Book.SourceLang) + p := terminology.FamilyParams{ + Enabled: sEnabled, HeadFinal: headFinal, + Affix: make(map[string]terminology.FamilyAffix, len(fm.Affix)), + MinMembers: fm.MinMembers, MaxMembers: fm.MaxMembers, ContainmentRunes: fm.ContainmentRunes, + } + for typ, a := range fm.Affix { + if !terminology.CandidateTypes[typ] { + return fmt.Errorf("pipeline: the family morphology of source %q names type %q, which no candidate can carry (accepted: %s) — a typo here would parse fine and leave the family channel silently half-off", + r.Book.SourceLang, typ, strings.Join(terminology.TypeNames(terminology.CandidateTypes), "|")) + } + p.Affix[typ] = terminology.FamilyAffix{Suffix: a.Suffix, MinRunes: a.MinRunes} + } + r.familyParams = p + return nil +} + // loadClassifierTemplate loads the pair's §2 classifier prompt when classify_types is on. Off → nil, and the // classifier phase is inert. Called from loadTerminologyTemplate: the classifier only exists as a phase of // the terminology gate. @@ -179,7 +227,11 @@ func (r *Runner) buildBankCandidates(mined []miner.Term, observed []terminology. Freq: m.Freq, SinceCh: m.SinceCh, Aliases: m.Aliases, Evidence: m.Evidence, }) } - cands := terminology.Merge(ms, observed) + // The vote is counted per target-form CONVENTION, not per byte string (§G5): «Море истинной ци» and + // «море истинной ци» are one decision, and splitting their evidence hands the §C2-3 frequency factor to + // whichever spelling a chunk happened to repeat. The normalizer is the SAME one the post-check matches + // against, so the fold cannot disagree with the check that reads the result. + cands := terminology.Merge(ms, observed, text.NormalizeTargetForm) _, kwicPer, kwicWidth := r.terminologyOpts() cands = terminology.AttachKWIC(cands, chunks, kwicPer, kwicWidth) @@ -262,20 +314,31 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te } } - // §1: co-batch grade/rank series so the role picks ONE generic head for the whole set. The pair-data — - // whether the source forms rune-morpheme series and where the head sits — comes from the language layer, - // so the batcher stays pair-agnostic and an alphabetic source takes a byte-identical, series-free path. + // §1 + §G1: co-batch each grade/rank SERIES and each term FAMILY so the role picks ONE generic head, and + // ONE shared root, for the whole set. The pair-data — whether the source forms rune-morpheme series, + // where the head sits, and which side of a surface carries a family's root — comes from the language + // layer, so the batcher stays pair-agnostic and a source with neither takes a byte-identical path. sEnabled, headFinal := lang.SeriesMorphology(r.Book.SourceLang) seriesID := terminology.DetectSeries(cands, terminology.SeriesParams{Enabled: sEnabled, HeadFinal: headFinal}) - batches := terminology.Batch(cands, batchRunes, seriesID) + fp := r.familyParams + fams := terminology.DetectFamilies(cands, fp) + unitID, ustats := terminology.MergeUnits(seriesID, fams, fp) + batches := terminology.Batch(cands, batchRunes, unitID) res.Batches = len(batches) - // §1 keeps a series (or a lone candidate) WHOLE even past the budget — splitting a series would defeat the - // one-head co-batching it exists for — so a co-batched grade set or an evidence-heavy single term can render + res.Families, res.FamiliesRefused, res.FamiliesHeld = len(fams), ustats.Refused, ustats.Held + if ustats.Refused > 0 || ustats.Held > 0 { + // Either guard leaves a family split across calls — the very defect this channel exists to close — so + // both are named rather than left to be inferred from a bank that disagrees with itself. + r.Log.WarnContext(ctx, "terminology: some families were NOT co-batched whole — the member cap refused the merge, or a series with an unrelated root kept its members; those families can still disagree with themselves across calls", + "book", r.Book.BookID, "refused_by_cap", ustats.Refused, "held_by_series", ustats.Held, "max_members", fp.MaxMembers) + } + // §1/§G1 keep a unit (or a lone candidate) WHOLE even past the budget — splitting one would defeat the + // co-batching it exists for — so a co-batched grade set or an evidence-heavy single term can render // over the cap. That is deliberate but NOT silent: an over-cap unit strains the model's output ceiling (the // cap-8000 mine), so name it while the run can still be watched. for i, b := range batches { if br := terminology.BatchRunes(b); br > batchRunes { - r.Log.WarnContext(ctx, "terminology: a batch renders OVER the size budget and is sent WHOLE (a series is co-batched by design, §1; a lone candidate cannot be split) — watch the model's output cap on this call", + r.Log.WarnContext(ctx, "terminology: a batch renders OVER the size budget and is sent WHOLE (a series/family is co-batched by design, §1/§G1; a lone candidate cannot be split) — watch the model's output cap on this call", "book", r.Book.BookID, "batch", i, "terms", len(b), "batch_runes", br, "budget_runes", batchRunes) } } @@ -293,13 +356,26 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te res.EstimateUSD, res.CostUSD, res.CumUSD, res.Fresh = run.estimateUSD, run.costUSD, run.cumUSD, run.fresh out := map[string]string{} + res.Conf = map[string]int{} for i, b := range batches { + if i >= run.attempted { + break // the budget or a ceiling stopped the pass here; runBankRoleBatches already said so + } if run.texts[i] == "" { + // An EMPTY completion on a call that was actually made. It used to `continue` in silence, which is + // how a paid batch that returned nothing became indistinguishable from «the role declined these + // terms» — a DECISION in §C2-7 — with the run exiting 0. + r.Log.WarnContext(ctx, "terminology: a paid batch came back with an EMPTY completion; its terms stay unconsolidated", + "book", r.Book.BookID, "batch", i, "asked", len(b), "answered", 0) continue } - got, st := terminology.ParseReply(run.texts[i], candKeys(b), text.NormalizeSourceKey, r.targetScript) + got, conf, st := terminology.ParseReply(run.texts[i], candKeys(b), text.NormalizeSourceKey, r.targetScript) res.BadLines += st.Bad res.OffLanguage += st.OffLanguage + // Asked-vs-answered per batch: the one number that separates «the model skipped half the block» from + // «the parser refused half the lines», and neither is visible in a total. + r.Log.InfoContext(ctx, "terminology render batch", "book", r.Book.BookID, + "batch", i, "asked", len(b), "answered", len(got), "bad_lines", st.Bad, "reply_chars", len(run.texts[i])) // A batch answered largely in another language is the measured cold-start failure, not noise: say it // while the run is happening, naming the lines, because the terms themselves just stay auto. if st.OffLanguage > 0 { @@ -317,6 +393,9 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te for k, v := range got { out[k] = v } + for k, v := range conf { + res.Conf[k] = v + } } for _, c := range cands { v, answered := out[c.Key] @@ -341,6 +420,21 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te r.Log.WarnContext(ctx, "terminology: consolidated renderings contradict the SIGNED bank; they stay unverified (⟨проверить⟩) and are the first rows to review at the stop", "book", r.Book.BookID, "conflicts", len(conflicts), "terms", strings.Join(named, "; ")) } + // §G2: the same check with the run's OWN consolidations on the right-hand side. On a live bank this is + // the MAJORITY of the contradictions (18 of 149, research/24 §A4) and until now nobody looked: the canon + // check can only see rows the owner already signed, and a book being consolidated for the first time has + // almost none. $0, evidence-side, never a gate. + if self := terminology.ConsolidationConflicts(cands, out); len(self) > 0 { + res.SelfConflicts = len(self) + res.Contradictions = map[string][]string{} + named := make([]string, 0, len(self)) + for _, cf := range self { + named = append(named, fmt.Sprintf("%s→%q drops %s→%q", cf.Src, cf.Dst, cf.PartSrc, cf.PartDst)) + res.Contradictions[cf.Src] = append(res.Contradictions[cf.Src], fmt.Sprintf("%s→%q", cf.PartSrc, cf.PartDst)) + } + r.Log.WarnContext(ctx, "terminology: consolidations of THIS run contradict each other — a compound's rendering drops the rendering the same reply gave its own part; they stay unverified and are review rows at the stop", + "book", r.Book.BookID, "conflicts", len(self), "terms", strings.Join(named, "; ")) + } // The $0 label screen (§2 warm-run hygiene): a name/place row whose rendering was clearly TRANSLATED is a // label/rendering disagreement worth a human's eye. It is a review FLAG, never a gate, and explicitly not // a safety net for the transliteration harm — the classifier phase is what prevents that (see @@ -356,7 +450,9 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te r.Log.InfoContext(ctx, "terminology finished", "book", r.Book.BookID, "consolidated", res.Consolidated, "declined", res.Declined, "unanswered", res.Unanswered, "reclassified", res.Reclassified, "bad_lines", res.BadLines, "off_language", res.OffLanguage, - "canon_conflicts", res.CanonConflicts, "cost_usd", fmt.Sprintf("%.6f", res.CostUSD), + "canon_conflicts", res.CanonConflicts, "self_conflicts", res.SelfConflicts, + "families", res.Families, "families_refused", res.FamiliesRefused, "families_held", res.FamiliesHeld, + "cost_usd", fmt.Sprintf("%.6f", res.CostUSD), "classify_cost_usd", fmt.Sprintf("%.6f", res.ClassifyCostUSD)) return out, classified, res, nil } @@ -400,11 +496,21 @@ func (r *Runner) runClassifier(ctx context.Context, snapID string, cands []termi out := map[string]string{} bad := 0 for i, b := range batches { + if i >= run.attempted { + break // the budget stopped the pass here, and it already said so + } if run.texts[i] == "" { + // The same silence the render phase carried: a paid classify batch that returned NOTHING left every + // one of its terms on the draft heuristic type — the mistyping this phase exists to remove — and the + // only trace was that `bad` stayed 0, which reads as a clean pass. + r.Log.WarnContext(ctx, "terminology classify: a paid batch came back with an EMPTY completion; its terms keep the draft heuristic type", + "book", r.Book.BookID, "batch", i, "asked", len(b), "answered", 0) continue } got, st := terminology.ParseTypes(run.texts[i], candKeys(b), text.NormalizeSourceKey) bad += st.Bad + r.Log.InfoContext(ctx, "terminology classify batch", "book", r.Book.BookID, + "batch", i, "asked", len(b), "answered", len(got), "bad_lines", st.Bad) for k, v := range got { out[k] = v } @@ -571,7 +677,11 @@ type bankRolePlan struct { // bankRoleRun is what one role's pass produced: each batch's reply text (in batch order, "" for a batch the // budget cut or that failed soft), plus the cost accounting for the report. type bankRoleRun struct { - texts []string + texts []string + // attempted is how many batches the pass actually reached before a budget ceiling or a soft denial stopped + // it. Without it "" is ambiguous — an EMPTY completion the run paid for and a batch never called look the + // same — and the caller cannot warn about the first without crying wolf about the second. + attempted int estimateUSD float64 costUSD float64 cumUSD float64 @@ -597,7 +707,8 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban } r.Log.InfoContext(ctx, "terminology "+logKind+": estimate before any call", "book", r.Book.BookID, "role", plan.role, "batches", len(batches), "model", st.Model, "reasoning", st.Reasoning, - "estimate_usd", fmt.Sprintf("%.6f", run.estimateUSD), "budget_usd", plan.budgetUSD, "version", terminologyVersion) + "estimate_usd", fmt.Sprintf("%.6f", run.estimateUSD), "budget_usd", plan.budgetUSD, + "version", terminologyVersion, "bank_data", lang.BankDataVersion()) spent, err := r.Store.RoleSpentUSD(r.Book.BookID, plan.role) if err != nil { @@ -638,6 +749,7 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban run.fresh = run.fresh || att.freshCall spent += att.runCost run.texts[i] = att.text + run.attempted = i + 1 } return run, nil } diff --git a/backend/internal/pipeline/waverun.go b/backend/internal/pipeline/waverun.go index 4813d8d7..e9e70ce2 100644 --- a/backend/internal/pipeline/waverun.go +++ b/backend/internal/pipeline/waverun.go @@ -127,6 +127,8 @@ func (r *Runner) translateBookWaves(ctx context.Context, chunks []chunk.Chunk, s return nil, err } + r.reportEvicted(ctx, "draft") + // --- the bank-mining stop: bank-mining stop boundary (auto-continues when mining is not configured) --- if stopped, err := r.runBankMiningStop(ctx, chunks, draftSnapshot, editWave); err != nil { return nil, err diff --git a/backend/internal/terminology/arbitration_test.go b/backend/internal/terminology/arbitration_test.go new file mode 100644 index 00000000..9d60c8b1 --- /dev/null +++ b/backend/internal/terminology/arbitration_test.go @@ -0,0 +1,215 @@ +package terminology + +import ( + "strings" + "testing" +) + +// arbitration_test.go: the §G2/§G3/§G5 half of the fix-pack — the run reading its OWN output (self +// contradictions), the confidence field, and the vote counted per convention instead of per byte string. + +// lowerFold is a stand-in for the target-form normalizer the pipeline injects: enough of it (case and +// whitespace) to exercise the fold without importing a target's morphology into this package. +func lowerFold(s string) string { return strings.Join(strings.Fields(strings.ToLower(s)), " ") } + +// TestConsolidationConflictsCatchesTheRunContradictingItself is the §G2 case measured on the live bank: the +// same reply consolidated a part and then rendered the compound without it. The canon check cannot see this +// — nothing here is signed — and until this pack nothing else looked either. +func TestConsolidationConflictsCatchesTheRunContradictingItself(t *testing.T) { + cands := []Candidate{ + {Key: "元海", Src: "元海"}, + {Key: "元海空窍", Src: "元海空窍"}, + {Key: "空窍", Src: "空窍"}, + {Key: "青茅山", Src: "青茅山"}, + } + out := map[string]string{ + "元海": "море истинной ци", + "空窍": "апертура", + "元海空窍": "апертура Первозданного моря", // carries «апертура», DROPS «истинной ци» + "青茅山": "гора Цинмао", + } + got := ConsolidationConflicts(cands, out) + if len(got) != 1 { + t.Fatalf("exactly one contradiction (the compound against 元海), got %+v", got) + } + if got[0].Src != "元海空窍" || got[0].PartSrc != "元海" { + t.Fatalf("the conflict must name the compound and the part it dropped: %+v", got[0]) + } + // A compound that DOES carry both parts is not a conflict, case endings and all. + out["元海空窍"] = "апертура моря истинной ци" + if got := ConsolidationConflicts(cands, out); len(got) != 0 { + t.Fatalf("a rendering carrying every part is consistent, got %+v", got) + } + // And a term nobody consolidated cannot contradict anything (silence is not a decision). + delete(out, "元海") + out["元海空窍"] = "апертура Первозданного моря" + if got := ConsolidationConflicts(cands, out); len(got) != 0 { + t.Fatalf("an unconsolidated part is not a contradiction, got %+v", got) + } +} + +// TestParseReplyTakesTheConfidenceFieldBeforeRejoining is the §G3 parser trap, both directions: a THIRD +// field of digits is the role's confidence and must come off before the rendering is re-joined, while a +// rendering that merely ENDS in a number is one field and must survive whole. +func TestParseReplyTakesTheConfidenceFieldBeforeRejoining(t *testing.T) { + id := func(s string) string { return s } + keys := []string{"方源", "花家", "一零八峰", "忘却"} + reply := strings.Join([]string{ + "方源\tФан Юань\t95", + "花家\tДом Хуа\t40", // a stray double space AND a confidence: both handled, or the name is halved + "一零八峰\tПик 108", // a rendering ending in a number, single space → one field, untouched + "忘却\tзабвение\t900", // a declared column that is not a confidence: dropped and counted, never glued + }, "\n") + got, conf, st := ParseReply(reply, keys, id, nil) + if got["方源"] != "Фан Юань" || conf["方源"] != 95 { + t.Fatalf("want the rendering without the confidence and the confidence beside it: %q / %d", got["方源"], conf["方源"]) + } + if got["花家"] != "Дом Хуа" || conf["花家"] != 40 { + t.Fatalf("a split rendering must be re-joined AFTER the confidence comes off: %q / %d", got["花家"], conf["花家"]) + } + if got["一零八峰"] != "Пик 108" { + t.Fatalf("a rendering that legitimately ends in a number must survive whole, got %q", got["一零八峰"]) + } + if _, has := conf["一零八峰"]; has { + t.Fatalf("there was no confidence field on that line: %v", conf) + } + if got["忘却"] != "забвение" || st.BadConfidence != 1 { + t.Fatalf("an unreadable confidence column is counted, never glued onto the rendering: %q / %d", got["忘却"], st.BadConfidence) + } + if st.Bad != 0 { + t.Fatalf("a bad confidence does not refuse the rendering, got %d bad", st.Bad) + } + // The whole class the third column opened: anything the model writes there that is not a bare 0..100 — + // «95%», «0.9», «высокая» — used to ride into the rendering, pass wellFormedLemma, pass the answer-language + // screen, and enter the bank as this book's canon with bad_lines reading 0. + for _, tail := range []string{"95%", "95 %", "0.9", "~90", "высокая"} { + g, c, s2 := ParseReply("师父\tнаставник\t"+tail, []string{"师父"}, id, nil) + if g["师父"] != "наставник" { + t.Fatalf("tail %q was glued onto the rendering: %q", tail, g["师父"]) + } + if _, has := c["师父"]; has { + t.Fatalf("tail %q must not be read as a confidence: %v", tail, c) + } + if s2.BadConfidence != 1 { + t.Fatalf("tail %q must be COUNTED, or the prompt's third column can fail silently: %+v", tail, s2) + } + } + // THE SPACE-RUN PATH, which every case above misses: all of them are TAB-delimited, so the confidence is + // stripped by the positional column reader and the space-run branch is never executed. Its own stripping + // could be removed without a single test noticing — and then a model that answers with spaces instead of + // tabs (the reason that tolerance exists at all) banks «Фан Юань 85» as this book's canon. + spaced, sconf, sst := ParseReply("方源 Фан Юань 85", []string{"方源"}, id, nil) + if spaced["方源"] != "Фан Юань" { + t.Fatalf("a space-delimited line must lose its confidence field, not bank it: %q", spaced["方源"]) + } + if sconf["方源"] != 85 { + t.Fatalf("and the confidence must be read: %v", sconf) + } + if sst.Bad != 0 { + t.Fatalf("the line is usable, got %d bad", sst.Bad) + } + // A two-field reply — the shape every existing pair prompt asks for — is unchanged and carries no + // confidence at all, so the field is additive rather than a new requirement. + plain, conf2, _ := ParseReply("方源\tФан Юань", []string{"方源"}, id, nil) + if plain["方源"] != "Фан Юань" || len(conf2) != 0 { + t.Fatalf("a two-field reply must parse as before with no confidence: %q / %v", plain["方源"], conf2) + } +} + +// TestParseReplyAcceptsAMangledDeclineSentinel: declining is a DECISION (§C2-7), and a decline the parser +// refuses is counted as a broken line instead — the term then reads as "the model produced garbage" rather +// than "the model said it could not tell". +func TestParseReplyAcceptsAMangledDeclineSentinel(t *testing.T) { + id := func(s string) string { return s } + keys := []string{"方源", "花家", "青茅山"} + reply := "方源\t" + NoDst + "\n花家\t«" + NoDst + "»\n青茅山\t" + NoDst + "." + got, _, st := ParseReply(reply, keys, id, nil) + for _, k := range keys { + v, answered := got[k] + if !answered || v != "" { + t.Fatalf("%s must read as an explicit decline, got %q (answered=%v)", k, v, answered) + } + } + if st.Bad != 0 { + t.Fatalf("a decline is not a parse failure, got %d bad lines", st.Bad) + } + // Narrow on purpose: a rendering that merely MENTIONS something is not a decline. + other, _, _ := ParseReply("方源\tвариант "+NoDst, []string{"方源"}, id, nil) + if other["方源"] == "" { + t.Fatal("only a rendering that IS the sentinel declines; a prefix match would swallow real answers") + } +} + +// TestFoldVariantsCountsConventionsAndShowsRawForms is §G5. Before it, «Море истинной ци» and «море +// истинной ци» were two variants: the §C2-3 frequency factor scored the consensus on half its evidence, and +// the spread column reported a contest where there was a spelling difference. +func TestFoldVariantsCountsConventionsAndShowsRawForms(t *testing.T) { + observed := []Observed{{Key: "元海", Src: "元海", Proposals: []Proposal{ + {Dst: "море истинной ци", Chunks: 3}, + {Dst: "Море истинной ци", Chunks: 4}, // same convention, different case + {Dst: "Первозданное море", Chunks: 5}, + }}} + got := Merge(nil, observed, lowerFold) + if len(got) != 1 { + t.Fatalf("one surface, one candidate: %+v", got) + } + c := got[0] + if c.Conventions() != 2 { + t.Fatalf("two decisions were made, not three: %+v", c.Variants) + } + if c.Spread() != 3 { + t.Fatalf("the drafts still wrote it three ways and the owner must see that: spread=%d", c.Spread()) + } + var folded *Variant + for i := range c.Variants { + if strings.EqualFold(c.Variants[i].Dst, "море истинной ци") { + folded = &c.Variants[i] + } + } + if folded == nil { + t.Fatalf("the folded class must be present as a RAW form, not a normalized one: %+v", c.Variants) + } + if folded.Chunks != 7 { + t.Fatalf("the vote is the sum of the class, got %d", folded.Chunks) + } + if folded.Dst != "Море истинной ци" { + t.Fatalf("the representative is the most-proposed RAW form, got %q", folded.Dst) + } + // And the fold decides the ranking: 7 folded chunks now outweigh the 5 that used to win on 4-vs-5. + ScoreVariants(&c, ScoreOpts{}) + if !strings.EqualFold(c.Best(), "море истинной ци") { + t.Fatalf("the consensus must stop losing to its own spelling variant, got %q", c.Best()) + } +} + +// TestFoldVariantsKeepsViaProvenanceHonest: a rendering that ALSO arrived on the candidate's own key is +// direct. First-wins used to label such a consensus «proposed for » purely because the alias row was +// seen first — presenting an agreed rendering as a mis-clustered alias's guess. +func TestFoldVariantsKeepsViaProvenanceHonest(t *testing.T) { + mined := []Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子", "小方"}}} + observed := []Observed{ + {Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 1}}}, // through an alias, FIRST + {Key: "方源", Src: "方源", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 6}}}, // and directly + {Key: "小方", Src: "小方", Proposals: []Proposal{{Dst: "малыш Фан", Chunks: 2}}}, // alias only + } + for _, c := range Merge(mined, observed, lowerFold) { + if c.Key != "方源" { + continue + } + for _, v := range c.Variants { + switch v.Dst { + case "Фан Юань": + if v.Via != "" { + t.Fatalf("a rendering also proposed on the term's own key is DIRECT, got via=%q", v.Via) + } + if v.Chunks != 7 { + t.Fatalf("the alias vote still counts toward the total, got %d", v.Chunks) + } + case "малыш Фан": + if v.Via != "小方" { + t.Fatalf("a rendering that ONLY ever came through an alias must keep saying so, got %q", v.Via) + } + } + } + } +} diff --git a/backend/internal/terminology/classify.go b/backend/internal/terminology/classify.go index e1639633..7da2a5f0 100644 --- a/backend/internal/terminology/classify.go +++ b/backend/internal/terminology/classify.go @@ -1,6 +1,9 @@ package terminology -import "strings" +import ( + "sort" + "strings" +) // classify.go: the TYPE re-derivation channel (bank-quality §2, D39.68). The draft type heuristic is wrong // 12–22% (D39.65 row 36a), and type ∈ {name,place} routes a candidate to transliteration — so a realia @@ -9,11 +12,30 @@ import "strings" // WITH the rendering arrives too late to de-bias it). This file is the pure half — the reply parser and the // honest $0 label screen; the class DEFINITIONS live in the pair's classifier prompt, no pair literal here. -// Types is the engine's closed type vocabulary. The names are engine identifiers (the miner emits them, the -// glossary stores them, emissionEligible gates on them), so they are Go constants here, not pair data; the -// pair's prompt explains each class in its own language with its own examples. +// Types is the engine's closed type vocabulary for the CLASSIFIER's answer. The names are engine identifiers +// (the miner emits them, the glossary stores them, emissionEligible gates on them), so they are Go constants +// here, not pair data; the pair's prompt explains each class in its own language with its own examples. var Types = map[string]bool{"name": true, "place": true, "title": true, "term": true} +// CandidateTypes is the wider set a CANDIDATE can actually carry, and it is not the same set — which is a +// live seam, not pedantry. The banknote channel accepts `nickname` from a draft (pipeline.bankTypeOK), that +// type rides into Observed and onto a draft-side-only candidate, and the classifier never overwrites a term +// it was not asked about. So anything keyed on a candidate's type — the family rules, first — must be keyed +// on THIS set: validating against Types alone rejects a legitimate pair rule for `nickname` while the type +// keeps arriving, i.e. refuses the fix and keeps the defect. +var CandidateTypes = map[string]bool{"name": true, "place": true, "title": true, "term": true, "nickname": true} + +// TypeNames lists a type set in a stable order, so an error message can say what it actually accepts instead +// of a hardcoded list that drifts from the map beside it. +func TypeNames(set map[string]bool) []string { + out := make([]string, 0, len(set)) + for t := range set { + out = append(out, t) + } + sort.Strings(out) + return out +} + // ParseTypes turns a classifier reply into key → corrected type, keeping only the terms we asked about and // only the closed vocabulary. Same tolerant two-field line format and the same accounting discipline as // ParseReply: an unusable or off-vocabulary line is COUNTED (st.Bad), never silently dropped, so a paid call diff --git a/backend/internal/terminology/family_test.go b/backend/internal/terminology/family_test.go new file mode 100644 index 00000000..b85b7929 --- /dev/null +++ b/backend/internal/terminology/family_test.go @@ -0,0 +1,348 @@ +package terminology + +import ( + "fmt" + "reflect" + "sort" + "strings" + "testing" +) + +// family_test.go: the §G1 family channel. What it has to prove is not "groups form" but the four properties +// the measurement asked for — a family stays whole across a batch boundary, a variable-length rank line +// stays whole, an equal-length series does not regress, and neither of them turns into a blob. + +// zhFamily mirrors the han rows the bank-data plane ships (internal/lang/bankdata/family-morphology.txt): +// names lead with the clan morpheme, realia end with the generic head, both need two runes of root, a family +// is two surfaces, a unit is at most 24. +var zhFamily = FamilyParams{ + Enabled: true, HeadFinal: true, + Affix: map[string]FamilyAffix{ + "name": {MinRunes: 2}, + "nickname": {MinRunes: 2}, + "place": {Suffix: true, MinRunes: 2}, + "title": {Suffix: true, MinRunes: 2}, + "term": {Suffix: true, MinRunes: 2}, + }, + MinMembers: 2, MaxMembers: 24, ContainmentRunes: 2, +} + +func typed(key, typ string, freq int) Candidate { + return Candidate{Key: key, Src: key, Type: typ, Freq: freq} +} + +// unitsOf inverts a unit map into sorted member lists, for readable assertions. +func unitsOf(unitID map[string]int) map[int][]string { + out := map[int][]string{} + for k, id := range unitID { + out[id] = append(out[id], k) + } + for _, v := range out { + sort.Strings(v) + } + return out +} + +func sameUnit(t *testing.T, unitID map[string]int, keys ...string) int { + t.Helper() + id := unitID[keys[0]] + if id == 0 { + t.Fatalf("%s is in no unit: %v", keys[0], unitsOf(unitID)) + } + for _, k := range keys[1:] { + if unitID[k] != id { + t.Fatalf("%s must share a unit with %s (%d vs %d): %v", k, keys[0], unitID[k], id, unitsOf(unitID)) + } + } + return id +} + +// TestDetectFamiliesGroupsBySharedRootAsymmetrically is the measured case (research/24 §B4): the 古月 family +// is eight surfaces of DIFFERENT lengths, which DetectSeries cannot see, and the drafts rendered it two ways +// on either side of a batch boundary. The clan morpheme LEADS a name and the generic head ENDS a realia +// term, and the asymmetry is data — so the same string in the other position forms nothing. +func TestDetectFamiliesGroupsBySharedRootAsymmetrically(t *testing.T) { + cands := []Candidate{ + typed("古月方源", "name", 40), typed("古月正", "name", 9), typed("古月赤练", "name", 5), + typed("元海空窍", "term", 7), typed("天海空窍", "term", 4), // realia sharing the trailing 海空窍 + typed("方源古月", "name", 2), // the clan morpheme TRAILING a name: not a family under the name rule + } + fams := DetectFamilies(cands, zhFamily) + byAnchor := map[string][]string{} + for _, f := range fams { + byAnchor[f.Anchor] = f.Keys + } + if got := byAnchor["古月"]; len(got) != 3 { + t.Fatalf("the clan family must hold its three names, got %v (all: %v)", got, byAnchor) + } + if got := byAnchor["海空窍"]; len(got) != 2 { + t.Fatalf("two realia sharing the trailing 海空窍 are one family, got %v", got) + } + // The asymmetry is real: 方源古月 shares 古月 as a SUFFIX, and the name rule reads prefixes. + for _, f := range fams { + if f.Anchor == "古月" { + for _, k := range f.Keys { + if k == "方源古月" { + t.Fatal("a name sharing the clan morpheme as a SUFFIX must not join the prefix family") + } + } + } + } +} + +// TestFamilyKeepsAVariableLengthRankLineWhole is the acceptance criterion of the fix-pack's own §2(а): the +// 1..12 rank line must be ONE batch unit. 一转…九转 is an equal-length series; 十一转 and 十二转 are three +// runes long and so fall outside it, and on the live corpus they landed in another batch — a rank scale +// split in half is exactly the chimera the co-batching exists to prevent. +func TestFamilyKeepsAVariableLengthRankLineWhole(t *testing.T) { + var cands []Candidate + for _, k := range []string{"一转", "二转", "三转", "四转", "五转", "六转", "七转", "八转", "九转", "十一转", "十二转"} { + cands = append(cands, typed(k, "term", 5)) + } + cands = append(cands, typed("中间", "term", 90)) // an unrelated term, so a whole-list unit would prove nothing + sort.Slice(cands, func(i, j int) bool { return cands[i].Key < cands[j].Key }) + + seriesID := DetectSeries(cands, zhSeries) + if seriesID["十一转"] != 0 { + t.Fatal("test premise broken: a three-rune surface cannot be in an equal-length series") + } + unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily) + if st.Refused != 0 || st.Held != 0 { + t.Fatalf("nothing here should hit a guard: %+v", st) + } + id := sameUnit(t, unitID, "一转", "二转", "九转", "十一转", "十二转") + if unitID["中间"] == id { + t.Fatal("an unrelated term must not be swept into the rank unit") + } + // And the unit survives the batcher under a budget so tight every singleton is its own batch. + batches := Batch(cands, 10, unitID) + var rank []Candidate + for _, b := range batches { + for _, c := range b { + if unitID[c.Key] == id { + rank = b + } + } + } + if len(rank) != 11 { + t.Fatalf("the whole 1..12 line must ride ONE call, got %d: %v", len(rank), rank) + } +} + +// TestFamilyDoesNotRegressAnEqualLengthSeries: the other half of the same criterion. 甲等/乙等/丙等 are two +// runes, so no two-rune root is a PROPER affix of them and the family channel has nothing to say; the series +// must come through untouched. +func TestFamilyDoesNotRegressAnEqualLengthSeries(t *testing.T) { + cands := []Candidate{typed("甲等", "term", 3), typed("乙等", "term", 3), typed("丙等", "term", 3), typed("丁等", "term", 3), typed("中间", "term", 50)} + seriesID := DetectSeries(cands, zhSeries) + unitID, _ := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily) + id := sameUnit(t, unitID, "甲等", "乙等", "丙等", "丁等") + for k, got := range unitID { + if got == id && !strings.HasSuffix(k, "等") { + t.Fatalf("%s joined the grade unit and shares no morpheme with it: %v", k, unitsOf(unitID)) + } + } +} + +// TestContainmentPairSharesAUnit is §2(б): two MINED surfaces where one contains the other (元海 ⊂ 元海空窍) +// had neither a Related record (Merge computes those for banknote-only rows) nor any atomicity in the +// batcher — so the compound could be consolidated in one call and its own part in another, which is how a +// rendering stops carrying the element it is built from. +func TestContainmentPairSharesAUnit(t *testing.T) { + cands := []Candidate{typed("元海", "term", 30), typed("元海空窍", "term", 8), typed("青茅山", "place", 12)} + unitID, _ := MergeUnits(nil, DetectFamilies(cands, zhFamily), zhFamily) + sameUnit(t, unitID, "元海", "元海空窍") + if unitID["青茅山"] != 0 { + t.Fatalf("an unrelated surface must stay a singleton: %v", unitsOf(unitID)) + } + // The anchor bound is real: a ONE-rune surface may not anchor the channel, or a generic morpheme sweeps + // half the bank into one call. + short := []Candidate{typed("蛊", "term", 900), typed("蛊虫", "term", 40), typed("蛊师", "term", 30), typed("月光蛊", "term", 10)} + if unit, _ := MergeUnits(nil, DetectFamilies(short, zhFamily), zhFamily); unit["蛊"] != 0 { + t.Fatalf("a one-rune anchor is below ContainmentRunes and must form no unit: %v", unitsOf(unit)) + } +} + +// TestFamilyMergeRefusesAnOversizeUnit: the merge is capped and the refusal is REPORTED, not silent — a +// family split across two calls is the defect this channel exists to close, so the caller has to be able to +// say it happened. +func TestFamilyMergeRefusesAnOversizeUnit(t *testing.T) { + var cands []Candidate + for _, k := range []string{"古月方源", "古月正", "古月赤练", "古月山寨"} { + cands = append(cands, typed(k, "name", 5)) + } + tight := zhFamily + tight.MaxMembers = 3 + unitID, st := MergeUnits(nil, DetectFamilies(cands, tight), tight) + if st.Refused == 0 { + t.Fatalf("a four-member family under a three-member cap must be refused: %v", unitsOf(unitID)) + } + for _, v := range unitsOf(unitID) { + if len(v) > tight.MaxMembers { + t.Fatalf("a unit past the cap was built anyway: %v", v) + } + } +} + +// TestFamilyChannelOffIsTheSeriesMap guards the generality answer: a source that declares no family data +// takes the byte-identical path it took before the channel existed — the SAME map object, so nothing +// downstream can even observe a difference. +func TestFamilyChannelOffIsTheSeriesMap(t *testing.T) { + cands := []Candidate{typed("古月方源", "name", 5), typed("古月正", "name", 5), typed("甲等", "term", 3), typed("乙等", "term", 3), typed("丙等", "term", 3)} + seriesID := DetectSeries(cands, zhSeries) + off := FamilyParams{} // no data → inert + if fams := DetectFamilies(cands, off); fams != nil { + t.Fatalf("an undeclared family channel must detect nothing, got %v", fams) + } + unitID, st := MergeUnits(seriesID, DetectFamilies(cands, off), off) + if st != (MergeStats{}) { + t.Fatalf("an inert channel gives nothing up, got %+v", st) + } + // The SAME map object, not a copy: with the channel off nothing downstream can even observe a difference. + if reflect.ValueOf(unitID).Pointer() != reflect.ValueOf(seriesID).Pointer() { + t.Fatalf("the inert path must hand back the series map itself: %v vs %v", unitID, seriesID) + } + for k, v := range seriesID { + if unitID[k] != v { + t.Fatalf("series membership changed with the channel off: %v vs %v", unitID, seriesID) + } + } +} + +// TestFamiliesOnBankFullFixtureAreBoundedAndWhole runs the channel over the real corpus DetectSeries was +// calibrated on. Two live properties, both of which a synthetic fixture cannot show: the 古月 family — the +// one the drafts actually split across a batch boundary — comes out as ONE unit, and no unit grows past the +// declared bound, so the co-batching cannot quietly turn into "the whole bank in one call". +func TestFamiliesOnBankFullFixtureAreBoundedAndWhole(t *testing.T) { + cands := loadBankFullSurfaces(t) + seriesID := DetectSeries(cands, zhSeries) + unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily) + units := unitsOf(unitID) + + clan := unitID["古月"] + if clan == 0 { + t.Fatalf("the clan surface must be in a unit with the names built on it: %v", units) + } + clanMembers := 0 + for _, k := range units[clan] { + if strings.HasPrefix(k, "古月") { + clanMembers++ + } + } + if clanMembers < 10 { + t.Fatalf("the 古月 family is the measured chimera and must ride one call, got %d of %v", clanMembers, units[clan]) + } + for id, keys := range units { + if len(keys) > zhFamily.MaxMembers { + t.Fatalf("unit %d grew past the declared bound (%d): %v", id, zhFamily.MaxMembers, keys) + } + } + // EXACT numbers, not bounds. Two mutations survived a bounds-only version of this test: swapping the + // relatedness test to substring containment (which splits 丙等 from 丙等资质 — the §B4 chimera) and + // inverting the family ranking (which changes which candidates share a CALL, i.e. the money). Both leave + // every inequality above satisfied, so only the composition can catch them. + if len(units) != 14 || largestUnit(units) != 20 || st.Refused != 0 || st.Held != 2 { + t.Fatalf("the corpus shape moved: %d units, largest %d, %+v — if this is intended, re-measure and update the numbers, do not relax the test", + len(units), largestUnit(units), st) + } + // The rank line and the grade family, whole, by composition. + sameUnit(t, unitID, "一转", "九转", "一转蛊师", "九转境界") + sameUnit(t, unitID, "丙等", "丙等资质", "甲等", "甲等资质") + // And the exact SHAPE of the partition. The aggregate counts above survive an inverted family ranking — + // which silently re-cuts which candidates share a CALL, i.e. what the run buys — so the sizes are pinned. + sizes := make([]int, 0, len(units)) + for _, keys := range units { + sizes = append(sizes, len(keys)) + } + sort.Sort(sort.Reverse(sort.IntSlice(sizes))) + want := []int{20, 13, 10, 10, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2} + if fmt.Sprint(sizes) != fmt.Sprint(want) { + t.Fatalf("the partition SHAPE moved: %v want %v — re-measure before relaxing this", sizes, want) + } + t.Logf("bankfull: %d candidates → %d units, largest %d, gave up %+v", + len(cands), len(units), largestUnit(units), st) +} + +// TestGradeFamilyMergesWithItsSeriesThroughASharedRune is the case that made the relatedness test wrong the +// first time round, kept as its own pin: the 等资质 series and the {丙等, 丙等资, 丙等资质} family share no +// substring in either direction, but they plainly share the morpheme 等 — and 丙等 landing in a different +// call from the 丙等资质 built on it is the research/24 §B4 chimera, term for term. +func TestGradeFamilyMergesWithItsSeriesThroughASharedRune(t *testing.T) { + var cands []Candidate + for _, k := range []string{"甲等", "乙等", "丙等", "甲等资质", "乙等资质", "丙等资质", "丙等资"} { + cands = append(cands, typed(k, "term", 5)) + } + unitID, st := MergeUnits(DetectSeries(cands, zhSeries), DetectFamilies(cands, zhFamily), zhFamily) + if st.Held != 0 { + t.Fatalf("a family sharing 等 with the series must not be held back: %+v (%v)", st, unitsOf(unitID)) + } + sameUnit(t, unitID, "丙等", "丙等资质", "甲等", "甲等资质") +} + +func largestUnit(units map[int][]string) int { + n := 0 + for _, keys := range units { + if len(keys) > n { + n = len(keys) + } + } + return n +} + +// TestSeriesHoldsOnlyItsOwnMembers is the guard's honest boundary, and it exists because the first cut of it +// was wrong in exactly the way that matters. «Series stronger than family» must hold back the SERIES — not +// dissolve the family around it. Here the clan 古月 has five names, three of which happen to form a 雄-series; +// the series' root (雄) has nothing to do with the clan morpheme, so the three keep their own unit, and the +// other two must still be co-batched as the clan. Refusing the whole merge instead leaves the clan as five +// singletons — the batch-boundary chimera of research/24 §B4, arriving through the guard meant to prevent it. +func TestSeriesHoldsOnlyItsOwnMembers(t *testing.T) { + cands := []Candidate{ + typed("古月方源", "name", 40), typed("古月正", "name", 9), + typed("古月大雄", "name", 5), typed("古月二雄", "name", 5), typed("古月三雄", "name", 5), + } + seriesID := DetectSeries(cands, zhSeries) + if seriesID["古月大雄"] == 0 { + t.Fatal("test premise broken: the three *雄 names must form an equal-length series") + } + unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily) + if st.Held == 0 { + t.Fatalf("holding part of a family back is a split the owner must be told about: %+v", st) + } + // The series keeps its three members … + sameUnit(t, unitID, "古月大雄", "古月二雄", "古月三雄") + // … and the REST of the clan is still one unit, not two singletons. + clan := sameUnit(t, unitID, "古月方源", "古月正") + if clan == unitID["古月大雄"] { + t.Fatalf("an unrelated series must not be swallowed by the family: %v", unitsOf(unitID)) + } +} + +// TestFamilyMergesAcrossASeriesWhenTheRootIsShared is the other side of the same guard: when the series' own +// root IS the family's anchor (the 转 case), nothing is held back and the whole line rides one call. +func TestFamilyMergesAcrossASeriesWhenTheRootIsShared(t *testing.T) { + var cands []Candidate + for _, k := range []string{"一转", "二转", "三转", "四转", "五转", "十一转"} { + cands = append(cands, typed(k, "term", 5)) + } + _, st := MergeUnits(DetectSeries(cands, zhSeries), DetectFamilies(cands, zhFamily), zhFamily) + if st.Held != 0 { + t.Fatalf("a family sharing the series' own root must not be held back: %+v", st) + } +} + +// TestFamilyRulesCoverEveryTypeACandidateCanCarry: the family rules are keyed on a candidate's TYPE, and the +// set a candidate can carry is wider than the classifier's answer vocabulary — the banknote channel accepts +// `nickname` from a draft and that type rides onto a draft-side-only candidate. A type with no rule forms no +// families, silently, which is a whole class of the bank going un-co-batched for no stated reason. +func TestFamilyRulesCoverEveryTypeACandidateCanCarry(t *testing.T) { + for _, typ := range TypeNames(CandidateTypes) { + if _, has := zhFamily.Affix[typ]; !has { + t.Fatalf("type %q can reach a candidate and has no family rule — declare one or say why in the data file", typ) + } + } + // And it is live, not just declared: two nicknames sharing the clan morpheme are one family. + cands := []Candidate{typed("方小子", "nickname", 5), typed("方小鬼", "nickname", 4)} + if fams := DetectFamilies(cands, zhFamily); len(fams) == 0 { + t.Fatalf("a nickname family must form: %v", fams) + } +} diff --git a/backend/internal/terminology/script_test.go b/backend/internal/terminology/script_test.go index 49bd8a54..f00feed9 100644 --- a/backend/internal/terminology/script_test.go +++ b/backend/internal/terminology/script_test.go @@ -68,7 +68,7 @@ func TestParseReplyRefusesForeignLanguageRenderings(t *testing.T) { keys := []string{"修行", "元海", "转", "是为", "资质"} reply := "修行\tcultivation\n元海\tPrimordial Sea\n转\trealm\n是为\tis\n资质\tталант" - got, st := ParseReply(reply, keys, id, unicode.Scripts["Cyrillic"]) + got, _, st := ParseReply(reply, keys, id, unicode.Scripts["Cyrillic"]) if len(got) != 1 || got["资质"] != "талант" { t.Fatalf("only the target-language line may be banked, got %#v", got) } @@ -80,7 +80,7 @@ func TestParseReplyRefusesForeignLanguageRenderings(t *testing.T) { } // Without a declared script the same reply is banked whole — the state the engine was in before, kept // visible so the cost of an undeclared script is a test, not a surprise. - if inert, st := ParseReply(reply, keys, id, nil); len(inert) != 5 || st.OffLanguage != 0 { + if inert, _, st := ParseReply(reply, keys, id, nil); len(inert) != 5 || st.OffLanguage != 0 { t.Fatalf("with no script the screen must be inert (that is what config refuses for an enabled gate), got %#v", inert) } } @@ -95,7 +95,7 @@ func TestOffLanguageSamplesAreBounded(t *testing.T) { keys = append(keys, k) lines = append(lines, k+"\tforeign") } - _, st := ParseReply(strings.Join(lines, "\n"), keys, id, unicode.Scripts["Cyrillic"]) + _, _, st := ParseReply(strings.Join(lines, "\n"), keys, id, unicode.Scripts["Cyrillic"]) if st.OffLanguage != len(keys) { t.Fatalf("every foreign line must be counted, got %d of %d", st.OffLanguage, len(keys)) } diff --git a/backend/internal/terminology/series.go b/backend/internal/terminology/series.go index 59774934..ddc2cbfe 100644 --- a/backend/internal/terminology/series.go +++ b/backend/internal/terminology/series.go @@ -1,6 +1,9 @@ package terminology -import "sort" +import ( + "sort" + "strings" +) // series.go: the SERIES channel (bank-quality §1, D39.68). A grade/rank series — 甲等/乙等/丙等, 一转…九转 — // shares one generic HEAD and varies only on a modifier. Measured (D39.65 row 21): the drafts disagree on the @@ -106,32 +109,456 @@ func blankAt(rs []rune, pos int) string { return string(out) } -// orderBySeries returns the candidates with every series' members made contiguous, anchored at the position -// of the series' first member in the input order; non-series candidates keep their place. The input is -// already key-sorted, so the result is deterministic and disturbs the key order minimally. -func orderBySeries(cands []Candidate, seriesID map[string]int) []Candidate { - if len(seriesID) == 0 { +// orderByUnit returns the candidates with every batch unit's members made contiguous, anchored at the +// position of the unit's first member in the input order; a candidate in no unit keeps its place. The input +// is already key-sorted, so the result is deterministic and disturbs the key order minimally. +func orderByUnit(cands []Candidate, unitID map[string]int) []Candidate { + if len(unitID) == 0 { return cands } byID := map[int][]Candidate{} for _, c := range cands { - if id := seriesID[c.Key]; id != 0 { + if id := unitID[c.Key]; id != 0 { byID[id] = append(byID[id], c) } } out := make([]Candidate, 0, len(cands)) emitted := map[int]bool{} for _, c := range cands { - id := seriesID[c.Key] + id := unitID[c.Key] if id == 0 { out = append(out, c) continue } if emitted[id] { - continue // a later member of an already-emitted series + continue // a later member of an already-emitted unit } out = append(out, byID[id]...) emitted[id] = true } return out } + +// --- the FAMILY channel (fix-pack §G1) --------------------------------------------------------------- +// +// A SERIES is the narrow case: equal-length surfaces differing in ONE non-head rune. It leaves the wider +// one open, and the wider one is where the measured damage is (research/24 §B4): the 古月-family — eight +// surfaces of DIFFERENT lengths sharing the clan morpheme — came back «Гу Юэ» in batch 0 and «Гуюэ» in +// batch 2 from ONE model, a split that correlates with the batch boundary perfectly. Nothing in the +// terminologist can fix a disagreement it is never shown; only co-batching can. +// +// A family is anchored on a shared ROOT MORPHEME, and the side that root sits on is ASYMMETRIC by class — +// a name leads with its clan surname, a realia term ends with its generic head. Both the side and the +// required root length are pair/script DATA (lang.FamilyMorphology), so a source with no rows leaves the +// channel inert and takes the byte-identical, family-free path. + +// FamilyAffix is one type's family rule: which side carries the shared root and how long it must be. +type FamilyAffix struct { + Suffix bool + MinRunes int +} + +// FamilyParams is the pair-data that governs the family channel. Like SeriesParams it arrives as a VALUE: +// the package stays pure and pair-agnostic, and the pipeline resolves it from the source's declared +// morphology. +type FamilyParams struct { + // Enabled gates the whole channel — true only for DENSE scripts, for the same reason the series channel + // is: one rune ≈ one morpheme, so a shared affix is a shared MORPHEME and not a coincidence of spelling. + Enabled bool + // HeadFinal says the generic head is the trailing morpheme. It is used to read a SERIES' own root when + // deciding whether a family may join it (see MergeUnits). + HeadFinal bool + // Affix is the per-type rule (name|place|title|term); a type absent here forms no families. + Affix map[string]FamilyAffix + // MinMembers is the smallest set that counts as a family (<2 → 2). + MinMembers int + // MaxMembers bounds the batch unit a family merge may produce; 0 = unbounded. A unit past it is a review + // set nobody reads and an output cap nobody planned, so the merge is refused and counted. + MaxMembers int + // ContainmentRunes is the shortest candidate that may anchor the COMPOSITIONAL channel: a candidate that + // is itself a substring of another candidate (元海 ⊂ 元海空窍 — both mined, so neither Related nor the + // batcher held them together before this pack). Bounding the anchor is what keeps one generic morpheme + // from sweeping half the bank into a single call. 0 → the channel is off. + ContainmentRunes int +} + +// Family is one detected family: the shared ANCHOR morpheme and the candidate keys carrying it (key-sorted). +type Family struct { + Anchor string + Keys []string +} + +// DetectFamilies returns the families of a candidate list, strongest first (larger set, then longer — i.e. +// more specific — anchor, then first-seen). It does NOT assign membership: a candidate legitimately shows +// up in several families, and which of them becomes a batch unit is decided in MergeUnits, where the +// existing series are already on the table. Deterministic; nothing here iterates a map for output. +// +// There is no transitive closure over runes: an anchor is a WHOLE root morpheme of at least MinRunes (or a +// whole candidate, for the compositional channel), never "these two differ in one position", which is the +// rule that produced an 11-surface blob mixing three families and the protagonist's name. +func DetectFamilies(cands []Candidate, p FamilyParams) []Family { + if !p.Enabled || len(cands) < 2 { + return nil + } + min := p.MinMembers + if min < 2 { + min = 2 + } + type gkey struct{ side, anchor string } + members := map[gkey][]string{} + seen := map[gkey]map[string]bool{} + var order []gkey + add := func(g gkey, key string) { + m := seen[g] + if m == nil { + m = map[string]bool{} + seen[g] = m + order = append(order, g) + } + if !m[key] { + m[key] = true + members[g] = append(members[g], key) + } + } + isCand := make(map[string]bool, len(cands)) + for _, c := range cands { + if c.Key != "" { + isCand[c.Key] = true + } + } + for _, c := range cands { + if c.Key == "" { + continue + } + rule, ok := p.Affix[typeOr(c.Type)] + if !ok || rule.MinRunes <= 0 { + continue + } + side := "prefix" + if rule.Suffix { + side = "suffix" + } + rs := []rune(c.Key) + // PROPER affixes only (l < len): a surface is not a member of a family whose root IS the surface — + // it is that root, and it joins below, once some other surface actually carries it. + for l := rule.MinRunes; l < len(rs); l++ { + anchor := string(rs[:l]) + if rule.Suffix { + anchor = string(rs[len(rs)-l:]) + } + g := gkey{side, anchor} + add(g, c.Key) + if isCand[anchor] { + // The surface that IS the root belongs to its own family. Leaving it out is how the head of a + // family ends up in a different call from the family it heads. + // + // ⚠ Bounded: an anchor is at least MinRunes long, so a ONE-rune head that is itself a bank term + // (蛊, 转, 窍) is NOT reachable this way and does stay in another call from its family. That is + // a data decision, not a Go one — family_containment_runes 1 turns it on — and it is left off + // because a single generic morpheme anchors half the bank; the cost is measured at the cold run. + add(g, anchor) + } + } + } + if p.ContainmentRunes > 0 { + for _, c := range cands { + if c.Key == "" || len([]rune(c.Key)) < p.ContainmentRunes { + continue + } + g := gkey{"contains", c.Key} + for _, d := range cands { + if d.Key == "" || d.Key == c.Key { + continue + } + if strings.Contains(d.Key, c.Key) { + add(g, c.Key) + add(g, d.Key) + } + } + } + } + all := make([]Family, 0, len(order)) + for _, g := range order { + if len(members[g]) < min { + continue + } + keys := append([]string(nil), members[g]...) + sort.Strings(keys) + all = append(all, Family{Anchor: g.anchor, Keys: keys}) + } + sort.SliceStable(all, func(i, j int) bool { + if len(all[i].Keys) != len(all[j].Keys) { + return len(all[i].Keys) > len(all[j].Keys) + } + ai, aj := len([]rune(all[i].Anchor)), len([]rune(all[j].Anchor)) + if ai != aj { + return ai > aj // the more SPECIFIC root wins the earlier merge + } + return all[i].Anchor < all[j].Anchor + }) + // Dedupe AFTER the ranking, so the survivor of a duplicated set is the strongest description of it. The + // affix and containment channels legitimately describe the same set from two directions (元海空窍/天海空窍 + // share the suffix 空窍 AND the longer 海空窍), and one of them is enough: a duplicate merges nothing the + // first did not, but it double-counts in the merge's give-up statistics — and a number an operator reads + // has to mean what it says. + out := make([]Family, 0, len(all)) + seenSet := map[string]bool{} + for _, f := range all { + set := strings.Join(f.Keys, "\x00") + if seenSet[set] { + continue + } + seenSet[set] = true + out = append(out, f) + } + return out +} + +// MergeUnits folds the series and the families into ONE batch-unit map — the map Batch packs against. +// +// Membership is a PARTITION (a candidate is in at most one unit), built by merging whole units rather than +// by claiming candidates: an exclusive assignment cannot express the case the fix-pack exists for. The +// rank line 一转…九转 is a series; 十一转 is three runes long and so is not, and under exclusive assignment +// it stays outside forever — the batch boundary splits a rank scale in half. Merging the family that spans +// both puts the whole 1..12 line in one call, which is the stated criterion. +// +// Two guards keep the merge from becoming a blob, and BOTH are counted rather than silent — a guard that +// splits a family is doing the same damage the channel exists to prevent, so it has to be visible: +// - SERIES PRECEDENCE: a series whose own root is unrelated to a family's anchor keeps its members; the +// family then forms WITHOUT them (转-case: root 转, anchors 转 / 一转 — related, so they merge instead). +// - MaxMembers: a merge that would produce a unit larger than the pair declares is refused. +// +// With no families (the channel off, or nothing detected) the series map is returned UNCHANGED, so a source +// with no family data takes the byte-identical path. +func MergeUnits(seriesID map[string]int, fams []Family, p FamilyParams) (unitID map[string]int, stats MergeStats) { + if len(fams) == 0 { + return seriesID, MergeStats{} + } + u := newUnitSet() + cappedSet := map[string]bool{} // key sets whose merge the member cap turned down + bySeries := map[int][]string{} + for k, id := range seriesID { + if id != 0 { + bySeries[id] = append(bySeries[id], k) + } + } + ids := make([]int, 0, len(bySeries)) + for id := range bySeries { + ids = append(ids, id) + } + sort.Ints(ids) + for _, id := range ids { + mem := bySeries[id] + sort.Strings(mem) + for _, k := range mem[1:] { + u.union(mem[0], k) + } + if root := commonAffix(mem, p.HeadFinal); root != "" { + r := u.find(mem[0]) + u.roots[r] = append(u.roots[r], root) + } + } + for _, f := range fams { + var targets []string + seen := map[string]bool{} + for _, k := range f.Keys { + r := u.find(k) + if seen[r] { + continue + } + seen[r] = true + targets = append(targets, r) + } + if len(targets) < 2 { + continue // already one unit + } + // SERIES PRECEDENCE, applied per TARGET rather than to the whole family. A series whose root is + // unrelated to this anchor is HELD BACK — and only it. Refusing the whole merge instead would + // dissolve the family around it: the clan 古月, minus three of its names that happen to form a + // 雄-series, would go back to being five singletons, which is the batch-boundary chimera this + // channel exists to close, arriving through the guard that was supposed to protect a series. + var join []string + size := 0 + for _, r := range targets { + blocked := false + for _, sr := range u.roots[r] { + if !sharesMorpheme(f.Anchor, sr) { + blocked = true + break + } + } + if blocked { + continue + } + join = append(join, r) + size += u.size[r] + } + if len(join) < 2 { + continue + } + if p.MaxMembers > 0 && size > p.MaxMembers { + cappedSet[strings.Join(f.Keys, "\x00")] = true + continue + } + for _, r := range join[1:] { + u.union(join[0], r) + } + } + // The give-up statistics are read off the FINAL partition, never off the attempts. A guard that fired + // mid-way is not a family the owner meets split: a later merge routinely puts those keys back together, + // and counting the attempt reports a split that does not exist. Measured on the coldrun-a distillation: + // counting attempts claimed three, of which one (蛊师 against 一转蛊师) had ended up in ONE unit anyway. + // A number an operator reads has to mean what it says — the standard this file sets for itself. + final := u.ids() + for _, f := range fams { + if !splitAcrossUnits(f, final) { + continue + } + if cappedSet[strings.Join(f.Keys, "\x00")] { + stats.Refused++ + continue + } + stats.Held++ + } + return final, stats +} + +// splitAcrossUnits reports whether a family's keys ended up in more than one batch unit (a key in no unit +// counts as its own). +func splitAcrossUnits(f Family, unitID map[string]int) bool { + first, seen := 0, false + for _, k := range f.Keys { + id := unitID[k] + if !seen { + first, seen = id, true + continue + } + if id != first || id == 0 { + return true + } + } + return false +} + +// MergeStats is what the unit merge had to give up. Both numbers mean "a family the owner will meet split +// across two calls", which is the defect the channel exists to close — so neither may be silent. +type MergeStats struct { + // Refused counts merges the member cap turned down. + Refused int + // Held counts families a SERIES held part of back: the series' own root is unrelated to the family's + // anchor, so the series keeps its members and the family forms without them. + Held int +} + +// sharesMorpheme reports whether two anchors share a root MORPHEME — the "делят одну корневую морфему" test +// behind the series-vs-family precedence. In a dense script a morpheme is a rune, so that is the test. +// +// Substring containment is NOT it, and the difference was measured, not reasoned: on the coldrun-a bank the +// containment version held back five families, every one of them wrongly. The 等资质 series (甲等资质/乙等资质/ +// 丙等资质, root 等资质) sits beside the family {丙等, 丙等资, 丙等资质}, anchored 丙等. Neither string contains +// the other, so containment called them unrelated and split the grade 丙等 away from the graded aptitude +// built on it — while they plainly share 等, which is the very morpheme whose rendering has to agree. The +// blob these guards protect against is bounded by MaxMembers, not by making relatedness narrow. +func sharesMorpheme(a, b string) bool { + if a == "" || b == "" { + return true + } + return sharedRunes(a, b) > 0 +} + +// commonAffix returns the longest common trailing (headFinal) or leading morpheme of the keys — a series' +// own root, read off its members so DetectSeries keeps its signature. +func commonAffix(keys []string, headFinal bool) string { + if len(keys) == 0 { + return "" + } + best := []rune(keys[0]) + for _, k := range keys[1:] { + rs := []rune(k) + n := 0 + for n < len(best) && n < len(rs) { + if headFinal { + if best[len(best)-1-n] != rs[len(rs)-1-n] { + break + } + } else if best[n] != rs[n] { + break + } + n++ + } + if headFinal { + best = best[len(best)-n:] + } else { + best = best[:n] + } + if len(best) == 0 { + return "" + } + } + return string(best) +} + +// unitSet is the disjoint-set the merge runs on: parent/size by key, plus the series roots a unit carries +// (the precedence guard reads them). Roots are the smallest member key, so the structure is deterministic +// without any map iteration reaching the output. +type unitSet struct { + parent map[string]string + size map[string]int + roots map[string][]string +} + +func newUnitSet() *unitSet { + return &unitSet{parent: map[string]string{}, size: map[string]int{}, roots: map[string][]string{}} +} + +func (u *unitSet) find(k string) string { + p, ok := u.parent[k] + if !ok { + u.parent[k], u.size[k] = k, 1 + return k + } + if p == k { + return k + } + r := u.find(p) + u.parent[k] = r + return r +} + +func (u *unitSet) union(a, b string) { + ra, rb := u.find(a), u.find(b) + if ra == rb { + return + } + if rb < ra { // the smaller key is always the root → the same input yields the same structure + ra, rb = rb, ra + } + u.parent[rb] = ra + u.size[ra] += u.size[rb] + u.roots[ra] = append(u.roots[ra], u.roots[rb]...) + delete(u.size, rb) + delete(u.roots, rb) +} + +// ids numbers the units of ≥2 members, in the order of their smallest key, so the map is a pure function of +// the input. +func (u *unitSet) ids() map[string]int { + roots := make([]string, 0, len(u.size)) + for r, n := range u.size { + if n > 1 { + roots = append(roots, r) + } + } + sort.Strings(roots) + num := make(map[string]int, len(roots)) + for i, r := range roots { + num[r] = i + 1 + } + out := make(map[string]int, len(u.parent)) + for k := range u.parent { + if id := num[u.find(k)]; id != 0 { + out[k] = id + } + } + return out +} diff --git a/backend/internal/terminology/terminology.go b/backend/internal/terminology/terminology.go index 1b7fbd8c..59fc45f3 100644 --- a/backend/internal/terminology/terminology.go +++ b/backend/internal/terminology/terminology.go @@ -80,8 +80,12 @@ type Neighbour struct { // Variant is one observed rendering with its consolidation score. type Variant struct { - Dst string - Chunks int + Dst string + Chunks int + // Forms is how many DISTINCT raw renderings folded into this one (see foldVariants). 1 for an unfolded + // variant; it is what keeps Spread() meaning "how many ways did the drafts write it" once the vote is + // counted per CONVENTION rather than per byte string. + Forms int Score float64 Signals []string // which factors fired, in fixed order — the score's audit trail // Via names the surface that actually proposed this rendering when it is NOT the candidate's own — @@ -125,7 +129,12 @@ type Candidate struct { // // Output order is by Key (never map order). Every mined candidate appears exactly once; every observed key // either merges into one or appears once in the reverse section. -func Merge(mined []Mined, observed []Observed) []Candidate { +// +// foldDst is the TARGET-form normalizer the vote is counted under (see foldVariants); nil folds only +// byte-identical renderings, which is what this did before the fix-pack. It is injected rather than +// imported so the package keeps no target knowledge — the ё/case/whitespace folds a Russian target needs +// are facts about that target, not about this algorithm. +func Merge(mined []Mined, observed []Observed, foldDst func(string) string) []Candidate { byKey := make(map[string]*Candidate, len(mined)) order := make([]string, 0, len(mined)+len(observed)) // aliasOwner maps every alias surface to its owning mined key, so an aliased proposal merges. @@ -206,31 +215,89 @@ func Merge(mined []Mined, observed []Observed) []Candidate { out := make([]Candidate, 0, len(order)) for _, k := range order { c := *byKey[k] - c.Variants = foldVariants(c.Variants) + c.Variants = foldVariants(c.Variants, foldDst) out = append(out, c) } sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) return out } -// foldVariants sums the chunk counts of IDENTICAL renderings. It matters on the alias path: 方源 and its -// alias 方小子 arrive as two Observed rows, and both may carry the same rendering — appended raw that is -// two variants of one word, which lies twice. Spread() would report 2 (the disagreement column the owner -// reads to decide whether a term is contested) for a term nobody disagreed about, and the §C2-3 frequency -// factor would score the consensus rendering on HALF its evidence, which can hand the win to a genuine -// competitor. Deterministic: first-seen order is preserved, so the fold introduces no new ordering. -func foldVariants(vs []Variant) []Variant { - if len(vs) < 2 { +// foldVariants sums the chunk counts of renderings that are the SAME CONVENTION and shows the raw form. +// +// It matters on the alias path: 方源 and its alias 方小子 arrive as two Observed rows, and both may carry the +// same rendering — appended raw that is two variants of one word, which lies twice. It matters just as much +// on orthography (fix-pack §G5, research/24 §A5): «Море истинной ци» and «море истинной ци» are one +// convention written two ways, and folding only byte-identical strings splits the §C2-3 frequency factor +// across them — the consensus rendering is then scored on half its evidence and a genuine competitor can +// take the win. So the VOTE is counted under foldDst (the target-form normalizer), while the form the owner +// and the model see stays the RAW one the drafts actually wrote. +// +// Two counts survive the fold and mean different things: Spread() (Σ Forms) is how many ways the drafts +// wrote the term, Conventions() is how many of those are genuinely different decisions. +// +// Via is aggregated honestly rather than by first-wins. A rendering that ALSO arrived on the candidate's own +// key is direct — labelling it «proposed for » because the alias happened to be seen first would +// present a consensus as a mis-clustered alias's guess. Only when EVERY contribution came through aliases +// is the provenance kept, and then it names all of them. +// +// Deterministic: first-seen order of the folded classes is preserved, and the representative is the raw form +// with the most chunks, ties going to the one seen first. +func foldVariants(vs []Variant, foldDst func(string) string) []Variant { + if len(vs) == 0 { return vs } + if foldDst == nil { + foldDst = func(s string) string { return s } + } + type form struct { + dst string + chunks int + } + type class struct { + forms []form + byForm map[string]int + chunks int + direct bool + vias []string + seenVia map[string]bool + } at := make(map[string]int, len(vs)) - out := make([]Variant, 0, len(vs)) + classes := make([]*class, 0, len(vs)) for _, v := range vs { - if i, seen := at[v.Dst]; seen { - out[i].Chunks += v.Chunks - continue + k := foldDst(v.Dst) + i, seen := at[k] + if !seen { + i = len(classes) + at[k] = i + classes = append(classes, &class{byForm: map[string]int{}, seenVia: map[string]bool{}}) + } + cl := classes[i] + cl.chunks += v.Chunks + if j, had := cl.byForm[v.Dst]; had { + cl.forms[j].chunks += v.Chunks + } else { + cl.byForm[v.Dst] = len(cl.forms) + cl.forms = append(cl.forms, form{dst: v.Dst, chunks: v.Chunks}) + } + if v.Via == "" { + cl.direct = true + } else if !cl.seenVia[v.Via] { + cl.seenVia[v.Via] = true + cl.vias = append(cl.vias, v.Via) + } + } + out := make([]Variant, 0, len(classes)) + for _, cl := range classes { + best := 0 + for j := 1; j < len(cl.forms); j++ { + if cl.forms[j].chunks > cl.forms[best].chunks { + best = j + } + } + v := Variant{Dst: cl.forms[best].dst, Chunks: cl.chunks, Forms: len(cl.forms)} + if !cl.direct { + v.Via = strings.Join(cl.vias, ", ") } - at[v.Dst] = len(out) out = append(out, v) } return out @@ -596,8 +663,27 @@ func (c Candidate) Best() string { } // Spread is the disagreement signal: how many DISTINCT renderings the drafts produced for this surface. -// One term coming back three ways is the drift a canon closes — the reason to sign it at all. -func (c Candidate) Spread() int { return len(c.Variants) } +// One term coming back three ways is the drift a canon closes — the reason to sign it at all. It counts RAW +// forms (Forms per folded class), so the fix-pack's convention fold did not silently shrink the column the +// owner reads to decide whether a term is contested. A variant built outside Merge carries no Forms and +// counts as one. +func (c Candidate) Spread() int { + n := 0 + for _, v := range c.Variants { + if v.Forms > 1 { + n += v.Forms + continue + } + n++ + } + return n +} + +// Conventions is the disagreement that actually matters: how many distinct DECISIONS the drafts made, after +// renderings that differ only in target form (case, ё, spacing) are folded together. Spread 3 with +// Conventions 1 is one canon written three ways — a normalization nit; Spread 3 with Conventions 3 is a real +// contest, and the two must not read the same at the stop. +func (c Candidate) Conventions() int { return len(c.Variants) } // --- the wire: request table and reply parser --------------------------------------------------------- @@ -611,6 +697,62 @@ const NoDst = "⟦TM-NO-DST⟧" // tolerance the banknote parser applies, for the same reason: models substitute spaces for tabs. var fieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`) +// columnSplit is the UNAMBIGUOUS half of that tolerance — a tab or a pipe is a delimiter the model chose and +// cannot occur inside a rendering, so a line carrying one has declared its own columns. spaceRun folds the +// stray spacing inside such a column. Both exist because guessing where the columns are, on a line that says +// where they are, is how a fix for one mis-split rendering corrupts the lines that were never mis-split. +var ( + columnSplit = regexp.MustCompile(`\t|\s*\|\s*`) + spaceRun = regexp.MustCompile(` {2,}`) +) + +// replyColumns splits one reply line into src, rendering and the CONFIDENCE column, and says whether a +// third column was present but unreadable as a confidence. +// +// A tab/pipe line is positional: field three IS the confidence column the pair's prompt declared, so a value +// that is not a bare 0…100 is a malformed CONFIDENCE — counted, and never glued onto the rendering. That +// glue is the failure this shape exists to prevent: «наставник» + «95%» silently became the rendering +// «наставник 95%», passed every downstream screen (it is well-formed, it is in the target script, it is not +// an echo) and would enter the bank as this book's canon with every counter reading clean. +// +// A line delimited only by a run of ≥2 spaces is ambiguous — that tolerance is exactly what splits «Фан␣␣ +// Юань» — so there the rendering is re-joined and only a trailing bare number is taken as a confidence. +func replyColumns(line string) (src, dst string, conf int, badConf, ok bool) { + conf = -1 + if cols := splitOn(columnSplit, line); len(cols) >= 2 { + src, dst = cols[0], strings.TrimSpace(spaceRun.ReplaceAllString(cols[1], " ")) + if len(cols) >= 3 { + if v, good := confidenceField(cols[2]); good { + conf = v + } else { + badConf = true + } + } + return src, dst, conf, badConf, dst != "" + } + f := splitOn(fieldSplit, line) + if len(f) < 2 { + return "", "", -1, false, false + } + if len(f) >= 3 { + if v, good := confidenceField(f[len(f)-1]); good { + conf, f = v, f[:len(f)-1] + } + } + return f[0], strings.TrimSpace(strings.Join(f[1:], " ")), conf, false, true +} + +// splitOn splits a line on re, dropping empty and whitespace-only fields. +func splitOn(re *regexp.Regexp, line string) []string { + var out []string + for _, p := range re.Split(strings.TrimSpace(line), -1) { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + // splitFields splits one reply line into its non-empty trimmed fields. func splitFields(line string) []string { var parts []string @@ -811,6 +953,49 @@ func CanonConflicts(cands []Candidate, consolidated map[string]string, ns []Neig return out } +// ConsolidationConflict is one consolidated rendering that contradicts ANOTHER consolidation of the SAME +// run — the compositional rule broken inside one reply rather than against a signed row. +type ConsolidationConflict struct { + Src string // the containing candidate's source surface + Dst string // what the role consolidated it to + PartSrc string // the source surface contained in it, also consolidated this run + PartDst string // the rendering its own dst fails to carry +} + +// ConsolidationConflicts is CanonConflicts' thin brother, and it exists because the canon check can only see +// what the owner already signed. On a live bank that is the minority: 18 of 149 contradictions the same run +// produced were between its OWN consolidations (research/24 §A4) — 元海空窍 rendered without the 元海 this +// very reply had just fixed — and nothing looked at them, because the role runs once and never reads itself. +// A step whose entire purpose is ONE consistent canon must not be the step that quietly breaks it. +// +// Same test as the canon side (containment + lexemeSubset, so Russian case endings do not false-flag), same +// discipline: it REPORTS, never rewrites and never fails a run. $0 — it is arithmetic over a map that +// already exists. Deterministic: candidates are key-ordered on both sides and nothing here iterates a map. +func ConsolidationConflicts(cands []Candidate, consolidated map[string]string) []ConsolidationConflict { + if len(consolidated) < 2 { + return nil + } + var out []ConsolidationConflict + for _, c := range cands { + dst := consolidated[c.Key] + if dst == "" || c.Key == "" { + continue + } + have := wordSet(dst) + for _, p := range cands { + pd := consolidated[p.Key] + if pd == "" || p.Key == "" || p.Key == c.Key || !strings.Contains(c.Key, p.Key) { + continue + } + if lexemeSubset(wordSet(pd), have) { + continue + } + out = append(out, ConsolidationConflict{Src: c.Src, Dst: dst, PartSrc: p.Src, PartDst: pd}) + } + } + return out +} + // sharedRunes counts the DISTINCT runes two sources have in common. func sharedRunes(a, b string) int { if a == "" || b == "" { @@ -838,6 +1023,10 @@ type ReplyStats struct { Bad int OffLanguage int OffLanguageSamples []string // first few refused lines verbatim, so the warning names them + // BadConfidence counts lines whose declared CONFIDENCE column was not a bare 0…100 («95%», «0.9», + // «высокая»). The rendering is kept — it is fine — but the value is dropped rather than glued onto it, + // and the count is what tells an operator the prompt's third column is not landing. + BadConfidence int } const offLanguageSampleCap = 8 @@ -850,25 +1039,27 @@ const offLanguageSampleCap = 8 // // `target` is the target language's script (nil → the answer-language check is inert; see OffLanguage). // -// Returns the accepted map plus the tally of unusable lines, which the caller logs: silence about a reply -// the parser could not read is how a paid call turns into an empty bank with no signal. -func ParseReply(reply string, expected []string, normalize func(string) string, target *unicode.RangeTable) (map[string]string, ReplyStats) { +// Returns the accepted map, the role's own stated CONFIDENCE per key (absent when the reply carried none), +// and the tally of unusable lines, which the caller logs: silence about a reply the parser could not read is +// how a paid call turns into an empty bank with no signal. +func ParseReply(reply string, expected []string, normalize func(string) string, target *unicode.RangeTable) (map[string]string, map[string]int, ReplyStats) { want := make(map[string]bool, len(expected)) for _, k := range expected { want[k] = true } out := map[string]string{} + conf := map[string]int{} var st ReplyStats for _, ln := range strings.Split(reply, "\n") { if strings.TrimSpace(ln) == "" || strings.HasPrefix(strings.TrimSpace(ln), "#") { continue } - f := splitFields(ln) - if len(f) < 2 { + rawSrc, dst, c, badConf, ok := replyColumns(ln) + if !ok { st.Bad++ continue } - key := normalize(f[0]) + key := normalize(rawSrc) if !want[key] { st.Bad++ continue @@ -876,15 +1067,23 @@ func ParseReply(reply string, expected []string, normalize func(string) string, if _, dup := out[key]; dup { continue // first answer wins; a model repeating itself is not a second opinion } + if badConf { + // The rendering is usable; the confidence column is not. Counted rather than refused — and never + // folded into the rendering, which is the whole point. + st.BadConfidence++ + } // Everything after the FIRST field is the rendering, re-joined on single spaces. The splitter is // deliberately tolerant (a tab, a run of ≥2 spaces, a padded pipe) because models substitute one for // another — but that tolerance also splits INSIDE a rendering the moment a model writes «Фан Юань» // with a stray double space, and taking f[1] alone would then bank the half-name «Фан» silently: // it passes wellFormedLemma, so no counter would ever mention it. The role's reply is specified as - // exactly two fields, so there is no third column to lose by re-joining. - dst := strings.TrimSpace(strings.Join(f[1:], " ")) - if dst == NoDst { + // exactly two fields, so there is no third column to lose by re-joining — see replyColumns for how the + // third (confidence) column and a mis-split rendering are told apart. + if isNoDst(dst) { out[key] = "" + if c >= 0 { + conf[key] = c + } continue } if !wellFormedLemma(dst) { @@ -905,18 +1104,60 @@ func ParseReply(reply string, expected []string, normalize func(string) string, st.Bad++ st.OffLanguage++ if len(st.OffLanguageSamples) < offLanguageSampleCap { - st.OffLanguageSamples = append(st.OffLanguageSamples, f[0]+" → "+dst) + st.OffLanguageSamples = append(st.OffLanguageSamples, rawSrc+" → "+dst) } continue } out[key] = dst + if c >= 0 { + conf[key] = c + } } - return out, st + return out, conf, st } -// Batch splits candidates into groups whose rendered size stays under maxRunes. A series (a key present in -// seriesID; pass nil to disable the channel) is kept WHOLE in one batch so the terminologist picks one -// generic head for it (§1); a series or a single candidate larger than the budget still gets its own batch +// confidenceField reads the role's stated confidence off a reply FIELD: 1–3 digits in 0…100 and nothing +// else. Out of that range it is not a confidence and stays part of the rendering. +// +// ⚠ What this number may and may not do is RATIFIED (D39.102): it orders the review list «least sure +// first» — an ordinal INSIDE one model's reply — and nothing more. It is not a weight, not a threshold, not +// comparable between models, and never decides which rendering wins; the measured AUC (0.77–0.85) supports +// exactly the ordering claim and nothing stronger. +func confidenceField(s string) (int, bool) { + s = strings.TrimSpace(s) + if len(s) == 0 || len(s) > 3 { + return 0, false + } + n := 0 + for _, r := range s { + if r < '0' || r > '9' { + return 0, false + } + n = n*10 + int(r-'0') + } + if n > 100 { + return 0, false + } + return n, true +} + +// isNoDst recognises the decline sentinel, including the forms a model mangles it into — a trailing period, +// surrounding quotes. Narrow on purpose: the bracketed engine token must still OPEN the rendering and only +// punctuation may follow, so no natural rendering can match. Without it a decline written «⟦TM-NO-DST⟧.» +// fails wellFormedLemma-then-bad-line and the term is counted as a PARSE failure rather than as the +// decision §C2-7 says it is. +func isNoDst(dst string) bool { + t := strings.Trim(strings.TrimSpace(dst), `"'«»“”`) + if !strings.HasPrefix(t, NoDst) { + return false + } + return strings.Trim(strings.TrimSpace(t[len(NoDst):]), `.,;:!?"'«»“”`) == "" +} + +// Batch splits candidates into groups whose rendered size stays under maxRunes. A batch UNIT — a series or +// a family, i.e. a key present in unitID; pass nil to disable both channels — is kept WHOLE in one batch so +// the terminologist picks one generic head or one shared root for it (§1, §G1); a unit or a single +// candidate larger than the budget still gets its own batch // rather than being split or silently dropped (the caller WARNs on such an over-cap unit — see BatchRunes — // because the whole unit still bills against one output cap). The batches are then ordered by descending // source frequency, so a budget ceiling drops the least-frequent terms instead of the lexicographic tail @@ -928,14 +1169,14 @@ func ParseReply(reply string, expected []string, normalize func(string) string, // into the request hash, so a batch's POSITION is part of its address. If a later run's frequencies reorder // the batches their ordinals shift and the affected batches re-pay — the accepted price of value-ordering, // and no worse than the content edit that would already re-pay them. -func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candidate { +func Batch(cands []Candidate, maxRunes int, unitID map[string]int) [][]Candidate { if len(cands) == 0 { return nil } if maxRunes <= 0 { return [][]Candidate{cands} } - ordered := orderBySeries(cands, seriesID) + ordered := orderByUnit(cands, unitID) var out [][]Candidate var cur []Candidate size := 0 @@ -946,9 +1187,9 @@ func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candida } } for i := 0; i < len(ordered); { - j := i + 1 // extend over a whole series block; a singleton is a unit of one - if id := seriesID[ordered[i].Key]; id != 0 { - for j < len(ordered) && seriesID[ordered[j].Key] == id { + j := i + 1 // extend over a whole unit block; a singleton is a unit of one + if id := unitID[ordered[i].Key]; id != 0 { + for j < len(ordered) && unitID[ordered[j].Key] == id { j++ } } @@ -967,7 +1208,7 @@ func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candida } // BatchRunes reports the rendered size RenderBatch produces for a whole batch — the same measure Batch packs -// against. The terminologist uses it to WARN when a co-batched series or a lone candidate exceeds the budget: +// against. The terminologist uses it to WARN when a co-batched unit or a lone candidate exceeds the budget: // §1 keeps such a unit WHOLE rather than split it, so it is a real over-cap call the operator must see coming // (the cap-8000 output mine, PROBES §1), not a silent overrun. func BatchRunes(batch []Candidate) int { return unitRunes(batch) } diff --git a/backend/internal/terminology/terminology_test.go b/backend/internal/terminology/terminology_test.go index 8f20fcc0..02150b03 100644 --- a/backend/internal/terminology/terminology_test.go +++ b/backend/internal/terminology/terminology_test.go @@ -21,7 +21,7 @@ func TestMergeExactAndAliasCollapseButContainmentDoesNot(t *testing.T) { {Key: "方小子", Src: "方小子", Type: "nickname", Proposals: []Proposal{{Dst: "малец Фан", Chunks: 1}}}, {Key: "古月方源", Src: "古月方源", Type: "name", Proposals: []Proposal{{Dst: "Гуюэ Фан Юань", Chunks: 2}}}, } - got := Merge(mined, observed) + got := Merge(mined, observed, nil) byKey := map[string]Candidate{} for _, c := range got { @@ -71,7 +71,7 @@ func TestMergeFoldsIdenticalRenderingsOfOneEntity(t *testing.T) { {Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 2}, {Dst: "малец Фан", Chunks: 1}}}, } var c Candidate - for _, got := range Merge(mined, observed) { + for _, got := range Merge(mined, observed, nil) { if got.Key == "方源" { // select by key: the alias surface is now a candidate of its own too c = got } @@ -98,7 +98,7 @@ func TestMergeFoldsIdenticalRenderingsOfOneEntity(t *testing.T) { func TestMergeKeepsAnAliasSwallowedSurface(t *testing.T) { mined := []Mined{{Key: "葛家", Src: "葛家", Type: "name", Freq: 40, Aliases: []string{"族长"}}} observed := []Observed{{Key: "族长", Src: "族长", Type: "title", Proposals: []Proposal{{Dst: "глава клана", Chunks: 3}}}} - got := Merge(mined, observed) + got := Merge(mined, observed, nil) var title, cluster *Candidate for i := range got { @@ -137,9 +137,9 @@ func TestMergeIsDeterministic(t *testing.T) { {Key: "z", Src: "z", Proposals: []Proposal{{Dst: "Z", Chunks: 1}}}, {Key: "a", Src: "a", Proposals: []Proposal{{Dst: "A", Chunks: 1}}}, } - first := keysOf(Merge(mined, observed)) + first := keysOf(Merge(mined, observed, nil)) for i := 0; i < 20; i++ { // map iteration would show up here, not on one run - if got := keysOf(Merge(mined, observed)); !reflect.DeepEqual(got, first) { + if got := keysOf(Merge(mined, observed, nil)); !reflect.DeepEqual(got, first) { t.Fatalf("merge is order-unstable: %v vs %v", got, first) } } @@ -340,13 +340,13 @@ func TestCanonConflictsFlagsOnlyRealContradictions(t *testing.T) { // stray double space inside a rendering must not truncate it (the splitter is tolerant BY DESIGN, and that // tolerance cuts both ways), and a batch must actually render within the budget it was split for. func TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds(t *testing.T) { - got, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil) + got, _, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil) if st.Bad != 0 || got["方源"] != "Фан Юань" { t.Fatalf("a double space inside a rendering must not bank half a name: %v (bad=%d)", got, st.Bad) } // A rendering that IS the source is not a rendering: a model echoing the term back would otherwise be // banked as this book's canon and shown to the editor as «方源 ⟨проверить⟩». - echo, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil) + echo, _, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil) if _, banked := echo["方源"]; banked || st.Bad != 1 { t.Fatalf("an echoed source must be refused and counted: %v (bad=%d)", echo, st.Bad) } @@ -380,7 +380,7 @@ func TestParseReplyAcceptsOnlyAskedTerms(t *testing.T) { "忘却\t" + NoDst, "мусор", }, "\n") - got, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil) + got, _, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil) want := map[string]string{"方源": "Фан Юань", "花家": "Дом Хуа", "青茅山": "гора Цинмао", "忘却": ""} if !reflect.DeepEqual(got, want) { t.Fatalf("parse = %#v, want %#v", got, want) @@ -398,7 +398,7 @@ func TestParseReplyNormalizesTheKey(t *testing.T) { // The caller passes the engine's source-key normalizer; a reply written in another orthography must // still land on its term. lower := strings.ToLower - got, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil) + got, _, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil) if got["fang"] != "Фан" { t.Fatalf("the reply key must be normalized by the caller's function, got %#v", got) } diff --git a/backend/prompts/zh-ru/terminologist.md b/backend/prompts/zh-ru/terminologist.md index bf94c55a..4fb5d574 100644 --- a/backend/prompts/zh-ru/terminologist.md +++ b/backend/prompts/zh-ru/terminologist.md @@ -1,5 +1,8 @@ +в D39.46/47/52 и отчётах полигон-пакетов. Файл — ДАННЫЕ: в снапшот не фолдится, текст не пинят тесты. +v3 (08.08, фикс-пак банка §G3): третье поле — уверенность 0–100. Она НЕ арбитр: движок сортирует ею лист +ревью «сначала неуверенное» (ordinal внутри одного ответа, AUC 0.77–0.85 research/24 §B3) и ничего больше — +ни весов, ни порогов, ни сравнения между моделями (запрещённые формы зафиксированы D39.102). --> Ты — терминолог издательского перевода с языка «{{source_lang}}» на язык «{{target_lang}}». Книга: «{{title}}». Жанр: {{genre}}. Аудитория: {{audience}}. @@ -33,12 +36,18 @@ - Если по данным контекстам ты не можешь выбрать перевод уверенно — верни ⟦TM-NO-DST⟧. Это нормальный ответ: непереведённый термин останется на ручную подпись, а выдуманный попадёт в книгу. -Формат ответа — по одной строке на термин, ровно два поля, разделённые СИМВОЛОМ ТАБУЛЯЦИИ: -первое поле — термин исходника, второе — перевод. +Формат ответа — по одной строке на термин, ровно ТРИ поля, разделённые СИМВОЛОМ ТАБУЛЯЦИИ: +первое поле — термин исходника, второе — перевод, третье — твоя уверенность в этом переводе, +целое число от 0 до 100. -Пример строки ответа (символ между полями — настоящая табуляция): +Уверенность — это оценка ТВОЕГО решения по данным контекстам, а не оценка термина: 90 — контексты +однозначны и вариант очевиден; 40 — контекстов мало или они противоречивы, и ты выбрал наименее плохой. +Она ни на что не влияет, кроме порядка, в котором человек будет просматривать список: сначала он посмотрит +то, в чём ты сам не уверен. Занижать её «на всякий случай» так же бесполезно, как завышать. -师父 наставник +Пример строки ответа (символы между полями — настоящие табуляции): + +师父 наставник 95 Никаких заголовков, нумерации, комментариев и markdown. Термины, которых нет в списке, не добавляй.