Land the bank cluster fixpack: family co-batching as pair data, arbitration observability, banknote parser rejoin, auto-bank drop diff, confidence field, after adversarial acceptance and a fix round

This commit is contained in:
Claude (backend session) 2026-08-08 21:03:31 +03:00
parent 7384c57f1a
commit bccf2d8842
27 changed files with 2934 additions and 200 deletions

View file

@ -5,6 +5,7 @@ import (
"fmt"
"io"
"sort"
"strconv"
"strings"
"unicode/utf8"
@ -103,14 +104,36 @@ func renderBankStopRows(w io.Writer, rows []pipeline.BankStopRow) {
if len(rows) == 0 {
return
}
shown := rows
// THE ONLY use the confidence is allowed to have (D39.102): the capped stdout view is the REVIEW surface,
// so it is ordered least-sure-first — an ordinal inside one model's reply and nothing more. It is not a
// weight, not a threshold, not comparable between models, and it never decides which rendering wins; the
// full sidecar keeps the source-key order, because that is the reference document.
shown := append([]pipeline.BankStopRow(nil), rows...)
sort.SliceStable(shown, func(i, j int) bool { return reviewRank(shown[i]) < reviewRank(shown[j]) })
if len(shown) > bankStopStdoutCap {
shown = shown[:bankStopStdoutCap]
}
fmt.Fprintln(w, "")
fmt.Fprintln(w, " src proposed dst origin freq spread")
fmt.Fprintln(w, " (least-confident first — the role's own ordering of what to read)")
fmt.Fprintln(w, " src proposed dst origin freq spread conv conf")
for _, r := range shown {
fmt.Fprintf(w, " %-20s %-25s %-9s %5d %6d\n", trunc(r.Src, 20), trunc(dashIfEmpty(r.Dst), 25), r.Origin, r.Freq, r.Spread)
fmt.Fprintf(w, " %-20s %-25s %-9s %5d %6d %5d %5s\n", trunc(r.Src, 20), trunc(dashIfEmpty(r.Dst), 25),
r.Origin, r.Freq, r.Spread, r.Conventions, confCell(r.Conf))
// WHY this row has this dst — the question the artifacts could not answer before the fix-pack. The
// signals are the ranking factors that fired for the winning variant; "invented" says the rendering is
// the role's own, not one the drafts proposed (legitimate — it saw the whole book — and worth a look).
if len(r.Signals) > 0 || r.Invented {
why := strings.Join(r.Signals, ", ")
if r.Invented {
why = strings.TrimPrefix(why+" · invented (no draft proposed it)", " · ")
}
fmt.Fprintf(w, " why: %s\n", trunc(why, 90))
}
if len(r.Contradicts) > 0 {
// A contradiction against this run's OWN consolidations: the compound dropped the rendering the
// same reply gave its part. Loudest line of the row — it is a canon breaking inside one call.
fmt.Fprintf(w, " ⚠ contradicts: %s\n", trunc(strings.Join(r.Contradicts, "; "), 80))
}
if len(r.Variants) > 1 {
// The disagreement is the reason to sign: one term coming back several ways is exactly what a
// canon exists to close, and hiding it behind one winner throws that reason away.
@ -126,6 +149,29 @@ func renderBankStopRows(w io.Writer, rows []pipeline.BankStopRow) {
fmt.Fprintln(w, "")
}
// reviewRank orders the review surface: rows the role was least sure of first, rows it said nothing about
// after them (silence is not a low score), and rows it never consolidated last.
func reviewRank(r pipeline.BankStopRow) int {
switch {
case r.Dst == "":
return 1000
case r.Conf < 0:
return 200
default:
return r.Conf
}
}
// confCell renders the role's stated confidence, or "—" when the reply carried none (a negative). Printing
// a bare 0 for both would hide the single most important row on the sheet — the one the role itself said it
// was least sure of — behind the rows it never mentioned.
func confCell(conf int) string {
if conf < 0 {
return "—"
}
return strconv.Itoa(conf)
}
// trunc shortens s to n runes with an ellipsis (rune-safe, so a CJK surface is never split mid-character).
func trunc(s string, n int) string {
rs := []rune(s)

View file

@ -290,3 +290,34 @@ func TestRenderSignatureStopShowsTheTable(t *testing.T) {
t.Fatalf("a term with no proposed dst must show a dash:\n%s", b2.String())
}
}
// TestRenderSignatureStopOrdersTheReviewLeastSureFirst pins the ONE use the role's stated confidence is
// allowed to have (D39.102): it orders the review list, an ordinal inside one model's reply, and nothing
// else. Without it the number is printed and never used — a column that claims to answer «what do I read
// first» and does not. The full sidecar keeps source-key order; only this capped view is the review surface.
func TestRenderSignatureStopOrdersTheReviewLeastSureFirst(t *testing.T) {
rows := []pipeline.BankStopRow{
{Src: "уверенный", Dst: "д", Origin: "mined", Freq: 9, Conf: 95},
{Src: "неконсолидированный", Origin: "mined", Freq: 9, Conf: -1},
{Src: "неуверенный", Dst: "д", Origin: "mined", Freq: 9, Conf: 20},
{Src: "безмолвный", Dst: "д", Origin: "mined", Freq: 9, Conf: -1},
}
var b bytes.Buffer
renderSignatureStop(&b, &pipeline.WaveSignatureStop{Terms: len(rows), SignaturePath: "p", Rows: rows})
out := b.String()
order := []string{"неуверенный", "уверенный", "безмолвный", "неконсолидированный"}
at := -1
for _, name := range order {
i := strings.Index(out, name)
if i < 0 {
t.Fatalf("row %q missing:\n%s", name, out)
}
if i < at {
t.Fatalf("review order broken at %q — want least-confident first, then silence, then unconsolidated:\n%s", name, out)
}
at = i
}
if !strings.Contains(out, "least-confident first") {
t.Fatalf("the banner must say what its order means, or a sorted table reads as arbitrary:\n%s", out)
}
}

View file

@ -47,8 +47,16 @@ partition "ВОЛНА ДРАФТА [OK] (∥ workers, по draft-чанкам)"
if (Майнинг вкл. И дельта непуста? [OK]) then (да)
:МАЙНИНГ-СТОП: MineBank по нормализованному источнику
против контраста (сид и reject-set исключены) →
signature map → прогон ОСТАНОВЛЕН (exit 3);
против контраста (сид и reject-set исключены);
:ТЕРМИНОЛОГ [OK]: Merge(WHICHWHAT) → KWIC → ранг §C2-3
(голос по ФОЛДУ целевой формы, сырые формы в показе) →
батчи ЮНИТАМИ: серия (равнодлинная) + СЕМЬЯ
(общий корень: имя-префикс либо реалия-суффикс, данные скрипта)
и containment-пара — юнит целиком в ОДИН вызов, кап членов;
:$0-экраны: канон-конфликты · КОНФЛИКТЫ СВОИХ ЖЕ
консолидаций · label-mismatch → стоп-таблица
(сигналы топ-варианта · invented · уверенность · конвенций);
:signature map → прогон ОСТАНОВЛЕН (exit 3);
:Владелец: каждый терм → promote в mined_delta (approved+dst)
ИЛИ decline в mined_rejects;
:Ре-ран: драфт $0-резюм · пустая дельта → авто-континью;

View file

@ -0,0 +1,250 @@
package lang
import (
"crypto/sha256"
"embed"
"encoding/hex"
"fmt"
"sort"
"strconv"
"strings"
"sync"
)
// bankdata.go: the BANK-ASSEMBLY data plane — language data read ONLY when the terminologist assembles its
// batches, and by nothing that a wave touches.
//
// WHY A PLANE OF ITS OWN, and why it is NOT in data/ beside script-series.txt. Both existing data planes
// fold into the WAVE snapshot: the embedded one through EmbeddedVersion, the pack one through
// Pack.Version(). That is right for bytes that ride the wire or resolve a verdict — editing them must be a
// loud --resnapshot. Family morphology does neither: it decides only WHICH candidates the terminologist
// shows in ONE call. Folding it into the wave would re-bill a whole book's draft for a knob that cannot
// change a single byte of what that draft bought — and it would contradict the ratified stance that the
// terminology axis stays OFF the snapshot (config.TerminologyGate, terminologyVersion,
// TestTerminologyIsNotSnapshotFolded).
//
// It is not unversioned, though: an edit here changes a batch's CONTENT, which is inside the bank-role
// RequestHash, so the affected batches re-pay by mechanism (cents), and BankDataVersion() is logged with
// the run so a signature map stays attributable to the data that produced it.
//go:embed bankdata/family-morphology.txt
var bankFS embed.FS
// bankDataAlgoVersion tags the bank plane's hashing RECIPE (like embedAlgoVersion tags the embedded one).
// Bump it only when the framing or the file set changes; a DATA edit already moves the hash via the bytes.
const bankDataAlgoVersion = "bankdata-v1"
// bankDataFiles are the plane's files, in a FIXED order (the order is folded into the hash).
var bankDataFiles = []string{"bankdata/family-morphology.txt"}
// FamilyAffix is one engine type's family rule: which side of the surface carries the shared root morpheme
// and how many runes of it must be shared before two surfaces count as one family.
type FamilyAffix struct {
Suffix bool // the root is the TRAILING morpheme (head-final realia); false → the LEADING one (names)
MinRunes int // ≤0 → this type forms no families
}
// FamilyMorph is a source language's resolved family-channel data. The zero value is INERT — a language
// written in no dense script, or in a dense script with no rows, co-batches exactly as it did before this
// channel existed.
type FamilyMorph struct {
Affix map[string]FamilyAffix // engine type (name|place|title|term) → its rule
MinMembers int // smallest set that counts as a family
MaxMembers int // largest batch unit a family merge may produce (0 = unbounded)
ContainmentRunes int // shortest candidate that may anchor the compositional channel
}
// Enabled reports whether the language declares enough to form a family at all.
func (f FamilyMorph) Enabled() bool {
return f.MinMembers > 0 && (len(f.Affix) > 0 || f.ContainmentRunes > 0)
}
// scriptFamily is one script's authored rows, before a language merges its dense scripts.
type scriptFamily struct {
affix map[string]FamilyAffix
minMembers, maxMembers, containmentRunes int
}
var (
familyOnce sync.Once
familyByScript map[string]*scriptFamily
bankDataOnce sync.Once
bankDataVal string
)
func loadFamilyMorphology() {
familyOnce.Do(func() {
m, err := parseFamilyMorphology(mustBankData(bankDataFiles[0]))
if err != nil {
panic(fmt.Sprintf("lang: embedded %s is corrupt: %v", bankDataFiles[0], err))
}
familyByScript = m
})
}
// BankDataVersion is the content hash of the bank-assembly plane (recipe tag + each file's path and bytes,
// same framing as EmbeddedVersion). It is deliberately NOT in the wave snapshot — see the file comment —
// and is logged with the run instead, so "which data produced this signature map" stays answerable.
func BankDataVersion() string {
bankDataOnce.Do(func() {
h := sha256.New()
h.Write([]byte(bankDataAlgoVersion))
for _, name := range bankDataFiles {
h.Write([]byte("\x00" + name + "\x00"))
h.Write(mustBankData(name))
}
bankDataVal = bankDataAlgoVersion + "-" + hex.EncodeToString(h.Sum(nil))[:12]
})
return bankDataVal
}
// FamilyMorphology resolves the family-channel data for a SOURCE language by unioning the rows of its
// DENSE scripts (the same gate the series channel uses: one rune ≈ one morpheme is what makes a shared
// affix a shared MORPHEME rather than a coincidence). Merge rules, all on the conservative side:
// - two dense scripts declaring the same type with DIFFERENT sides disagree about the language's
// morphology, so that type forms no families rather than one guessed direction;
// - otherwise the STRICTER rule wins (the longer required root), the strictest min_members, and the
// SMALLEST max_members, so adding a script never loosens what another script asked for.
func FamilyMorphology(lng string) FamilyMorph {
loadLangScripts()
loadFamilyMorphology()
var out FamilyMorph
names := make([]string, 0, 4)
for name := range langScriptBy[strings.ToLower(strings.TrimSpace(lng))] {
if cjkScriptNames[name] {
names = append(names, name)
}
}
sort.Strings(names) // deterministic merge order (the rules are order-free, the iteration must be too)
conflict := map[string]bool{}
for _, name := range names {
sf := familyByScript[name]
if sf == nil {
continue
}
if out.Affix == nil {
out.Affix = map[string]FamilyAffix{}
}
for typ, a := range sf.affix {
cur, had := out.Affix[typ]
switch {
case !had:
out.Affix[typ] = a
case cur.Suffix != a.Suffix:
conflict[typ] = true
case a.MinRunes > cur.MinRunes:
out.Affix[typ] = a
}
}
if sf.minMembers > out.MinMembers {
out.MinMembers = sf.minMembers
}
if sf.containmentRunes > out.ContainmentRunes {
out.ContainmentRunes = sf.containmentRunes
}
if sf.maxMembers > 0 && (out.MaxMembers == 0 || sf.maxMembers < out.MaxMembers) {
out.MaxMembers = sf.maxMembers
}
}
for typ := range conflict {
delete(out.Affix, typ)
}
return out
}
// parseFamilyMorphology reads the plane's rows into script → rules. Pure, so a test can drive it with a
// synthetic table and prove the channel is DATA. Fail-loud on an unknown key / side / non-positive integer:
// a malformed row is a corrupt asset, and a silently-inert family channel is exactly the drift this
// codebase refuses (the Palladius typo precedent).
func parseFamilyMorphology(b []byte) (map[string]*scriptFamily, error) {
out := map[string]*scriptFamily{}
at := func(script string) *scriptFamily {
if out[script] == nil {
out[script] = &scriptFamily{affix: map[string]FamilyAffix{}}
}
return out[script]
}
posInt := func(i int, key, raw string) (int, error) {
n, err := strconv.Atoi(strings.TrimSpace(raw))
if err != nil || n <= 0 {
return 0, fmt.Errorf("line %d: %s must be a positive integer (%q)", i+1, key, raw)
}
return n, nil
}
line := 0
for i, raw := range strings.Split(string(b), "\n") {
t := strings.TrimSpace(strings.TrimRight(raw, "\r"))
if t == "" || strings.HasPrefix(t, "#") {
continue
}
line = i
f := strings.Split(t, "\t")
for j := range f {
f[j] = strings.TrimSpace(f[j])
}
if len(f) < 3 || f[1] == "" {
return nil, fmt.Errorf("line %d: want `key<TAB>script<TAB>…`, got %q", i+1, t)
}
// The SCRIPT column is checked against the range registry. A typo parses fine and then simply never
// matches a language's declared scripts — the channel goes silently inert for the pair that asked for
// it, which is the exact silent-empty-table class the pack loader refuses everywhere else.
if _, known := scriptRanges[f[1]]; !known {
return nil, fmt.Errorf("line %d: unknown script %q — it resolves to no Unicode range, so every row naming it would be silently ignored (add it to scriptRanges, or fix the spelling)", i+1, f[1])
}
switch f[0] {
case "family_affix":
if len(f) != 5 || f[2] == "" {
return nil, fmt.Errorf("line %d: family_affix wants `family_affix<TAB>script<TAB>type<TAB>prefix|suffix<TAB>min_root_runes`, got %q", i+1, t)
}
n, err := posInt(i, "min_root_runes", f[4])
if err != nil {
return nil, err
}
switch f[3] {
case "prefix":
at(f[1]).affix[f[2]] = FamilyAffix{MinRunes: n}
case "suffix":
at(f[1]).affix[f[2]] = FamilyAffix{Suffix: true, MinRunes: n}
default:
return nil, fmt.Errorf("line %d: side must be prefix|suffix, got %q", i+1, f[3])
}
case "family_min_members", "family_max_members", "family_containment_runes":
if len(f) != 3 {
return nil, fmt.Errorf("line %d: %s wants `%s<TAB>script<TAB>n`, got %q", i+1, f[0], f[0], t)
}
n, err := posInt(i, f[0], f[2])
if err != nil {
return nil, err
}
switch f[0] {
case "family_min_members":
at(f[1]).minMembers = n
case "family_max_members":
at(f[1]).maxMembers = n
default:
at(f[1]).containmentRunes = n
}
default:
return nil, fmt.Errorf("line %d: unknown key %q (want family_affix|family_min_members|family_max_members|family_containment_runes)", i+1, f[0])
}
}
// A HALF table is a corrupt one, for the same reason: family_affix with no family_min_members leaves
// Enabled() false, so the rows are read, parsed, and then quietly do nothing.
for name, sf := range out {
switch {
case sf.minMembers == 0:
return nil, fmt.Errorf("script %q declares family rules but no family_min_members — the channel would load and stay inert (line %d region)", name, line+1)
case len(sf.affix) == 0 && sf.containmentRunes == 0:
return nil, fmt.Errorf("script %q declares family_min_members but neither family_affix nor family_containment_runes — nothing can form a family", name)
}
}
return out, nil
}
func mustBankData(name string) []byte {
b, err := bankFS.ReadFile(name)
if err != nil {
panic(fmt.Sprintf("lang: missing bank-data asset %q: %v", name, err))
}
return b
}

View file

@ -0,0 +1,37 @@
# FAMILY morphology per writing SCRIPT (bank fix-pack §G1, research/24 §B4): which candidates of a dense
# script belong to ONE term FAMILY and must therefore travel in ONE terminologist call. Measured: the same
# model rendered the 古月-family one way in batch 0 and another way in batch 2 — a chimera that correlates
# with the batch boundary perfectly, on a family DetectSeries cannot co-batch (its members differ in length).
#
# A family is anchored on a shared ROOT MORPHEME, and where that root sits is ASYMMETRIC by class: a NAME
# carries its clan/surname morpheme in FRONT (古月方源, 古月正), while a realia term carries its generic head
# LAST (head-final writing: *-海空窍, *-寨). The asymmetry is DATA, not a Go branch, so a pair whose names
# are head-final (or whose script is head-initial) declares its own rows and needs no engine edit.
#
# Rows (TAB-separated), all keyed by SCRIPT name (the names lang/script.go resolves to Unicode ranges):
# family_affix<TAB>script<TAB>type<TAB>prefix|suffix<TAB>min_root_runes
# one rule per engine type (name|place|title|term). A type with no row never forms a family.
# family_min_members<TAB>script<TAB>n — below n a "family" is a coincidence, not a convention.
# family_max_members<TAB>script<TAB>n — the blob backstop: a merge that would push a unit past n
# is refused and COUNTED. Measured on the coldrun-a bank
# distillation (150 surfaces): 24 refuses nothing and the
# largest real unit is 20 — the 古月 clan with the surfaces
# built on it; 16 splits that unit, and a split family is
# the very chimera this channel exists to prevent.
# family_containment_runes<TAB>script<TAB>n — the compositional channel: a candidate that is itself a
# substring of another candidate (元海 ⊂ 元海空窍) joins its
# unit. n bounds the anchor so a single generic morpheme
# cannot sweep half the bank into one call.
#
# A script with NO rows here leaves the family channel INERT (the batcher then behaves exactly as before
# this file existed), so a pair that has not been measured never silently changes what it buys.
family_affix han name prefix 2
# `nickname` is a person-surface the BANKNOTE channel accepts from a draft, so a candidate can carry it and
# it needs a rule of its own — otherwise a whole legitimate type forms no families, silently.
family_affix han nickname prefix 2
family_affix han place suffix 2
family_affix han title suffix 2
family_affix han term suffix 2
family_min_members han 2
family_max_members han 24
family_containment_runes han 2

View file

@ -0,0 +1,106 @@
package lang
import (
"strings"
"testing"
)
// TestBankPlaneIsOutsideTheWaveFold is the load-bearing property of the whole plane, and the reason it is a
// separate embed rather than another row in data/script-series.txt: editing family morphology must NOT move
// the wave snapshot. EmbeddedVersion folds into every book's snapshot, so a row added there would re-bill a
// whole draft wave for a knob that only decides which candidates share a terminologist CALL — the axis
// config.TerminologyGate is deliberately kept off the snapshot for.
func TestBankPlaneIsOutsideTheWaveFold(t *testing.T) {
for _, f := range embeddedDataFiles() {
if strings.Contains(f.name, "family") {
t.Fatalf("the bank plane's %s reached the WAVE-folded embed set — editing it would re-snapshot every book", f.name)
}
}
// Both versions exist and are distinct namespaces, so a log line naming one can never be read as the other.
if v := BankDataVersion(); !strings.HasPrefix(v, bankDataAlgoVersion+"-") {
t.Fatalf("BankDataVersion must be tagged with its own recipe, got %q", v)
}
if strings.HasPrefix(BankDataVersion(), embedAlgoVersion) {
t.Fatal("the two data planes must not share a version namespace")
}
}
// TestFamilyMorphologyIsData drives the parser with a synthetic table: a script's family rules — including
// the name/realia asymmetry and the head side — are DATA, so a pair that is not in the repo declares its own
// and needs no Go edit. The script here is a REGISTERED one that ships no family rows of its own — the first
// version of this test declared «devanagari», which resolves to no Unicode range and was therefore dropped
// in silence, so the flagship generality test proved nothing at all.
func TestFamilyMorphologyIsData(t *testing.T) {
got, err := parseFamilyMorphology([]byte(strings.Join([]string{
"# a source whose NAMES are head-final and whose realia lead with the root",
"family_affix\thangul\tname\tsuffix\t3",
"family_affix\thangul\tterm\tprefix\t2",
"family_min_members\thangul\t4",
"family_containment_runes\thangul\t3",
}, "\n")))
if err != nil {
t.Fatal(err)
}
sf := got["hangul"]
if sf == nil {
t.Fatalf("the declared script must be present: %v", got)
}
if a := sf.affix["name"]; !a.Suffix || a.MinRunes != 3 {
t.Fatalf("a head-final name rule is one data row, got %+v", a)
}
if a := sf.affix["term"]; a.Suffix || a.MinRunes != 2 {
t.Fatalf("the asymmetry is per-type data, got %+v", a)
}
if sf.minMembers != 4 || sf.containmentRunes != 3 || sf.maxMembers != 0 {
t.Fatalf("bounds come from the rows, absent ones stay unset: %+v", sf)
}
}
// TestFamilyMorphologyRefusesCorruptRows: a malformed row is a corrupt asset, and the failure mode it would
// otherwise produce — a channel that parses fine and silently forms nothing — is the exact drift the pack
// loader already refuses for the Palladius tables.
func TestFamilyMorphologyRefusesCorruptRows(t *testing.T) {
for _, bad := range []string{
"family_affix\than\tname\tmiddle\t2", // no such side
"family_affix\than\tname\tprefix\t0", // a zero-rune root matches everything
"family_affix\than\tname\tprefix", // truncated row
"family_minmembers\than\t2", // typo'd key
"family_min_members\than\tlots", // non-integer
// A typo'd SCRIPT resolves to no range, so every row naming it is silently ignored and the pair that
// asked for the channel gets an inert one.
"family_affix\thann\tname\tprefix\t2\nfamily_min_members\thann\t2",
// A HALF table: rules with no quorum, or a quorum with nothing to form a family from. Both parse and
// then do nothing, which is the failure this pack declared intolerable everywhere else.
"family_affix\than\tname\tprefix\t2",
"family_min_members\than\t2",
} {
if _, err := parseFamilyMorphology([]byte(bad)); err == nil {
t.Fatalf("a corrupt row must fail loud: %q", bad)
}
}
}
// TestFamilyMorphologyForZhIsLive pins the shipped table through the resolver a book actually calls: zh is
// written in a dense script that declares rules, an alphabetic source is inert, and the ja case shows the
// union — kana declare nothing, so the language inherits han's rules rather than losing them.
func TestFamilyMorphologyForZhIsLive(t *testing.T) {
zh := FamilyMorphology("zh")
if !zh.Enabled() {
t.Fatal("zh is the measured pair and must have a live family channel")
}
if a := zh.Affix["name"]; a.Suffix || a.MinRunes < 2 {
t.Fatalf("a han name family leads with the clan morpheme: %+v", a)
}
if a := zh.Affix["term"]; !a.Suffix || a.MinRunes < 2 {
t.Fatalf("a han realia family ends with the generic head: %+v", a)
}
if zh.MinMembers < 2 || zh.MaxMembers <= 0 || zh.ContainmentRunes < 2 {
t.Fatalf("the bounds must all be declared: %+v", zh)
}
if en := FamilyMorphology("en"); en.Enabled() {
t.Fatalf("an alphabetic source forms no morpheme families: %+v", en)
}
if ja := FamilyMorphology("ja"); !ja.Enabled() || ja.Affix["term"] != zh.Affix["term"] {
t.Fatalf("a language written in han inherits han's rules through the union: %+v", ja)
}
}

View file

@ -202,6 +202,16 @@ type PickedEntry struct {
// no real path changes (pack-16 tail).
func (p PickedEntry) valid() bool { return p.entry != nil }
// Src is the bank row's SOURCE surface — the only thing about a record a caller outside the bank can NAME.
// It exists because n_evicted counted budget drops without ever saying WHICH rows the model did not get to
// see, and a bare count is not something an operator can act on. Empty for a zero value.
func (p PickedEntry) Src() string {
if p.entry == nil {
return ""
}
return p.entry.src
}
// TrustGateEvent records a longest-match suppression the DISPOSITION-gate REFUSED (D39 layer 4):
// a longer but LOWER-trust key (draft/ambiguous, e.g. draft 四代族长) that would have deleted a nested
// HIGHER-trust key (approved/confirmed, e.g. approved 族长). The draft is editor-excluded (the block

View file

@ -240,7 +240,14 @@ func DeltaYAML(mined []Term, proposals map[string][]DstProposal) (string, error)
// The join key is the miner's own normalized surface, so a proposal written with different
// orthography still lands on its term (the same normalization the seed-surface exclusion uses).
if props := proposals[text.NormalizeSourceKey(m.Src)]; len(props) > 0 {
st.Dst = props[0].Dst
// The dst comes from a proposal made ON THIS SURFACE. A rendering that reached the term through an
// ALIAS of its cluster is evidence — it is listed in the note with the surface that proposed it —
// but the miner's clustering is itself unverified (measured: a surname cluster holding a clan
// title), and promoting an alias's word to the term's own rendering is the clustering DECISION,
// which belongs to the owner's signature and to nothing else.
if props[0].Via == "" {
st.Dst = props[0].Dst
}
note = appendProposalNote(note, props)
}
// TWO-MODE EMISSION (§C2-7, ratified and until pack-20 unbuilt): a term that arrives with a
@ -272,19 +279,40 @@ func appendProposalNote(note string, props []DstProposal) string {
b.WriteString(note)
b.WriteString("; ")
}
fmt.Fprintf(&b, "dst PROPOSED by the translator (banknote, %d chunk(s)) — NOT approved, sign or replace it", props[0].Chunks)
if props[0].Via != "" {
// Nothing was proposed on this surface itself: every rendering came through an alias, so the row
// carries no dst and the note says who proposed what for which surface.
b.WriteString("dst PROPOSED only through cluster alias(es) — NOT attached, sign or replace it: ")
for i, p := range props {
if i > 0 {
b.WriteString(", ")
}
fmt.Fprintf(&b, "%q ×%d%s", p.Dst, p.Chunks, viaNote(p.Via))
}
return b.String()
}
fmt.Fprintf(&b, "dst PROPOSED by the translator (banknote, %d chunk(s)) — NOT approved, sign or replace it",
props[0].Chunks)
if len(props) > 1 {
b.WriteString("; other proposals: ")
for i, p := range props[1:] {
if i > 0 {
b.WriteString(", ")
}
fmt.Fprintf(&b, "%q ×%d", p.Dst, p.Chunks)
fmt.Fprintf(&b, "%q ×%d%s", p.Dst, p.Chunks, viaNote(p.Via))
}
}
return b.String()
}
// viaNote renders the alias provenance of a proposal, or "" for one made on the term's own surface.
func viaNote(via string) string {
if via == "" {
return ""
}
return " [proposed for " + via + "]"
}
// appendConsolidatedNote records that the dst on this row is the TERMINOLOGIST's consolidation, not a
// per-chunk guess and not a signature. The distinction is the whole trust story of the auto mode: a
// `draft` row reaches the wire with an unverified marker, so the sign map must say, in the artifact
@ -374,4 +402,9 @@ type DstProposal struct {
Dst string
Type string
Chunks int
// Via names the surface that actually proposed this rendering when it is not the term's own — an ALIAS
// the miner clustered onto it. It travels into the note because the alternative is a sign map that
// presents a rendering proposed FOR ANOTHER SURFACE as a rendering of this one, and the clustering that
// joined them is itself unverified (the polygon's surname cluster holding a clan title).
Via string
}

View file

@ -0,0 +1,383 @@
package pipeline
import (
"bytes"
"context"
"log/slog"
"os"
"path/filepath"
"strings"
"testing"
"textmachine/backend/internal/config"
"textmachine/backend/internal/miner"
"textmachine/backend/internal/terminology"
"textmachine/backend/internal/text"
)
// bankfixpack_test.go: the fix-pack's pipeline-level seams — the arbitration record on the stop table (§G3),
// the signature map's alias join (§G3), the auto-bank rewrite diff (row 130) and the eviction list.
// TestBankStopTableCarriesTheArbitrationRecord pins §A7's finding closed: «why does this term have THIS
// dst» has to be answerable from the artifacts. The row now carries the winning variant's ranking signals,
// whether the rendering is one the drafts ever proposed, how many CONVENTIONS they actually offered (as
// against raw forms), and the role's own confidence.
func TestBankStopTableCarriesTheArbitrationRecord(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
if isTerminologyBody(body) {
// A rendering NO draft proposed, with a confidence field — both the invented flag and the
// confidence column have to survive the parser.
return "方源\tГуюэ Фан Юань\t40", "stop"
}
// The drafts disagree on the CASE only: one convention written two ways.
return "Фан Юань пришёл." + "\n" + bankSeparator +
"\n方源\tФан Юань\tname\n方源\tфан юань\tname", "stop"
})
defer srv.Close()
r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}))
defer r.Close()
stop := runToSignatureStop(t, r)
var row *BankStopRow
for i := range stop.Rows {
if stop.Rows[i].Src == "方源" {
row = &stop.Rows[i]
}
}
if row == nil {
t.Fatalf("方源 missing from the table: %+v", stop.Rows)
}
if row.Conf != 40 {
t.Fatalf("the role's stated confidence must reach the review table, got %d", row.Conf)
}
if !row.Invented {
t.Fatalf("a rendering no draft proposed must say so: %+v", *row)
}
if row.Conventions != 1 {
t.Fatalf("two spellings of one rendering are ONE convention, got %d (%v)", row.Conventions, row.Variants)
}
if row.Spread < 2 {
t.Fatalf("the owner must still see that the drafts wrote it two ways, got spread=%d", row.Spread)
}
raw, err := os.ReadFile(stop.TablePath)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"conventions=", "confidence=40", "INVENTED"} {
if !strings.Contains(string(raw), want) {
t.Fatalf("the sidecar must carry %q:\n%s", want, raw)
}
}
}
// TestBankStopRowCarriesTheWinningVariantsSignals pins the OTHER half of §A7's answer: not only WHAT was
// chosen but WHY. The signals are the §C2-3 factors that fired for the variant the ranking put first, and
// they were already computed and thrown away before this pack — so the failure mode is a row that looks
// complete and explains nothing.
func TestBankStopRowCarriesTheWinningVariantsSignals(t *testing.T) {
c := terminology.Candidate{Key: "元海", Src: "元海", Type: "term", Variants: []terminology.Variant{
{Dst: "море истинной ци", Chunks: 5, Forms: 1},
{Dst: "первозданное море", Chunks: 1, Forms: 1},
}}
terminology.ScoreVariants(&c, terminology.ScoreOpts{})
if len(c.Variants[0].Signals) == 0 {
t.Fatalf("test premise broken: the winner must have fired at least one factor: %+v", c.Variants)
}
rows := bankStopRows([]terminology.Candidate{c}, map[string]string{"元海": "море истинной ци"}, terminologyResult{})
if len(rows[0].Signals) == 0 {
t.Fatalf("the winning variant's audit trail must reach the row: %+v", rows[0])
}
if strings.Join(rows[0].Signals, ",") != strings.Join(c.Variants[0].Signals, ",") {
t.Fatalf("the row must carry the TOP variant's signals, got %v want %v", rows[0].Signals, c.Variants[0].Signals)
}
if !strings.Contains(renderBankStopTable(rows), "why: ") {
t.Fatalf("and the sidecar must print them:\n%s", renderBankStopTable(rows))
}
// A row with no consolidation carries no confidence rather than a zero one: 0 means «the role said it was
// not sure at all», which is the first row to review, not the same as silence.
if rows[0].Conf >= 0 {
t.Fatalf("an absent confidence must stay absent, got %d", rows[0].Conf)
}
}
// TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply guards the seam between the two «this batch bought
// nothing» signals. A batch the budget never reached has an empty reply text for a completely different
// reason, and warning about an EMPTY COMPLETION there would cry wolf on every budget-bounded run — the
// budget already said what happened, one line earlier.
func TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply(t *testing.T) {
var logBuf bytes.Buffer
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
if isTerminologyBody(body) {
return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop"
}
return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
})
defer srv.Close()
// batch_runes tiny → several batches; a budget sized under one batch → the pass stops at the first.
probe := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, batchRunes: 400}))
_ = runToSignatureStop(t, probe)
if probe.lastTerminology.Batches < 2 {
t.Fatalf("the fixture must actually batch, got %d", probe.lastTerminology.Batches)
}
perBatch := probe.lastTerminology.EstimateUSD / float64(probe.lastTerminology.Batches)
probe.Close()
r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{
terminology: true, batchRunes: 400, budgetUSD: perBatch * 1.5,
}))
defer r.Close()
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
_ = runToSignatureStop(t, r)
out := logBuf.String()
if !strings.Contains(out, "budget would be exceeded") {
t.Fatalf("the fixture must actually hit the budget:\n%s", out)
}
if strings.Contains(out, "EMPTY completion") {
t.Fatalf("a batch the budget never reached is not a paid batch that came back empty:\n%s", out)
}
}
// TestSelfContradictionIsReportedAtTheStop is §G2 end to end: the run consolidates a part and then renders
// the compound without it, and both the log and the row say so. The canon check cannot see this — the book
// has no signed rows at all — which is precisely why 18 of the 149 contradictions on the live bank were
// invisible.
func TestSelfContradictionIsReportedAtTheStop(t *testing.T) {
var logBuf bytes.Buffer
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
if isTerminologyBody(body) {
// 花家 → «клан Хуа», and the compound built on it drops «Хуа» entirely.
return "花家\tклан Хуа\n花家的人\tлюди клана", "stop"
}
return "Фан Юань пришёл." + "\n" + bankSeparator +
"\n花家\tклан Хуа\tname\n花家的人\tлюди клана\tterm", "stop"
})
defer srv.Close()
r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}))
defer r.Close()
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
_ = runToSignatureStop(t, r)
if r.lastTerminology.SelfConflicts == 0 {
t.Fatalf("the compound contradicts the part the same reply consolidated: %+v", r.lastTerminology)
}
if !strings.Contains(logBuf.String(), "contradict each other") {
t.Fatalf("a run breaking its own canon must say so:\n%s", logBuf.String())
}
var flagged bool
for _, row := range r.lastBankStopRows {
if row.Src == "花家的人" && len(row.Contradicts) > 0 {
flagged = true
}
}
if !flagged {
t.Fatalf("the contradicting row must be marked in the review table: %+v", r.lastBankStopRows)
}
}
// TestSignatureMapJoinsAliasProposals is the acceptance finding of §G3: a rendering that reached the
// terminologist through an ALIAS of a cluster showed up in the stop table (Merge routes it to the owner)
// and was ABSENT from the owner's row in the signature map, which joined on the alias's own key. The map
// and the table then described the same bank differently, for exactly the terms the miner clustered.
func TestSignatureMapJoinsAliasProposals(t *testing.T) {
mined := []terminology.Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子"}}}
observed := []terminology.Observed{
{Key: "方小子", Src: "方小子", Type: "name", Proposals: []terminology.Proposal{{Dst: "малыш Фан", Chunks: 3}}},
}
cands := terminology.Merge(mined, observed, text.NormalizeTargetForm)
props := proposalsFromCandidates(cands)
if got := props["方源"]; len(got) == 0 || got[0].Dst != "малыш Фан" {
t.Fatalf("an alias-routed proposal must land on the OWNER's key, got %+v", props)
}
if props["方源"][0].Via != "方小子" {
t.Fatalf("and it must say which surface actually proposed it, got %q", props["方源"][0].Via)
}
yaml, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, props)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(yaml, "малыш Фан") {
t.Fatalf("the signature map must carry the rendering the table showed:\n%s", yaml)
}
if !strings.Contains(yaml, "proposed for 方小子") {
t.Fatalf("and it must not present an alias's guess as the term's own:\n%s", yaml)
}
// AND it must not become the term's dst. Merge routes an alias's rendering to the cluster owner so the
// ranking sees all the evidence; promoting it to the owner's rendering is the clustering DECISION, which
// belongs to a signature. Here NOTHING was proposed on 方源 itself, so the row carries no dst at all.
if strings.Contains(yaml, "dst: малыш Фан") {
t.Fatalf("an alias's rendering became the term's dst — it reaches the wire with no model and no gate:\n%s", yaml)
}
// With a DIRECT proposal present, that one is the dst and the alias stays evidence in the note.
observed = append(observed, terminology.Observed{
Key: "方源", Src: "方源", Type: "name", Proposals: []terminology.Proposal{{Dst: "Фан Юань", Chunks: 1}},
})
direct := proposalsFromCandidates(terminology.Merge(mined, observed, text.NormalizeTargetForm))
yaml2, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, direct)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(yaml2, "dst: Фан Юань") {
t.Fatalf("a rendering proposed on the term's own surface must be its dst:\n%s", yaml2)
}
if !strings.Contains(yaml2, "малыш Фан") {
t.Fatalf("and the alias proposal must still be listed as evidence:\n%s", yaml2)
}
}
// TestAutoBankRewriteNamesDroppedTerms is the $0 minimum of backlog row 130. The auto-bank file is rewritten
// WHOLE from a top-N-capped delta, so a term that was in the bank for twenty chapters can vanish from it
// mid-run with nothing in the artifacts saying so. The pack does not change that behaviour — accumulating is
// the owner's STOP decision, because it moves memory_version — it makes the loss visible.
func TestAutoBankRewriteNamesDroppedTerms(t *testing.T) {
var logBuf bytes.Buffer
r := &Runner{
Log: slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})),
Book: &config.Book{BookID: "test-book", ProjectDB: filepath.Join(t.TempDir(), "project.db")},
}
ctx := context.Background()
first := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "青茅山", Type: "place", Freq: 12}}
if err := r.writeAutoBank(ctx, first, nil, nil); err != nil {
t.Fatal(err)
}
if logBuf.Len() != 0 {
t.Fatalf("the first write has nothing to diff against:\n%s", logBuf.String())
}
// The next run's delta no longer holds 青茅山 — the rank cap cut it, and it silently leaves the bank.
second := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "花家", Type: "name", Freq: 9}}
if err := r.writeAutoBank(ctx, second, nil, nil); err != nil {
t.Fatal(err)
}
out := logBuf.String()
if !strings.Contains(out, "青茅山") {
t.Fatalf("a term dropped by the rewrite must be NAMED:\n%s", out)
}
if strings.Contains(out, "方源") {
t.Fatalf("a term that is still there is not a loss:\n%s", out)
}
// The boundary is intact: the file is the new delta, not a merge of both runs.
raw, err := os.ReadFile(r.autoBankPath())
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "青茅山") {
t.Fatalf("the pack must REPORT the drop, not accumulate the file (that moves memory_version):\n%s", raw)
}
// And the owner's OWN two verbs are not a loss. Promoting a term into the mined-delta (it becomes a seed
// surface) or declining it in mined-rejects removes it from the next delta ON PURPOSE — warning about
// that fires a false alarm on the normal signature cycle and advises the owner to do what he just did.
logBuf.Reset()
third := []miner.Term{{Src: "方源", Type: "name", Freq: 40}}
handled := map[string]bool{text.NormalizeSourceKey("花家"): true}
if err := r.writeAutoBank(ctx, third, nil, handled); err != nil {
t.Fatal(err)
}
if strings.Contains(logBuf.String(), "花家") {
t.Fatalf("a term the owner promoted or declined is not a silent loss:\n%s", logBuf.String())
}
}
// TestAutoBankDiffSurvivesTheEngineOwnRows is the row-130 warning through the path production takes: two
// real auto-mode runs, a term that leaves the delta between them, and the warning that has to name it.
//
// It has to be end-to-end, because the defect lives in the CALL SITE, not in the helper. From the second
// auto-mode run the stored glossary also holds the engine's own unsigned rows — seedGlossary re-seeds the
// auto-bank at start (seeding.go:83-93) — so passing the raw seed makes every term the engine ever proposed
// read as «the owner decided about this». The diff then empties and the warning is structurally dead in
// production while every unit test stays green. Risk 2, the self-exclusion trap, arriving at the one place
// in this file that did not filter for it.
func TestAutoBankDiffSurvivesTheEngineOwnRows(t *testing.T) {
var logBuf bytes.Buffer
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
if isTerminologyBody(body) {
return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop"
}
return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
})
defer srv.Close()
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
// Run 1: the auto mode mines 花家 among others, writes the auto-bank and seeds it into the store.
r1 := newRunner(t, bookPath)
if _, err := r1.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
var had bool
for _, src := range r1.autoBankSurfaces() {
if src == "花家" {
had = true
}
}
if !had {
t.Fatalf("test premise broken: the first run must bank 花家, got %v", r1.autoBankSurfaces())
}
r1.Close()
// The book changes and 花家 stops occurring, so it leaves the delta — standing in for the rank cap doing
// the same thing mid-book, which is the case backlog row 130 is about.
writeFile(t, filepath.Join(filepath.Dir(bookPath), "source.txt"),
strings.Repeat("方源来到青茅山。方源很强。青茅山很高。", 6))
r2 := newRunner(t, bookPath)
defer r2.Close()
r2.Resnapshot = true // the source moved, and this fixture is about the bank diff, not the snapshot gate
r2.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
if _, err := r2.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
// The store now holds the engine's own unsigned rows — the state that used to swallow the warning.
seed, err := r2.Store.GlossaryForBook("test-book")
if err != nil {
t.Fatal(err)
}
engineRows := 0
for _, e := range seed {
if e.Source == "mined" && e.Status != "approved" {
engineRows++
}
}
if engineRows == 0 {
t.Fatal("test premise broken: the store must hold the engine's own unsigned rows by the second run")
}
out := logBuf.String()
if !strings.Contains(out, "NOT in the one this run just wrote") || !strings.Contains(out, "花家") {
t.Fatalf("the row-130 warning must NAME the term the rewrite dropped, engine rows in the seed and all:\n%s", out)
}
}
// TestEvictedBankRowsAreNamed: n_evicted counted the rows the injection budget dropped and never said which,
// so «the model had no canon for this term» was an unactionable integer. The names go to the log rather than
// to a new column — a per-row detail column is a schema migration, and this repository has already paid for
// a non-idempotent one (backlog row 49а).
func TestEvictedBankRowsAreNamed(t *testing.T) {
var logBuf bytes.Buffer
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
// A budget of one token cannot fit even the first glossary line, so every matched row is evicted.
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
source: suzukiSource, glossarySeed: suzukiSeed, glossaryTokenBudget: 1,
})
r := newRunner(t, bookPath)
defer r.Close()
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
if _, err := r.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
rs, err := r.Store.GetRetrievalState("test-book", 1, 0)
if err != nil || rs == nil {
t.Fatalf("retrieval_state: %v %v", rs, err)
}
if rs.NEvicted == 0 {
t.Fatal("test premise broken: a one-token budget must evict the matched row")
}
out := logBuf.String()
if !strings.Contains(out, "鈴木") {
t.Fatalf("the evicted rows must be NAMED, not counted:\n%s", out)
}
}

View file

@ -51,8 +51,11 @@ const bankMaxLines = 20
// re-resolves paid text, so it must be a loud --resnapshot.
// - bankParseVersion governs parseBanknote — which candidate lines are accepted. Its output is EVIDENCE
// for the owner's signature, never a checkpoint and never a wire byte; every run recomputes it from
// the stored answers (bankObservedForBook). NOT folded, logged with the run — the same contract
// gates.terminology and gates.voice already carry, and for the same reason.
// the stored answers (bankObservedForBook). NOT folded, and LOGGED with the run at the bank-mining stop
// (mining.go, "draft-side proposals folded") — the same contract gates.terminology and gates.voice
// carry, and for the same reason. The logging is what makes staying out of the snapshot honest rather
// than merely cheap: without a consumer the tag is a comment, and two maps produced by different rules
// are indistinguishable after the fact.
//
// Folding the parse rule (as a single parser_version did) would re-bill a whole draft wave for a change
// that cannot alter one byte of what was bought — which is a standing incentive to leave the rule wrong.
@ -60,8 +63,11 @@ const (
bankSliceVersion = "banknote-slice-v1"
// v2 (backlog 19 / D39.53 default): the src rule stopped being "contains a Han ideograph" (a language
// FAMILY hardcoded in the engine) and became "occurs in the book"; v2.1 adds the column-order recovery
// that same invariant makes possible.
bankParseVersion = "banknote-parse-v2.1-src-attested+column-order"
// that same invariant makes possible; v2.2 (backlog 19 fix-pack) makes the reading DELIMITER-AWARE — a
// tab/pipe line is positional, only a space-run line re-joins a split rendering. Bumping this is not
// decoration: the tag is what attributes a stored signature map to the rule that produced it, and a rule
// change under an unchanged tag is exactly the drift the split into two versions exists to prevent.
bankParseVersion = "banknote-parse-v2.2-src-attested+column-order+delimiter-aware"
)
// bankDerivedNS is the derived-checkpoint id NAMESPACE (§4б EXACT formula) — the "tm-<name>-v1" prefix
@ -79,6 +85,54 @@ var bankTypeOK = map[string]bool{"name": true, "place": true, "title": true, "te
// with Python's unicode \s around the pipe is exact on the term corpus.)
var bankFieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`)
// bankColumnSplit is the UNAMBIGUOUS half of that tolerance: a tab or a pipe is a delimiter the model chose,
// never something that can occur inside a rendering. A line carrying one has declared its own columns.
var bankColumnSplit = regexp.MustCompile(`\t|\s*\|\s*`)
// bankSpaceRun collapses the internal whitespace of a column read positionally, so «Фан␣␣Юань» arrives as
// one clean rendering instead of carrying the model's stray spacing into the bank.
var bankSpaceRun = regexp.MustCompile(` {2,}`)
// bankColumns splits one banknote line into (src + rendering fields, type). See parseBanknote for why the
// delimiter decides which of the two readings applies.
func bankColumns(line string) ([]string, string) {
typ := "term"
if cols := nonEmptyFields(bankColumnSplit, line); len(cols) >= 2 {
if len(cols) >= 3 {
if t := strings.ToLower(cols[2]); bankTypeOK[t] {
typ = t
}
}
// BOTH columns are collapsed, not just the second: the order-recovery below may swap them, and the
// column that becomes the rendering must arrive clean either way.
return []string{collapseRuns(cols[0]), collapseRuns(cols[1])}, typ
}
fields := nonEmptyFields(bankFieldSplit, line)
if len(fields) >= 3 {
if last := strings.ToLower(fields[len(fields)-1]); bankTypeOK[last] {
typ = last
fields = fields[:len(fields)-1]
}
}
return fields, typ
}
// collapseRuns folds a run of spaces inside a positionally-read column into one.
func collapseRuns(s string) string {
return strings.TrimSpace(bankSpaceRun.ReplaceAllString(s, " "))
}
// nonEmptyFields splits a line on re with empty and whitespace-only fields dropped.
func nonEmptyFields(re *regexp.Regexp, line string) []string {
var out []string
for _, p := range re.Split(strings.TrimSpace(line), -1) {
if p = strings.TrimSpace(p); p != "" {
out = append(out, p)
}
}
return out
}
// bankEntry is one parsed candidate line (evidence for the bank-mining stop (§C); NOT written to the store until owner
// signoff). Dst is the model's proposed translation — the direct dst delivery the co-occurrence
// miner could not extract (蛊→гу, non-seed 龙公→Лун Гун).
@ -92,7 +146,11 @@ type bankEntry struct {
type bankFlags struct {
NLines int // accepted (well-formed) banknote lines
ParseFail bool // any malformed residual line (src not in the book / <2 fields) that was NOT a tolerated truncation
Truncated bool // the LAST line was cut by generation length (tolerated, not a parse fail)
// Truncated marks a block the run did NOT complete: the separator was there, so the model opened the
// channel, but the generation ended on a non-stop finish and the stop-only gate refuses it. Read off the
// finish reason, not from the parser (which never sees such a block). ⚠ It cannot see a cut that removed
// the separator itself — on the reference that is the majority of length-cut chunks — so it is a floor.
Truncated bool
}
// splitBanknote slices the banknote block off the raw model output BEFORE any gate/editor (integration
@ -198,12 +256,17 @@ func (r *Runner) bankSrcAttested(chunkSource string) func(string) bool {
}
}
// parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry).
// Tolerant of a TRUNCATED final line when truncatedGeneration is set (§B3-5): a short last line under
// truncation is flagged banknote_truncated, not counted as a parse fail. Any other malformed line
// (fewer than 2 fields, or a src NOT attested in the source) sets banknote_parse_fail. Deterministic, no
// time/rand: srcAttested is a pure function of the book text (bankSourceIndex).
func parseBanknote(block string, truncatedGeneration bool, srcAttested func(string) bool) ([]bankEntry, bankFlags) {
// parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry). A
// malformed line (fewer than 2 fields, or a src NOT attested in the source) sets banknote_parse_fail.
// Deterministic, no time/rand: srcAttested is a pure function of the book text (bankSourceIndex).
//
// It carried a `truncatedGeneration` tolerance for a short LAST line (§B3-5, faithful to banknote.py). The
// ratified finish=stop-only gate means this parser is never handed a truncated block at all, so the only
// caller pinned the argument to false and the branch could not execute in production while its comment said
// it could. The truncation FACT is now read where it actually exists — off the finish reason, before the
// gate refuses the block (applyBanknoteWithEntries) — and the unreachable branch is gone rather than left
// as a tolerance nobody can reach.
func parseBanknote(block string, srcAttested func(string) bool) ([]bankEntry, bankFlags) {
var entries []bankEntry
var flags bankFlags
if strings.TrimSpace(block) == "" {
@ -216,31 +279,25 @@ func parseBanknote(block string, truncatedGeneration bool, srcAttested func(stri
}
}
bad := 0
for i, ln := range lines {
raw := bankFieldSplit.Split(strings.TrimSpace(ln), -1)
var parts []string
for _, p := range raw {
if p = strings.TrimSpace(p); p != "" {
parts = append(parts, p)
}
}
if len(parts) < 2 {
// A short LAST line under a truncated generation is a tolerated cut, not a failure.
if i == len(lines)-1 && truncatedGeneration {
flags.Truncated = true
continue
}
for _, ln := range lines {
// THE DELIMITER CARRIES INFORMATION, and reading the columns without it is what made the first cut of
// this fix a regression on the frozen reference (6 lines over 56 real blocks).
//
// A line the model delimited with a TAB or a PIPE has told us where its columns are, so it is read
// POSITIONALLY: src, rendering, type — exactly as it was before the fix-pack. A third column outside
// the closed type vocabulary («clan», «proverb», «onomatopoeia» — 1.25% of the reference) is then a
// benign unknown type, as it always was; guessing it into the rendering corrupts a term the owner is
// about to sign, and on a REVERSED line it made the src unfindable and dropped the line entirely.
//
// Only a line with NO tab and NO pipe is ambiguous, because the ≥2-space tolerance exists for models
// that substitute spaces for tabs — and that same tolerance is what splits «Фан␣␣Юань» into two
// fields. There, and only there, is the type identified by the closed vocabulary on the LAST field
// and everything between re-joined, which is the discipline terminology.ParseReply already applies.
fields, typ := bankColumns(ln)
if len(fields) < 2 {
bad++
continue
}
src, dst := parts[0], parts[1]
typ := "term"
if len(parts) >= 3 {
typ = strings.ToLower(parts[2])
}
if !bankTypeOK[typ] {
typ = "term"
}
// The src rule: a bank line must name something the BOOK contains. A surface no chunk of the source
// carries is either a hallucination or a mis-split line — not evidence about this book, and letting
// it through would put an invented term on the owner's sign map.
@ -249,14 +306,18 @@ func parseBanknote(block string, truncatedGeneration bool, srcAttested func(stri
// is recovered instead of discarded (measured: one chunk emitted all 12 of its lines reversed,
// including a term the owner had signed by hand). Order is decided, never guessed: the declared
// order wins whenever it is attested, the reversal is taken only when it is the ONLY reading the
// book supports, and a line neither reading supports is still a parse fail.
// book supports, and a line neither reading supports is still a parse fail. The src is one field on
// either reading — a source surface has no spaces to be split on — so the rendering is whatever
// remains, which is what makes the recovery survive a split rendering instead of failing on it.
src, dst := fields[0], strings.Join(fields[1:], " ")
revSrc, revDst := fields[len(fields)-1], strings.Join(fields[:len(fields)-1], " ")
switch {
case srcAttested == nil:
bad++
continue
case srcAttested(src): // declared order — including the ambiguous case where both sides are attested
case srcAttested(dst):
src, dst = dst, src
case srcAttested(revSrc):
src, dst = revSrc, revDst
default:
bad++
continue
@ -335,11 +396,20 @@ func (r *Runner) applyBanknoteWithEntries(role, rawText, finish, chunkSource str
if block == "" {
return rawText, "", bankFlags{}, nil // no separator → no strip, no derived checkpoint (byte-identical path)
}
if finish == "stop" {
// finish=stop-only gate: accept candidates + count telemetry only for a complete generation.
// truncatedGeneration=false is unreachable-otherwise in prod (a truncated gen is finish≠stop).
entries, flags = parseBanknote(block, false, r.bankSrcAttested(chunkSource))
if finish != "stop" {
// The finish=stop-only gate (§4б, ratified) refuses the block, and that refusal is the whole content
// of banknote_truncated: a block that IS there under a finish the run did not complete is a block the
// bank never got. (Length is the usual cause and the one the column was named for; a refusal or a
// filter cut lands here too, and for the owner it means the same thing — the WHAT of this chunk is
// missing, and the counters must not read as if it were clean.)
//
// The flag used to be derived INSIDE the parser from a truncatedGeneration argument the only caller
// pinned to false, so the column could not be 1 in production and the comment beside it said the
// opposite. Reading it off the finish reason needs no parse of untrusted text, no schema change, and
// leaves the gate exactly where it was ratified: nothing is accepted, the fact is recorded.
return cleanText, cleanText, bankFlags{Truncated: true}, nil
}
entries, flags = parseBanknote(block, r.bankSrcAttested(chunkSource))
return cleanText, cleanText, flags, entries
}
@ -465,8 +535,9 @@ func (f *bankFold) observed() []terminology.Observed {
}
// bankObservedByKey folds the durable per-chunk banknote rows into the draft-side view of each source
// surface. Both consumers read it: the signature-map join (proposalsFromObserved) and the terminologist's
// merge, which also needs the SURFACE so a proposal for a term the miner never found is not lost.
// surface. Both consumers read it: the terminologist's merge (whose candidates the signature-map join
// then reads, proposalsFromCandidates) — which also needs the SURFACE, so a proposal for a term the miner
// never found is not lost.
func bankObservedByKey(states []store.RetrievalState, target *unicode.RangeTable) []terminology.Observed {
f := newBankFold(target)
addRetrievalStates(f, states)

View file

@ -40,7 +40,7 @@ func TestSplitBanknoteSlicesBlock(t *testing.T) {
if strings.Contains(clean, bankSeparator) || strings.Contains(clean, "方源") {
t.Fatalf("the banknote block leaked into the clean translation: %q", clean)
}
ents, flags := parseBanknote(block, false, attestedIn(srcAll))
ents, flags := parseBanknote(block, attestedIn(srcAll))
if len(ents) != 2 || flags.ParseFail || flags.Truncated || flags.NLines != 2 {
t.Fatalf("parse = %+v flags=%+v, want 2 clean entries", ents, flags)
}
@ -56,7 +56,7 @@ func TestParseBanknoteTolerantFieldSplit(t *testing.T) {
// tab, ≥2 spaces, and pipe are all accepted delimiters (a model that emits spaces instead of a
// real TAB still parses). type falls back to "term" when absent or unknown.
block := "方源\tФан Юань\tname\n蛊师 гу-мастер title\n青茅山 | гора Цинмао | place\n古月\tГу Юэ"
ents, flags := parseBanknote(block, false, attestedIn(srcAll))
ents, flags := parseBanknote(block, attestedIn(srcAll))
if flags.ParseFail {
t.Fatalf("tolerant split should not fail: %+v", flags)
}
@ -82,7 +82,7 @@ func TestParseBanknoteSrcMustBeAttested(t *testing.T) {
"San Michon\tСан-Мишон\tplace", // multi-word latin src
"リン\tРин\tname", // kana src
}, "\n")
ents, flags := parseBanknote(block, false, attestedIn(src))
ents, flags := parseBanknote(block, attestedIn(src))
if flags.ParseFail {
t.Fatalf("every src here is in the source; none may be a parse fail: %+v", flags)
}
@ -91,7 +91,7 @@ func TestParseBanknoteSrcMustBeAttested(t *testing.T) {
}
// The other half of the rule: a plausible, well-formed, Han-bearing line the book never contains.
invented := "天魔宗\tСекта Небесного Демона\tplace"
ents, flags = parseBanknote(invented, false, attestedIn(src))
ents, flags = parseBanknote(invented, attestedIn(src))
if !flags.ParseFail || len(ents) != 0 {
t.Fatalf("an invented src must be a parse fail and yield nothing, got %+v %+v", ents, flags)
}
@ -111,7 +111,7 @@ func TestParseBanknoteRecoversColumnOrder(t *testing.T) {
"花海\tМоре цветов\tplace", // declared order
"выдумка\tтоже выдумка\tterm", // neither side attested → still a parse fail
}, "\n")
ents, flags := parseBanknote(block, false, attestedIn(src))
ents, flags := parseBanknote(block, attestedIn(src))
if !flags.ParseFail {
t.Fatalf("the unattested line must still fail: %+v", flags)
}
@ -127,7 +127,7 @@ func TestParseBanknoteRecoversColumnOrder(t *testing.T) {
// Ambiguity is resolved by the DECLARED order, never by a guess: when the book attests both fields
// (a source that quotes the target language, or a same-script pair), the model's order stands.
both := "San Michon\tСан-Мишон\tplace"
ents, flags = parseBanknote(both, false, attestedIn(src+" Сан-Мишон"))
ents, flags = parseBanknote(both, attestedIn(src+" Сан-Мишон"))
if flags.ParseFail || len(ents) != 1 || ents[0].Src != "San Michon" {
t.Fatalf("both-attested must keep the declared order, got %+v %+v", ents, flags)
}
@ -157,22 +157,21 @@ func TestBankSourceIndexParity(t *testing.T) {
}
}
func TestParseBanknoteTruncationTolerated(t *testing.T) {
// The LAST line cut by generation length is tolerated (banknote_truncated), not a parse fail —
// but ONLY under truncated_generation; the same short line otherwise IS a parse fail.
// TestParseBanknoteShortLastLineIsAParseFail: the parser's truncation TOLERANCE is gone with the fix-pack,
// and this pins what replaced it. A short last line reaches this parser only under finish=stop — a complete
// generation that simply emitted a broken line — and that is a parse fail, not a tolerated cut. The
// truncation FACT now lives where it is actually known, one level up, off the finish reason
// (TestBanknoteTruncatedIsReachable).
func TestParseBanknoteShortLastLineIsAParseFail(t *testing.T) {
block := "方源\tФан Юань\tname\n蛊" // last line has no dst
entsT, flagsT := parseBanknote(block, true, attestedIn(srcAll))
if !flagsT.Truncated || flagsT.ParseFail || len(entsT) != 1 {
t.Fatalf("truncated=true: want 1 entry + truncated flag + no parse_fail, got %+v %+v", entsT, flagsT)
}
entsF, flagsF := parseBanknote(block, false, attestedIn(srcAll))
if flagsF.Truncated || !flagsF.ParseFail || len(entsF) != 1 {
t.Fatalf("truncated=false: the short last line must be a parse_fail, got %+v %+v", entsF, flagsF)
ents, flags := parseBanknote(block, attestedIn(srcAll))
if flags.Truncated || !flags.ParseFail || len(ents) != 1 {
t.Fatalf("a complete generation's short line is a parse_fail, got %+v %+v", ents, flags)
}
}
func TestParseBanknoteEmptyBlock(t *testing.T) {
ents, flags := parseBanknote("", false, attestedIn(srcAll))
ents, flags := parseBanknote("", attestedIn(srcAll))
if len(ents) != 0 || flags.ParseFail || flags.Truncated || flags.NLines != 0 {
t.Fatalf("empty block must yield no entries and clean flags, got %+v %+v", ents, flags)
}
@ -277,3 +276,109 @@ func TestBankTokenBudgetIsDerived(t *testing.T) {
t.Fatalf("the line cap was raised on the mini-run measurement (2 of 9 blocks pressed against 12), got %d", bankMaxLines)
}
}
// TestParseBanknoteRejoinsASplitRendering is backlog row 129. The field splitter tolerates a run of ≥2
// spaces because models substitute spaces for tabs — and that same tolerance splits INSIDE a rendering the
// moment a model writes «Фан␣␣Юань». Taking parts[1] alone banked the half-name and pushed the tail into
// the type column, where it failed the closed vocabulary and silently became "term": a mangled surface on
// the owner's sign map with parse_fail still reading 0. The terminologist's parser was fixed for exactly
// this; the banknote's was not.
func TestParseBanknoteRejoinsASplitRendering(t *testing.T) {
attested := attestedIn(srcAll)
// BOTH shapes: with the type column present, and as a bare two-column line.
entries, flags := parseBanknote("方源\tФан Юань\tname\n蛊师\tмастер гу", attested)
if flags.ParseFail {
t.Fatalf("neither line is malformed: %+v", flags)
}
if len(entries) != 2 {
t.Fatalf("want both lines, got %+v", entries)
}
if entries[0].Dst != "Фан Юань" || entries[0].Type != "name" {
t.Fatalf("a split rendering must be re-joined and the type column still read: %+v", entries[0])
}
if entries[1].Dst != "мастер гу" || entries[1].Type != "term" {
t.Fatalf("with no type column the whole tail is the rendering: %+v", entries[1])
}
// THE SPACE-RUN PATH is the only one where the re-join can be observed at all: on a TAB line the
// rendering is one column, so join(fields[1:]) and fields[1] are the same string and the re-join could be
// reverted with every test still green. Here they differ, and «Фан» alone is what used to be banked.
spaced, sflags := parseBanknote("方源 Фан Юань name", attested)
if sflags.ParseFail || len(spaced) != 1 || spaced[0].Dst != "Фан Юань" || spaced[0].Type != "name" {
t.Fatalf("a space-delimited line must re-join its rendering: %+v %+v", spaced, sflags)
}
if two, _ := parseBanknote("蛊师 мастер гу", attested); len(two) != 1 || two[0].Dst != "мастер гу" {
t.Fatalf("and so must a space-delimited line with no type column: %+v", two)
}
// The recognised shapes are unchanged: a real three-field line is still src/dst/type.
plain, _ := parseBanknote("青茅山\tгора Цинмао\tplace", attested)
if len(plain) != 1 || plain[0].Dst != "гора Цинмао" || plain[0].Type != "place" {
t.Fatalf("the ordinary three-field line must be untouched: %+v", plain)
}
// A TAB-delimited third column outside the type vocabulary is a benign unknown TYPE, exactly as it was
// before the fix-pack — NOT a piece of the rendering. Guessing it into the rendering is what the first cut
// of this fix did, and it corrupted four terms and dropped two on the frozen coldrun-a reference.
odd, _ := parseBanknote("古月\tклан Гуюэ\tсемья", attested)
if len(odd) != 1 || odd[0].Dst != "клан Гуюэ" || odd[0].Type != "term" {
t.Fatalf("an unknown TYPE column must be discarded, not glued onto the rendering: %+v", odd)
}
// And the column-order recovery still works, including over a rendering the model spaced out.
rev, _ := parseBanknote("Фан Юань\t方源\tname", attested)
if len(rev) != 1 || rev[0].Src != "方源" || rev[0].Dst != "Фан Юань" {
t.Fatalf("a reversed line must still be recovered: %+v", rev)
}
}
// TestParseBanknoteReadsDeclaredColumnsPositionally is the pin the frozen reference bought. A tab or a pipe
// is a delimiter the model CHOSE — it cannot occur inside a rendering — so such a line has declared its own
// columns and is read by position, as it was before the fix-pack. Only a line delimited by a run of spaces
// is ambiguous, and only there may the type be guessed from the closed vocabulary.
//
// Measured cost of getting this wrong: replaying 56 real banknote blocks of ~/books/gu-zhenren/coldrun-a
// through the first cut of the fix gave 6 lines against HEAD and +0 — four renderings corrupted by a glued
// type word (clan/event/proverb/onomatopoeia — 1.25% of the reference carries a type outside the
// vocabulary) and two lines dropped outright, with banknote_parse_fail unchanged in every one of them.
func TestParseBanknoteReadsDeclaredColumnsPositionally(t *testing.T) {
attested := attestedIn(srcAll)
for _, tc := range []struct{ line, src, dst, typ string }{
// An unknown third column: discarded as an unknown type, never glued (the four corrupted terms).
{"古月\tГу Юэ\tclan", "古月", "Гу Юэ", "term"},
{"方源|Фан Юань|proverb", "方源", "Фан Юань", "term"},
// The same line REVERSED: the src is still found by attestation, and the unknown type still discarded.
// The first cut looked for the src in the last field, found «clan», and dropped the line (the two lost).
{"Гу Юэ\t古月\tclan", "古月", "Гу Юэ", "term"},
// A known type column is read as a type, in both orders.
{"青茅山\tгора Цинмао\tplace", "青茅山", "гора Цинмао", "place"},
// And a rendering the model spaced out inside a REAL column arrives whole and clean.
{"方源\tФан Юань\tname", "方源", "Фан Юань", "name"},
} {
got, flags := parseBanknote(tc.line, attested)
if len(got) != 1 {
t.Fatalf("%q: want one entry, got %+v (parse_fail=%v)", tc.line, got, flags.ParseFail)
}
if got[0].Src != tc.src || got[0].Dst != tc.dst || got[0].Type != tc.typ {
t.Fatalf("%q: got %+v, want src=%q dst=%q type=%q", tc.line, got[0], tc.src, tc.dst, tc.typ)
}
}
}
// TestBanknoteTruncatedIsReachable closes the hygiene finding beside row 129: banknote_truncated was
// derived inside the parser from an argument the only production caller pinned to false, so the column was
// structurally 0 while its comment claimed otherwise. The refusal itself is the signal — a block present
// under a non-stop finish was cut by generation length — and reading it off the finish reason changes
// nothing about the ratified stop-only gate: still no candidates, still no telemetry lines.
func TestBanknoteTruncatedIsReachable(t *testing.T) {
r := &Runner{bankSrc: newBankSourceIndex([]chunk.Chunk{{Text: srcAll}})}
raw := "Незаконченный перевод\n" + bankSeparator + "\n方源\tФан"
_, _, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "length", srcAll)
if !flags.Truncated {
t.Fatalf("a block under a non-stop finish IS the truncation the column was meant to record: %+v", flags)
}
if len(entries) != 0 || flags.NLines != 0 {
t.Fatalf("the stop-only gate still accepts nothing: %+v / %+v", entries, flags)
}
// A complete generation is unaffected.
_, _, ok, got := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", srcAll)
if ok.Truncated || len(got) != 1 {
t.Fatalf("a complete generation is not truncated and its block is parsed: %+v / %+v", ok, got)
}
}

View file

@ -1,9 +1,12 @@
package pipeline
import (
"context"
"encoding/json"
"fmt"
"maps"
"slices"
"sort"
"strings"
"textmachine/backend/internal/checks"
@ -130,6 +133,7 @@ func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel memban
}
rs.NSpoilerBlocked = len(sel.Rejected)
rs.NEvicted = len(sel.Evicted)
r.recordEvicted(sel.Evicted)
// Loud record of the disposition-gated suppressor (D39 layer 4): a longer lower-trust key refused
// from eating a nested higher-trust one — the term-drift code root, now visible instead of a
// silent drop (research/13 §7). Detail carries the refused suppressor→protected pairs for a human.
@ -164,6 +168,72 @@ func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel memban
return r.Store.UpsertRetrievalState(rs)
}
// recordEvicted accumulates WHICH bank rows the injection budget dropped, book-wide, so the wave can name
// them once instead of leaving n_evicted as a number nobody can act on.
//
// The list is kept in memory and reported to the LOG rather than stored beside the counter, deliberately: a
// per-row detail column would be a schema migration, and this repository has already paid for a
// non-idempotent one (backlog row 49а). The count keeps its column; the names ride the plane an operator
// reads when deciding whether to raise gates.glossary token budget or prune the seed.
func (r *Runner) recordEvicted(evicted []membank.PickedEntry) {
if len(evicted) == 0 {
return
}
r.evictMu.Lock()
defer r.evictMu.Unlock()
if r.evictedRows == nil {
r.evictedRows = map[string]int{}
}
for _, p := range evicted {
if src := p.Src(); src != "" {
r.evictedRows[src]++
}
}
}
// evictedNameCap bounds the names one warning carries: the point is to name the rows that lose most often,
// not to reprint the bank.
const evictedNameCap = 20
// reportEvicted drains the accumulator and names the rows the budget dropped most often. Called once per
// wave, so a book that never overflows its injection budget stays silent.
//
// ⚠ Scope, stated: only the DRAFT wave feeds it, because n_evicted is a draft-wave column
// (persistRetrievalState) — the edit wave runs its own Select whose evictions this pack does not record.
func (r *Runner) reportEvicted(ctx context.Context, wave string) {
r.evictMu.Lock()
rows := r.evictedRows
r.evictedRows = nil
r.evictMu.Unlock()
if len(rows) == 0 {
return
}
type row struct {
src string
n int
}
list := make([]row, 0, len(rows))
for src, n := range rows {
list = append(list, row{src, n})
}
sort.Slice(list, func(i, j int) bool {
if list[i].n != list[j].n {
return list[i].n > list[j].n
}
return list[i].src < list[j].src
})
named := make([]string, 0, evictedNameCap)
for _, e := range list {
if len(named) == evictedNameCap {
break
}
named = append(named, fmt.Sprintf("%s ×%d", e.src, e.n))
}
r.Log.WarnContext(ctx, "memory: the injection token budget DROPPED bank rows before the model saw them — these terms had no canon on the wire for those units",
"book", r.Book.BookID, "wave", wave, "rows", len(list), "shown", len(named),
"budget_tokens", r.Pipeline.Context.GlossaryTokenBudget, "terms", strings.Join(named, ", "))
}
// boolToStoreInt maps a bool telemetry flag to the store's 0/1 integer column form.
func boolToStoreInt(b bool) int {
if b {

View file

@ -86,10 +86,17 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
// exists. They arrive as EVIDENCE — nothing proposed enters the bank without a signature. A read failure
// degrades to the WHICH-only map rather than blocking the stop: the map is what the owner needs, the
// dst is a bonus.
// The parse rule is LOGGED with the run, which is the whole contract that lets it stay out of the
// snapshot: it cannot change a paid byte, but it decides which candidate lines became evidence, so a
// signature map has to be attributable to the rule that produced it. Until the fix-pack the constant was
// declared "logged with the run" and read by nothing — the tag was a comment, not a mechanism, and two
// artifacts produced by different rules were indistinguishable.
observed, offLanguage, oerr := r.bankObservedForBook()
if oerr != nil {
r.Log.WarnContext(ctx, "bank-mining: could not read the banknote proposals; the signature map falls back to WHICH-only (bare terms)", "err", oerr)
}
r.Log.InfoContext(ctx, "bank-mining: draft-side proposals folded", "book", r.Book.BookID,
"surfaces", len(observed), "parse_version", bankParseVersion, "slice_version", bankSliceVersion)
if offLanguage > 0 {
r.Log.WarnContext(ctx, "bank-mining: draft-side proposals were written in another script and are NOT offered for signature",
"book", r.Book.BookID, "dropped", offLanguage, "target_script", r.Pipeline.Gates.Terminology.TargetScript)
@ -97,8 +104,12 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
// The TERMINOLOGIST (pack-20, D39.42): merge both channels, gather each candidate's source contexts,
// rank the renderings the drafts produced (§C2-3), and — when the gate is on — consolidate the whole
// bank in a handful of batched calls. With the gate off this is $0 assembly whose only consumer is the
// stop's own table, and the emitted delta is byte-identical to before.
// bank in a handful of batched calls. With the gate off this is $0 assembly, but NOT a no-op for the
// artifact: since the fix-pack the delta's dst comes from these ranked, target-form-folded candidates
// rather than from the raw draft-side order, so a term whose drafts disagreed can carry a different
// proposal than it did before — see proposalsFromCandidates. That is bank CONTENT, so it moves
// memory_version and the edit-wave snapshot with it (bank-only → $0 re-pin for every unit the term does
// not occur in). The draft wave is untouched: these rows are Source:"mined" and base-excluded.
tchunks := make([]terminology.Chunk, len(minerChunks))
for i, c := range minerChunks {
tchunks[i] = terminology.Chunk{Chapter: c.Chapter, ChunkIdx: c.ChunkIdx, NSource: c.NSource}
@ -146,7 +157,7 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
mined = attachConsolidatedDst(mined, consolidated)
// Non-empty delta → write the owner signature map (the mined seed-delta YAML) and STOP before the edit wave.
proposals := proposalsFromObserved(observed)
proposals := proposalsFromCandidates(cands)
withDst := 0
for _, m := range mined {
if m.Dst != "" || len(proposals[text.NormalizeSourceKey(m.Src)]) > 0 {
@ -163,7 +174,7 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
// The RICH table (D39.36's «стоп с таблицей»: src · dst · frequency · variant spread · evidence). It is
// written as a sidecar on EVERY run, signed or not, because it is also the auto mode's record of what
// the book decided on its own — and it is capped on stdout, never in the file (emitRankCap is 200).
rows := bankStopRows(cands, consolidated)
rows := bankStopRows(cands, consolidated, tres)
if werr := os.WriteFile(r.bankStopTablePath(), []byte(renderBankStopTable(rows)), 0o644); werr != nil {
r.Log.WarnContext(ctx, "bank-mining: could not write the stop table sidecar (the signature map is unaffected)", "err", werr)
}
@ -180,7 +191,7 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
// re-seeding through the ordinary seedGlossary path (rather than a second, private write) is what
// keeps ONE definition of what the bank is: every guard the seed path owns — the collision checks,
// the reject filter, the fail-louds — applies to the engine's rows exactly as to the owner's.
if err := r.writeAutoBank(mined, proposals); err != nil {
if err := r.writeAutoBank(ctx, mined, proposals, ownerHandled(unsignedEngineSurfaces(seed), rejects)); err != nil {
return false, err
}
if err := r.seedGlossary(ctx); err != nil {
@ -251,34 +262,98 @@ type BankStopRow struct {
Variants []string // "rendering ×N", best-ranked first
Contexts []string // source KWIC
Evidence []string
// The §G3 arbitration record: until this pack, «why does this term have THIS dst» was unanswerable from
// the artifacts (research/24 §A7). Every field below is read off work the ranking already did.
//
// Conventions is how many genuinely different DECISIONS the drafts made, once renderings differing only
// in target form are folded (Spread counts the raw forms). Signals are the §C2-3 factors that fired for
// the TOP-ranked variant — the winner's audit trail, printed for the row rather than per variant,
// because the row is what the owner signs. Invented says the consolidated rendering is NOT one the
// drafts proposed: legitimate (the role sees the whole book, the drafts saw fragments) and exactly the
// class to read first. Conf is the role's own stated confidence, which orders the review list and
// nothing else (D39.102) — NEGATIVE when the reply carried none, because «the role said it was 0% sure»
// is the most important row on the sheet and «the role said nothing» is not a row at all.
// Contradicts names THIS RUN's other consolidations the rendering breaks (§G2).
Conventions int
Signals []string
Invented bool
Conf int
Contradicts []string
}
// bankStopRows projects the merged candidates into the operator table, best-ranked variants first.
// Deterministic: cands is key-ordered and nothing here iterates a map for output.
func bankStopRows(cands []terminology.Candidate, consolidated map[string]string) []BankStopRow {
func bankStopRows(cands []terminology.Candidate, consolidated map[string]string, tres terminologyResult) []BankStopRow {
out := make([]BankStopRow, 0, len(cands))
for _, c := range cands {
dst := consolidated[c.Key]
row := BankStopRow{
Src: c.Src, Dst: consolidated[c.Key], Origin: string(c.Origin), Type: c.Type,
Freq: c.Freq, Spread: c.Spread(), Contexts: c.KWIC, Evidence: c.Evidence,
Src: c.Src, Dst: dst, Origin: string(c.Origin), Type: c.Type,
Freq: c.Freq, Spread: c.Spread(), Conventions: c.Conventions(),
Contexts: c.KWIC, Evidence: c.Evidence,
Conf: confOrAbsent(tres.Conf, c.Key), Contradicts: tres.Contradictions[c.Src],
}
for _, v := range c.Variants {
row.Variants = append(row.Variants, fmt.Sprintf("%s ×%d", v.Dst, v.Chunks))
for i, v := range c.Variants {
label := fmt.Sprintf("%s ×%d", v.Dst, v.Chunks)
if v.Via != "" {
label += " (proposed for " + v.Via + ")"
}
row.Variants = append(row.Variants, label)
if i == 0 {
row.Signals = v.Signals
}
}
row.Invented = dst != "" && !proposedByDrafts(dst, c.Variants)
out = append(out, row)
}
return out
}
// confOrAbsent reads the role's stated confidence for a key, or -1 when the reply carried none. A plain
// zero would merge the two, and they are opposites: one is the first row to review, the other is silence.
func confOrAbsent(conf map[string]int, key string) int {
if v, ok := conf[key]; ok {
return v
}
return -1
}
// proposedByDrafts reports whether the consolidated rendering is one the drafts actually produced, compared
// under the same target-form fold the vote is counted with — so a case or ё difference is not reported as
// an invention.
func proposedByDrafts(dst string, vs []terminology.Variant) bool {
want := text.NormalizeTargetForm(dst)
for _, v := range vs {
if text.NormalizeTargetForm(v.Dst) == want {
return true
}
}
return false
}
// renderBankStopTable serializes the FULL table for the sidecar. One block per term, the same shape the
// stdout banner prints — so the capped view and the file cannot describe the bank differently.
func renderBankStopTable(rows []BankStopRow) string {
var b strings.Builder
fmt.Fprintf(&b, "BANK VERIFICATION TABLE — %d term(s)\n", len(rows))
b.WriteString("src · proposed dst · origin · type · freq · variant spread · evidence · source contexts\n\n")
b.WriteString("src · proposed dst · origin · type · freq · variant spread · conventions · confidence ·\n")
b.WriteString("why (the ranking factors that won) · contradictions · drafts · evidence · source contexts\n\n")
for _, r := range rows {
fmt.Fprintf(&b, "%s\t%s\n", r.Src, dashIfEmpty(r.Dst))
fmt.Fprintf(&b, " origin=%s type=%s freq=%d spread=%d\n", r.Origin, dashIfEmpty(r.Type), r.Freq, r.Spread)
fmt.Fprintf(&b, " origin=%s type=%s freq=%d spread=%d conventions=%d", r.Origin, dashIfEmpty(r.Type), r.Freq, r.Spread, r.Conventions)
if r.Conf >= 0 {
fmt.Fprintf(&b, " confidence=%d", r.Conf)
}
if r.Invented {
b.WriteString(" INVENTED(no draft proposed it)")
}
b.WriteString("\n")
if len(r.Signals) > 0 {
fmt.Fprintf(&b, " why: %s\n", strings.Join(r.Signals, ", "))
}
if len(r.Contradicts) > 0 {
fmt.Fprintf(&b, " CONTRADICTS this run's own: %s\n", strings.Join(r.Contradicts, "; "))
}
if len(r.Variants) > 0 {
fmt.Fprintf(&b, " drafts: %s\n", strings.Join(r.Variants, " | "))
}
@ -348,17 +423,40 @@ func reverseSectionTerms(cands []terminology.Candidate, seed []store.GlossaryEnt
return out, eligible
}
// proposalsFromObserved re-shapes the folded draft-side view into the signature-map join's input. One
// definition of the fold (bankObservedByKey) feeds both consumers, so the map the owner signs and the
// table the terminologist read can never disagree about what a chunk proposed.
func proposalsFromObserved(obs []terminology.Observed) map[string][]miner.DstProposal {
out := make(map[string][]miner.DstProposal, len(obs))
for _, o := range obs {
list := make([]miner.DstProposal, 0, len(o.Proposals))
for _, p := range o.Proposals {
list = append(list, miner.DstProposal{Dst: p.Dst, Type: p.Type, Chunks: p.Chunks})
// proposalsFromCandidates re-shapes the MERGED candidates into the signature-map join's input.
//
// It reads the candidates rather than the raw draft-side view, and that is the fix (§G3, acceptance
// finding): a proposal that arrived under an ALIAS of a cluster is routed to the cluster's owner by
// Merge — the stop table therefore showed it — while the signature map joined on the alias's own key and
// the owner's row never mentioned it. The map the owner signs then disagreed with the table he was reading
// it against, for exactly the terms the miner clustered. One source for both removes the divergence
// structurally instead of keeping two joins in step by hand.
//
// The list arrives §C2-3-ranked and target-form folded, so the note's alternatives are the ones the table
// shows, in the order it shows them.
func proposalsFromCandidates(cands []terminology.Candidate) map[string][]miner.DstProposal {
out := make(map[string][]miner.DstProposal, len(cands))
for _, c := range cands {
if len(c.Variants) == 0 {
continue
}
out[o.Key] = list
// DIRECT proposals first, alias-routed ones after — and DeltaYAML takes the term's dst from a DIRECT
// one only. Merge routes an alias's rendering to the cluster owner so the ranking sees all of the
// entity's evidence; letting that rendering become the OWNER's dst is a different act entirely. It
// would put «Малыш Фан» on 方源 with no model involved, on the $0 path, with the terminology gate OFF —
// the exact harm Variant.Via was introduced to prevent, arriving through the fix that introduced Via.
list := make([]miner.DstProposal, 0, len(c.Variants))
for _, v := range c.Variants {
if v.Via == "" {
list = append(list, miner.DstProposal{Dst: v.Dst, Type: c.Type, Chunks: v.Chunks})
}
}
for _, v := range c.Variants {
if v.Via != "" {
list = append(list, miner.DstProposal{Dst: v.Dst, Type: c.Type, Chunks: v.Chunks, Via: v.Via})
}
}
out[c.Key] = list
}
return out
}
@ -448,7 +546,19 @@ func unsignedEngineSurfaces(rows []store.GlossaryEntry) []store.GlossaryEntry {
// writeAutoBank persists the unsigned rows the auto mode decided to carry forward, as the same seed-YAML
// schema everything else in this pipeline speaks (so `tmctl seed-lint` reads it, and a row can be moved
// into the owner's delta by copy-paste). Deterministic: the mined list is already sorted by src.
func (r *Runner) writeAutoBank(mined []miner.Term, proposals map[string][]miner.DstProposal) error {
//
// It DIFFS the file it is about to replace, and that is the $0 minimum of backlog row 130. The file is
// rewritten WHOLE from this run's delta, and the delta is capped at the miner's top-200 (emitRankCap,
// applied BEFORE the emission filters, so seed and declined terms do not free their slots). The two
// mechanisms were ratified separately and their INTERACTION never was: as a book grows, a term that was in
// the bank for twenty chapters silently vanishes from it mid-run, with nothing in the artifacts saying so.
// Naming the losers costs nothing and moves no bytes.
//
// The BOUNDARY is deliberate and is the owner's STOP: this reports, it does not accumulate. Merging the old
// file into the new one would change what the bank CONTAINS, which moves memory_version and re-prices the
// edit wave — a decision, not a hygiene fix.
func (r *Runner) writeAutoBank(ctx context.Context, mined []miner.Term, proposals map[string][]miner.DstProposal, signed map[string]bool) error {
before := r.autoBankSurfaces()
body, err := miner.DeltaYAML(mined, proposals)
if err != nil {
return fmt.Errorf("pipeline: marshal auto-bank: %w", err)
@ -456,9 +566,72 @@ func (r *Runner) writeAutoBank(mined []miner.Term, proposals map[string][]miner.
if err := os.WriteFile(r.autoBankPath(), []byte(body), 0o644); err != nil {
return fmt.Errorf("pipeline: write auto-bank %s: %w", r.autoBankPath(), err)
}
if len(before) == 0 {
return nil
}
now := make(map[string]bool, len(mined))
for _, m := range mined {
now[text.NormalizeSourceKey(m.Src)] = true
}
var gone []string
for _, src := range before {
key := text.NormalizeSourceKey(src)
if now[key] || signed[key] {
// signed[] is the owner's two verbs — PROMOTED into the mined-delta or DECLINED in mined-rejects.
// Both remove the term from this run's delta on purpose, and reporting them as a silent loss would
// fire a false alarm on the normal signature cycle — advising the owner to do what he just did.
continue
}
gone = append(gone, src)
}
if len(gone) > 0 {
sort.Strings(gone)
r.Log.WarnContext(ctx, "bank-mining: terms that were in the auto-bank are NOT in the one this run just wrote — the file is rewritten whole from a top-N-capped delta, so a term the book still uses can drop out of the bank mid-run; if one of these matters, promote it into the owner's mined-delta file (it is then a seed surface and cannot be cut again)",
"book", r.Book.BookID, "dropped", len(gone), "kept", len(mined), "rank_cap", miner.EmitRankCap(),
"terms", strings.Join(gone, ", "), "auto_bank", r.autoBankPath())
}
return nil
}
// ownerHandled is the surface set the OWNER has already decided about: every signed seed surface (a promoted
// term is one) plus every declined one. A term leaving the delta through either door is not a loss.
//
// ⚠ The caller MUST pass it through unsignedEngineSurfaces — risk 2, the self-exclusion trap this file
// documents twice and works around in two other places. From the SECOND auto-mode run the stored glossary
// also holds the engine's OWN unsigned rows (seedGlossary re-seeds the auto-bank at start), so a raw seed
// makes every term the engine ever proposed look owner-decided: the diff empties, and the row-130 warning —
// whose entire purpose is to name terms the rewrite dropped — can never fire again in production.
func ownerHandled(seed []store.GlossaryEntry, rejects map[string]bool) map[string]bool {
out := make(map[string]bool, len(seed)+len(rejects))
for _, e := range seed {
out[text.NormalizeSourceKey(e.Src)] = true
for _, a := range e.Aliases {
out[text.NormalizeSourceKey(a.Alias)] = true
}
}
for k := range rejects {
out[k] = true
}
return out
}
// autoBankSurfaces reads the src surfaces of the auto-bank file as it stands BEFORE this run rewrites it,
// in file order. Absent or unreadable → nil: the diff is observability, and failing a paid run because the
// PREVIOUS artifact cannot be parsed would be the tail wagging the dog.
func (r *Runner) autoBankSurfaces() []string {
entries, err := membank.LoadGlossarySeed(r.autoBankPath())
if err != nil {
return nil
}
out := make([]string, 0, len(entries))
for _, e := range entries {
if e.Src != "" {
out = append(out, e.Src)
}
}
return out
}
// minedRejectFile is the owner's mined-term reject list (Book.MinedRejects, R1-FL-B): the src surfaces the
// owner reviewed and DECLINED. It is a PROPOSAL filter only — rejects never enter the bank content, so this
// file is deliberately NOT folded into the snapshot (a reject affects the next mining PROPOSAL, not any

View file

@ -576,29 +576,40 @@ func TestTerminologistSpendIsInTheRunTotal(t *testing.T) {
}
}
// TestTerminologistEmptyReplyIsLoud: a paid batch that comes back unreadable (empty completion, prose,
// truncation) would otherwise be indistinguishable from «the role declined these terms» — which is a
// DECISION in §C2-7 — and the run would exit 0 having bought nothing.
// TestTerminologistEmptyReplyIsLoud: a paid batch that comes back unreadable would otherwise be
// indistinguishable from «the role declined these terms» — which is a DECISION in §C2-7 — and the run would
// exit 0 having bought nothing.
//
// TWO branches, and until the fix-pack only the first was covered while the test's NAME claimed the second
// (§G3): prose is a non-empty completion the parser rejects line by line, whereas an EMPTY completion never
// reached the parser at all and took a silent `continue`. The fixture below is the honest one.
func TestTerminologistEmptyReplyIsLoud(t *testing.T) {
var logBuf bytes.Buffer
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
if isTerminologyBody(body) {
return "Извините, я не понял задание.", "stop" // prose: no line the parser can use
}
return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
})
defer srv.Close()
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
for _, tc := range []struct{ name, reply, want string }{
{"prose", "Извините, я не понял задание.", "returned nothing the parser could use"},
{"empty completion", "", "came back with an EMPTY completion"},
} {
t.Run(tc.name, func(t *testing.T) {
var logBuf bytes.Buffer
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
if isTerminologyBody(body) {
return tc.reply, "stop"
}
return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
})
defer srv.Close()
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
r := newRunner(t, bookPath)
defer r.Close()
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
if _, err := r.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
if !strings.Contains(logBuf.String(), "returned nothing the parser could use") {
t.Fatalf("a paid batch that bought no terminology must say so:\n%s", logBuf.String())
r := newRunner(t, bookPath)
defer r.Close()
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
if _, err := r.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
if !strings.Contains(logBuf.String(), tc.want) {
t.Fatalf("a paid batch that bought no terminology must say so (%s):\n%s", tc.name, logBuf.String())
}
})
}
}

View file

@ -10,6 +10,8 @@ import (
"textmachine/backend/internal/obs"
"textmachine/backend/internal/store"
"textmachine/backend/internal/terminology"
"textmachine/backend/internal/text"
)
// minirun_fixes_test.go: the two seams the 25.07 mini-run proved were producing a fact and losing it.
@ -165,7 +167,11 @@ func TestBanknoteProposalsArePersistedAndJoined(t *testing.T) {
if len(props) != 2 {
t.Fatalf("want 2 stored proposals, got %d: %s", len(props), rs.BanknoteDetail)
}
byKey := proposalsFromObserved(bankObservedByKey([]store.RetrievalState{*rs}, nil))
// The signature-map join reads the MERGED candidates (fix-pack §G3), so the assertion follows the same
// path production takes: a proposal must land on the miner's normalized surface after the merge, not
// only in the raw draft-side view.
observed := bankObservedByKey([]store.RetrievalState{*rs}, nil)
byKey := proposalsFromCandidates(terminology.Merge(nil, observed, text.NormalizeTargetForm))
if got := byKey[props[0].SrcKey]; len(got) == 0 || got[0].Dst != props[0].Dst {
t.Fatalf("the fold must key proposals by the miner's normalized surface, got %+v", byKey)
}

View file

@ -19,6 +19,7 @@ import (
"textmachine/backend/internal/llm"
"textmachine/backend/internal/membank"
"textmachine/backend/internal/store"
"textmachine/backend/internal/terminology"
)
// runner.go: runner setup — Runner and its wiring (config stack, store in the right
@ -78,10 +79,21 @@ type Runner struct {
// .target_script (config validates the name). nil when the book declares none — the answer-language
// screen is then inert, which loadTargetScript says out loud if any channel could have used it.
targetScript *unicode.RangeTable
// familyParams is the §G1 family channel resolved from the source's declared morphology, once, at load
// time (loadFamilyParams). The zero value is a disabled channel — the batcher then co-batches series
// only, exactly as before the channel existed. Like the rest of the terminology axis it is NOT
// snapshot-folded: it changes which candidates share a CALL, never a wave byte.
familyParams terminology.FamilyParams
// rateGuards is the per-model wave-concurrency guard set (WS1 §1б), built once in the precompute pass and
// read-only in the waves — a transport axis, never snapshot-folded. nil until buildRateGuards.
rateGuards map[string]*rateGuard
// evictedRows accumulates WHICH bank rows the injection budget dropped (src → how many units), so the
// wave can NAME them once rather than leave n_evicted as an unactionable count. Written from N draft
// workers, hence the mutex; drained by reportEvicted.
evictMu sync.Mutex
evictedRows map[string]int
// bankSrc is the banknote parser's "is this surface in the book?" index (backlog 19), built once from
// the chunk manifest in the precompute pass. nil outside a book run → the parser then judges a line by
// the chunk text it was handed, and accepts nothing without one.

View file

@ -117,6 +117,9 @@ type projectOpts struct {
// rebillConsentUSD writes book.yaml's `rebill_consent_usd` override (D20.2-Q2). 0 = omit the key, so
// the book takes the ratified default threshold and every pre-existing fixture is byte-unchanged.
rebillConsentUSD float64
// glossaryTokenBudget overrides context.glossary_token_budget. 0 keeps the fixture's historical 800, so
// every pre-existing project is byte-identical; a tiny value is how the EVICTION path is reachable at all.
glossaryTokenBudget int
}
func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string {
@ -130,6 +133,9 @@ func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string {
if o.bookUSD == 0 {
o.bookUSD = 1.0
}
if o.glossaryTokenBudget == 0 {
o.glossaryTokenBudget = 800
}
dir := t.TempDir()
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
@ -163,12 +169,12 @@ core: C1
version: 1
defaults: { max_output_ratio: 2.0, min_max_tokens: %d }
retries: { regenerate_before_escalate: %d }
context: { glossary_injection: selective, glossary_token_budget: 800 }
context: { glossary_injection: selective, glossary_token_budget: %d }
waves: { workers: %d }
stages:
- { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
- { name: edit, role: editor, model: fake-model, prompt_override: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
%s`, o.minMaxTokens, o.regenerate, o.waveWorkers, gatesBlock))
%s`, o.minMaxTokens, o.regenerate, o.glossaryTokenBudget, o.waveWorkers, gatesBlock))
sourceName := "source.txt"
if len(o.epub) > 0 {

View file

@ -57,7 +57,7 @@ const terminologyStageName = "terminology"
// terminologyVersion versions the ASSEMBLY algorithm (merge → KWIC → §C2-3 ranking → batching → parse). It
// is deliberately NOT snapshot-folded — see config.TerminologyGate — but it is logged with the run so a
// signature map can be attributed to the algorithm that produced it.
const terminologyVersion = "terminology-v2-merge+kwic+c2-3+series"
const terminologyVersion = "terminology-v3-merge+kwic+c2-3+series+families+formfold"
// Engine defaults for the block sizing. They bound ONE call's input; the whole book is covered by batching.
const (
@ -83,14 +83,29 @@ type terminologyResult struct {
// OBSERVABILITY, never a gate: the rows stay unverified either way, and this is what tells the owner
// which of them to look at first.
CanonConflicts int
CostUSD float64 // what THIS run's RENDER phase paid
CumUSD float64 // what the render calls cost in total (a replayed checkpoint is $0 now, not then)
Fresh bool // at least one call actually reached the provider this run
EstimateUSD float64 // the pre-call projection, logged before any money moves
// SelfConflicts counts consolidations that contradict ANOTHER consolidation of the same run (§G2) — the
// majority class on a live bank, and the one nothing looked at before this pack. Contradictions carries
// the same finding per key, so the stop table can mark the rows instead of printing a bare total.
SelfConflicts int
Contradictions map[string][]string
// Conf is the role's own stated confidence per key. It sorts the review list «least sure first» and does
// nothing else — never a weight, a threshold or a cross-model comparison (D39.102).
Conf map[string]int
CostUSD float64 // what THIS run's RENDER phase paid
CumUSD float64 // what the render calls cost in total (a replayed checkpoint is $0 now, not then)
Fresh bool // at least one call actually reached the provider this run
EstimateUSD float64 // the pre-call projection, logged before any money moves
// Reclassified is how many candidate types the §2 classifier phase actually changed; ClassifyCostUSD is
// what that phase paid this run. Both zero when classify_types is off.
Reclassified int
ClassifyCostUSD float64
// Families is how many family GROUPS §G1 detected; FamiliesRefused how many of their merges the member
// cap turned down, and FamiliesHeld how many a series held part of back. Both of the latter mean the same
// thing to the owner — a family met split across two calls — and both are zero when the source declares
// no family data.
Families int
FamiliesRefused int
FamiliesHeld int
}
// loadTerminologyTemplate loads the pair's terminologist prompt when the gate is on. Mirrors
@ -104,9 +119,42 @@ func (r *Runner) loadTerminologyTemplate() error {
return err
}
r.terminologyTemplate = tpl
if err := r.loadFamilyParams(); err != nil {
return err
}
return r.loadClassifierTemplate()
}
// loadFamilyParams resolves the §G1 family channel from the SOURCE language's declared morphology, once, at
// LOAD time — before any billing, like every other gate precondition. A source with no family data yields a
// disabled channel and the batcher behaves exactly as it did before the channel existed.
//
// It is also where a data typo dies: the file names engine TYPES, and lang cannot check them against the
// engine's closed vocabulary without depending on this layer. A rule for a type nothing emits would parse
// fine and leave the channel quietly half-off — the same silent-empty-table class the pack loader refuses.
func (r *Runner) loadFamilyParams() error {
fm := lang.FamilyMorphology(r.Book.SourceLang)
if !fm.Enabled() {
r.familyParams = terminology.FamilyParams{}
return nil
}
sEnabled, headFinal := lang.SeriesMorphology(r.Book.SourceLang)
p := terminology.FamilyParams{
Enabled: sEnabled, HeadFinal: headFinal,
Affix: make(map[string]terminology.FamilyAffix, len(fm.Affix)),
MinMembers: fm.MinMembers, MaxMembers: fm.MaxMembers, ContainmentRunes: fm.ContainmentRunes,
}
for typ, a := range fm.Affix {
if !terminology.CandidateTypes[typ] {
return fmt.Errorf("pipeline: the family morphology of source %q names type %q, which no candidate can carry (accepted: %s) — a typo here would parse fine and leave the family channel silently half-off",
r.Book.SourceLang, typ, strings.Join(terminology.TypeNames(terminology.CandidateTypes), "|"))
}
p.Affix[typ] = terminology.FamilyAffix{Suffix: a.Suffix, MinRunes: a.MinRunes}
}
r.familyParams = p
return nil
}
// loadClassifierTemplate loads the pair's §2 classifier prompt when classify_types is on. Off → nil, and the
// classifier phase is inert. Called from loadTerminologyTemplate: the classifier only exists as a phase of
// the terminology gate.
@ -179,7 +227,11 @@ func (r *Runner) buildBankCandidates(mined []miner.Term, observed []terminology.
Freq: m.Freq, SinceCh: m.SinceCh, Aliases: m.Aliases, Evidence: m.Evidence,
})
}
cands := terminology.Merge(ms, observed)
// The vote is counted per target-form CONVENTION, not per byte string (§G5): «Море истинной ци» and
// «море истинной ци» are one decision, and splitting their evidence hands the §C2-3 frequency factor to
// whichever spelling a chunk happened to repeat. The normalizer is the SAME one the post-check matches
// against, so the fold cannot disagree with the check that reads the result.
cands := terminology.Merge(ms, observed, text.NormalizeTargetForm)
_, kwicPer, kwicWidth := r.terminologyOpts()
cands = terminology.AttachKWIC(cands, chunks, kwicPer, kwicWidth)
@ -262,20 +314,31 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
}
}
// §1: co-batch grade/rank series so the role picks ONE generic head for the whole set. The pair-data —
// whether the source forms rune-morpheme series and where the head sits — comes from the language layer,
// so the batcher stays pair-agnostic and an alphabetic source takes a byte-identical, series-free path.
// §1 + §G1: co-batch each grade/rank SERIES and each term FAMILY so the role picks ONE generic head, and
// ONE shared root, for the whole set. The pair-data — whether the source forms rune-morpheme series,
// where the head sits, and which side of a surface carries a family's root — comes from the language
// layer, so the batcher stays pair-agnostic and a source with neither takes a byte-identical path.
sEnabled, headFinal := lang.SeriesMorphology(r.Book.SourceLang)
seriesID := terminology.DetectSeries(cands, terminology.SeriesParams{Enabled: sEnabled, HeadFinal: headFinal})
batches := terminology.Batch(cands, batchRunes, seriesID)
fp := r.familyParams
fams := terminology.DetectFamilies(cands, fp)
unitID, ustats := terminology.MergeUnits(seriesID, fams, fp)
batches := terminology.Batch(cands, batchRunes, unitID)
res.Batches = len(batches)
// §1 keeps a series (or a lone candidate) WHOLE even past the budget — splitting a series would defeat the
// one-head co-batching it exists for — so a co-batched grade set or an evidence-heavy single term can render
res.Families, res.FamiliesRefused, res.FamiliesHeld = len(fams), ustats.Refused, ustats.Held
if ustats.Refused > 0 || ustats.Held > 0 {
// Either guard leaves a family split across calls — the very defect this channel exists to close — so
// both are named rather than left to be inferred from a bank that disagrees with itself.
r.Log.WarnContext(ctx, "terminology: some families were NOT co-batched whole — the member cap refused the merge, or a series with an unrelated root kept its members; those families can still disagree with themselves across calls",
"book", r.Book.BookID, "refused_by_cap", ustats.Refused, "held_by_series", ustats.Held, "max_members", fp.MaxMembers)
}
// §1/§G1 keep a unit (or a lone candidate) WHOLE even past the budget — splitting one would defeat the
// co-batching it exists for — so a co-batched grade set or an evidence-heavy single term can render
// over the cap. That is deliberate but NOT silent: an over-cap unit strains the model's output ceiling (the
// cap-8000 mine), so name it while the run can still be watched.
for i, b := range batches {
if br := terminology.BatchRunes(b); br > batchRunes {
r.Log.WarnContext(ctx, "terminology: a batch renders OVER the size budget and is sent WHOLE (a series is co-batched by design, §1; a lone candidate cannot be split) — watch the model's output cap on this call",
r.Log.WarnContext(ctx, "terminology: a batch renders OVER the size budget and is sent WHOLE (a series/family is co-batched by design, §1/§G1; a lone candidate cannot be split) — watch the model's output cap on this call",
"book", r.Book.BookID, "batch", i, "terms", len(b), "batch_runes", br, "budget_runes", batchRunes)
}
}
@ -293,13 +356,26 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
res.EstimateUSD, res.CostUSD, res.CumUSD, res.Fresh = run.estimateUSD, run.costUSD, run.cumUSD, run.fresh
out := map[string]string{}
res.Conf = map[string]int{}
for i, b := range batches {
if i >= run.attempted {
break // the budget or a ceiling stopped the pass here; runBankRoleBatches already said so
}
if run.texts[i] == "" {
// An EMPTY completion on a call that was actually made. It used to `continue` in silence, which is
// how a paid batch that returned nothing became indistinguishable from «the role declined these
// terms» — a DECISION in §C2-7 — with the run exiting 0.
r.Log.WarnContext(ctx, "terminology: a paid batch came back with an EMPTY completion; its terms stay unconsolidated",
"book", r.Book.BookID, "batch", i, "asked", len(b), "answered", 0)
continue
}
got, st := terminology.ParseReply(run.texts[i], candKeys(b), text.NormalizeSourceKey, r.targetScript)
got, conf, st := terminology.ParseReply(run.texts[i], candKeys(b), text.NormalizeSourceKey, r.targetScript)
res.BadLines += st.Bad
res.OffLanguage += st.OffLanguage
// Asked-vs-answered per batch: the one number that separates «the model skipped half the block» from
// «the parser refused half the lines», and neither is visible in a total.
r.Log.InfoContext(ctx, "terminology render batch", "book", r.Book.BookID,
"batch", i, "asked", len(b), "answered", len(got), "bad_lines", st.Bad, "reply_chars", len(run.texts[i]))
// A batch answered largely in another language is the measured cold-start failure, not noise: say it
// while the run is happening, naming the lines, because the terms themselves just stay auto.
if st.OffLanguage > 0 {
@ -317,6 +393,9 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
for k, v := range got {
out[k] = v
}
for k, v := range conf {
res.Conf[k] = v
}
}
for _, c := range cands {
v, answered := out[c.Key]
@ -341,6 +420,21 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
r.Log.WarnContext(ctx, "terminology: consolidated renderings contradict the SIGNED bank; they stay unverified (⟨проверить⟩) and are the first rows to review at the stop",
"book", r.Book.BookID, "conflicts", len(conflicts), "terms", strings.Join(named, "; "))
}
// §G2: the same check with the run's OWN consolidations on the right-hand side. On a live bank this is
// the MAJORITY of the contradictions (18 of 149, research/24 §A4) and until now nobody looked: the canon
// check can only see rows the owner already signed, and a book being consolidated for the first time has
// almost none. $0, evidence-side, never a gate.
if self := terminology.ConsolidationConflicts(cands, out); len(self) > 0 {
res.SelfConflicts = len(self)
res.Contradictions = map[string][]string{}
named := make([]string, 0, len(self))
for _, cf := range self {
named = append(named, fmt.Sprintf("%s→%q drops %s→%q", cf.Src, cf.Dst, cf.PartSrc, cf.PartDst))
res.Contradictions[cf.Src] = append(res.Contradictions[cf.Src], fmt.Sprintf("%s→%q", cf.PartSrc, cf.PartDst))
}
r.Log.WarnContext(ctx, "terminology: consolidations of THIS run contradict each other — a compound's rendering drops the rendering the same reply gave its own part; they stay unverified and are review rows at the stop",
"book", r.Book.BookID, "conflicts", len(self), "terms", strings.Join(named, "; "))
}
// The $0 label screen (§2 warm-run hygiene): a name/place row whose rendering was clearly TRANSLATED is a
// label/rendering disagreement worth a human's eye. It is a review FLAG, never a gate, and explicitly not
// a safety net for the transliteration harm — the classifier phase is what prevents that (see
@ -356,7 +450,9 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
r.Log.InfoContext(ctx, "terminology finished", "book", r.Book.BookID,
"consolidated", res.Consolidated, "declined", res.Declined, "unanswered", res.Unanswered,
"reclassified", res.Reclassified, "bad_lines", res.BadLines, "off_language", res.OffLanguage,
"canon_conflicts", res.CanonConflicts, "cost_usd", fmt.Sprintf("%.6f", res.CostUSD),
"canon_conflicts", res.CanonConflicts, "self_conflicts", res.SelfConflicts,
"families", res.Families, "families_refused", res.FamiliesRefused, "families_held", res.FamiliesHeld,
"cost_usd", fmt.Sprintf("%.6f", res.CostUSD),
"classify_cost_usd", fmt.Sprintf("%.6f", res.ClassifyCostUSD))
return out, classified, res, nil
}
@ -400,11 +496,21 @@ func (r *Runner) runClassifier(ctx context.Context, snapID string, cands []termi
out := map[string]string{}
bad := 0
for i, b := range batches {
if i >= run.attempted {
break // the budget stopped the pass here, and it already said so
}
if run.texts[i] == "" {
// The same silence the render phase carried: a paid classify batch that returned NOTHING left every
// one of its terms on the draft heuristic type — the mistyping this phase exists to remove — and the
// only trace was that `bad` stayed 0, which reads as a clean pass.
r.Log.WarnContext(ctx, "terminology classify: a paid batch came back with an EMPTY completion; its terms keep the draft heuristic type",
"book", r.Book.BookID, "batch", i, "asked", len(b), "answered", 0)
continue
}
got, st := terminology.ParseTypes(run.texts[i], candKeys(b), text.NormalizeSourceKey)
bad += st.Bad
r.Log.InfoContext(ctx, "terminology classify batch", "book", r.Book.BookID,
"batch", i, "asked", len(b), "answered", len(got), "bad_lines", st.Bad)
for k, v := range got {
out[k] = v
}
@ -571,7 +677,11 @@ type bankRolePlan struct {
// bankRoleRun is what one role's pass produced: each batch's reply text (in batch order, "" for a batch the
// budget cut or that failed soft), plus the cost accounting for the report.
type bankRoleRun struct {
texts []string
texts []string
// attempted is how many batches the pass actually reached before a budget ceiling or a soft denial stopped
// it. Without it "" is ambiguous — an EMPTY completion the run paid for and a batch never called look the
// same — and the caller cannot warn about the first without crying wolf about the second.
attempted int
estimateUSD float64
costUSD float64
cumUSD float64
@ -597,7 +707,8 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban
}
r.Log.InfoContext(ctx, "terminology "+logKind+": estimate before any call",
"book", r.Book.BookID, "role", plan.role, "batches", len(batches), "model", st.Model, "reasoning", st.Reasoning,
"estimate_usd", fmt.Sprintf("%.6f", run.estimateUSD), "budget_usd", plan.budgetUSD, "version", terminologyVersion)
"estimate_usd", fmt.Sprintf("%.6f", run.estimateUSD), "budget_usd", plan.budgetUSD,
"version", terminologyVersion, "bank_data", lang.BankDataVersion())
spent, err := r.Store.RoleSpentUSD(r.Book.BookID, plan.role)
if err != nil {
@ -638,6 +749,7 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban
run.fresh = run.fresh || att.freshCall
spent += att.runCost
run.texts[i] = att.text
run.attempted = i + 1
}
return run, nil
}

View file

@ -127,6 +127,8 @@ func (r *Runner) translateBookWaves(ctx context.Context, chunks []chunk.Chunk, s
return nil, err
}
r.reportEvicted(ctx, "draft")
// --- the bank-mining stop: bank-mining stop boundary (auto-continues when mining is not configured) ---
if stopped, err := r.runBankMiningStop(ctx, chunks, draftSnapshot, editWave); err != nil {
return nil, err

View file

@ -0,0 +1,215 @@
package terminology
import (
"strings"
"testing"
)
// arbitration_test.go: the §G2/§G3/§G5 half of the fix-pack — the run reading its OWN output (self
// contradictions), the confidence field, and the vote counted per convention instead of per byte string.
// lowerFold is a stand-in for the target-form normalizer the pipeline injects: enough of it (case and
// whitespace) to exercise the fold without importing a target's morphology into this package.
func lowerFold(s string) string { return strings.Join(strings.Fields(strings.ToLower(s)), " ") }
// TestConsolidationConflictsCatchesTheRunContradictingItself is the §G2 case measured on the live bank: the
// same reply consolidated a part and then rendered the compound without it. The canon check cannot see this
// — nothing here is signed — and until this pack nothing else looked either.
func TestConsolidationConflictsCatchesTheRunContradictingItself(t *testing.T) {
cands := []Candidate{
{Key: "元海", Src: "元海"},
{Key: "元海空窍", Src: "元海空窍"},
{Key: "空窍", Src: "空窍"},
{Key: "青茅山", Src: "青茅山"},
}
out := map[string]string{
"元海": "море истинной ци",
"空窍": "апертура",
"元海空窍": "апертура Первозданного моря", // carries «апертура», DROPS «истинной ци»
"青茅山": "гора Цинмао",
}
got := ConsolidationConflicts(cands, out)
if len(got) != 1 {
t.Fatalf("exactly one contradiction (the compound against 元海), got %+v", got)
}
if got[0].Src != "元海空窍" || got[0].PartSrc != "元海" {
t.Fatalf("the conflict must name the compound and the part it dropped: %+v", got[0])
}
// A compound that DOES carry both parts is not a conflict, case endings and all.
out["元海空窍"] = "апертура моря истинной ци"
if got := ConsolidationConflicts(cands, out); len(got) != 0 {
t.Fatalf("a rendering carrying every part is consistent, got %+v", got)
}
// And a term nobody consolidated cannot contradict anything (silence is not a decision).
delete(out, "元海")
out["元海空窍"] = "апертура Первозданного моря"
if got := ConsolidationConflicts(cands, out); len(got) != 0 {
t.Fatalf("an unconsolidated part is not a contradiction, got %+v", got)
}
}
// TestParseReplyTakesTheConfidenceFieldBeforeRejoining is the §G3 parser trap, both directions: a THIRD
// field of digits is the role's confidence and must come off before the rendering is re-joined, while a
// rendering that merely ENDS in a number is one field and must survive whole.
func TestParseReplyTakesTheConfidenceFieldBeforeRejoining(t *testing.T) {
id := func(s string) string { return s }
keys := []string{"方源", "花家", "一零八峰", "忘却"}
reply := strings.Join([]string{
"方源\tФан Юань\t95",
"花家\tДом Хуа\t40", // a stray double space AND a confidence: both handled, or the name is halved
"一零八峰\tПик 108", // a rendering ending in a number, single space → one field, untouched
"忘却\tзабвение\t900", // a declared column that is not a confidence: dropped and counted, never glued
}, "\n")
got, conf, st := ParseReply(reply, keys, id, nil)
if got["方源"] != "Фан Юань" || conf["方源"] != 95 {
t.Fatalf("want the rendering without the confidence and the confidence beside it: %q / %d", got["方源"], conf["方源"])
}
if got["花家"] != "Дом Хуа" || conf["花家"] != 40 {
t.Fatalf("a split rendering must be re-joined AFTER the confidence comes off: %q / %d", got["花家"], conf["花家"])
}
if got["一零八峰"] != "Пик 108" {
t.Fatalf("a rendering that legitimately ends in a number must survive whole, got %q", got["一零八峰"])
}
if _, has := conf["一零八峰"]; has {
t.Fatalf("there was no confidence field on that line: %v", conf)
}
if got["忘却"] != "забвение" || st.BadConfidence != 1 {
t.Fatalf("an unreadable confidence column is counted, never glued onto the rendering: %q / %d", got["忘却"], st.BadConfidence)
}
if st.Bad != 0 {
t.Fatalf("a bad confidence does not refuse the rendering, got %d bad", st.Bad)
}
// The whole class the third column opened: anything the model writes there that is not a bare 0..100 —
// «95%», «0.9», «высокая» — used to ride into the rendering, pass wellFormedLemma, pass the answer-language
// screen, and enter the bank as this book's canon with bad_lines reading 0.
for _, tail := range []string{"95%", "95 %", "0.9", "~90", "высокая"} {
g, c, s2 := ParseReply("师父\tнаставник\t"+tail, []string{"师父"}, id, nil)
if g["师父"] != "наставник" {
t.Fatalf("tail %q was glued onto the rendering: %q", tail, g["师父"])
}
if _, has := c["师父"]; has {
t.Fatalf("tail %q must not be read as a confidence: %v", tail, c)
}
if s2.BadConfidence != 1 {
t.Fatalf("tail %q must be COUNTED, or the prompt's third column can fail silently: %+v", tail, s2)
}
}
// THE SPACE-RUN PATH, which every case above misses: all of them are TAB-delimited, so the confidence is
// stripped by the positional column reader and the space-run branch is never executed. Its own stripping
// could be removed without a single test noticing — and then a model that answers with spaces instead of
// tabs (the reason that tolerance exists at all) banks «Фан Юань 85» as this book's canon.
spaced, sconf, sst := ParseReply("方源 Фан Юань 85", []string{"方源"}, id, nil)
if spaced["方源"] != "Фан Юань" {
t.Fatalf("a space-delimited line must lose its confidence field, not bank it: %q", spaced["方源"])
}
if sconf["方源"] != 85 {
t.Fatalf("and the confidence must be read: %v", sconf)
}
if sst.Bad != 0 {
t.Fatalf("the line is usable, got %d bad", sst.Bad)
}
// A two-field reply — the shape every existing pair prompt asks for — is unchanged and carries no
// confidence at all, so the field is additive rather than a new requirement.
plain, conf2, _ := ParseReply("方源\tФан Юань", []string{"方源"}, id, nil)
if plain["方源"] != "Фан Юань" || len(conf2) != 0 {
t.Fatalf("a two-field reply must parse as before with no confidence: %q / %v", plain["方源"], conf2)
}
}
// TestParseReplyAcceptsAMangledDeclineSentinel: declining is a DECISION (§C2-7), and a decline the parser
// refuses is counted as a broken line instead — the term then reads as "the model produced garbage" rather
// than "the model said it could not tell".
func TestParseReplyAcceptsAMangledDeclineSentinel(t *testing.T) {
id := func(s string) string { return s }
keys := []string{"方源", "花家", "青茅山"}
reply := "方源\t" + NoDst + "\n花家\t«" + NoDst + "»\n青茅山\t" + NoDst + "."
got, _, st := ParseReply(reply, keys, id, nil)
for _, k := range keys {
v, answered := got[k]
if !answered || v != "" {
t.Fatalf("%s must read as an explicit decline, got %q (answered=%v)", k, v, answered)
}
}
if st.Bad != 0 {
t.Fatalf("a decline is not a parse failure, got %d bad lines", st.Bad)
}
// Narrow on purpose: a rendering that merely MENTIONS something is not a decline.
other, _, _ := ParseReply("方源\tвариант "+NoDst, []string{"方源"}, id, nil)
if other["方源"] == "" {
t.Fatal("only a rendering that IS the sentinel declines; a prefix match would swallow real answers")
}
}
// TestFoldVariantsCountsConventionsAndShowsRawForms is §G5. Before it, «Море истинной ци» and «море
// истинной ци» were two variants: the §C2-3 frequency factor scored the consensus on half its evidence, and
// the spread column reported a contest where there was a spelling difference.
func TestFoldVariantsCountsConventionsAndShowsRawForms(t *testing.T) {
observed := []Observed{{Key: "元海", Src: "元海", Proposals: []Proposal{
{Dst: "море истинной ци", Chunks: 3},
{Dst: "Море истинной ци", Chunks: 4}, // same convention, different case
{Dst: "Первозданное море", Chunks: 5},
}}}
got := Merge(nil, observed, lowerFold)
if len(got) != 1 {
t.Fatalf("one surface, one candidate: %+v", got)
}
c := got[0]
if c.Conventions() != 2 {
t.Fatalf("two decisions were made, not three: %+v", c.Variants)
}
if c.Spread() != 3 {
t.Fatalf("the drafts still wrote it three ways and the owner must see that: spread=%d", c.Spread())
}
var folded *Variant
for i := range c.Variants {
if strings.EqualFold(c.Variants[i].Dst, "море истинной ци") {
folded = &c.Variants[i]
}
}
if folded == nil {
t.Fatalf("the folded class must be present as a RAW form, not a normalized one: %+v", c.Variants)
}
if folded.Chunks != 7 {
t.Fatalf("the vote is the sum of the class, got %d", folded.Chunks)
}
if folded.Dst != "Море истинной ци" {
t.Fatalf("the representative is the most-proposed RAW form, got %q", folded.Dst)
}
// And the fold decides the ranking: 7 folded chunks now outweigh the 5 that used to win on 4-vs-5.
ScoreVariants(&c, ScoreOpts{})
if !strings.EqualFold(c.Best(), "море истинной ци") {
t.Fatalf("the consensus must stop losing to its own spelling variant, got %q", c.Best())
}
}
// TestFoldVariantsKeepsViaProvenanceHonest: a rendering that ALSO arrived on the candidate's own key is
// direct. First-wins used to label such a consensus «proposed for <alias>» purely because the alias row was
// seen first — presenting an agreed rendering as a mis-clustered alias's guess.
func TestFoldVariantsKeepsViaProvenanceHonest(t *testing.T) {
mined := []Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子", "小方"}}}
observed := []Observed{
{Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 1}}}, // through an alias, FIRST
{Key: "方源", Src: "方源", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 6}}}, // and directly
{Key: "小方", Src: "小方", Proposals: []Proposal{{Dst: "малыш Фан", Chunks: 2}}}, // alias only
}
for _, c := range Merge(mined, observed, lowerFold) {
if c.Key != "方源" {
continue
}
for _, v := range c.Variants {
switch v.Dst {
case "Фан Юань":
if v.Via != "" {
t.Fatalf("a rendering also proposed on the term's own key is DIRECT, got via=%q", v.Via)
}
if v.Chunks != 7 {
t.Fatalf("the alias vote still counts toward the total, got %d", v.Chunks)
}
case "малыш Фан":
if v.Via != "小方" {
t.Fatalf("a rendering that ONLY ever came through an alias must keep saying so, got %q", v.Via)
}
}
}
}
}

View file

@ -1,6 +1,9 @@
package terminology
import "strings"
import (
"sort"
"strings"
)
// classify.go: the TYPE re-derivation channel (bank-quality §2, D39.68). The draft type heuristic is wrong
// 1222% (D39.65 row 36a), and type ∈ {name,place} routes a candidate to transliteration — so a realia
@ -9,11 +12,30 @@ import "strings"
// WITH the rendering arrives too late to de-bias it). This file is the pure half — the reply parser and the
// honest $0 label screen; the class DEFINITIONS live in the pair's classifier prompt, no pair literal here.
// Types is the engine's closed type vocabulary. The names are engine identifiers (the miner emits them, the
// glossary stores them, emissionEligible gates on them), so they are Go constants here, not pair data; the
// pair's prompt explains each class in its own language with its own examples.
// Types is the engine's closed type vocabulary for the CLASSIFIER's answer. The names are engine identifiers
// (the miner emits them, the glossary stores them, emissionEligible gates on them), so they are Go constants
// here, not pair data; the pair's prompt explains each class in its own language with its own examples.
var Types = map[string]bool{"name": true, "place": true, "title": true, "term": true}
// CandidateTypes is the wider set a CANDIDATE can actually carry, and it is not the same set — which is a
// live seam, not pedantry. The banknote channel accepts `nickname` from a draft (pipeline.bankTypeOK), that
// type rides into Observed and onto a draft-side-only candidate, and the classifier never overwrites a term
// it was not asked about. So anything keyed on a candidate's type — the family rules, first — must be keyed
// on THIS set: validating against Types alone rejects a legitimate pair rule for `nickname` while the type
// keeps arriving, i.e. refuses the fix and keeps the defect.
var CandidateTypes = map[string]bool{"name": true, "place": true, "title": true, "term": true, "nickname": true}
// TypeNames lists a type set in a stable order, so an error message can say what it actually accepts instead
// of a hardcoded list that drifts from the map beside it.
func TypeNames(set map[string]bool) []string {
out := make([]string, 0, len(set))
for t := range set {
out = append(out, t)
}
sort.Strings(out)
return out
}
// ParseTypes turns a classifier reply into key → corrected type, keeping only the terms we asked about and
// only the closed vocabulary. Same tolerant two-field line format and the same accounting discipline as
// ParseReply: an unusable or off-vocabulary line is COUNTED (st.Bad), never silently dropped, so a paid call

View file

@ -0,0 +1,348 @@
package terminology
import (
"fmt"
"reflect"
"sort"
"strings"
"testing"
)
// family_test.go: the §G1 family channel. What it has to prove is not "groups form" but the four properties
// the measurement asked for — a family stays whole across a batch boundary, a variable-length rank line
// stays whole, an equal-length series does not regress, and neither of them turns into a blob.
// zhFamily mirrors the han rows the bank-data plane ships (internal/lang/bankdata/family-morphology.txt):
// names lead with the clan morpheme, realia end with the generic head, both need two runes of root, a family
// is two surfaces, a unit is at most 24.
var zhFamily = FamilyParams{
Enabled: true, HeadFinal: true,
Affix: map[string]FamilyAffix{
"name": {MinRunes: 2},
"nickname": {MinRunes: 2},
"place": {Suffix: true, MinRunes: 2},
"title": {Suffix: true, MinRunes: 2},
"term": {Suffix: true, MinRunes: 2},
},
MinMembers: 2, MaxMembers: 24, ContainmentRunes: 2,
}
func typed(key, typ string, freq int) Candidate {
return Candidate{Key: key, Src: key, Type: typ, Freq: freq}
}
// unitsOf inverts a unit map into sorted member lists, for readable assertions.
func unitsOf(unitID map[string]int) map[int][]string {
out := map[int][]string{}
for k, id := range unitID {
out[id] = append(out[id], k)
}
for _, v := range out {
sort.Strings(v)
}
return out
}
func sameUnit(t *testing.T, unitID map[string]int, keys ...string) int {
t.Helper()
id := unitID[keys[0]]
if id == 0 {
t.Fatalf("%s is in no unit: %v", keys[0], unitsOf(unitID))
}
for _, k := range keys[1:] {
if unitID[k] != id {
t.Fatalf("%s must share a unit with %s (%d vs %d): %v", k, keys[0], unitID[k], id, unitsOf(unitID))
}
}
return id
}
// TestDetectFamiliesGroupsBySharedRootAsymmetrically is the measured case (research/24 §B4): the 古月 family
// is eight surfaces of DIFFERENT lengths, which DetectSeries cannot see, and the drafts rendered it two ways
// on either side of a batch boundary. The clan morpheme LEADS a name and the generic head ENDS a realia
// term, and the asymmetry is data — so the same string in the other position forms nothing.
func TestDetectFamiliesGroupsBySharedRootAsymmetrically(t *testing.T) {
cands := []Candidate{
typed("古月方源", "name", 40), typed("古月正", "name", 9), typed("古月赤练", "name", 5),
typed("元海空窍", "term", 7), typed("天海空窍", "term", 4), // realia sharing the trailing 海空窍
typed("方源古月", "name", 2), // the clan morpheme TRAILING a name: not a family under the name rule
}
fams := DetectFamilies(cands, zhFamily)
byAnchor := map[string][]string{}
for _, f := range fams {
byAnchor[f.Anchor] = f.Keys
}
if got := byAnchor["古月"]; len(got) != 3 {
t.Fatalf("the clan family must hold its three names, got %v (all: %v)", got, byAnchor)
}
if got := byAnchor["海空窍"]; len(got) != 2 {
t.Fatalf("two realia sharing the trailing 海空窍 are one family, got %v", got)
}
// The asymmetry is real: 方源古月 shares 古月 as a SUFFIX, and the name rule reads prefixes.
for _, f := range fams {
if f.Anchor == "古月" {
for _, k := range f.Keys {
if k == "方源古月" {
t.Fatal("a name sharing the clan morpheme as a SUFFIX must not join the prefix family")
}
}
}
}
}
// TestFamilyKeepsAVariableLengthRankLineWhole is the acceptance criterion of the fix-pack's own §2(а): the
// 1..12 rank line must be ONE batch unit. 一转…九转 is an equal-length series; 十一转 and 十二转 are three
// runes long and so fall outside it, and on the live corpus they landed in another batch — a rank scale
// split in half is exactly the chimera the co-batching exists to prevent.
func TestFamilyKeepsAVariableLengthRankLineWhole(t *testing.T) {
var cands []Candidate
for _, k := range []string{"一转", "二转", "三转", "四转", "五转", "六转", "七转", "八转", "九转", "十一转", "十二转"} {
cands = append(cands, typed(k, "term", 5))
}
cands = append(cands, typed("中间", "term", 90)) // an unrelated term, so a whole-list unit would prove nothing
sort.Slice(cands, func(i, j int) bool { return cands[i].Key < cands[j].Key })
seriesID := DetectSeries(cands, zhSeries)
if seriesID["十一转"] != 0 {
t.Fatal("test premise broken: a three-rune surface cannot be in an equal-length series")
}
unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
if st.Refused != 0 || st.Held != 0 {
t.Fatalf("nothing here should hit a guard: %+v", st)
}
id := sameUnit(t, unitID, "一转", "二转", "九转", "十一转", "十二转")
if unitID["中间"] == id {
t.Fatal("an unrelated term must not be swept into the rank unit")
}
// And the unit survives the batcher under a budget so tight every singleton is its own batch.
batches := Batch(cands, 10, unitID)
var rank []Candidate
for _, b := range batches {
for _, c := range b {
if unitID[c.Key] == id {
rank = b
}
}
}
if len(rank) != 11 {
t.Fatalf("the whole 1..12 line must ride ONE call, got %d: %v", len(rank), rank)
}
}
// TestFamilyDoesNotRegressAnEqualLengthSeries: the other half of the same criterion. 甲等/乙等/丙等 are two
// runes, so no two-rune root is a PROPER affix of them and the family channel has nothing to say; the series
// must come through untouched.
func TestFamilyDoesNotRegressAnEqualLengthSeries(t *testing.T) {
cands := []Candidate{typed("甲等", "term", 3), typed("乙等", "term", 3), typed("丙等", "term", 3), typed("丁等", "term", 3), typed("中间", "term", 50)}
seriesID := DetectSeries(cands, zhSeries)
unitID, _ := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
id := sameUnit(t, unitID, "甲等", "乙等", "丙等", "丁等")
for k, got := range unitID {
if got == id && !strings.HasSuffix(k, "等") {
t.Fatalf("%s joined the grade unit and shares no morpheme with it: %v", k, unitsOf(unitID))
}
}
}
// TestContainmentPairSharesAUnit is §2(б): two MINED surfaces where one contains the other (元海 ⊂ 元海空窍)
// had neither a Related record (Merge computes those for banknote-only rows) nor any atomicity in the
// batcher — so the compound could be consolidated in one call and its own part in another, which is how a
// rendering stops carrying the element it is built from.
func TestContainmentPairSharesAUnit(t *testing.T) {
cands := []Candidate{typed("元海", "term", 30), typed("元海空窍", "term", 8), typed("青茅山", "place", 12)}
unitID, _ := MergeUnits(nil, DetectFamilies(cands, zhFamily), zhFamily)
sameUnit(t, unitID, "元海", "元海空窍")
if unitID["青茅山"] != 0 {
t.Fatalf("an unrelated surface must stay a singleton: %v", unitsOf(unitID))
}
// The anchor bound is real: a ONE-rune surface may not anchor the channel, or a generic morpheme sweeps
// half the bank into one call.
short := []Candidate{typed("蛊", "term", 900), typed("蛊虫", "term", 40), typed("蛊师", "term", 30), typed("月光蛊", "term", 10)}
if unit, _ := MergeUnits(nil, DetectFamilies(short, zhFamily), zhFamily); unit["蛊"] != 0 {
t.Fatalf("a one-rune anchor is below ContainmentRunes and must form no unit: %v", unitsOf(unit))
}
}
// TestFamilyMergeRefusesAnOversizeUnit: the merge is capped and the refusal is REPORTED, not silent — a
// family split across two calls is the defect this channel exists to close, so the caller has to be able to
// say it happened.
func TestFamilyMergeRefusesAnOversizeUnit(t *testing.T) {
var cands []Candidate
for _, k := range []string{"古月方源", "古月正", "古月赤练", "古月山寨"} {
cands = append(cands, typed(k, "name", 5))
}
tight := zhFamily
tight.MaxMembers = 3
unitID, st := MergeUnits(nil, DetectFamilies(cands, tight), tight)
if st.Refused == 0 {
t.Fatalf("a four-member family under a three-member cap must be refused: %v", unitsOf(unitID))
}
for _, v := range unitsOf(unitID) {
if len(v) > tight.MaxMembers {
t.Fatalf("a unit past the cap was built anyway: %v", v)
}
}
}
// TestFamilyChannelOffIsTheSeriesMap guards the generality answer: a source that declares no family data
// takes the byte-identical path it took before the channel existed — the SAME map object, so nothing
// downstream can even observe a difference.
func TestFamilyChannelOffIsTheSeriesMap(t *testing.T) {
cands := []Candidate{typed("古月方源", "name", 5), typed("古月正", "name", 5), typed("甲等", "term", 3), typed("乙等", "term", 3), typed("丙等", "term", 3)}
seriesID := DetectSeries(cands, zhSeries)
off := FamilyParams{} // no data → inert
if fams := DetectFamilies(cands, off); fams != nil {
t.Fatalf("an undeclared family channel must detect nothing, got %v", fams)
}
unitID, st := MergeUnits(seriesID, DetectFamilies(cands, off), off)
if st != (MergeStats{}) {
t.Fatalf("an inert channel gives nothing up, got %+v", st)
}
// The SAME map object, not a copy: with the channel off nothing downstream can even observe a difference.
if reflect.ValueOf(unitID).Pointer() != reflect.ValueOf(seriesID).Pointer() {
t.Fatalf("the inert path must hand back the series map itself: %v vs %v", unitID, seriesID)
}
for k, v := range seriesID {
if unitID[k] != v {
t.Fatalf("series membership changed with the channel off: %v vs %v", unitID, seriesID)
}
}
}
// TestFamiliesOnBankFullFixtureAreBoundedAndWhole runs the channel over the real corpus DetectSeries was
// calibrated on. Two live properties, both of which a synthetic fixture cannot show: the 古月 family — the
// one the drafts actually split across a batch boundary — comes out as ONE unit, and no unit grows past the
// declared bound, so the co-batching cannot quietly turn into "the whole bank in one call".
func TestFamiliesOnBankFullFixtureAreBoundedAndWhole(t *testing.T) {
cands := loadBankFullSurfaces(t)
seriesID := DetectSeries(cands, zhSeries)
unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
units := unitsOf(unitID)
clan := unitID["古月"]
if clan == 0 {
t.Fatalf("the clan surface must be in a unit with the names built on it: %v", units)
}
clanMembers := 0
for _, k := range units[clan] {
if strings.HasPrefix(k, "古月") {
clanMembers++
}
}
if clanMembers < 10 {
t.Fatalf("the 古月 family is the measured chimera and must ride one call, got %d of %v", clanMembers, units[clan])
}
for id, keys := range units {
if len(keys) > zhFamily.MaxMembers {
t.Fatalf("unit %d grew past the declared bound (%d): %v", id, zhFamily.MaxMembers, keys)
}
}
// EXACT numbers, not bounds. Two mutations survived a bounds-only version of this test: swapping the
// relatedness test to substring containment (which splits 丙等 from 丙等资质 — the §B4 chimera) and
// inverting the family ranking (which changes which candidates share a CALL, i.e. the money). Both leave
// every inequality above satisfied, so only the composition can catch them.
if len(units) != 14 || largestUnit(units) != 20 || st.Refused != 0 || st.Held != 2 {
t.Fatalf("the corpus shape moved: %d units, largest %d, %+v — if this is intended, re-measure and update the numbers, do not relax the test",
len(units), largestUnit(units), st)
}
// The rank line and the grade family, whole, by composition.
sameUnit(t, unitID, "一转", "九转", "一转蛊师", "九转境界")
sameUnit(t, unitID, "丙等", "丙等资质", "甲等", "甲等资质")
// And the exact SHAPE of the partition. The aggregate counts above survive an inverted family ranking —
// which silently re-cuts which candidates share a CALL, i.e. what the run buys — so the sizes are pinned.
sizes := make([]int, 0, len(units))
for _, keys := range units {
sizes = append(sizes, len(keys))
}
sort.Sort(sort.Reverse(sort.IntSlice(sizes)))
want := []int{20, 13, 10, 10, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2}
if fmt.Sprint(sizes) != fmt.Sprint(want) {
t.Fatalf("the partition SHAPE moved: %v want %v — re-measure before relaxing this", sizes, want)
}
t.Logf("bankfull: %d candidates → %d units, largest %d, gave up %+v",
len(cands), len(units), largestUnit(units), st)
}
// TestGradeFamilyMergesWithItsSeriesThroughASharedRune is the case that made the relatedness test wrong the
// first time round, kept as its own pin: the 等资质 series and the {丙等, 丙等资, 丙等资质} family share no
// substring in either direction, but they plainly share the morpheme 等 — and 丙等 landing in a different
// call from the 丙等资质 built on it is the research/24 §B4 chimera, term for term.
func TestGradeFamilyMergesWithItsSeriesThroughASharedRune(t *testing.T) {
var cands []Candidate
for _, k := range []string{"甲等", "乙等", "丙等", "甲等资质", "乙等资质", "丙等资质", "丙等资"} {
cands = append(cands, typed(k, "term", 5))
}
unitID, st := MergeUnits(DetectSeries(cands, zhSeries), DetectFamilies(cands, zhFamily), zhFamily)
if st.Held != 0 {
t.Fatalf("a family sharing 等 with the series must not be held back: %+v (%v)", st, unitsOf(unitID))
}
sameUnit(t, unitID, "丙等", "丙等资质", "甲等", "甲等资质")
}
func largestUnit(units map[int][]string) int {
n := 0
for _, keys := range units {
if len(keys) > n {
n = len(keys)
}
}
return n
}
// TestSeriesHoldsOnlyItsOwnMembers is the guard's honest boundary, and it exists because the first cut of it
// was wrong in exactly the way that matters. «Series stronger than family» must hold back the SERIES — not
// dissolve the family around it. Here the clan 古月 has five names, three of which happen to form a 雄-series;
// the series' root (雄) has nothing to do with the clan morpheme, so the three keep their own unit, and the
// other two must still be co-batched as the clan. Refusing the whole merge instead leaves the clan as five
// singletons — the batch-boundary chimera of research/24 §B4, arriving through the guard meant to prevent it.
func TestSeriesHoldsOnlyItsOwnMembers(t *testing.T) {
cands := []Candidate{
typed("古月方源", "name", 40), typed("古月正", "name", 9),
typed("古月大雄", "name", 5), typed("古月二雄", "name", 5), typed("古月三雄", "name", 5),
}
seriesID := DetectSeries(cands, zhSeries)
if seriesID["古月大雄"] == 0 {
t.Fatal("test premise broken: the three *雄 names must form an equal-length series")
}
unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
if st.Held == 0 {
t.Fatalf("holding part of a family back is a split the owner must be told about: %+v", st)
}
// The series keeps its three members …
sameUnit(t, unitID, "古月大雄", "古月二雄", "古月三雄")
// … and the REST of the clan is still one unit, not two singletons.
clan := sameUnit(t, unitID, "古月方源", "古月正")
if clan == unitID["古月大雄"] {
t.Fatalf("an unrelated series must not be swallowed by the family: %v", unitsOf(unitID))
}
}
// TestFamilyMergesAcrossASeriesWhenTheRootIsShared is the other side of the same guard: when the series' own
// root IS the family's anchor (the 转 case), nothing is held back and the whole line rides one call.
func TestFamilyMergesAcrossASeriesWhenTheRootIsShared(t *testing.T) {
var cands []Candidate
for _, k := range []string{"一转", "二转", "三转", "四转", "五转", "十一转"} {
cands = append(cands, typed(k, "term", 5))
}
_, st := MergeUnits(DetectSeries(cands, zhSeries), DetectFamilies(cands, zhFamily), zhFamily)
if st.Held != 0 {
t.Fatalf("a family sharing the series' own root must not be held back: %+v", st)
}
}
// TestFamilyRulesCoverEveryTypeACandidateCanCarry: the family rules are keyed on a candidate's TYPE, and the
// set a candidate can carry is wider than the classifier's answer vocabulary — the banknote channel accepts
// `nickname` from a draft and that type rides onto a draft-side-only candidate. A type with no rule forms no
// families, silently, which is a whole class of the bank going un-co-batched for no stated reason.
func TestFamilyRulesCoverEveryTypeACandidateCanCarry(t *testing.T) {
for _, typ := range TypeNames(CandidateTypes) {
if _, has := zhFamily.Affix[typ]; !has {
t.Fatalf("type %q can reach a candidate and has no family rule — declare one or say why in the data file", typ)
}
}
// And it is live, not just declared: two nicknames sharing the clan morpheme are one family.
cands := []Candidate{typed("方小子", "nickname", 5), typed("方小鬼", "nickname", 4)}
if fams := DetectFamilies(cands, zhFamily); len(fams) == 0 {
t.Fatalf("a nickname family must form: %v", fams)
}
}

View file

@ -68,7 +68,7 @@ func TestParseReplyRefusesForeignLanguageRenderings(t *testing.T) {
keys := []string{"修行", "元海", "转", "是为", "资质"}
reply := "修行\tcultivation\n元海\tPrimordial Sea\n转\trealm\n是为\tis\n资质\tталант"
got, st := ParseReply(reply, keys, id, unicode.Scripts["Cyrillic"])
got, _, st := ParseReply(reply, keys, id, unicode.Scripts["Cyrillic"])
if len(got) != 1 || got["资质"] != "талант" {
t.Fatalf("only the target-language line may be banked, got %#v", got)
}
@ -80,7 +80,7 @@ func TestParseReplyRefusesForeignLanguageRenderings(t *testing.T) {
}
// Without a declared script the same reply is banked whole — the state the engine was in before, kept
// visible so the cost of an undeclared script is a test, not a surprise.
if inert, st := ParseReply(reply, keys, id, nil); len(inert) != 5 || st.OffLanguage != 0 {
if inert, _, st := ParseReply(reply, keys, id, nil); len(inert) != 5 || st.OffLanguage != 0 {
t.Fatalf("with no script the screen must be inert (that is what config refuses for an enabled gate), got %#v", inert)
}
}
@ -95,7 +95,7 @@ func TestOffLanguageSamplesAreBounded(t *testing.T) {
keys = append(keys, k)
lines = append(lines, k+"\tforeign")
}
_, st := ParseReply(strings.Join(lines, "\n"), keys, id, unicode.Scripts["Cyrillic"])
_, _, st := ParseReply(strings.Join(lines, "\n"), keys, id, unicode.Scripts["Cyrillic"])
if st.OffLanguage != len(keys) {
t.Fatalf("every foreign line must be counted, got %d of %d", st.OffLanguage, len(keys))
}

View file

@ -1,6 +1,9 @@
package terminology
import "sort"
import (
"sort"
"strings"
)
// series.go: the SERIES channel (bank-quality §1, D39.68). A grade/rank series — 甲等/乙等/丙等, 一转…九转 —
// shares one generic HEAD and varies only on a modifier. Measured (D39.65 row 21): the drafts disagree on the
@ -106,32 +109,456 @@ func blankAt(rs []rune, pos int) string {
return string(out)
}
// orderBySeries returns the candidates with every series' members made contiguous, anchored at the position
// of the series' first member in the input order; non-series candidates keep their place. The input is
// already key-sorted, so the result is deterministic and disturbs the key order minimally.
func orderBySeries(cands []Candidate, seriesID map[string]int) []Candidate {
if len(seriesID) == 0 {
// orderByUnit returns the candidates with every batch unit's members made contiguous, anchored at the
// position of the unit's first member in the input order; a candidate in no unit keeps its place. The input
// is already key-sorted, so the result is deterministic and disturbs the key order minimally.
func orderByUnit(cands []Candidate, unitID map[string]int) []Candidate {
if len(unitID) == 0 {
return cands
}
byID := map[int][]Candidate{}
for _, c := range cands {
if id := seriesID[c.Key]; id != 0 {
if id := unitID[c.Key]; id != 0 {
byID[id] = append(byID[id], c)
}
}
out := make([]Candidate, 0, len(cands))
emitted := map[int]bool{}
for _, c := range cands {
id := seriesID[c.Key]
id := unitID[c.Key]
if id == 0 {
out = append(out, c)
continue
}
if emitted[id] {
continue // a later member of an already-emitted series
continue // a later member of an already-emitted unit
}
out = append(out, byID[id]...)
emitted[id] = true
}
return out
}
// --- the FAMILY channel (fix-pack §G1) ---------------------------------------------------------------
//
// A SERIES is the narrow case: equal-length surfaces differing in ONE non-head rune. It leaves the wider
// one open, and the wider one is where the measured damage is (research/24 §B4): the 古月-family — eight
// surfaces of DIFFERENT lengths sharing the clan morpheme — came back «Гу Юэ» in batch 0 and «Гуюэ» in
// batch 2 from ONE model, a split that correlates with the batch boundary perfectly. Nothing in the
// terminologist can fix a disagreement it is never shown; only co-batching can.
//
// A family is anchored on a shared ROOT MORPHEME, and the side that root sits on is ASYMMETRIC by class —
// a name leads with its clan surname, a realia term ends with its generic head. Both the side and the
// required root length are pair/script DATA (lang.FamilyMorphology), so a source with no rows leaves the
// channel inert and takes the byte-identical, family-free path.
// FamilyAffix is one type's family rule: which side carries the shared root and how long it must be.
type FamilyAffix struct {
Suffix bool
MinRunes int
}
// FamilyParams is the pair-data that governs the family channel. Like SeriesParams it arrives as a VALUE:
// the package stays pure and pair-agnostic, and the pipeline resolves it from the source's declared
// morphology.
type FamilyParams struct {
// Enabled gates the whole channel — true only for DENSE scripts, for the same reason the series channel
// is: one rune ≈ one morpheme, so a shared affix is a shared MORPHEME and not a coincidence of spelling.
Enabled bool
// HeadFinal says the generic head is the trailing morpheme. It is used to read a SERIES' own root when
// deciding whether a family may join it (see MergeUnits).
HeadFinal bool
// Affix is the per-type rule (name|place|title|term); a type absent here forms no families.
Affix map[string]FamilyAffix
// MinMembers is the smallest set that counts as a family (<2 → 2).
MinMembers int
// MaxMembers bounds the batch unit a family merge may produce; 0 = unbounded. A unit past it is a review
// set nobody reads and an output cap nobody planned, so the merge is refused and counted.
MaxMembers int
// ContainmentRunes is the shortest candidate that may anchor the COMPOSITIONAL channel: a candidate that
// is itself a substring of another candidate (元海 ⊂ 元海空窍 — both mined, so neither Related nor the
// batcher held them together before this pack). Bounding the anchor is what keeps one generic morpheme
// from sweeping half the bank into a single call. 0 → the channel is off.
ContainmentRunes int
}
// Family is one detected family: the shared ANCHOR morpheme and the candidate keys carrying it (key-sorted).
type Family struct {
Anchor string
Keys []string
}
// DetectFamilies returns the families of a candidate list, strongest first (larger set, then longer — i.e.
// more specific — anchor, then first-seen). It does NOT assign membership: a candidate legitimately shows
// up in several families, and which of them becomes a batch unit is decided in MergeUnits, where the
// existing series are already on the table. Deterministic; nothing here iterates a map for output.
//
// There is no transitive closure over runes: an anchor is a WHOLE root morpheme of at least MinRunes (or a
// whole candidate, for the compositional channel), never "these two differ in one position", which is the
// rule that produced an 11-surface blob mixing three families and the protagonist's name.
func DetectFamilies(cands []Candidate, p FamilyParams) []Family {
if !p.Enabled || len(cands) < 2 {
return nil
}
min := p.MinMembers
if min < 2 {
min = 2
}
type gkey struct{ side, anchor string }
members := map[gkey][]string{}
seen := map[gkey]map[string]bool{}
var order []gkey
add := func(g gkey, key string) {
m := seen[g]
if m == nil {
m = map[string]bool{}
seen[g] = m
order = append(order, g)
}
if !m[key] {
m[key] = true
members[g] = append(members[g], key)
}
}
isCand := make(map[string]bool, len(cands))
for _, c := range cands {
if c.Key != "" {
isCand[c.Key] = true
}
}
for _, c := range cands {
if c.Key == "" {
continue
}
rule, ok := p.Affix[typeOr(c.Type)]
if !ok || rule.MinRunes <= 0 {
continue
}
side := "prefix"
if rule.Suffix {
side = "suffix"
}
rs := []rune(c.Key)
// PROPER affixes only (l < len): a surface is not a member of a family whose root IS the surface —
// it is that root, and it joins below, once some other surface actually carries it.
for l := rule.MinRunes; l < len(rs); l++ {
anchor := string(rs[:l])
if rule.Suffix {
anchor = string(rs[len(rs)-l:])
}
g := gkey{side, anchor}
add(g, c.Key)
if isCand[anchor] {
// The surface that IS the root belongs to its own family. Leaving it out is how the head of a
// family ends up in a different call from the family it heads.
//
// ⚠ Bounded: an anchor is at least MinRunes long, so a ONE-rune head that is itself a bank term
// (蛊, 转, 窍) is NOT reachable this way and does stay in another call from its family. That is
// a data decision, not a Go one — family_containment_runes 1 turns it on — and it is left off
// because a single generic morpheme anchors half the bank; the cost is measured at the cold run.
add(g, anchor)
}
}
}
if p.ContainmentRunes > 0 {
for _, c := range cands {
if c.Key == "" || len([]rune(c.Key)) < p.ContainmentRunes {
continue
}
g := gkey{"contains", c.Key}
for _, d := range cands {
if d.Key == "" || d.Key == c.Key {
continue
}
if strings.Contains(d.Key, c.Key) {
add(g, c.Key)
add(g, d.Key)
}
}
}
}
all := make([]Family, 0, len(order))
for _, g := range order {
if len(members[g]) < min {
continue
}
keys := append([]string(nil), members[g]...)
sort.Strings(keys)
all = append(all, Family{Anchor: g.anchor, Keys: keys})
}
sort.SliceStable(all, func(i, j int) bool {
if len(all[i].Keys) != len(all[j].Keys) {
return len(all[i].Keys) > len(all[j].Keys)
}
ai, aj := len([]rune(all[i].Anchor)), len([]rune(all[j].Anchor))
if ai != aj {
return ai > aj // the more SPECIFIC root wins the earlier merge
}
return all[i].Anchor < all[j].Anchor
})
// Dedupe AFTER the ranking, so the survivor of a duplicated set is the strongest description of it. The
// affix and containment channels legitimately describe the same set from two directions (元海空窍/天海空窍
// share the suffix 空窍 AND the longer 海空窍), and one of them is enough: a duplicate merges nothing the
// first did not, but it double-counts in the merge's give-up statistics — and a number an operator reads
// has to mean what it says.
out := make([]Family, 0, len(all))
seenSet := map[string]bool{}
for _, f := range all {
set := strings.Join(f.Keys, "\x00")
if seenSet[set] {
continue
}
seenSet[set] = true
out = append(out, f)
}
return out
}
// MergeUnits folds the series and the families into ONE batch-unit map — the map Batch packs against.
//
// Membership is a PARTITION (a candidate is in at most one unit), built by merging whole units rather than
// by claiming candidates: an exclusive assignment cannot express the case the fix-pack exists for. The
// rank line 一转…九转 is a series; 十一转 is three runes long and so is not, and under exclusive assignment
// it stays outside forever — the batch boundary splits a rank scale in half. Merging the family that spans
// both puts the whole 1..12 line in one call, which is the stated criterion.
//
// Two guards keep the merge from becoming a blob, and BOTH are counted rather than silent — a guard that
// splits a family is doing the same damage the channel exists to prevent, so it has to be visible:
// - SERIES PRECEDENCE: a series whose own root is unrelated to a family's anchor keeps its members; the
// family then forms WITHOUT them (转-case: root 转, anchors 转 / 一转 — related, so they merge instead).
// - MaxMembers: a merge that would produce a unit larger than the pair declares is refused.
//
// With no families (the channel off, or nothing detected) the series map is returned UNCHANGED, so a source
// with no family data takes the byte-identical path.
func MergeUnits(seriesID map[string]int, fams []Family, p FamilyParams) (unitID map[string]int, stats MergeStats) {
if len(fams) == 0 {
return seriesID, MergeStats{}
}
u := newUnitSet()
cappedSet := map[string]bool{} // key sets whose merge the member cap turned down
bySeries := map[int][]string{}
for k, id := range seriesID {
if id != 0 {
bySeries[id] = append(bySeries[id], k)
}
}
ids := make([]int, 0, len(bySeries))
for id := range bySeries {
ids = append(ids, id)
}
sort.Ints(ids)
for _, id := range ids {
mem := bySeries[id]
sort.Strings(mem)
for _, k := range mem[1:] {
u.union(mem[0], k)
}
if root := commonAffix(mem, p.HeadFinal); root != "" {
r := u.find(mem[0])
u.roots[r] = append(u.roots[r], root)
}
}
for _, f := range fams {
var targets []string
seen := map[string]bool{}
for _, k := range f.Keys {
r := u.find(k)
if seen[r] {
continue
}
seen[r] = true
targets = append(targets, r)
}
if len(targets) < 2 {
continue // already one unit
}
// SERIES PRECEDENCE, applied per TARGET rather than to the whole family. A series whose root is
// unrelated to this anchor is HELD BACK — and only it. Refusing the whole merge instead would
// dissolve the family around it: the clan 古月, minus three of its names that happen to form a
// 雄-series, would go back to being five singletons, which is the batch-boundary chimera this
// channel exists to close, arriving through the guard that was supposed to protect a series.
var join []string
size := 0
for _, r := range targets {
blocked := false
for _, sr := range u.roots[r] {
if !sharesMorpheme(f.Anchor, sr) {
blocked = true
break
}
}
if blocked {
continue
}
join = append(join, r)
size += u.size[r]
}
if len(join) < 2 {
continue
}
if p.MaxMembers > 0 && size > p.MaxMembers {
cappedSet[strings.Join(f.Keys, "\x00")] = true
continue
}
for _, r := range join[1:] {
u.union(join[0], r)
}
}
// The give-up statistics are read off the FINAL partition, never off the attempts. A guard that fired
// mid-way is not a family the owner meets split: a later merge routinely puts those keys back together,
// and counting the attempt reports a split that does not exist. Measured on the coldrun-a distillation:
// counting attempts claimed three, of which one (蛊师 against 一转蛊师) had ended up in ONE unit anyway.
// A number an operator reads has to mean what it says — the standard this file sets for itself.
final := u.ids()
for _, f := range fams {
if !splitAcrossUnits(f, final) {
continue
}
if cappedSet[strings.Join(f.Keys, "\x00")] {
stats.Refused++
continue
}
stats.Held++
}
return final, stats
}
// splitAcrossUnits reports whether a family's keys ended up in more than one batch unit (a key in no unit
// counts as its own).
func splitAcrossUnits(f Family, unitID map[string]int) bool {
first, seen := 0, false
for _, k := range f.Keys {
id := unitID[k]
if !seen {
first, seen = id, true
continue
}
if id != first || id == 0 {
return true
}
}
return false
}
// MergeStats is what the unit merge had to give up. Both numbers mean "a family the owner will meet split
// across two calls", which is the defect the channel exists to close — so neither may be silent.
type MergeStats struct {
// Refused counts merges the member cap turned down.
Refused int
// Held counts families a SERIES held part of back: the series' own root is unrelated to the family's
// anchor, so the series keeps its members and the family forms without them.
Held int
}
// sharesMorpheme reports whether two anchors share a root MORPHEME — the "делят одну корневую морфему" test
// behind the series-vs-family precedence. In a dense script a morpheme is a rune, so that is the test.
//
// Substring containment is NOT it, and the difference was measured, not reasoned: on the coldrun-a bank the
// containment version held back five families, every one of them wrongly. The 等资质 series (甲等资质/乙等资质/
// 丙等资质, root 等资质) sits beside the family {丙等, 丙等资, 丙等资质}, anchored 丙等. Neither string contains
// the other, so containment called them unrelated and split the grade 丙等 away from the graded aptitude
// built on it — while they plainly share 等, which is the very morpheme whose rendering has to agree. The
// blob these guards protect against is bounded by MaxMembers, not by making relatedness narrow.
func sharesMorpheme(a, b string) bool {
if a == "" || b == "" {
return true
}
return sharedRunes(a, b) > 0
}
// commonAffix returns the longest common trailing (headFinal) or leading morpheme of the keys — a series'
// own root, read off its members so DetectSeries keeps its signature.
func commonAffix(keys []string, headFinal bool) string {
if len(keys) == 0 {
return ""
}
best := []rune(keys[0])
for _, k := range keys[1:] {
rs := []rune(k)
n := 0
for n < len(best) && n < len(rs) {
if headFinal {
if best[len(best)-1-n] != rs[len(rs)-1-n] {
break
}
} else if best[n] != rs[n] {
break
}
n++
}
if headFinal {
best = best[len(best)-n:]
} else {
best = best[:n]
}
if len(best) == 0 {
return ""
}
}
return string(best)
}
// unitSet is the disjoint-set the merge runs on: parent/size by key, plus the series roots a unit carries
// (the precedence guard reads them). Roots are the smallest member key, so the structure is deterministic
// without any map iteration reaching the output.
type unitSet struct {
parent map[string]string
size map[string]int
roots map[string][]string
}
func newUnitSet() *unitSet {
return &unitSet{parent: map[string]string{}, size: map[string]int{}, roots: map[string][]string{}}
}
func (u *unitSet) find(k string) string {
p, ok := u.parent[k]
if !ok {
u.parent[k], u.size[k] = k, 1
return k
}
if p == k {
return k
}
r := u.find(p)
u.parent[k] = r
return r
}
func (u *unitSet) union(a, b string) {
ra, rb := u.find(a), u.find(b)
if ra == rb {
return
}
if rb < ra { // the smaller key is always the root → the same input yields the same structure
ra, rb = rb, ra
}
u.parent[rb] = ra
u.size[ra] += u.size[rb]
u.roots[ra] = append(u.roots[ra], u.roots[rb]...)
delete(u.size, rb)
delete(u.roots, rb)
}
// ids numbers the units of ≥2 members, in the order of their smallest key, so the map is a pure function of
// the input.
func (u *unitSet) ids() map[string]int {
roots := make([]string, 0, len(u.size))
for r, n := range u.size {
if n > 1 {
roots = append(roots, r)
}
}
sort.Strings(roots)
num := make(map[string]int, len(roots))
for i, r := range roots {
num[r] = i + 1
}
out := make(map[string]int, len(u.parent))
for k := range u.parent {
if id := num[u.find(k)]; id != 0 {
out[k] = id
}
}
return out
}

View file

@ -80,8 +80,12 @@ type Neighbour struct {
// Variant is one observed rendering with its consolidation score.
type Variant struct {
Dst string
Chunks int
Dst string
Chunks int
// Forms is how many DISTINCT raw renderings folded into this one (see foldVariants). 1 for an unfolded
// variant; it is what keeps Spread() meaning "how many ways did the drafts write it" once the vote is
// counted per CONVENTION rather than per byte string.
Forms int
Score float64
Signals []string // which factors fired, in fixed order — the score's audit trail
// Via names the surface that actually proposed this rendering when it is NOT the candidate's own —
@ -125,7 +129,12 @@ type Candidate struct {
//
// Output order is by Key (never map order). Every mined candidate appears exactly once; every observed key
// either merges into one or appears once in the reverse section.
func Merge(mined []Mined, observed []Observed) []Candidate {
//
// foldDst is the TARGET-form normalizer the vote is counted under (see foldVariants); nil folds only
// byte-identical renderings, which is what this did before the fix-pack. It is injected rather than
// imported so the package keeps no target knowledge — the ё/case/whitespace folds a Russian target needs
// are facts about that target, not about this algorithm.
func Merge(mined []Mined, observed []Observed, foldDst func(string) string) []Candidate {
byKey := make(map[string]*Candidate, len(mined))
order := make([]string, 0, len(mined)+len(observed))
// aliasOwner maps every alias surface to its owning mined key, so an aliased proposal merges.
@ -206,31 +215,89 @@ func Merge(mined []Mined, observed []Observed) []Candidate {
out := make([]Candidate, 0, len(order))
for _, k := range order {
c := *byKey[k]
c.Variants = foldVariants(c.Variants)
c.Variants = foldVariants(c.Variants, foldDst)
out = append(out, c)
}
sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
return out
}
// foldVariants sums the chunk counts of IDENTICAL renderings. It matters on the alias path: 方源 and its
// alias 方小子 arrive as two Observed rows, and both may carry the same rendering — appended raw that is
// two variants of one word, which lies twice. Spread() would report 2 (the disagreement column the owner
// reads to decide whether a term is contested) for a term nobody disagreed about, and the §C2-3 frequency
// factor would score the consensus rendering on HALF its evidence, which can hand the win to a genuine
// competitor. Deterministic: first-seen order is preserved, so the fold introduces no new ordering.
func foldVariants(vs []Variant) []Variant {
if len(vs) < 2 {
// foldVariants sums the chunk counts of renderings that are the SAME CONVENTION and shows the raw form.
//
// It matters on the alias path: 方源 and its alias 方小子 arrive as two Observed rows, and both may carry the
// same rendering — appended raw that is two variants of one word, which lies twice. It matters just as much
// on orthography (fix-pack §G5, research/24 §A5): «Море истинной ци» and «море истинной ци» are one
// convention written two ways, and folding only byte-identical strings splits the §C2-3 frequency factor
// across them — the consensus rendering is then scored on half its evidence and a genuine competitor can
// take the win. So the VOTE is counted under foldDst (the target-form normalizer), while the form the owner
// and the model see stays the RAW one the drafts actually wrote.
//
// Two counts survive the fold and mean different things: Spread() (Σ Forms) is how many ways the drafts
// wrote the term, Conventions() is how many of those are genuinely different decisions.
//
// Via is aggregated honestly rather than by first-wins. A rendering that ALSO arrived on the candidate's own
// key is direct — labelling it «proposed for <alias>» because the alias happened to be seen first would
// present a consensus as a mis-clustered alias's guess. Only when EVERY contribution came through aliases
// is the provenance kept, and then it names all of them.
//
// Deterministic: first-seen order of the folded classes is preserved, and the representative is the raw form
// with the most chunks, ties going to the one seen first.
func foldVariants(vs []Variant, foldDst func(string) string) []Variant {
if len(vs) == 0 {
return vs
}
if foldDst == nil {
foldDst = func(s string) string { return s }
}
type form struct {
dst string
chunks int
}
type class struct {
forms []form
byForm map[string]int
chunks int
direct bool
vias []string
seenVia map[string]bool
}
at := make(map[string]int, len(vs))
out := make([]Variant, 0, len(vs))
classes := make([]*class, 0, len(vs))
for _, v := range vs {
if i, seen := at[v.Dst]; seen {
out[i].Chunks += v.Chunks
continue
k := foldDst(v.Dst)
i, seen := at[k]
if !seen {
i = len(classes)
at[k] = i
classes = append(classes, &class{byForm: map[string]int{}, seenVia: map[string]bool{}})
}
cl := classes[i]
cl.chunks += v.Chunks
if j, had := cl.byForm[v.Dst]; had {
cl.forms[j].chunks += v.Chunks
} else {
cl.byForm[v.Dst] = len(cl.forms)
cl.forms = append(cl.forms, form{dst: v.Dst, chunks: v.Chunks})
}
if v.Via == "" {
cl.direct = true
} else if !cl.seenVia[v.Via] {
cl.seenVia[v.Via] = true
cl.vias = append(cl.vias, v.Via)
}
}
out := make([]Variant, 0, len(classes))
for _, cl := range classes {
best := 0
for j := 1; j < len(cl.forms); j++ {
if cl.forms[j].chunks > cl.forms[best].chunks {
best = j
}
}
v := Variant{Dst: cl.forms[best].dst, Chunks: cl.chunks, Forms: len(cl.forms)}
if !cl.direct {
v.Via = strings.Join(cl.vias, ", ")
}
at[v.Dst] = len(out)
out = append(out, v)
}
return out
@ -596,8 +663,27 @@ func (c Candidate) Best() string {
}
// Spread is the disagreement signal: how many DISTINCT renderings the drafts produced for this surface.
// One term coming back three ways is the drift a canon closes — the reason to sign it at all.
func (c Candidate) Spread() int { return len(c.Variants) }
// One term coming back three ways is the drift a canon closes — the reason to sign it at all. It counts RAW
// forms (Forms per folded class), so the fix-pack's convention fold did not silently shrink the column the
// owner reads to decide whether a term is contested. A variant built outside Merge carries no Forms and
// counts as one.
func (c Candidate) Spread() int {
n := 0
for _, v := range c.Variants {
if v.Forms > 1 {
n += v.Forms
continue
}
n++
}
return n
}
// Conventions is the disagreement that actually matters: how many distinct DECISIONS the drafts made, after
// renderings that differ only in target form (case, ё, spacing) are folded together. Spread 3 with
// Conventions 1 is one canon written three ways — a normalization nit; Spread 3 with Conventions 3 is a real
// contest, and the two must not read the same at the stop.
func (c Candidate) Conventions() int { return len(c.Variants) }
// --- the wire: request table and reply parser ---------------------------------------------------------
@ -611,6 +697,62 @@ const NoDst = "⟦TM-NO-DST⟧"
// tolerance the banknote parser applies, for the same reason: models substitute spaces for tabs.
var fieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`)
// columnSplit is the UNAMBIGUOUS half of that tolerance — a tab or a pipe is a delimiter the model chose and
// cannot occur inside a rendering, so a line carrying one has declared its own columns. spaceRun folds the
// stray spacing inside such a column. Both exist because guessing where the columns are, on a line that says
// where they are, is how a fix for one mis-split rendering corrupts the lines that were never mis-split.
var (
columnSplit = regexp.MustCompile(`\t|\s*\|\s*`)
spaceRun = regexp.MustCompile(` {2,}`)
)
// replyColumns splits one reply line into src, rendering and the CONFIDENCE column, and says whether a
// third column was present but unreadable as a confidence.
//
// A tab/pipe line is positional: field three IS the confidence column the pair's prompt declared, so a value
// that is not a bare 0…100 is a malformed CONFIDENCE — counted, and never glued onto the rendering. That
// glue is the failure this shape exists to prevent: «наставник» + «95%» silently became the rendering
// «наставник 95%», passed every downstream screen (it is well-formed, it is in the target script, it is not
// an echo) and would enter the bank as this book's canon with every counter reading clean.
//
// A line delimited only by a run of ≥2 spaces is ambiguous — that tolerance is exactly what splits «Фан␣␣
// Юань» — so there the rendering is re-joined and only a trailing bare number is taken as a confidence.
func replyColumns(line string) (src, dst string, conf int, badConf, ok bool) {
conf = -1
if cols := splitOn(columnSplit, line); len(cols) >= 2 {
src, dst = cols[0], strings.TrimSpace(spaceRun.ReplaceAllString(cols[1], " "))
if len(cols) >= 3 {
if v, good := confidenceField(cols[2]); good {
conf = v
} else {
badConf = true
}
}
return src, dst, conf, badConf, dst != ""
}
f := splitOn(fieldSplit, line)
if len(f) < 2 {
return "", "", -1, false, false
}
if len(f) >= 3 {
if v, good := confidenceField(f[len(f)-1]); good {
conf, f = v, f[:len(f)-1]
}
}
return f[0], strings.TrimSpace(strings.Join(f[1:], " ")), conf, false, true
}
// splitOn splits a line on re, dropping empty and whitespace-only fields.
func splitOn(re *regexp.Regexp, line string) []string {
var out []string
for _, p := range re.Split(strings.TrimSpace(line), -1) {
if p = strings.TrimSpace(p); p != "" {
out = append(out, p)
}
}
return out
}
// splitFields splits one reply line into its non-empty trimmed fields.
func splitFields(line string) []string {
var parts []string
@ -811,6 +953,49 @@ func CanonConflicts(cands []Candidate, consolidated map[string]string, ns []Neig
return out
}
// ConsolidationConflict is one consolidated rendering that contradicts ANOTHER consolidation of the SAME
// run — the compositional rule broken inside one reply rather than against a signed row.
type ConsolidationConflict struct {
Src string // the containing candidate's source surface
Dst string // what the role consolidated it to
PartSrc string // the source surface contained in it, also consolidated this run
PartDst string // the rendering its own dst fails to carry
}
// ConsolidationConflicts is CanonConflicts' thin brother, and it exists because the canon check can only see
// what the owner already signed. On a live bank that is the minority: 18 of 149 contradictions the same run
// produced were between its OWN consolidations (research/24 §A4) — 元海空窍 rendered without the 元海 this
// very reply had just fixed — and nothing looked at them, because the role runs once and never reads itself.
// A step whose entire purpose is ONE consistent canon must not be the step that quietly breaks it.
//
// Same test as the canon side (containment + lexemeSubset, so Russian case endings do not false-flag), same
// discipline: it REPORTS, never rewrites and never fails a run. $0 — it is arithmetic over a map that
// already exists. Deterministic: candidates are key-ordered on both sides and nothing here iterates a map.
func ConsolidationConflicts(cands []Candidate, consolidated map[string]string) []ConsolidationConflict {
if len(consolidated) < 2 {
return nil
}
var out []ConsolidationConflict
for _, c := range cands {
dst := consolidated[c.Key]
if dst == "" || c.Key == "" {
continue
}
have := wordSet(dst)
for _, p := range cands {
pd := consolidated[p.Key]
if pd == "" || p.Key == "" || p.Key == c.Key || !strings.Contains(c.Key, p.Key) {
continue
}
if lexemeSubset(wordSet(pd), have) {
continue
}
out = append(out, ConsolidationConflict{Src: c.Src, Dst: dst, PartSrc: p.Src, PartDst: pd})
}
}
return out
}
// sharedRunes counts the DISTINCT runes two sources have in common.
func sharedRunes(a, b string) int {
if a == "" || b == "" {
@ -838,6 +1023,10 @@ type ReplyStats struct {
Bad int
OffLanguage int
OffLanguageSamples []string // first few refused lines verbatim, so the warning names them
// BadConfidence counts lines whose declared CONFIDENCE column was not a bare 0…100 («95%», «0.9»,
// «высокая»). The rendering is kept — it is fine — but the value is dropped rather than glued onto it,
// and the count is what tells an operator the prompt's third column is not landing.
BadConfidence int
}
const offLanguageSampleCap = 8
@ -850,25 +1039,27 @@ const offLanguageSampleCap = 8
//
// `target` is the target language's script (nil → the answer-language check is inert; see OffLanguage).
//
// Returns the accepted map plus the tally of unusable lines, which the caller logs: silence about a reply
// the parser could not read is how a paid call turns into an empty bank with no signal.
func ParseReply(reply string, expected []string, normalize func(string) string, target *unicode.RangeTable) (map[string]string, ReplyStats) {
// Returns the accepted map, the role's own stated CONFIDENCE per key (absent when the reply carried none),
// and the tally of unusable lines, which the caller logs: silence about a reply the parser could not read is
// how a paid call turns into an empty bank with no signal.
func ParseReply(reply string, expected []string, normalize func(string) string, target *unicode.RangeTable) (map[string]string, map[string]int, ReplyStats) {
want := make(map[string]bool, len(expected))
for _, k := range expected {
want[k] = true
}
out := map[string]string{}
conf := map[string]int{}
var st ReplyStats
for _, ln := range strings.Split(reply, "\n") {
if strings.TrimSpace(ln) == "" || strings.HasPrefix(strings.TrimSpace(ln), "#") {
continue
}
f := splitFields(ln)
if len(f) < 2 {
rawSrc, dst, c, badConf, ok := replyColumns(ln)
if !ok {
st.Bad++
continue
}
key := normalize(f[0])
key := normalize(rawSrc)
if !want[key] {
st.Bad++
continue
@ -876,15 +1067,23 @@ func ParseReply(reply string, expected []string, normalize func(string) string,
if _, dup := out[key]; dup {
continue // first answer wins; a model repeating itself is not a second opinion
}
if badConf {
// The rendering is usable; the confidence column is not. Counted rather than refused — and never
// folded into the rendering, which is the whole point.
st.BadConfidence++
}
// Everything after the FIRST field is the rendering, re-joined on single spaces. The splitter is
// deliberately tolerant (a tab, a run of ≥2 spaces, a padded pipe) because models substitute one for
// another — but that tolerance also splits INSIDE a rendering the moment a model writes «Фан Юань»
// with a stray double space, and taking f[1] alone would then bank the half-name «Фан» silently:
// it passes wellFormedLemma, so no counter would ever mention it. The role's reply is specified as
// exactly two fields, so there is no third column to lose by re-joining.
dst := strings.TrimSpace(strings.Join(f[1:], " "))
if dst == NoDst {
// exactly two fields, so there is no third column to lose by re-joining — see replyColumns for how the
// third (confidence) column and a mis-split rendering are told apart.
if isNoDst(dst) {
out[key] = ""
if c >= 0 {
conf[key] = c
}
continue
}
if !wellFormedLemma(dst) {
@ -905,18 +1104,60 @@ func ParseReply(reply string, expected []string, normalize func(string) string,
st.Bad++
st.OffLanguage++
if len(st.OffLanguageSamples) < offLanguageSampleCap {
st.OffLanguageSamples = append(st.OffLanguageSamples, f[0]+" → "+dst)
st.OffLanguageSamples = append(st.OffLanguageSamples, rawSrc+" → "+dst)
}
continue
}
out[key] = dst
if c >= 0 {
conf[key] = c
}
}
return out, st
return out, conf, st
}
// Batch splits candidates into groups whose rendered size stays under maxRunes. A series (a key present in
// seriesID; pass nil to disable the channel) is kept WHOLE in one batch so the terminologist picks one
// generic head for it (§1); a series or a single candidate larger than the budget still gets its own batch
// confidenceField reads the role's stated confidence off a reply FIELD: 13 digits in 0…100 and nothing
// else. Out of that range it is not a confidence and stays part of the rendering.
//
// ⚠ What this number may and may not do is RATIFIED (D39.102): it orders the review list «least sure
// first» — an ordinal INSIDE one model's reply — and nothing more. It is not a weight, not a threshold, not
// comparable between models, and never decides which rendering wins; the measured AUC (0.770.85) supports
// exactly the ordering claim and nothing stronger.
func confidenceField(s string) (int, bool) {
s = strings.TrimSpace(s)
if len(s) == 0 || len(s) > 3 {
return 0, false
}
n := 0
for _, r := range s {
if r < '0' || r > '9' {
return 0, false
}
n = n*10 + int(r-'0')
}
if n > 100 {
return 0, false
}
return n, true
}
// isNoDst recognises the decline sentinel, including the forms a model mangles it into — a trailing period,
// surrounding quotes. Narrow on purpose: the bracketed engine token must still OPEN the rendering and only
// punctuation may follow, so no natural rendering can match. Without it a decline written «⟦TM-NO-DST⟧.»
// fails wellFormedLemma-then-bad-line and the term is counted as a PARSE failure rather than as the
// decision §C2-7 says it is.
func isNoDst(dst string) bool {
t := strings.Trim(strings.TrimSpace(dst), `"'«»“”`)
if !strings.HasPrefix(t, NoDst) {
return false
}
return strings.Trim(strings.TrimSpace(t[len(NoDst):]), `.,;:!?"'«»“”`) == ""
}
// Batch splits candidates into groups whose rendered size stays under maxRunes. A batch UNIT — a series or
// a family, i.e. a key present in unitID; pass nil to disable both channels — is kept WHOLE in one batch so
// the terminologist picks one generic head or one shared root for it (§1, §G1); a unit or a single
// candidate larger than the budget still gets its own batch
// rather than being split or silently dropped (the caller WARNs on such an over-cap unit — see BatchRunes —
// because the whole unit still bills against one output cap). The batches are then ordered by descending
// source frequency, so a budget ceiling drops the least-frequent terms instead of the lexicographic tail
@ -928,14 +1169,14 @@ func ParseReply(reply string, expected []string, normalize func(string) string,
// into the request hash, so a batch's POSITION is part of its address. If a later run's frequencies reorder
// the batches their ordinals shift and the affected batches re-pay — the accepted price of value-ordering,
// and no worse than the content edit that would already re-pay them.
func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candidate {
func Batch(cands []Candidate, maxRunes int, unitID map[string]int) [][]Candidate {
if len(cands) == 0 {
return nil
}
if maxRunes <= 0 {
return [][]Candidate{cands}
}
ordered := orderBySeries(cands, seriesID)
ordered := orderByUnit(cands, unitID)
var out [][]Candidate
var cur []Candidate
size := 0
@ -946,9 +1187,9 @@ func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candida
}
}
for i := 0; i < len(ordered); {
j := i + 1 // extend over a whole series block; a singleton is a unit of one
if id := seriesID[ordered[i].Key]; id != 0 {
for j < len(ordered) && seriesID[ordered[j].Key] == id {
j := i + 1 // extend over a whole unit block; a singleton is a unit of one
if id := unitID[ordered[i].Key]; id != 0 {
for j < len(ordered) && unitID[ordered[j].Key] == id {
j++
}
}
@ -967,7 +1208,7 @@ func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candida
}
// BatchRunes reports the rendered size RenderBatch produces for a whole batch — the same measure Batch packs
// against. The terminologist uses it to WARN when a co-batched series or a lone candidate exceeds the budget:
// against. The terminologist uses it to WARN when a co-batched unit or a lone candidate exceeds the budget:
// §1 keeps such a unit WHOLE rather than split it, so it is a real over-cap call the operator must see coming
// (the cap-8000 output mine, PROBES §1), not a silent overrun.
func BatchRunes(batch []Candidate) int { return unitRunes(batch) }

View file

@ -21,7 +21,7 @@ func TestMergeExactAndAliasCollapseButContainmentDoesNot(t *testing.T) {
{Key: "方小子", Src: "方小子", Type: "nickname", Proposals: []Proposal{{Dst: "малец Фан", Chunks: 1}}},
{Key: "古月方源", Src: "古月方源", Type: "name", Proposals: []Proposal{{Dst: "Гуюэ Фан Юань", Chunks: 2}}},
}
got := Merge(mined, observed)
got := Merge(mined, observed, nil)
byKey := map[string]Candidate{}
for _, c := range got {
@ -71,7 +71,7 @@ func TestMergeFoldsIdenticalRenderingsOfOneEntity(t *testing.T) {
{Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 2}, {Dst: "малец Фан", Chunks: 1}}},
}
var c Candidate
for _, got := range Merge(mined, observed) {
for _, got := range Merge(mined, observed, nil) {
if got.Key == "方源" { // select by key: the alias surface is now a candidate of its own too
c = got
}
@ -98,7 +98,7 @@ func TestMergeFoldsIdenticalRenderingsOfOneEntity(t *testing.T) {
func TestMergeKeepsAnAliasSwallowedSurface(t *testing.T) {
mined := []Mined{{Key: "葛家", Src: "葛家", Type: "name", Freq: 40, Aliases: []string{"族长"}}}
observed := []Observed{{Key: "族长", Src: "族长", Type: "title", Proposals: []Proposal{{Dst: "глава клана", Chunks: 3}}}}
got := Merge(mined, observed)
got := Merge(mined, observed, nil)
var title, cluster *Candidate
for i := range got {
@ -137,9 +137,9 @@ func TestMergeIsDeterministic(t *testing.T) {
{Key: "z", Src: "z", Proposals: []Proposal{{Dst: "Z", Chunks: 1}}},
{Key: "a", Src: "a", Proposals: []Proposal{{Dst: "A", Chunks: 1}}},
}
first := keysOf(Merge(mined, observed))
first := keysOf(Merge(mined, observed, nil))
for i := 0; i < 20; i++ { // map iteration would show up here, not on one run
if got := keysOf(Merge(mined, observed)); !reflect.DeepEqual(got, first) {
if got := keysOf(Merge(mined, observed, nil)); !reflect.DeepEqual(got, first) {
t.Fatalf("merge is order-unstable: %v vs %v", got, first)
}
}
@ -340,13 +340,13 @@ func TestCanonConflictsFlagsOnlyRealContradictions(t *testing.T) {
// stray double space inside a rendering must not truncate it (the splitter is tolerant BY DESIGN, and that
// tolerance cuts both ways), and a batch must actually render within the budget it was split for.
func TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds(t *testing.T) {
got, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
got, _, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
if st.Bad != 0 || got["方源"] != "Фан Юань" {
t.Fatalf("a double space inside a rendering must not bank half a name: %v (bad=%d)", got, st.Bad)
}
// A rendering that IS the source is not a rendering: a model echoing the term back would otherwise be
// banked as this book's canon and shown to the editor as «方源 ⟨проверить⟩».
echo, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
echo, _, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
if _, banked := echo["方源"]; banked || st.Bad != 1 {
t.Fatalf("an echoed source must be refused and counted: %v (bad=%d)", echo, st.Bad)
}
@ -380,7 +380,7 @@ func TestParseReplyAcceptsOnlyAskedTerms(t *testing.T) {
"忘却\t" + NoDst,
"мусор",
}, "\n")
got, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil)
got, _, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil)
want := map[string]string{"方源": "Фан Юань", "花家": "Дом Хуа", "青茅山": "гора Цинмао", "忘却": ""}
if !reflect.DeepEqual(got, want) {
t.Fatalf("parse = %#v, want %#v", got, want)
@ -398,7 +398,7 @@ func TestParseReplyNormalizesTheKey(t *testing.T) {
// The caller passes the engine's source-key normalizer; a reply written in another orthography must
// still land on its term.
lower := strings.ToLower
got, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil)
got, _, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil)
if got["fang"] != "Фан" {
t.Fatalf("the reply key must be normalized by the caller's function, got %#v", got)
}

View file

@ -1,5 +1,8 @@
<!-- v2 (26.07): KWIC-контексты, узкое правило транскрипции, пример формата ⟦TM-CANON⟧ — обоснования
в D39.46/47/52 и отчётах полигон-пакетов. Файл — ДАННЫЕ: в снапшот не фолдится, текст не пинят тесты. -->
в D39.46/47/52 и отчётах полигон-пакетов. Файл — ДАННЫЕ: в снапшот не фолдится, текст не пинят тесты.
v3 (08.08, фикс-пак банка §G3): третье поле — уверенность 0100. Она НЕ арбитр: движок сортирует ею лист
ревью «сначала неуверенное» (ordinal внутри одного ответа, AUC 0.770.85 research/24 §B3) и ничего больше —
ни весов, ни порогов, ни сравнения между моделями (запрещённые формы зафиксированы D39.102). -->
Ты — терминолог издательского перевода с языка «{{source_lang}}» на язык «{{target_lang}}».
Книга: «{{title}}». Жанр: {{genre}}. Аудитория: {{audience}}.
@ -33,12 +36,18 @@
- Если по данным контекстам ты не можешь выбрать перевод уверенно — верни ⟦TM-NO-DST⟧. Это нормальный
ответ: непереведённый термин останется на ручную подпись, а выдуманный попадёт в книгу.
Формат ответа — по одной строке на термин, ровно два поля, разделённые СИМВОЛОМ ТАБУЛЯЦИИ:
первое поле — термин исходника, второе — перевод.
Формат ответа — по одной строке на термин, ровно ТРИ поля, разделённые СИМВОЛОМ ТАБУЛЯЦИИ:
первое поле — термин исходника, второе — перевод, третье — твоя уверенность в этом переводе,
целое число от 0 до 100.
Пример строки ответа (символ между полями — настоящая табуляция):
Уверенность — это оценка ТВОЕГО решения по данным контекстам, а не оценка термина: 90 — контексты
однозначны и вариант очевиден; 40 — контекстов мало или они противоречивы, и ты выбрал наименее плохой.
Она ни на что не влияет, кроме порядка, в котором человек будет просматривать список: сначала он посмотрит
то, в чём ты сам не уверен. Занижать её «на всякий случай» так же бесполезно, как завышать.
师父 наставник
Пример строки ответа (символы между полями — настоящие табуляции):
师父 наставник 95
Никаких заголовков, нумерации, комментариев и markdown. Термины, которых нет в списке, не добавляй.