Land the bank cluster fixpack: family co-batching as pair data, arbitration observability, banknote parser rejoin, auto-bank drop diff, confidence field, after adversarial acceptance and a fix round
This commit is contained in:
parent
7384c57f1a
commit
bccf2d8842
27 changed files with 2934 additions and 200 deletions
|
|
@ -5,6 +5,7 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
|
|
@ -103,14 +104,36 @@ func renderBankStopRows(w io.Writer, rows []pipeline.BankStopRow) {
|
|||
if len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
shown := rows
|
||||
// THE ONLY use the confidence is allowed to have (D39.102): the capped stdout view is the REVIEW surface,
|
||||
// so it is ordered least-sure-first — an ordinal inside one model's reply and nothing more. It is not a
|
||||
// weight, not a threshold, not comparable between models, and it never decides which rendering wins; the
|
||||
// full sidecar keeps the source-key order, because that is the reference document.
|
||||
shown := append([]pipeline.BankStopRow(nil), rows...)
|
||||
sort.SliceStable(shown, func(i, j int) bool { return reviewRank(shown[i]) < reviewRank(shown[j]) })
|
||||
if len(shown) > bankStopStdoutCap {
|
||||
shown = shown[:bankStopStdoutCap]
|
||||
}
|
||||
fmt.Fprintln(w, "")
|
||||
fmt.Fprintln(w, " src proposed dst origin freq spread")
|
||||
fmt.Fprintln(w, " (least-confident first — the role's own ordering of what to read)")
|
||||
fmt.Fprintln(w, " src proposed dst origin freq spread conv conf")
|
||||
for _, r := range shown {
|
||||
fmt.Fprintf(w, " %-20s %-25s %-9s %5d %6d\n", trunc(r.Src, 20), trunc(dashIfEmpty(r.Dst), 25), r.Origin, r.Freq, r.Spread)
|
||||
fmt.Fprintf(w, " %-20s %-25s %-9s %5d %6d %5d %5s\n", trunc(r.Src, 20), trunc(dashIfEmpty(r.Dst), 25),
|
||||
r.Origin, r.Freq, r.Spread, r.Conventions, confCell(r.Conf))
|
||||
// WHY this row has this dst — the question the artifacts could not answer before the fix-pack. The
|
||||
// signals are the ranking factors that fired for the winning variant; "invented" says the rendering is
|
||||
// the role's own, not one the drafts proposed (legitimate — it saw the whole book — and worth a look).
|
||||
if len(r.Signals) > 0 || r.Invented {
|
||||
why := strings.Join(r.Signals, ", ")
|
||||
if r.Invented {
|
||||
why = strings.TrimPrefix(why+" · invented (no draft proposed it)", " · ")
|
||||
}
|
||||
fmt.Fprintf(w, " why: %s\n", trunc(why, 90))
|
||||
}
|
||||
if len(r.Contradicts) > 0 {
|
||||
// A contradiction against this run's OWN consolidations: the compound dropped the rendering the
|
||||
// same reply gave its part. Loudest line of the row — it is a canon breaking inside one call.
|
||||
fmt.Fprintf(w, " ⚠ contradicts: %s\n", trunc(strings.Join(r.Contradicts, "; "), 80))
|
||||
}
|
||||
if len(r.Variants) > 1 {
|
||||
// The disagreement is the reason to sign: one term coming back several ways is exactly what a
|
||||
// canon exists to close, and hiding it behind one winner throws that reason away.
|
||||
|
|
@ -126,6 +149,29 @@ func renderBankStopRows(w io.Writer, rows []pipeline.BankStopRow) {
|
|||
fmt.Fprintln(w, "")
|
||||
}
|
||||
|
||||
// reviewRank orders the review surface: rows the role was least sure of first, rows it said nothing about
|
||||
// after them (silence is not a low score), and rows it never consolidated last.
|
||||
func reviewRank(r pipeline.BankStopRow) int {
|
||||
switch {
|
||||
case r.Dst == "":
|
||||
return 1000
|
||||
case r.Conf < 0:
|
||||
return 200
|
||||
default:
|
||||
return r.Conf
|
||||
}
|
||||
}
|
||||
|
||||
// confCell renders the role's stated confidence, or "—" when the reply carried none (a negative). Printing
|
||||
// a bare 0 for both would hide the single most important row on the sheet — the one the role itself said it
|
||||
// was least sure of — behind the rows it never mentioned.
|
||||
func confCell(conf int) string {
|
||||
if conf < 0 {
|
||||
return "—"
|
||||
}
|
||||
return strconv.Itoa(conf)
|
||||
}
|
||||
|
||||
// trunc shortens s to n runes with an ellipsis (rune-safe, so a CJK surface is never split mid-character).
|
||||
func trunc(s string, n int) string {
|
||||
rs := []rune(s)
|
||||
|
|
|
|||
|
|
@ -290,3 +290,34 @@ func TestRenderSignatureStopShowsTheTable(t *testing.T) {
|
|||
t.Fatalf("a term with no proposed dst must show a dash:\n%s", b2.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderSignatureStopOrdersTheReviewLeastSureFirst pins the ONE use the role's stated confidence is
|
||||
// allowed to have (D39.102): it orders the review list, an ordinal inside one model's reply, and nothing
|
||||
// else. Without it the number is printed and never used — a column that claims to answer «what do I read
|
||||
// first» and does not. The full sidecar keeps source-key order; only this capped view is the review surface.
|
||||
func TestRenderSignatureStopOrdersTheReviewLeastSureFirst(t *testing.T) {
|
||||
rows := []pipeline.BankStopRow{
|
||||
{Src: "уверенный", Dst: "д", Origin: "mined", Freq: 9, Conf: 95},
|
||||
{Src: "неконсолидированный", Origin: "mined", Freq: 9, Conf: -1},
|
||||
{Src: "неуверенный", Dst: "д", Origin: "mined", Freq: 9, Conf: 20},
|
||||
{Src: "безмолвный", Dst: "д", Origin: "mined", Freq: 9, Conf: -1},
|
||||
}
|
||||
var b bytes.Buffer
|
||||
renderSignatureStop(&b, &pipeline.WaveSignatureStop{Terms: len(rows), SignaturePath: "p", Rows: rows})
|
||||
out := b.String()
|
||||
order := []string{"неуверенный", "уверенный", "безмолвный", "неконсолидированный"}
|
||||
at := -1
|
||||
for _, name := range order {
|
||||
i := strings.Index(out, name)
|
||||
if i < 0 {
|
||||
t.Fatalf("row %q missing:\n%s", name, out)
|
||||
}
|
||||
if i < at {
|
||||
t.Fatalf("review order broken at %q — want least-confident first, then silence, then unconsolidated:\n%s", name, out)
|
||||
}
|
||||
at = i
|
||||
}
|
||||
if !strings.Contains(out, "least-confident first") {
|
||||
t.Fatalf("the banner must say what its order means, or a sorted table reads as arbitrary:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,8 +47,16 @@ partition "ВОЛНА ДРАФТА [OK] (∥ workers, по draft-чанкам)"
|
|||
|
||||
if (Майнинг вкл. И дельта непуста? [OK]) then (да)
|
||||
:МАЙНИНГ-СТОП: MineBank по нормализованному источнику
|
||||
против контраста (сид и reject-set исключены) →
|
||||
signature map → прогон ОСТАНОВЛЕН (exit 3);
|
||||
против контраста (сид и reject-set исключены);
|
||||
:ТЕРМИНОЛОГ [OK]: Merge(WHICH∪WHAT) → KWIC → ранг §C2-3
|
||||
(голос по ФОЛДУ целевой формы, сырые формы в показе) →
|
||||
батчи ЮНИТАМИ: серия (равнодлинная) + СЕМЬЯ
|
||||
(общий корень: имя-префикс либо реалия-суффикс, данные скрипта)
|
||||
и containment-пара — юнит целиком в ОДИН вызов, кап членов;
|
||||
:$0-экраны: канон-конфликты · КОНФЛИКТЫ СВОИХ ЖЕ
|
||||
консолидаций · label-mismatch → стоп-таблица
|
||||
(сигналы топ-варианта · invented · уверенность · конвенций);
|
||||
:signature map → прогон ОСТАНОВЛЕН (exit 3);
|
||||
:Владелец: каждый терм → promote в mined_delta (approved+dst)
|
||||
ИЛИ decline в mined_rejects;
|
||||
:Ре-ран: драфт $0-резюм · пустая дельта → авто-континью;
|
||||
|
|
|
|||
250
backend/internal/lang/bankdata.go
Normal file
250
backend/internal/lang/bankdata.go
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
package lang
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"embed"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// bankdata.go: the BANK-ASSEMBLY data plane — language data read ONLY when the terminologist assembles its
|
||||
// batches, and by nothing that a wave touches.
|
||||
//
|
||||
// WHY A PLANE OF ITS OWN, and why it is NOT in data/ beside script-series.txt. Both existing data planes
|
||||
// fold into the WAVE snapshot: the embedded one through EmbeddedVersion, the pack one through
|
||||
// Pack.Version(). That is right for bytes that ride the wire or resolve a verdict — editing them must be a
|
||||
// loud --resnapshot. Family morphology does neither: it decides only WHICH candidates the terminologist
|
||||
// shows in ONE call. Folding it into the wave would re-bill a whole book's draft for a knob that cannot
|
||||
// change a single byte of what that draft bought — and it would contradict the ratified stance that the
|
||||
// terminology axis stays OFF the snapshot (config.TerminologyGate, terminologyVersion,
|
||||
// TestTerminologyIsNotSnapshotFolded).
|
||||
//
|
||||
// It is not unversioned, though: an edit here changes a batch's CONTENT, which is inside the bank-role
|
||||
// RequestHash, so the affected batches re-pay by mechanism (cents), and BankDataVersion() is logged with
|
||||
// the run so a signature map stays attributable to the data that produced it.
|
||||
|
||||
//go:embed bankdata/family-morphology.txt
|
||||
var bankFS embed.FS
|
||||
|
||||
// bankDataAlgoVersion tags the bank plane's hashing RECIPE (like embedAlgoVersion tags the embedded one).
|
||||
// Bump it only when the framing or the file set changes; a DATA edit already moves the hash via the bytes.
|
||||
const bankDataAlgoVersion = "bankdata-v1"
|
||||
|
||||
// bankDataFiles are the plane's files, in a FIXED order (the order is folded into the hash).
|
||||
var bankDataFiles = []string{"bankdata/family-morphology.txt"}
|
||||
|
||||
// FamilyAffix is one engine type's family rule: which side of the surface carries the shared root morpheme
|
||||
// and how many runes of it must be shared before two surfaces count as one family.
|
||||
type FamilyAffix struct {
|
||||
Suffix bool // the root is the TRAILING morpheme (head-final realia); false → the LEADING one (names)
|
||||
MinRunes int // ≤0 → this type forms no families
|
||||
}
|
||||
|
||||
// FamilyMorph is a source language's resolved family-channel data. The zero value is INERT — a language
|
||||
// written in no dense script, or in a dense script with no rows, co-batches exactly as it did before this
|
||||
// channel existed.
|
||||
type FamilyMorph struct {
|
||||
Affix map[string]FamilyAffix // engine type (name|place|title|term) → its rule
|
||||
MinMembers int // smallest set that counts as a family
|
||||
MaxMembers int // largest batch unit a family merge may produce (0 = unbounded)
|
||||
ContainmentRunes int // shortest candidate that may anchor the compositional channel
|
||||
}
|
||||
|
||||
// Enabled reports whether the language declares enough to form a family at all.
|
||||
func (f FamilyMorph) Enabled() bool {
|
||||
return f.MinMembers > 0 && (len(f.Affix) > 0 || f.ContainmentRunes > 0)
|
||||
}
|
||||
|
||||
// scriptFamily is one script's authored rows, before a language merges its dense scripts.
|
||||
type scriptFamily struct {
|
||||
affix map[string]FamilyAffix
|
||||
minMembers, maxMembers, containmentRunes int
|
||||
}
|
||||
|
||||
var (
|
||||
familyOnce sync.Once
|
||||
familyByScript map[string]*scriptFamily
|
||||
bankDataOnce sync.Once
|
||||
bankDataVal string
|
||||
)
|
||||
|
||||
func loadFamilyMorphology() {
|
||||
familyOnce.Do(func() {
|
||||
m, err := parseFamilyMorphology(mustBankData(bankDataFiles[0]))
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("lang: embedded %s is corrupt: %v", bankDataFiles[0], err))
|
||||
}
|
||||
familyByScript = m
|
||||
})
|
||||
}
|
||||
|
||||
// BankDataVersion is the content hash of the bank-assembly plane (recipe tag + each file's path and bytes,
|
||||
// same framing as EmbeddedVersion). It is deliberately NOT in the wave snapshot — see the file comment —
|
||||
// and is logged with the run instead, so "which data produced this signature map" stays answerable.
|
||||
func BankDataVersion() string {
|
||||
bankDataOnce.Do(func() {
|
||||
h := sha256.New()
|
||||
h.Write([]byte(bankDataAlgoVersion))
|
||||
for _, name := range bankDataFiles {
|
||||
h.Write([]byte("\x00" + name + "\x00"))
|
||||
h.Write(mustBankData(name))
|
||||
}
|
||||
bankDataVal = bankDataAlgoVersion + "-" + hex.EncodeToString(h.Sum(nil))[:12]
|
||||
})
|
||||
return bankDataVal
|
||||
}
|
||||
|
||||
// FamilyMorphology resolves the family-channel data for a SOURCE language by unioning the rows of its
|
||||
// DENSE scripts (the same gate the series channel uses: one rune ≈ one morpheme is what makes a shared
|
||||
// affix a shared MORPHEME rather than a coincidence). Merge rules, all on the conservative side:
|
||||
// - two dense scripts declaring the same type with DIFFERENT sides disagree about the language's
|
||||
// morphology, so that type forms no families rather than one guessed direction;
|
||||
// - otherwise the STRICTER rule wins (the longer required root), the strictest min_members, and the
|
||||
// SMALLEST max_members, so adding a script never loosens what another script asked for.
|
||||
func FamilyMorphology(lng string) FamilyMorph {
|
||||
loadLangScripts()
|
||||
loadFamilyMorphology()
|
||||
var out FamilyMorph
|
||||
names := make([]string, 0, 4)
|
||||
for name := range langScriptBy[strings.ToLower(strings.TrimSpace(lng))] {
|
||||
if cjkScriptNames[name] {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names) // deterministic merge order (the rules are order-free, the iteration must be too)
|
||||
conflict := map[string]bool{}
|
||||
for _, name := range names {
|
||||
sf := familyByScript[name]
|
||||
if sf == nil {
|
||||
continue
|
||||
}
|
||||
if out.Affix == nil {
|
||||
out.Affix = map[string]FamilyAffix{}
|
||||
}
|
||||
for typ, a := range sf.affix {
|
||||
cur, had := out.Affix[typ]
|
||||
switch {
|
||||
case !had:
|
||||
out.Affix[typ] = a
|
||||
case cur.Suffix != a.Suffix:
|
||||
conflict[typ] = true
|
||||
case a.MinRunes > cur.MinRunes:
|
||||
out.Affix[typ] = a
|
||||
}
|
||||
}
|
||||
if sf.minMembers > out.MinMembers {
|
||||
out.MinMembers = sf.minMembers
|
||||
}
|
||||
if sf.containmentRunes > out.ContainmentRunes {
|
||||
out.ContainmentRunes = sf.containmentRunes
|
||||
}
|
||||
if sf.maxMembers > 0 && (out.MaxMembers == 0 || sf.maxMembers < out.MaxMembers) {
|
||||
out.MaxMembers = sf.maxMembers
|
||||
}
|
||||
}
|
||||
for typ := range conflict {
|
||||
delete(out.Affix, typ)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseFamilyMorphology reads the plane's rows into script → rules. Pure, so a test can drive it with a
|
||||
// synthetic table and prove the channel is DATA. Fail-loud on an unknown key / side / non-positive integer:
|
||||
// a malformed row is a corrupt asset, and a silently-inert family channel is exactly the drift this
|
||||
// codebase refuses (the Palladius typo precedent).
|
||||
func parseFamilyMorphology(b []byte) (map[string]*scriptFamily, error) {
|
||||
out := map[string]*scriptFamily{}
|
||||
at := func(script string) *scriptFamily {
|
||||
if out[script] == nil {
|
||||
out[script] = &scriptFamily{affix: map[string]FamilyAffix{}}
|
||||
}
|
||||
return out[script]
|
||||
}
|
||||
posInt := func(i int, key, raw string) (int, error) {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(raw))
|
||||
if err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("line %d: %s must be a positive integer (%q)", i+1, key, raw)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
line := 0
|
||||
for i, raw := range strings.Split(string(b), "\n") {
|
||||
t := strings.TrimSpace(strings.TrimRight(raw, "\r"))
|
||||
if t == "" || strings.HasPrefix(t, "#") {
|
||||
continue
|
||||
}
|
||||
line = i
|
||||
f := strings.Split(t, "\t")
|
||||
for j := range f {
|
||||
f[j] = strings.TrimSpace(f[j])
|
||||
}
|
||||
if len(f) < 3 || f[1] == "" {
|
||||
return nil, fmt.Errorf("line %d: want `key<TAB>script<TAB>…`, got %q", i+1, t)
|
||||
}
|
||||
// The SCRIPT column is checked against the range registry. A typo parses fine and then simply never
|
||||
// matches a language's declared scripts — the channel goes silently inert for the pair that asked for
|
||||
// it, which is the exact silent-empty-table class the pack loader refuses everywhere else.
|
||||
if _, known := scriptRanges[f[1]]; !known {
|
||||
return nil, fmt.Errorf("line %d: unknown script %q — it resolves to no Unicode range, so every row naming it would be silently ignored (add it to scriptRanges, or fix the spelling)", i+1, f[1])
|
||||
}
|
||||
switch f[0] {
|
||||
case "family_affix":
|
||||
if len(f) != 5 || f[2] == "" {
|
||||
return nil, fmt.Errorf("line %d: family_affix wants `family_affix<TAB>script<TAB>type<TAB>prefix|suffix<TAB>min_root_runes`, got %q", i+1, t)
|
||||
}
|
||||
n, err := posInt(i, "min_root_runes", f[4])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch f[3] {
|
||||
case "prefix":
|
||||
at(f[1]).affix[f[2]] = FamilyAffix{MinRunes: n}
|
||||
case "suffix":
|
||||
at(f[1]).affix[f[2]] = FamilyAffix{Suffix: true, MinRunes: n}
|
||||
default:
|
||||
return nil, fmt.Errorf("line %d: side must be prefix|suffix, got %q", i+1, f[3])
|
||||
}
|
||||
case "family_min_members", "family_max_members", "family_containment_runes":
|
||||
if len(f) != 3 {
|
||||
return nil, fmt.Errorf("line %d: %s wants `%s<TAB>script<TAB>n`, got %q", i+1, f[0], f[0], t)
|
||||
}
|
||||
n, err := posInt(i, f[0], f[2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch f[0] {
|
||||
case "family_min_members":
|
||||
at(f[1]).minMembers = n
|
||||
case "family_max_members":
|
||||
at(f[1]).maxMembers = n
|
||||
default:
|
||||
at(f[1]).containmentRunes = n
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("line %d: unknown key %q (want family_affix|family_min_members|family_max_members|family_containment_runes)", i+1, f[0])
|
||||
}
|
||||
}
|
||||
// A HALF table is a corrupt one, for the same reason: family_affix with no family_min_members leaves
|
||||
// Enabled() false, so the rows are read, parsed, and then quietly do nothing.
|
||||
for name, sf := range out {
|
||||
switch {
|
||||
case sf.minMembers == 0:
|
||||
return nil, fmt.Errorf("script %q declares family rules but no family_min_members — the channel would load and stay inert (line %d region)", name, line+1)
|
||||
case len(sf.affix) == 0 && sf.containmentRunes == 0:
|
||||
return nil, fmt.Errorf("script %q declares family_min_members but neither family_affix nor family_containment_runes — nothing can form a family", name)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func mustBankData(name string) []byte {
|
||||
b, err := bankFS.ReadFile(name)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("lang: missing bank-data asset %q: %v", name, err))
|
||||
}
|
||||
return b
|
||||
}
|
||||
37
backend/internal/lang/bankdata/family-morphology.txt
Normal file
37
backend/internal/lang/bankdata/family-morphology.txt
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# FAMILY morphology per writing SCRIPT (bank fix-pack §G1, research/24 §B4): which candidates of a dense
|
||||
# script belong to ONE term FAMILY and must therefore travel in ONE terminologist call. Measured: the same
|
||||
# model rendered the 古月-family one way in batch 0 and another way in batch 2 — a chimera that correlates
|
||||
# with the batch boundary perfectly, on a family DetectSeries cannot co-batch (its members differ in length).
|
||||
#
|
||||
# A family is anchored on a shared ROOT MORPHEME, and where that root sits is ASYMMETRIC by class: a NAME
|
||||
# carries its clan/surname morpheme in FRONT (古月方源, 古月正), while a realia term carries its generic head
|
||||
# LAST (head-final writing: *-海空窍, *-寨). The asymmetry is DATA, not a Go branch, so a pair whose names
|
||||
# are head-final (or whose script is head-initial) declares its own rows and needs no engine edit.
|
||||
#
|
||||
# Rows (TAB-separated), all keyed by SCRIPT name (the names lang/script.go resolves to Unicode ranges):
|
||||
# family_affix<TAB>script<TAB>type<TAB>prefix|suffix<TAB>min_root_runes
|
||||
# one rule per engine type (name|place|title|term). A type with no row never forms a family.
|
||||
# family_min_members<TAB>script<TAB>n — below n a "family" is a coincidence, not a convention.
|
||||
# family_max_members<TAB>script<TAB>n — the blob backstop: a merge that would push a unit past n
|
||||
# is refused and COUNTED. Measured on the coldrun-a bank
|
||||
# distillation (150 surfaces): 24 refuses nothing and the
|
||||
# largest real unit is 20 — the 古月 clan with the surfaces
|
||||
# built on it; 16 splits that unit, and a split family is
|
||||
# the very chimera this channel exists to prevent.
|
||||
# family_containment_runes<TAB>script<TAB>n — the compositional channel: a candidate that is itself a
|
||||
# substring of another candidate (元海 ⊂ 元海空窍) joins its
|
||||
# unit. n bounds the anchor so a single generic morpheme
|
||||
# cannot sweep half the bank into one call.
|
||||
#
|
||||
# A script with NO rows here leaves the family channel INERT (the batcher then behaves exactly as before
|
||||
# this file existed), so a pair that has not been measured never silently changes what it buys.
|
||||
family_affix han name prefix 2
|
||||
# `nickname` is a person-surface the BANKNOTE channel accepts from a draft, so a candidate can carry it and
|
||||
# it needs a rule of its own — otherwise a whole legitimate type forms no families, silently.
|
||||
family_affix han nickname prefix 2
|
||||
family_affix han place suffix 2
|
||||
family_affix han title suffix 2
|
||||
family_affix han term suffix 2
|
||||
family_min_members han 2
|
||||
family_max_members han 24
|
||||
family_containment_runes han 2
|
||||
106
backend/internal/lang/bankdata_test.go
Normal file
106
backend/internal/lang/bankdata_test.go
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
package lang
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestBankPlaneIsOutsideTheWaveFold is the load-bearing property of the whole plane, and the reason it is a
|
||||
// separate embed rather than another row in data/script-series.txt: editing family morphology must NOT move
|
||||
// the wave snapshot. EmbeddedVersion folds into every book's snapshot, so a row added there would re-bill a
|
||||
// whole draft wave for a knob that only decides which candidates share a terminologist CALL — the axis
|
||||
// config.TerminologyGate is deliberately kept off the snapshot for.
|
||||
func TestBankPlaneIsOutsideTheWaveFold(t *testing.T) {
|
||||
for _, f := range embeddedDataFiles() {
|
||||
if strings.Contains(f.name, "family") {
|
||||
t.Fatalf("the bank plane's %s reached the WAVE-folded embed set — editing it would re-snapshot every book", f.name)
|
||||
}
|
||||
}
|
||||
// Both versions exist and are distinct namespaces, so a log line naming one can never be read as the other.
|
||||
if v := BankDataVersion(); !strings.HasPrefix(v, bankDataAlgoVersion+"-") {
|
||||
t.Fatalf("BankDataVersion must be tagged with its own recipe, got %q", v)
|
||||
}
|
||||
if strings.HasPrefix(BankDataVersion(), embedAlgoVersion) {
|
||||
t.Fatal("the two data planes must not share a version namespace")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamilyMorphologyIsData drives the parser with a synthetic table: a script's family rules — including
|
||||
// the name/realia asymmetry and the head side — are DATA, so a pair that is not in the repo declares its own
|
||||
// and needs no Go edit. The script here is a REGISTERED one that ships no family rows of its own — the first
|
||||
// version of this test declared «devanagari», which resolves to no Unicode range and was therefore dropped
|
||||
// in silence, so the flagship generality test proved nothing at all.
|
||||
func TestFamilyMorphologyIsData(t *testing.T) {
|
||||
got, err := parseFamilyMorphology([]byte(strings.Join([]string{
|
||||
"# a source whose NAMES are head-final and whose realia lead with the root",
|
||||
"family_affix\thangul\tname\tsuffix\t3",
|
||||
"family_affix\thangul\tterm\tprefix\t2",
|
||||
"family_min_members\thangul\t4",
|
||||
"family_containment_runes\thangul\t3",
|
||||
}, "\n")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sf := got["hangul"]
|
||||
if sf == nil {
|
||||
t.Fatalf("the declared script must be present: %v", got)
|
||||
}
|
||||
if a := sf.affix["name"]; !a.Suffix || a.MinRunes != 3 {
|
||||
t.Fatalf("a head-final name rule is one data row, got %+v", a)
|
||||
}
|
||||
if a := sf.affix["term"]; a.Suffix || a.MinRunes != 2 {
|
||||
t.Fatalf("the asymmetry is per-type data, got %+v", a)
|
||||
}
|
||||
if sf.minMembers != 4 || sf.containmentRunes != 3 || sf.maxMembers != 0 {
|
||||
t.Fatalf("bounds come from the rows, absent ones stay unset: %+v", sf)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamilyMorphologyRefusesCorruptRows: a malformed row is a corrupt asset, and the failure mode it would
|
||||
// otherwise produce — a channel that parses fine and silently forms nothing — is the exact drift the pack
|
||||
// loader already refuses for the Palladius tables.
|
||||
func TestFamilyMorphologyRefusesCorruptRows(t *testing.T) {
|
||||
for _, bad := range []string{
|
||||
"family_affix\than\tname\tmiddle\t2", // no such side
|
||||
"family_affix\than\tname\tprefix\t0", // a zero-rune root matches everything
|
||||
"family_affix\than\tname\tprefix", // truncated row
|
||||
"family_minmembers\than\t2", // typo'd key
|
||||
"family_min_members\than\tlots", // non-integer
|
||||
// A typo'd SCRIPT resolves to no range, so every row naming it is silently ignored and the pair that
|
||||
// asked for the channel gets an inert one.
|
||||
"family_affix\thann\tname\tprefix\t2\nfamily_min_members\thann\t2",
|
||||
// A HALF table: rules with no quorum, or a quorum with nothing to form a family from. Both parse and
|
||||
// then do nothing, which is the failure this pack declared intolerable everywhere else.
|
||||
"family_affix\than\tname\tprefix\t2",
|
||||
"family_min_members\than\t2",
|
||||
} {
|
||||
if _, err := parseFamilyMorphology([]byte(bad)); err == nil {
|
||||
t.Fatalf("a corrupt row must fail loud: %q", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamilyMorphologyForZhIsLive pins the shipped table through the resolver a book actually calls: zh is
|
||||
// written in a dense script that declares rules, an alphabetic source is inert, and the ja case shows the
|
||||
// union — kana declare nothing, so the language inherits han's rules rather than losing them.
|
||||
func TestFamilyMorphologyForZhIsLive(t *testing.T) {
|
||||
zh := FamilyMorphology("zh")
|
||||
if !zh.Enabled() {
|
||||
t.Fatal("zh is the measured pair and must have a live family channel")
|
||||
}
|
||||
if a := zh.Affix["name"]; a.Suffix || a.MinRunes < 2 {
|
||||
t.Fatalf("a han name family leads with the clan morpheme: %+v", a)
|
||||
}
|
||||
if a := zh.Affix["term"]; !a.Suffix || a.MinRunes < 2 {
|
||||
t.Fatalf("a han realia family ends with the generic head: %+v", a)
|
||||
}
|
||||
if zh.MinMembers < 2 || zh.MaxMembers <= 0 || zh.ContainmentRunes < 2 {
|
||||
t.Fatalf("the bounds must all be declared: %+v", zh)
|
||||
}
|
||||
if en := FamilyMorphology("en"); en.Enabled() {
|
||||
t.Fatalf("an alphabetic source forms no morpheme families: %+v", en)
|
||||
}
|
||||
if ja := FamilyMorphology("ja"); !ja.Enabled() || ja.Affix["term"] != zh.Affix["term"] {
|
||||
t.Fatalf("a language written in han inherits han's rules through the union: %+v", ja)
|
||||
}
|
||||
}
|
||||
|
|
@ -202,6 +202,16 @@ type PickedEntry struct {
|
|||
// no real path changes (pack-16 tail).
|
||||
func (p PickedEntry) valid() bool { return p.entry != nil }
|
||||
|
||||
// Src is the bank row's SOURCE surface — the only thing about a record a caller outside the bank can NAME.
|
||||
// It exists because n_evicted counted budget drops without ever saying WHICH rows the model did not get to
|
||||
// see, and a bare count is not something an operator can act on. Empty for a zero value.
|
||||
func (p PickedEntry) Src() string {
|
||||
if p.entry == nil {
|
||||
return ""
|
||||
}
|
||||
return p.entry.src
|
||||
}
|
||||
|
||||
// TrustGateEvent records a longest-match suppression the DISPOSITION-gate REFUSED (D39 layer 4):
|
||||
// a longer but LOWER-trust key (draft/ambiguous, e.g. draft 四代族长) that would have deleted a nested
|
||||
// HIGHER-trust key (approved/confirmed, e.g. approved 族长). The draft is editor-excluded (the block
|
||||
|
|
|
|||
|
|
@ -240,7 +240,14 @@ func DeltaYAML(mined []Term, proposals map[string][]DstProposal) (string, error)
|
|||
// The join key is the miner's own normalized surface, so a proposal written with different
|
||||
// orthography still lands on its term (the same normalization the seed-surface exclusion uses).
|
||||
if props := proposals[text.NormalizeSourceKey(m.Src)]; len(props) > 0 {
|
||||
st.Dst = props[0].Dst
|
||||
// The dst comes from a proposal made ON THIS SURFACE. A rendering that reached the term through an
|
||||
// ALIAS of its cluster is evidence — it is listed in the note with the surface that proposed it —
|
||||
// but the miner's clustering is itself unverified (measured: a surname cluster holding a clan
|
||||
// title), and promoting an alias's word to the term's own rendering is the clustering DECISION,
|
||||
// which belongs to the owner's signature and to nothing else.
|
||||
if props[0].Via == "" {
|
||||
st.Dst = props[0].Dst
|
||||
}
|
||||
note = appendProposalNote(note, props)
|
||||
}
|
||||
// TWO-MODE EMISSION (§C2-7, ratified and until pack-20 unbuilt): a term that arrives with a
|
||||
|
|
@ -272,19 +279,40 @@ func appendProposalNote(note string, props []DstProposal) string {
|
|||
b.WriteString(note)
|
||||
b.WriteString("; ")
|
||||
}
|
||||
fmt.Fprintf(&b, "dst PROPOSED by the translator (banknote, %d chunk(s)) — NOT approved, sign or replace it", props[0].Chunks)
|
||||
if props[0].Via != "" {
|
||||
// Nothing was proposed on this surface itself: every rendering came through an alias, so the row
|
||||
// carries no dst and the note says who proposed what for which surface.
|
||||
b.WriteString("dst PROPOSED only through cluster alias(es) — NOT attached, sign or replace it: ")
|
||||
for i, p := range props {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
fmt.Fprintf(&b, "%q ×%d%s", p.Dst, p.Chunks, viaNote(p.Via))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
fmt.Fprintf(&b, "dst PROPOSED by the translator (banknote, %d chunk(s)) — NOT approved, sign or replace it",
|
||||
props[0].Chunks)
|
||||
if len(props) > 1 {
|
||||
b.WriteString("; other proposals: ")
|
||||
for i, p := range props[1:] {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
fmt.Fprintf(&b, "%q ×%d", p.Dst, p.Chunks)
|
||||
fmt.Fprintf(&b, "%q ×%d%s", p.Dst, p.Chunks, viaNote(p.Via))
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// viaNote renders the alias provenance of a proposal, or "" for one made on the term's own surface.
|
||||
func viaNote(via string) string {
|
||||
if via == "" {
|
||||
return ""
|
||||
}
|
||||
return " [proposed for " + via + "]"
|
||||
}
|
||||
|
||||
// appendConsolidatedNote records that the dst on this row is the TERMINOLOGIST's consolidation, not a
|
||||
// per-chunk guess and not a signature. The distinction is the whole trust story of the auto mode: a
|
||||
// `draft` row reaches the wire with an unverified marker, so the sign map must say, in the artifact
|
||||
|
|
@ -374,4 +402,9 @@ type DstProposal struct {
|
|||
Dst string
|
||||
Type string
|
||||
Chunks int
|
||||
// Via names the surface that actually proposed this rendering when it is not the term's own — an ALIAS
|
||||
// the miner clustered onto it. It travels into the note because the alternative is a sign map that
|
||||
// presents a rendering proposed FOR ANOTHER SURFACE as a rendering of this one, and the clustering that
|
||||
// joined them is itself unverified (the polygon's surname cluster holding a clan title).
|
||||
Via string
|
||||
}
|
||||
|
|
|
|||
383
backend/internal/pipeline/bankfixpack_test.go
Normal file
383
backend/internal/pipeline/bankfixpack_test.go
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/config"
|
||||
"textmachine/backend/internal/miner"
|
||||
"textmachine/backend/internal/terminology"
|
||||
"textmachine/backend/internal/text"
|
||||
)
|
||||
|
||||
// bankfixpack_test.go: the fix-pack's pipeline-level seams — the arbitration record on the stop table (§G3),
|
||||
// the signature map's alias join (§G3), the auto-bank rewrite diff (row 130) and the eviction list.
|
||||
|
||||
// TestBankStopTableCarriesTheArbitrationRecord pins §A7's finding closed: «why does this term have THIS
|
||||
// dst» has to be answerable from the artifacts. The row now carries the winning variant's ranking signals,
|
||||
// whether the rendering is one the drafts ever proposed, how many CONVENTIONS they actually offered (as
|
||||
// against raw forms), and the role's own confidence.
|
||||
func TestBankStopTableCarriesTheArbitrationRecord(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
// A rendering NO draft proposed, with a confidence field — both the invented flag and the
|
||||
// confidence column have to survive the parser.
|
||||
return "方源\tГуюэ Фан Юань\t40", "stop"
|
||||
}
|
||||
// The drafts disagree on the CASE only: one convention written two ways.
|
||||
return "Фан Юань пришёл." + "\n" + bankSeparator +
|
||||
"\n方源\tФан Юань\tname\n方源\tфан юань\tname", "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}))
|
||||
defer r.Close()
|
||||
stop := runToSignatureStop(t, r)
|
||||
|
||||
var row *BankStopRow
|
||||
for i := range stop.Rows {
|
||||
if stop.Rows[i].Src == "方源" {
|
||||
row = &stop.Rows[i]
|
||||
}
|
||||
}
|
||||
if row == nil {
|
||||
t.Fatalf("方源 missing from the table: %+v", stop.Rows)
|
||||
}
|
||||
if row.Conf != 40 {
|
||||
t.Fatalf("the role's stated confidence must reach the review table, got %d", row.Conf)
|
||||
}
|
||||
if !row.Invented {
|
||||
t.Fatalf("a rendering no draft proposed must say so: %+v", *row)
|
||||
}
|
||||
if row.Conventions != 1 {
|
||||
t.Fatalf("two spellings of one rendering are ONE convention, got %d (%v)", row.Conventions, row.Variants)
|
||||
}
|
||||
if row.Spread < 2 {
|
||||
t.Fatalf("the owner must still see that the drafts wrote it two ways, got spread=%d", row.Spread)
|
||||
}
|
||||
raw, err := os.ReadFile(stop.TablePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"conventions=", "confidence=40", "INVENTED"} {
|
||||
if !strings.Contains(string(raw), want) {
|
||||
t.Fatalf("the sidecar must carry %q:\n%s", want, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBankStopRowCarriesTheWinningVariantsSignals pins the OTHER half of §A7's answer: not only WHAT was
|
||||
// chosen but WHY. The signals are the §C2-3 factors that fired for the variant the ranking put first, and
|
||||
// they were already computed and thrown away before this pack — so the failure mode is a row that looks
|
||||
// complete and explains nothing.
|
||||
func TestBankStopRowCarriesTheWinningVariantsSignals(t *testing.T) {
|
||||
c := terminology.Candidate{Key: "元海", Src: "元海", Type: "term", Variants: []terminology.Variant{
|
||||
{Dst: "море истинной ци", Chunks: 5, Forms: 1},
|
||||
{Dst: "первозданное море", Chunks: 1, Forms: 1},
|
||||
}}
|
||||
terminology.ScoreVariants(&c, terminology.ScoreOpts{})
|
||||
if len(c.Variants[0].Signals) == 0 {
|
||||
t.Fatalf("test premise broken: the winner must have fired at least one factor: %+v", c.Variants)
|
||||
}
|
||||
rows := bankStopRows([]terminology.Candidate{c}, map[string]string{"元海": "море истинной ци"}, terminologyResult{})
|
||||
if len(rows[0].Signals) == 0 {
|
||||
t.Fatalf("the winning variant's audit trail must reach the row: %+v", rows[0])
|
||||
}
|
||||
if strings.Join(rows[0].Signals, ",") != strings.Join(c.Variants[0].Signals, ",") {
|
||||
t.Fatalf("the row must carry the TOP variant's signals, got %v want %v", rows[0].Signals, c.Variants[0].Signals)
|
||||
}
|
||||
if !strings.Contains(renderBankStopTable(rows), "why: ") {
|
||||
t.Fatalf("and the sidecar must print them:\n%s", renderBankStopTable(rows))
|
||||
}
|
||||
// A row with no consolidation carries no confidence rather than a zero one: 0 means «the role said it was
|
||||
// not sure at all», which is the first row to review, not the same as silence.
|
||||
if rows[0].Conf >= 0 {
|
||||
t.Fatalf("an absent confidence must stay absent, got %d", rows[0].Conf)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply guards the seam between the two «this batch bought
|
||||
// nothing» signals. A batch the budget never reached has an empty reply text for a completely different
|
||||
// reason, and warning about an EMPTY COMPLETION there would cry wolf on every budget-bounded run — the
|
||||
// budget already said what happened, one line earlier.
|
||||
func TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply(t *testing.T) {
|
||||
var logBuf bytes.Buffer
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
// batch_runes tiny → several batches; a budget sized under one batch → the pass stops at the first.
|
||||
probe := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, batchRunes: 400}))
|
||||
_ = runToSignatureStop(t, probe)
|
||||
if probe.lastTerminology.Batches < 2 {
|
||||
t.Fatalf("the fixture must actually batch, got %d", probe.lastTerminology.Batches)
|
||||
}
|
||||
perBatch := probe.lastTerminology.EstimateUSD / float64(probe.lastTerminology.Batches)
|
||||
probe.Close()
|
||||
|
||||
r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{
|
||||
terminology: true, batchRunes: 400, budgetUSD: perBatch * 1.5,
|
||||
}))
|
||||
defer r.Close()
|
||||
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
_ = runToSignatureStop(t, r)
|
||||
out := logBuf.String()
|
||||
if !strings.Contains(out, "budget would be exceeded") {
|
||||
t.Fatalf("the fixture must actually hit the budget:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "EMPTY completion") {
|
||||
t.Fatalf("a batch the budget never reached is not a paid batch that came back empty:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSelfContradictionIsReportedAtTheStop is §G2 end to end: the run consolidates a part and then renders
|
||||
// the compound without it, and both the log and the row say so. The canon check cannot see this — the book
|
||||
// has no signed rows at all — which is precisely why 18 of the 149 contradictions on the live bank were
|
||||
// invisible.
|
||||
func TestSelfContradictionIsReportedAtTheStop(t *testing.T) {
|
||||
var logBuf bytes.Buffer
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
// 花家 → «клан Хуа», and the compound built on it drops «Хуа» entirely.
|
||||
return "花家\tклан Хуа\n花家的人\tлюди клана", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл." + "\n" + bankSeparator +
|
||||
"\n花家\tклан Хуа\tname\n花家的人\tлюди клана\tterm", "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}))
|
||||
defer r.Close()
|
||||
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
_ = runToSignatureStop(t, r)
|
||||
|
||||
if r.lastTerminology.SelfConflicts == 0 {
|
||||
t.Fatalf("the compound contradicts the part the same reply consolidated: %+v", r.lastTerminology)
|
||||
}
|
||||
if !strings.Contains(logBuf.String(), "contradict each other") {
|
||||
t.Fatalf("a run breaking its own canon must say so:\n%s", logBuf.String())
|
||||
}
|
||||
var flagged bool
|
||||
for _, row := range r.lastBankStopRows {
|
||||
if row.Src == "花家的人" && len(row.Contradicts) > 0 {
|
||||
flagged = true
|
||||
}
|
||||
}
|
||||
if !flagged {
|
||||
t.Fatalf("the contradicting row must be marked in the review table: %+v", r.lastBankStopRows)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSignatureMapJoinsAliasProposals is the acceptance finding of §G3: a rendering that reached the
|
||||
// terminologist through an ALIAS of a cluster showed up in the stop table (Merge routes it to the owner)
|
||||
// and was ABSENT from the owner's row in the signature map, which joined on the alias's own key. The map
|
||||
// and the table then described the same bank differently, for exactly the terms the miner clustered.
|
||||
func TestSignatureMapJoinsAliasProposals(t *testing.T) {
|
||||
mined := []terminology.Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子"}}}
|
||||
observed := []terminology.Observed{
|
||||
{Key: "方小子", Src: "方小子", Type: "name", Proposals: []terminology.Proposal{{Dst: "малыш Фан", Chunks: 3}}},
|
||||
}
|
||||
cands := terminology.Merge(mined, observed, text.NormalizeTargetForm)
|
||||
props := proposalsFromCandidates(cands)
|
||||
if got := props["方源"]; len(got) == 0 || got[0].Dst != "малыш Фан" {
|
||||
t.Fatalf("an alias-routed proposal must land on the OWNER's key, got %+v", props)
|
||||
}
|
||||
if props["方源"][0].Via != "方小子" {
|
||||
t.Fatalf("and it must say which surface actually proposed it, got %q", props["方源"][0].Via)
|
||||
}
|
||||
yaml, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, props)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(yaml, "малыш Фан") {
|
||||
t.Fatalf("the signature map must carry the rendering the table showed:\n%s", yaml)
|
||||
}
|
||||
if !strings.Contains(yaml, "proposed for 方小子") {
|
||||
t.Fatalf("and it must not present an alias's guess as the term's own:\n%s", yaml)
|
||||
}
|
||||
// AND it must not become the term's dst. Merge routes an alias's rendering to the cluster owner so the
|
||||
// ranking sees all the evidence; promoting it to the owner's rendering is the clustering DECISION, which
|
||||
// belongs to a signature. Here NOTHING was proposed on 方源 itself, so the row carries no dst at all.
|
||||
if strings.Contains(yaml, "dst: малыш Фан") {
|
||||
t.Fatalf("an alias's rendering became the term's dst — it reaches the wire with no model and no gate:\n%s", yaml)
|
||||
}
|
||||
// With a DIRECT proposal present, that one is the dst and the alias stays evidence in the note.
|
||||
observed = append(observed, terminology.Observed{
|
||||
Key: "方源", Src: "方源", Type: "name", Proposals: []terminology.Proposal{{Dst: "Фан Юань", Chunks: 1}},
|
||||
})
|
||||
direct := proposalsFromCandidates(terminology.Merge(mined, observed, text.NormalizeTargetForm))
|
||||
yaml2, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, direct)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(yaml2, "dst: Фан Юань") {
|
||||
t.Fatalf("a rendering proposed on the term's own surface must be its dst:\n%s", yaml2)
|
||||
}
|
||||
if !strings.Contains(yaml2, "малыш Фан") {
|
||||
t.Fatalf("and the alias proposal must still be listed as evidence:\n%s", yaml2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAutoBankRewriteNamesDroppedTerms is the $0 minimum of backlog row 130. The auto-bank file is rewritten
|
||||
// WHOLE from a top-N-capped delta, so a term that was in the bank for twenty chapters can vanish from it
|
||||
// mid-run with nothing in the artifacts saying so. The pack does not change that behaviour — accumulating is
|
||||
// the owner's STOP decision, because it moves memory_version — it makes the loss visible.
|
||||
func TestAutoBankRewriteNamesDroppedTerms(t *testing.T) {
|
||||
var logBuf bytes.Buffer
|
||||
r := &Runner{
|
||||
Log: slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})),
|
||||
Book: &config.Book{BookID: "test-book", ProjectDB: filepath.Join(t.TempDir(), "project.db")},
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
first := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "青茅山", Type: "place", Freq: 12}}
|
||||
if err := r.writeAutoBank(ctx, first, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if logBuf.Len() != 0 {
|
||||
t.Fatalf("the first write has nothing to diff against:\n%s", logBuf.String())
|
||||
}
|
||||
// The next run's delta no longer holds 青茅山 — the rank cap cut it, and it silently leaves the bank.
|
||||
second := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "花家", Type: "name", Freq: 9}}
|
||||
if err := r.writeAutoBank(ctx, second, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := logBuf.String()
|
||||
if !strings.Contains(out, "青茅山") {
|
||||
t.Fatalf("a term dropped by the rewrite must be NAMED:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "方源") {
|
||||
t.Fatalf("a term that is still there is not a loss:\n%s", out)
|
||||
}
|
||||
// The boundary is intact: the file is the new delta, not a merge of both runs.
|
||||
raw, err := os.ReadFile(r.autoBankPath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(raw), "青茅山") {
|
||||
t.Fatalf("the pack must REPORT the drop, not accumulate the file (that moves memory_version):\n%s", raw)
|
||||
}
|
||||
|
||||
// And the owner's OWN two verbs are not a loss. Promoting a term into the mined-delta (it becomes a seed
|
||||
// surface) or declining it in mined-rejects removes it from the next delta ON PURPOSE — warning about
|
||||
// that fires a false alarm on the normal signature cycle and advises the owner to do what he just did.
|
||||
logBuf.Reset()
|
||||
third := []miner.Term{{Src: "方源", Type: "name", Freq: 40}}
|
||||
handled := map[string]bool{text.NormalizeSourceKey("花家"): true}
|
||||
if err := r.writeAutoBank(ctx, third, nil, handled); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(logBuf.String(), "花家") {
|
||||
t.Fatalf("a term the owner promoted or declined is not a silent loss:\n%s", logBuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestAutoBankDiffSurvivesTheEngineOwnRows is the row-130 warning through the path production takes: two
|
||||
// real auto-mode runs, a term that leaves the delta between them, and the warning that has to name it.
|
||||
//
|
||||
// It has to be end-to-end, because the defect lives in the CALL SITE, not in the helper. From the second
|
||||
// auto-mode run the stored glossary also holds the engine's own unsigned rows — seedGlossary re-seeds the
|
||||
// auto-bank at start (seeding.go:83-93) — so passing the raw seed makes every term the engine ever proposed
|
||||
// read as «the owner decided about this». The diff then empties and the warning is structurally dead in
|
||||
// production while every unit test stays green. Risk 2, the self-exclusion trap, arriving at the one place
|
||||
// in this file that did not filter for it.
|
||||
func TestAutoBankDiffSurvivesTheEngineOwnRows(t *testing.T) {
|
||||
var logBuf bytes.Buffer
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
|
||||
|
||||
// Run 1: the auto mode mines 花家 among others, writes the auto-bank and seeds it into the store.
|
||||
r1 := newRunner(t, bookPath)
|
||||
if _, err := r1.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var had bool
|
||||
for _, src := range r1.autoBankSurfaces() {
|
||||
if src == "花家" {
|
||||
had = true
|
||||
}
|
||||
}
|
||||
if !had {
|
||||
t.Fatalf("test premise broken: the first run must bank 花家, got %v", r1.autoBankSurfaces())
|
||||
}
|
||||
r1.Close()
|
||||
|
||||
// The book changes and 花家 stops occurring, so it leaves the delta — standing in for the rank cap doing
|
||||
// the same thing mid-book, which is the case backlog row 130 is about.
|
||||
writeFile(t, filepath.Join(filepath.Dir(bookPath), "source.txt"),
|
||||
strings.Repeat("方源来到青茅山。方源很强。青茅山很高。", 6))
|
||||
|
||||
r2 := newRunner(t, bookPath)
|
||||
defer r2.Close()
|
||||
r2.Resnapshot = true // the source moved, and this fixture is about the bank diff, not the snapshot gate
|
||||
r2.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
if _, err := r2.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The store now holds the engine's own unsigned rows — the state that used to swallow the warning.
|
||||
seed, err := r2.Store.GlossaryForBook("test-book")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engineRows := 0
|
||||
for _, e := range seed {
|
||||
if e.Source == "mined" && e.Status != "approved" {
|
||||
engineRows++
|
||||
}
|
||||
}
|
||||
if engineRows == 0 {
|
||||
t.Fatal("test premise broken: the store must hold the engine's own unsigned rows by the second run")
|
||||
}
|
||||
out := logBuf.String()
|
||||
if !strings.Contains(out, "NOT in the one this run just wrote") || !strings.Contains(out, "花家") {
|
||||
t.Fatalf("the row-130 warning must NAME the term the rewrite dropped, engine rows in the seed and all:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvictedBankRowsAreNamed: n_evicted counted the rows the injection budget dropped and never said which,
|
||||
// so «the model had no canon for this term» was an unactionable integer. The names go to the log rather than
|
||||
// to a new column — a per-row detail column is a schema migration, and this repository has already paid for
|
||||
// a non-idempotent one (backlog row 49а).
|
||||
func TestEvictedBankRowsAreNamed(t *testing.T) {
|
||||
var logBuf bytes.Buffer
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, draftEdit)
|
||||
defer srv.Close()
|
||||
// A budget of one token cannot fit even the first glossary line, so every matched row is evicted.
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||||
source: suzukiSource, glossarySeed: suzukiSeed, glossaryTokenBudget: 1,
|
||||
})
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rs, err := r.Store.GetRetrievalState("test-book", 1, 0)
|
||||
if err != nil || rs == nil {
|
||||
t.Fatalf("retrieval_state: %v %v", rs, err)
|
||||
}
|
||||
if rs.NEvicted == 0 {
|
||||
t.Fatal("test premise broken: a one-token budget must evict the matched row")
|
||||
}
|
||||
out := logBuf.String()
|
||||
if !strings.Contains(out, "鈴木") {
|
||||
t.Fatalf("the evicted rows must be NAMED, not counted:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
|
@ -51,8 +51,11 @@ const bankMaxLines = 20
|
|||
// re-resolves paid text, so it must be a loud --resnapshot.
|
||||
// - bankParseVersion governs parseBanknote — which candidate lines are accepted. Its output is EVIDENCE
|
||||
// for the owner's signature, never a checkpoint and never a wire byte; every run recomputes it from
|
||||
// the stored answers (bankObservedForBook). NOT folded, logged with the run — the same contract
|
||||
// gates.terminology and gates.voice already carry, and for the same reason.
|
||||
// the stored answers (bankObservedForBook). NOT folded, and LOGGED with the run at the bank-mining stop
|
||||
// (mining.go, "draft-side proposals folded") — the same contract gates.terminology and gates.voice
|
||||
// carry, and for the same reason. The logging is what makes staying out of the snapshot honest rather
|
||||
// than merely cheap: without a consumer the tag is a comment, and two maps produced by different rules
|
||||
// are indistinguishable after the fact.
|
||||
//
|
||||
// Folding the parse rule (as a single parser_version did) would re-bill a whole draft wave for a change
|
||||
// that cannot alter one byte of what was bought — which is a standing incentive to leave the rule wrong.
|
||||
|
|
@ -60,8 +63,11 @@ const (
|
|||
bankSliceVersion = "banknote-slice-v1"
|
||||
// v2 (backlog 19 / D39.53 default): the src rule stopped being "contains a Han ideograph" (a language
|
||||
// FAMILY hardcoded in the engine) and became "occurs in the book"; v2.1 adds the column-order recovery
|
||||
// that same invariant makes possible.
|
||||
bankParseVersion = "banknote-parse-v2.1-src-attested+column-order"
|
||||
// that same invariant makes possible; v2.2 (backlog 19 fix-pack) makes the reading DELIMITER-AWARE — a
|
||||
// tab/pipe line is positional, only a space-run line re-joins a split rendering. Bumping this is not
|
||||
// decoration: the tag is what attributes a stored signature map to the rule that produced it, and a rule
|
||||
// change under an unchanged tag is exactly the drift the split into two versions exists to prevent.
|
||||
bankParseVersion = "banknote-parse-v2.2-src-attested+column-order+delimiter-aware"
|
||||
)
|
||||
|
||||
// bankDerivedNS is the derived-checkpoint id NAMESPACE (§4б EXACT formula) — the "tm-<name>-v1" prefix
|
||||
|
|
@ -79,6 +85,54 @@ var bankTypeOK = map[string]bool{"name": true, "place": true, "title": true, "te
|
|||
// with Python's unicode \s around the pipe is exact on the term corpus.)
|
||||
var bankFieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`)
|
||||
|
||||
// bankColumnSplit is the UNAMBIGUOUS half of that tolerance: a tab or a pipe is a delimiter the model chose,
|
||||
// never something that can occur inside a rendering. A line carrying one has declared its own columns.
|
||||
var bankColumnSplit = regexp.MustCompile(`\t|\s*\|\s*`)
|
||||
|
||||
// bankSpaceRun collapses the internal whitespace of a column read positionally, so «Фан␣␣Юань» arrives as
|
||||
// one clean rendering instead of carrying the model's stray spacing into the bank.
|
||||
var bankSpaceRun = regexp.MustCompile(` {2,}`)
|
||||
|
||||
// bankColumns splits one banknote line into (src + rendering fields, type). See parseBanknote for why the
|
||||
// delimiter decides which of the two readings applies.
|
||||
func bankColumns(line string) ([]string, string) {
|
||||
typ := "term"
|
||||
if cols := nonEmptyFields(bankColumnSplit, line); len(cols) >= 2 {
|
||||
if len(cols) >= 3 {
|
||||
if t := strings.ToLower(cols[2]); bankTypeOK[t] {
|
||||
typ = t
|
||||
}
|
||||
}
|
||||
// BOTH columns are collapsed, not just the second: the order-recovery below may swap them, and the
|
||||
// column that becomes the rendering must arrive clean either way.
|
||||
return []string{collapseRuns(cols[0]), collapseRuns(cols[1])}, typ
|
||||
}
|
||||
fields := nonEmptyFields(bankFieldSplit, line)
|
||||
if len(fields) >= 3 {
|
||||
if last := strings.ToLower(fields[len(fields)-1]); bankTypeOK[last] {
|
||||
typ = last
|
||||
fields = fields[:len(fields)-1]
|
||||
}
|
||||
}
|
||||
return fields, typ
|
||||
}
|
||||
|
||||
// collapseRuns folds a run of spaces inside a positionally-read column into one.
|
||||
func collapseRuns(s string) string {
|
||||
return strings.TrimSpace(bankSpaceRun.ReplaceAllString(s, " "))
|
||||
}
|
||||
|
||||
// nonEmptyFields splits a line on re with empty and whitespace-only fields dropped.
|
||||
func nonEmptyFields(re *regexp.Regexp, line string) []string {
|
||||
var out []string
|
||||
for _, p := range re.Split(strings.TrimSpace(line), -1) {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// bankEntry is one parsed candidate line (evidence for the bank-mining stop (§C); NOT written to the store until owner
|
||||
// signoff). Dst is the model's proposed translation — the direct dst delivery the co-occurrence
|
||||
// miner could not extract (蛊→гу, non-seed 龙公→Лун Гун).
|
||||
|
|
@ -92,7 +146,11 @@ type bankEntry struct {
|
|||
type bankFlags struct {
|
||||
NLines int // accepted (well-formed) banknote lines
|
||||
ParseFail bool // any malformed residual line (src not in the book / <2 fields) that was NOT a tolerated truncation
|
||||
Truncated bool // the LAST line was cut by generation length (tolerated, not a parse fail)
|
||||
// Truncated marks a block the run did NOT complete: the separator was there, so the model opened the
|
||||
// channel, but the generation ended on a non-stop finish and the stop-only gate refuses it. Read off the
|
||||
// finish reason, not from the parser (which never sees such a block). ⚠ It cannot see a cut that removed
|
||||
// the separator itself — on the reference that is the majority of length-cut chunks — so it is a floor.
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
// splitBanknote slices the banknote block off the raw model output BEFORE any gate/editor (integration
|
||||
|
|
@ -198,12 +256,17 @@ func (r *Runner) bankSrcAttested(chunkSource string) func(string) bool {
|
|||
}
|
||||
}
|
||||
|
||||
// parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry).
|
||||
// Tolerant of a TRUNCATED final line when truncatedGeneration is set (§B3-5): a short last line under
|
||||
// truncation is flagged banknote_truncated, not counted as a parse fail. Any other malformed line
|
||||
// (fewer than 2 fields, or a src NOT attested in the source) sets banknote_parse_fail. Deterministic, no
|
||||
// time/rand: srcAttested is a pure function of the book text (bankSourceIndex).
|
||||
func parseBanknote(block string, truncatedGeneration bool, srcAttested func(string) bool) ([]bankEntry, bankFlags) {
|
||||
// parseBanknote parses the tab-delimited block (integration seam feeding §C evidence + telemetry). A
|
||||
// malformed line (fewer than 2 fields, or a src NOT attested in the source) sets banknote_parse_fail.
|
||||
// Deterministic, no time/rand: srcAttested is a pure function of the book text (bankSourceIndex).
|
||||
//
|
||||
// It carried a `truncatedGeneration` tolerance for a short LAST line (§B3-5, faithful to banknote.py). The
|
||||
// ratified finish=stop-only gate means this parser is never handed a truncated block at all, so the only
|
||||
// caller pinned the argument to false and the branch could not execute in production while its comment said
|
||||
// it could. The truncation FACT is now read where it actually exists — off the finish reason, before the
|
||||
// gate refuses the block (applyBanknoteWithEntries) — and the unreachable branch is gone rather than left
|
||||
// as a tolerance nobody can reach.
|
||||
func parseBanknote(block string, srcAttested func(string) bool) ([]bankEntry, bankFlags) {
|
||||
var entries []bankEntry
|
||||
var flags bankFlags
|
||||
if strings.TrimSpace(block) == "" {
|
||||
|
|
@ -216,31 +279,25 @@ func parseBanknote(block string, truncatedGeneration bool, srcAttested func(stri
|
|||
}
|
||||
}
|
||||
bad := 0
|
||||
for i, ln := range lines {
|
||||
raw := bankFieldSplit.Split(strings.TrimSpace(ln), -1)
|
||||
var parts []string
|
||||
for _, p := range raw {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
parts = append(parts, p)
|
||||
}
|
||||
}
|
||||
if len(parts) < 2 {
|
||||
// A short LAST line under a truncated generation is a tolerated cut, not a failure.
|
||||
if i == len(lines)-1 && truncatedGeneration {
|
||||
flags.Truncated = true
|
||||
continue
|
||||
}
|
||||
for _, ln := range lines {
|
||||
// THE DELIMITER CARRIES INFORMATION, and reading the columns without it is what made the first cut of
|
||||
// this fix a regression on the frozen reference (−6 lines over 56 real blocks).
|
||||
//
|
||||
// A line the model delimited with a TAB or a PIPE has told us where its columns are, so it is read
|
||||
// POSITIONALLY: src, rendering, type — exactly as it was before the fix-pack. A third column outside
|
||||
// the closed type vocabulary («clan», «proverb», «onomatopoeia» — 1.25% of the reference) is then a
|
||||
// benign unknown type, as it always was; guessing it into the rendering corrupts a term the owner is
|
||||
// about to sign, and on a REVERSED line it made the src unfindable and dropped the line entirely.
|
||||
//
|
||||
// Only a line with NO tab and NO pipe is ambiguous, because the ≥2-space tolerance exists for models
|
||||
// that substitute spaces for tabs — and that same tolerance is what splits «Фан␣␣Юань» into two
|
||||
// fields. There, and only there, is the type identified by the closed vocabulary on the LAST field
|
||||
// and everything between re-joined, which is the discipline terminology.ParseReply already applies.
|
||||
fields, typ := bankColumns(ln)
|
||||
if len(fields) < 2 {
|
||||
bad++
|
||||
continue
|
||||
}
|
||||
src, dst := parts[0], parts[1]
|
||||
typ := "term"
|
||||
if len(parts) >= 3 {
|
||||
typ = strings.ToLower(parts[2])
|
||||
}
|
||||
if !bankTypeOK[typ] {
|
||||
typ = "term"
|
||||
}
|
||||
// The src rule: a bank line must name something the BOOK contains. A surface no chunk of the source
|
||||
// carries is either a hallucination or a mis-split line — not evidence about this book, and letting
|
||||
// it through would put an invented term on the owner's sign map.
|
||||
|
|
@ -249,14 +306,18 @@ func parseBanknote(block string, truncatedGeneration bool, srcAttested func(stri
|
|||
// is recovered instead of discarded (measured: one chunk emitted all 12 of its lines reversed,
|
||||
// including a term the owner had signed by hand). Order is decided, never guessed: the declared
|
||||
// order wins whenever it is attested, the reversal is taken only when it is the ONLY reading the
|
||||
// book supports, and a line neither reading supports is still a parse fail.
|
||||
// book supports, and a line neither reading supports is still a parse fail. The src is one field on
|
||||
// either reading — a source surface has no spaces to be split on — so the rendering is whatever
|
||||
// remains, which is what makes the recovery survive a split rendering instead of failing on it.
|
||||
src, dst := fields[0], strings.Join(fields[1:], " ")
|
||||
revSrc, revDst := fields[len(fields)-1], strings.Join(fields[:len(fields)-1], " ")
|
||||
switch {
|
||||
case srcAttested == nil:
|
||||
bad++
|
||||
continue
|
||||
case srcAttested(src): // declared order — including the ambiguous case where both sides are attested
|
||||
case srcAttested(dst):
|
||||
src, dst = dst, src
|
||||
case srcAttested(revSrc):
|
||||
src, dst = revSrc, revDst
|
||||
default:
|
||||
bad++
|
||||
continue
|
||||
|
|
@ -335,11 +396,20 @@ func (r *Runner) applyBanknoteWithEntries(role, rawText, finish, chunkSource str
|
|||
if block == "" {
|
||||
return rawText, "", bankFlags{}, nil // no separator → no strip, no derived checkpoint (byte-identical path)
|
||||
}
|
||||
if finish == "stop" {
|
||||
// finish=stop-only gate: accept candidates + count telemetry only for a complete generation.
|
||||
// truncatedGeneration=false is unreachable-otherwise in prod (a truncated gen is finish≠stop).
|
||||
entries, flags = parseBanknote(block, false, r.bankSrcAttested(chunkSource))
|
||||
if finish != "stop" {
|
||||
// The finish=stop-only gate (§4б, ratified) refuses the block, and that refusal is the whole content
|
||||
// of banknote_truncated: a block that IS there under a finish the run did not complete is a block the
|
||||
// bank never got. (Length is the usual cause and the one the column was named for; a refusal or a
|
||||
// filter cut lands here too, and for the owner it means the same thing — the WHAT of this chunk is
|
||||
// missing, and the counters must not read as if it were clean.)
|
||||
//
|
||||
// The flag used to be derived INSIDE the parser from a truncatedGeneration argument the only caller
|
||||
// pinned to false, so the column could not be 1 in production and the comment beside it said the
|
||||
// opposite. Reading it off the finish reason needs no parse of untrusted text, no schema change, and
|
||||
// leaves the gate exactly where it was ratified: nothing is accepted, the fact is recorded.
|
||||
return cleanText, cleanText, bankFlags{Truncated: true}, nil
|
||||
}
|
||||
entries, flags = parseBanknote(block, r.bankSrcAttested(chunkSource))
|
||||
return cleanText, cleanText, flags, entries
|
||||
}
|
||||
|
||||
|
|
@ -465,8 +535,9 @@ func (f *bankFold) observed() []terminology.Observed {
|
|||
}
|
||||
|
||||
// bankObservedByKey folds the durable per-chunk banknote rows into the draft-side view of each source
|
||||
// surface. Both consumers read it: the signature-map join (proposalsFromObserved) and the terminologist's
|
||||
// merge, which also needs the SURFACE so a proposal for a term the miner never found is not lost.
|
||||
// surface. Both consumers read it: the terminologist's merge (whose candidates the signature-map join
|
||||
// then reads, proposalsFromCandidates) — which also needs the SURFACE, so a proposal for a term the miner
|
||||
// never found is not lost.
|
||||
func bankObservedByKey(states []store.RetrievalState, target *unicode.RangeTable) []terminology.Observed {
|
||||
f := newBankFold(target)
|
||||
addRetrievalStates(f, states)
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ func TestSplitBanknoteSlicesBlock(t *testing.T) {
|
|||
if strings.Contains(clean, bankSeparator) || strings.Contains(clean, "方源") {
|
||||
t.Fatalf("the banknote block leaked into the clean translation: %q", clean)
|
||||
}
|
||||
ents, flags := parseBanknote(block, false, attestedIn(srcAll))
|
||||
ents, flags := parseBanknote(block, attestedIn(srcAll))
|
||||
if len(ents) != 2 || flags.ParseFail || flags.Truncated || flags.NLines != 2 {
|
||||
t.Fatalf("parse = %+v flags=%+v, want 2 clean entries", ents, flags)
|
||||
}
|
||||
|
|
@ -56,7 +56,7 @@ func TestParseBanknoteTolerantFieldSplit(t *testing.T) {
|
|||
// tab, ≥2 spaces, and pipe are all accepted delimiters (a model that emits spaces instead of a
|
||||
// real TAB still parses). type falls back to "term" when absent or unknown.
|
||||
block := "方源\tФан Юань\tname\n蛊师 гу-мастер title\n青茅山 | гора Цинмао | place\n古月\tГу Юэ"
|
||||
ents, flags := parseBanknote(block, false, attestedIn(srcAll))
|
||||
ents, flags := parseBanknote(block, attestedIn(srcAll))
|
||||
if flags.ParseFail {
|
||||
t.Fatalf("tolerant split should not fail: %+v", flags)
|
||||
}
|
||||
|
|
@ -82,7 +82,7 @@ func TestParseBanknoteSrcMustBeAttested(t *testing.T) {
|
|||
"San Michon\tСан-Мишон\tplace", // multi-word latin src
|
||||
"リン\tРин\tname", // kana src
|
||||
}, "\n")
|
||||
ents, flags := parseBanknote(block, false, attestedIn(src))
|
||||
ents, flags := parseBanknote(block, attestedIn(src))
|
||||
if flags.ParseFail {
|
||||
t.Fatalf("every src here is in the source; none may be a parse fail: %+v", flags)
|
||||
}
|
||||
|
|
@ -91,7 +91,7 @@ func TestParseBanknoteSrcMustBeAttested(t *testing.T) {
|
|||
}
|
||||
// The other half of the rule: a plausible, well-formed, Han-bearing line the book never contains.
|
||||
invented := "天魔宗\tСекта Небесного Демона\tplace"
|
||||
ents, flags = parseBanknote(invented, false, attestedIn(src))
|
||||
ents, flags = parseBanknote(invented, attestedIn(src))
|
||||
if !flags.ParseFail || len(ents) != 0 {
|
||||
t.Fatalf("an invented src must be a parse fail and yield nothing, got %+v %+v", ents, flags)
|
||||
}
|
||||
|
|
@ -111,7 +111,7 @@ func TestParseBanknoteRecoversColumnOrder(t *testing.T) {
|
|||
"花海\tМоре цветов\tplace", // declared order
|
||||
"выдумка\tтоже выдумка\tterm", // neither side attested → still a parse fail
|
||||
}, "\n")
|
||||
ents, flags := parseBanknote(block, false, attestedIn(src))
|
||||
ents, flags := parseBanknote(block, attestedIn(src))
|
||||
if !flags.ParseFail {
|
||||
t.Fatalf("the unattested line must still fail: %+v", flags)
|
||||
}
|
||||
|
|
@ -127,7 +127,7 @@ func TestParseBanknoteRecoversColumnOrder(t *testing.T) {
|
|||
// Ambiguity is resolved by the DECLARED order, never by a guess: when the book attests both fields
|
||||
// (a source that quotes the target language, or a same-script pair), the model's order stands.
|
||||
both := "San Michon\tСан-Мишон\tplace"
|
||||
ents, flags = parseBanknote(both, false, attestedIn(src+" Сан-Мишон"))
|
||||
ents, flags = parseBanknote(both, attestedIn(src+" Сан-Мишон"))
|
||||
if flags.ParseFail || len(ents) != 1 || ents[0].Src != "San Michon" {
|
||||
t.Fatalf("both-attested must keep the declared order, got %+v %+v", ents, flags)
|
||||
}
|
||||
|
|
@ -157,22 +157,21 @@ func TestBankSourceIndexParity(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestParseBanknoteTruncationTolerated(t *testing.T) {
|
||||
// The LAST line cut by generation length is tolerated (banknote_truncated), not a parse fail —
|
||||
// but ONLY under truncated_generation; the same short line otherwise IS a parse fail.
|
||||
// TestParseBanknoteShortLastLineIsAParseFail: the parser's truncation TOLERANCE is gone with the fix-pack,
|
||||
// and this pins what replaced it. A short last line reaches this parser only under finish=stop — a complete
|
||||
// generation that simply emitted a broken line — and that is a parse fail, not a tolerated cut. The
|
||||
// truncation FACT now lives where it is actually known, one level up, off the finish reason
|
||||
// (TestBanknoteTruncatedIsReachable).
|
||||
func TestParseBanknoteShortLastLineIsAParseFail(t *testing.T) {
|
||||
block := "方源\tФан Юань\tname\n蛊" // last line has no dst
|
||||
entsT, flagsT := parseBanknote(block, true, attestedIn(srcAll))
|
||||
if !flagsT.Truncated || flagsT.ParseFail || len(entsT) != 1 {
|
||||
t.Fatalf("truncated=true: want 1 entry + truncated flag + no parse_fail, got %+v %+v", entsT, flagsT)
|
||||
}
|
||||
entsF, flagsF := parseBanknote(block, false, attestedIn(srcAll))
|
||||
if flagsF.Truncated || !flagsF.ParseFail || len(entsF) != 1 {
|
||||
t.Fatalf("truncated=false: the short last line must be a parse_fail, got %+v %+v", entsF, flagsF)
|
||||
ents, flags := parseBanknote(block, attestedIn(srcAll))
|
||||
if flags.Truncated || !flags.ParseFail || len(ents) != 1 {
|
||||
t.Fatalf("a complete generation's short line is a parse_fail, got %+v %+v", ents, flags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBanknoteEmptyBlock(t *testing.T) {
|
||||
ents, flags := parseBanknote("", false, attestedIn(srcAll))
|
||||
ents, flags := parseBanknote("", attestedIn(srcAll))
|
||||
if len(ents) != 0 || flags.ParseFail || flags.Truncated || flags.NLines != 0 {
|
||||
t.Fatalf("empty block must yield no entries and clean flags, got %+v %+v", ents, flags)
|
||||
}
|
||||
|
|
@ -277,3 +276,109 @@ func TestBankTokenBudgetIsDerived(t *testing.T) {
|
|||
t.Fatalf("the line cap was raised on the mini-run measurement (2 of 9 blocks pressed against 12), got %d", bankMaxLines)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseBanknoteRejoinsASplitRendering is backlog row 129. The field splitter tolerates a run of ≥2
|
||||
// spaces because models substitute spaces for tabs — and that same tolerance splits INSIDE a rendering the
|
||||
// moment a model writes «Фан␣␣Юань». Taking parts[1] alone banked the half-name and pushed the tail into
|
||||
// the type column, where it failed the closed vocabulary and silently became "term": a mangled surface on
|
||||
// the owner's sign map with parse_fail still reading 0. The terminologist's parser was fixed for exactly
|
||||
// this; the banknote's was not.
|
||||
func TestParseBanknoteRejoinsASplitRendering(t *testing.T) {
|
||||
attested := attestedIn(srcAll)
|
||||
// BOTH shapes: with the type column present, and as a bare two-column line.
|
||||
entries, flags := parseBanknote("方源\tФан Юань\tname\n蛊师\tмастер гу", attested)
|
||||
if flags.ParseFail {
|
||||
t.Fatalf("neither line is malformed: %+v", flags)
|
||||
}
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("want both lines, got %+v", entries)
|
||||
}
|
||||
if entries[0].Dst != "Фан Юань" || entries[0].Type != "name" {
|
||||
t.Fatalf("a split rendering must be re-joined and the type column still read: %+v", entries[0])
|
||||
}
|
||||
if entries[1].Dst != "мастер гу" || entries[1].Type != "term" {
|
||||
t.Fatalf("with no type column the whole tail is the rendering: %+v", entries[1])
|
||||
}
|
||||
// THE SPACE-RUN PATH is the only one where the re-join can be observed at all: on a TAB line the
|
||||
// rendering is one column, so join(fields[1:]) and fields[1] are the same string and the re-join could be
|
||||
// reverted with every test still green. Here they differ, and «Фан» alone is what used to be banked.
|
||||
spaced, sflags := parseBanknote("方源 Фан Юань name", attested)
|
||||
if sflags.ParseFail || len(spaced) != 1 || spaced[0].Dst != "Фан Юань" || spaced[0].Type != "name" {
|
||||
t.Fatalf("a space-delimited line must re-join its rendering: %+v %+v", spaced, sflags)
|
||||
}
|
||||
if two, _ := parseBanknote("蛊师 мастер гу", attested); len(two) != 1 || two[0].Dst != "мастер гу" {
|
||||
t.Fatalf("and so must a space-delimited line with no type column: %+v", two)
|
||||
}
|
||||
// The recognised shapes are unchanged: a real three-field line is still src/dst/type.
|
||||
plain, _ := parseBanknote("青茅山\tгора Цинмао\tplace", attested)
|
||||
if len(plain) != 1 || plain[0].Dst != "гора Цинмао" || plain[0].Type != "place" {
|
||||
t.Fatalf("the ordinary three-field line must be untouched: %+v", plain)
|
||||
}
|
||||
// A TAB-delimited third column outside the type vocabulary is a benign unknown TYPE, exactly as it was
|
||||
// before the fix-pack — NOT a piece of the rendering. Guessing it into the rendering is what the first cut
|
||||
// of this fix did, and it corrupted four terms and dropped two on the frozen coldrun-a reference.
|
||||
odd, _ := parseBanknote("古月\tклан Гуюэ\tсемья", attested)
|
||||
if len(odd) != 1 || odd[0].Dst != "клан Гуюэ" || odd[0].Type != "term" {
|
||||
t.Fatalf("an unknown TYPE column must be discarded, not glued onto the rendering: %+v", odd)
|
||||
}
|
||||
// And the column-order recovery still works, including over a rendering the model spaced out.
|
||||
rev, _ := parseBanknote("Фан Юань\t方源\tname", attested)
|
||||
if len(rev) != 1 || rev[0].Src != "方源" || rev[0].Dst != "Фан Юань" {
|
||||
t.Fatalf("a reversed line must still be recovered: %+v", rev)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseBanknoteReadsDeclaredColumnsPositionally is the pin the frozen reference bought. A tab or a pipe
|
||||
// is a delimiter the model CHOSE — it cannot occur inside a rendering — so such a line has declared its own
|
||||
// columns and is read by position, as it was before the fix-pack. Only a line delimited by a run of spaces
|
||||
// is ambiguous, and only there may the type be guessed from the closed vocabulary.
|
||||
//
|
||||
// Measured cost of getting this wrong: replaying 56 real banknote blocks of ~/books/gu-zhenren/coldrun-a
|
||||
// through the first cut of the fix gave −6 lines against HEAD and +0 — four renderings corrupted by a glued
|
||||
// type word (clan/event/proverb/onomatopoeia — 1.25% of the reference carries a type outside the
|
||||
// vocabulary) and two lines dropped outright, with banknote_parse_fail unchanged in every one of them.
|
||||
func TestParseBanknoteReadsDeclaredColumnsPositionally(t *testing.T) {
|
||||
attested := attestedIn(srcAll)
|
||||
for _, tc := range []struct{ line, src, dst, typ string }{
|
||||
// An unknown third column: discarded as an unknown type, never glued (the four corrupted terms).
|
||||
{"古月\tГу Юэ\tclan", "古月", "Гу Юэ", "term"},
|
||||
{"方源|Фан Юань|proverb", "方源", "Фан Юань", "term"},
|
||||
// The same line REVERSED: the src is still found by attestation, and the unknown type still discarded.
|
||||
// The first cut looked for the src in the last field, found «clan», and dropped the line (the two lost).
|
||||
{"Гу Юэ\t古月\tclan", "古月", "Гу Юэ", "term"},
|
||||
// A known type column is read as a type, in both orders.
|
||||
{"青茅山\tгора Цинмао\tplace", "青茅山", "гора Цинмао", "place"},
|
||||
// And a rendering the model spaced out inside a REAL column arrives whole and clean.
|
||||
{"方源\tФан Юань\tname", "方源", "Фан Юань", "name"},
|
||||
} {
|
||||
got, flags := parseBanknote(tc.line, attested)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%q: want one entry, got %+v (parse_fail=%v)", tc.line, got, flags.ParseFail)
|
||||
}
|
||||
if got[0].Src != tc.src || got[0].Dst != tc.dst || got[0].Type != tc.typ {
|
||||
t.Fatalf("%q: got %+v, want src=%q dst=%q type=%q", tc.line, got[0], tc.src, tc.dst, tc.typ)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBanknoteTruncatedIsReachable closes the hygiene finding beside row 129: banknote_truncated was
|
||||
// derived inside the parser from an argument the only production caller pinned to false, so the column was
|
||||
// structurally 0 while its comment claimed otherwise. The refusal itself is the signal — a block present
|
||||
// under a non-stop finish was cut by generation length — and reading it off the finish reason changes
|
||||
// nothing about the ratified stop-only gate: still no candidates, still no telemetry lines.
|
||||
func TestBanknoteTruncatedIsReachable(t *testing.T) {
|
||||
r := &Runner{bankSrc: newBankSourceIndex([]chunk.Chunk{{Text: srcAll}})}
|
||||
raw := "Незаконченный перевод\n" + bankSeparator + "\n方源\tФан"
|
||||
_, _, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "length", srcAll)
|
||||
if !flags.Truncated {
|
||||
t.Fatalf("a block under a non-stop finish IS the truncation the column was meant to record: %+v", flags)
|
||||
}
|
||||
if len(entries) != 0 || flags.NLines != 0 {
|
||||
t.Fatalf("the stop-only gate still accepts nothing: %+v / %+v", entries, flags)
|
||||
}
|
||||
// A complete generation is unaffected.
|
||||
_, _, ok, got := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", srcAll)
|
||||
if ok.Truncated || len(got) != 1 {
|
||||
t.Fatalf("a complete generation is not truncated and its block is parsed: %+v / %+v", ok, got)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"textmachine/backend/internal/checks"
|
||||
|
|
@ -130,6 +133,7 @@ func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel memban
|
|||
}
|
||||
rs.NSpoilerBlocked = len(sel.Rejected)
|
||||
rs.NEvicted = len(sel.Evicted)
|
||||
r.recordEvicted(sel.Evicted)
|
||||
// Loud record of the disposition-gated suppressor (D39 layer 4): a longer lower-trust key refused
|
||||
// from eating a nested higher-trust one — the term-drift code root, now visible instead of a
|
||||
// silent drop (research/13 §7). Detail carries the refused suppressor→protected pairs for a human.
|
||||
|
|
@ -164,6 +168,72 @@ func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel memban
|
|||
return r.Store.UpsertRetrievalState(rs)
|
||||
}
|
||||
|
||||
// recordEvicted accumulates WHICH bank rows the injection budget dropped, book-wide, so the wave can name
|
||||
// them once instead of leaving n_evicted as a number nobody can act on.
|
||||
//
|
||||
// The list is kept in memory and reported to the LOG rather than stored beside the counter, deliberately: a
|
||||
// per-row detail column would be a schema migration, and this repository has already paid for a
|
||||
// non-idempotent one (backlog row 49а). The count keeps its column; the names ride the plane an operator
|
||||
// reads when deciding whether to raise gates.glossary token budget or prune the seed.
|
||||
func (r *Runner) recordEvicted(evicted []membank.PickedEntry) {
|
||||
if len(evicted) == 0 {
|
||||
return
|
||||
}
|
||||
r.evictMu.Lock()
|
||||
defer r.evictMu.Unlock()
|
||||
if r.evictedRows == nil {
|
||||
r.evictedRows = map[string]int{}
|
||||
}
|
||||
for _, p := range evicted {
|
||||
if src := p.Src(); src != "" {
|
||||
r.evictedRows[src]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// evictedNameCap bounds the names one warning carries: the point is to name the rows that lose most often,
|
||||
// not to reprint the bank.
|
||||
const evictedNameCap = 20
|
||||
|
||||
// reportEvicted drains the accumulator and names the rows the budget dropped most often. Called once per
|
||||
// wave, so a book that never overflows its injection budget stays silent.
|
||||
//
|
||||
// ⚠ Scope, stated: only the DRAFT wave feeds it, because n_evicted is a draft-wave column
|
||||
// (persistRetrievalState) — the edit wave runs its own Select whose evictions this pack does not record.
|
||||
func (r *Runner) reportEvicted(ctx context.Context, wave string) {
|
||||
r.evictMu.Lock()
|
||||
rows := r.evictedRows
|
||||
r.evictedRows = nil
|
||||
r.evictMu.Unlock()
|
||||
if len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
type row struct {
|
||||
src string
|
||||
n int
|
||||
}
|
||||
list := make([]row, 0, len(rows))
|
||||
for src, n := range rows {
|
||||
list = append(list, row{src, n})
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool {
|
||||
if list[i].n != list[j].n {
|
||||
return list[i].n > list[j].n
|
||||
}
|
||||
return list[i].src < list[j].src
|
||||
})
|
||||
named := make([]string, 0, evictedNameCap)
|
||||
for _, e := range list {
|
||||
if len(named) == evictedNameCap {
|
||||
break
|
||||
}
|
||||
named = append(named, fmt.Sprintf("%s ×%d", e.src, e.n))
|
||||
}
|
||||
r.Log.WarnContext(ctx, "memory: the injection token budget DROPPED bank rows before the model saw them — these terms had no canon on the wire for those units",
|
||||
"book", r.Book.BookID, "wave", wave, "rows", len(list), "shown", len(named),
|
||||
"budget_tokens", r.Pipeline.Context.GlossaryTokenBudget, "terms", strings.Join(named, ", "))
|
||||
}
|
||||
|
||||
// boolToStoreInt maps a bool telemetry flag to the store's 0/1 integer column form.
|
||||
func boolToStoreInt(b bool) int {
|
||||
if b {
|
||||
|
|
|
|||
|
|
@ -86,10 +86,17 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
|
|||
// exists. They arrive as EVIDENCE — nothing proposed enters the bank without a signature. A read failure
|
||||
// degrades to the WHICH-only map rather than blocking the stop: the map is what the owner needs, the
|
||||
// dst is a bonus.
|
||||
// The parse rule is LOGGED with the run, which is the whole contract that lets it stay out of the
|
||||
// snapshot: it cannot change a paid byte, but it decides which candidate lines became evidence, so a
|
||||
// signature map has to be attributable to the rule that produced it. Until the fix-pack the constant was
|
||||
// declared "logged with the run" and read by nothing — the tag was a comment, not a mechanism, and two
|
||||
// artifacts produced by different rules were indistinguishable.
|
||||
observed, offLanguage, oerr := r.bankObservedForBook()
|
||||
if oerr != nil {
|
||||
r.Log.WarnContext(ctx, "bank-mining: could not read the banknote proposals; the signature map falls back to WHICH-only (bare terms)", "err", oerr)
|
||||
}
|
||||
r.Log.InfoContext(ctx, "bank-mining: draft-side proposals folded", "book", r.Book.BookID,
|
||||
"surfaces", len(observed), "parse_version", bankParseVersion, "slice_version", bankSliceVersion)
|
||||
if offLanguage > 0 {
|
||||
r.Log.WarnContext(ctx, "bank-mining: draft-side proposals were written in another script and are NOT offered for signature",
|
||||
"book", r.Book.BookID, "dropped", offLanguage, "target_script", r.Pipeline.Gates.Terminology.TargetScript)
|
||||
|
|
@ -97,8 +104,12 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
|
|||
|
||||
// The TERMINOLOGIST (pack-20, D39.42): merge both channels, gather each candidate's source contexts,
|
||||
// rank the renderings the drafts produced (§C2-3), and — when the gate is on — consolidate the whole
|
||||
// bank in a handful of batched calls. With the gate off this is $0 assembly whose only consumer is the
|
||||
// stop's own table, and the emitted delta is byte-identical to before.
|
||||
// bank in a handful of batched calls. With the gate off this is $0 assembly, but NOT a no-op for the
|
||||
// artifact: since the fix-pack the delta's dst comes from these ranked, target-form-folded candidates
|
||||
// rather than from the raw draft-side order, so a term whose drafts disagreed can carry a different
|
||||
// proposal than it did before — see proposalsFromCandidates. That is bank CONTENT, so it moves
|
||||
// memory_version and the edit-wave snapshot with it (bank-only → $0 re-pin for every unit the term does
|
||||
// not occur in). The draft wave is untouched: these rows are Source:"mined" and base-excluded.
|
||||
tchunks := make([]terminology.Chunk, len(minerChunks))
|
||||
for i, c := range minerChunks {
|
||||
tchunks[i] = terminology.Chunk{Chapter: c.Chapter, ChunkIdx: c.ChunkIdx, NSource: c.NSource}
|
||||
|
|
@ -146,7 +157,7 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
|
|||
mined = attachConsolidatedDst(mined, consolidated)
|
||||
|
||||
// Non-empty delta → write the owner signature map (the mined seed-delta YAML) and STOP before the edit wave.
|
||||
proposals := proposalsFromObserved(observed)
|
||||
proposals := proposalsFromCandidates(cands)
|
||||
withDst := 0
|
||||
for _, m := range mined {
|
||||
if m.Dst != "" || len(proposals[text.NormalizeSourceKey(m.Src)]) > 0 {
|
||||
|
|
@ -163,7 +174,7 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
|
|||
// The RICH table (D39.36's «стоп с таблицей»: src · dst · frequency · variant spread · evidence). It is
|
||||
// written as a sidecar on EVERY run, signed or not, because it is also the auto mode's record of what
|
||||
// the book decided on its own — and it is capped on stdout, never in the file (emitRankCap is 200).
|
||||
rows := bankStopRows(cands, consolidated)
|
||||
rows := bankStopRows(cands, consolidated, tres)
|
||||
if werr := os.WriteFile(r.bankStopTablePath(), []byte(renderBankStopTable(rows)), 0o644); werr != nil {
|
||||
r.Log.WarnContext(ctx, "bank-mining: could not write the stop table sidecar (the signature map is unaffected)", "err", werr)
|
||||
}
|
||||
|
|
@ -180,7 +191,7 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
|
|||
// re-seeding through the ordinary seedGlossary path (rather than a second, private write) is what
|
||||
// keeps ONE definition of what the bank is: every guard the seed path owns — the collision checks,
|
||||
// the reject filter, the fail-louds — applies to the engine's rows exactly as to the owner's.
|
||||
if err := r.writeAutoBank(mined, proposals); err != nil {
|
||||
if err := r.writeAutoBank(ctx, mined, proposals, ownerHandled(unsignedEngineSurfaces(seed), rejects)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := r.seedGlossary(ctx); err != nil {
|
||||
|
|
@ -251,34 +262,98 @@ type BankStopRow struct {
|
|||
Variants []string // "rendering ×N", best-ranked first
|
||||
Contexts []string // source KWIC
|
||||
Evidence []string
|
||||
// The §G3 arbitration record: until this pack, «why does this term have THIS dst» was unanswerable from
|
||||
// the artifacts (research/24 §A7). Every field below is read off work the ranking already did.
|
||||
//
|
||||
// Conventions is how many genuinely different DECISIONS the drafts made, once renderings differing only
|
||||
// in target form are folded (Spread counts the raw forms). Signals are the §C2-3 factors that fired for
|
||||
// the TOP-ranked variant — the winner's audit trail, printed for the row rather than per variant,
|
||||
// because the row is what the owner signs. Invented says the consolidated rendering is NOT one the
|
||||
// drafts proposed: legitimate (the role sees the whole book, the drafts saw fragments) and exactly the
|
||||
// class to read first. Conf is the role's own stated confidence, which orders the review list and
|
||||
// nothing else (D39.102) — NEGATIVE when the reply carried none, because «the role said it was 0% sure»
|
||||
// is the most important row on the sheet and «the role said nothing» is not a row at all.
|
||||
// Contradicts names THIS RUN's other consolidations the rendering breaks (§G2).
|
||||
Conventions int
|
||||
Signals []string
|
||||
Invented bool
|
||||
Conf int
|
||||
Contradicts []string
|
||||
}
|
||||
|
||||
// bankStopRows projects the merged candidates into the operator table, best-ranked variants first.
|
||||
// Deterministic: cands is key-ordered and nothing here iterates a map for output.
|
||||
func bankStopRows(cands []terminology.Candidate, consolidated map[string]string) []BankStopRow {
|
||||
func bankStopRows(cands []terminology.Candidate, consolidated map[string]string, tres terminologyResult) []BankStopRow {
|
||||
out := make([]BankStopRow, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
dst := consolidated[c.Key]
|
||||
row := BankStopRow{
|
||||
Src: c.Src, Dst: consolidated[c.Key], Origin: string(c.Origin), Type: c.Type,
|
||||
Freq: c.Freq, Spread: c.Spread(), Contexts: c.KWIC, Evidence: c.Evidence,
|
||||
Src: c.Src, Dst: dst, Origin: string(c.Origin), Type: c.Type,
|
||||
Freq: c.Freq, Spread: c.Spread(), Conventions: c.Conventions(),
|
||||
Contexts: c.KWIC, Evidence: c.Evidence,
|
||||
Conf: confOrAbsent(tres.Conf, c.Key), Contradicts: tres.Contradictions[c.Src],
|
||||
}
|
||||
for _, v := range c.Variants {
|
||||
row.Variants = append(row.Variants, fmt.Sprintf("%s ×%d", v.Dst, v.Chunks))
|
||||
for i, v := range c.Variants {
|
||||
label := fmt.Sprintf("%s ×%d", v.Dst, v.Chunks)
|
||||
if v.Via != "" {
|
||||
label += " (proposed for " + v.Via + ")"
|
||||
}
|
||||
row.Variants = append(row.Variants, label)
|
||||
if i == 0 {
|
||||
row.Signals = v.Signals
|
||||
}
|
||||
}
|
||||
row.Invented = dst != "" && !proposedByDrafts(dst, c.Variants)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// confOrAbsent reads the role's stated confidence for a key, or -1 when the reply carried none. A plain
|
||||
// zero would merge the two, and they are opposites: one is the first row to review, the other is silence.
|
||||
func confOrAbsent(conf map[string]int, key string) int {
|
||||
if v, ok := conf[key]; ok {
|
||||
return v
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// proposedByDrafts reports whether the consolidated rendering is one the drafts actually produced, compared
|
||||
// under the same target-form fold the vote is counted with — so a case or ё difference is not reported as
|
||||
// an invention.
|
||||
func proposedByDrafts(dst string, vs []terminology.Variant) bool {
|
||||
want := text.NormalizeTargetForm(dst)
|
||||
for _, v := range vs {
|
||||
if text.NormalizeTargetForm(v.Dst) == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// renderBankStopTable serializes the FULL table for the sidecar. One block per term, the same shape the
|
||||
// stdout banner prints — so the capped view and the file cannot describe the bank differently.
|
||||
func renderBankStopTable(rows []BankStopRow) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "BANK VERIFICATION TABLE — %d term(s)\n", len(rows))
|
||||
b.WriteString("src · proposed dst · origin · type · freq · variant spread · evidence · source contexts\n\n")
|
||||
b.WriteString("src · proposed dst · origin · type · freq · variant spread · conventions · confidence ·\n")
|
||||
b.WriteString("why (the ranking factors that won) · contradictions · drafts · evidence · source contexts\n\n")
|
||||
for _, r := range rows {
|
||||
fmt.Fprintf(&b, "%s\t%s\n", r.Src, dashIfEmpty(r.Dst))
|
||||
fmt.Fprintf(&b, " origin=%s type=%s freq=%d spread=%d\n", r.Origin, dashIfEmpty(r.Type), r.Freq, r.Spread)
|
||||
fmt.Fprintf(&b, " origin=%s type=%s freq=%d spread=%d conventions=%d", r.Origin, dashIfEmpty(r.Type), r.Freq, r.Spread, r.Conventions)
|
||||
if r.Conf >= 0 {
|
||||
fmt.Fprintf(&b, " confidence=%d", r.Conf)
|
||||
}
|
||||
if r.Invented {
|
||||
b.WriteString(" INVENTED(no draft proposed it)")
|
||||
}
|
||||
b.WriteString("\n")
|
||||
if len(r.Signals) > 0 {
|
||||
fmt.Fprintf(&b, " why: %s\n", strings.Join(r.Signals, ", "))
|
||||
}
|
||||
if len(r.Contradicts) > 0 {
|
||||
fmt.Fprintf(&b, " CONTRADICTS this run's own: %s\n", strings.Join(r.Contradicts, "; "))
|
||||
}
|
||||
if len(r.Variants) > 0 {
|
||||
fmt.Fprintf(&b, " drafts: %s\n", strings.Join(r.Variants, " | "))
|
||||
}
|
||||
|
|
@ -348,17 +423,40 @@ func reverseSectionTerms(cands []terminology.Candidate, seed []store.GlossaryEnt
|
|||
return out, eligible
|
||||
}
|
||||
|
||||
// proposalsFromObserved re-shapes the folded draft-side view into the signature-map join's input. One
|
||||
// definition of the fold (bankObservedByKey) feeds both consumers, so the map the owner signs and the
|
||||
// table the terminologist read can never disagree about what a chunk proposed.
|
||||
func proposalsFromObserved(obs []terminology.Observed) map[string][]miner.DstProposal {
|
||||
out := make(map[string][]miner.DstProposal, len(obs))
|
||||
for _, o := range obs {
|
||||
list := make([]miner.DstProposal, 0, len(o.Proposals))
|
||||
for _, p := range o.Proposals {
|
||||
list = append(list, miner.DstProposal{Dst: p.Dst, Type: p.Type, Chunks: p.Chunks})
|
||||
// proposalsFromCandidates re-shapes the MERGED candidates into the signature-map join's input.
|
||||
//
|
||||
// It reads the candidates rather than the raw draft-side view, and that is the fix (§G3, acceptance
|
||||
// finding): a proposal that arrived under an ALIAS of a cluster is routed to the cluster's owner by
|
||||
// Merge — the stop table therefore showed it — while the signature map joined on the alias's own key and
|
||||
// the owner's row never mentioned it. The map the owner signs then disagreed with the table he was reading
|
||||
// it against, for exactly the terms the miner clustered. One source for both removes the divergence
|
||||
// structurally instead of keeping two joins in step by hand.
|
||||
//
|
||||
// The list arrives §C2-3-ranked and target-form folded, so the note's alternatives are the ones the table
|
||||
// shows, in the order it shows them.
|
||||
func proposalsFromCandidates(cands []terminology.Candidate) map[string][]miner.DstProposal {
|
||||
out := make(map[string][]miner.DstProposal, len(cands))
|
||||
for _, c := range cands {
|
||||
if len(c.Variants) == 0 {
|
||||
continue
|
||||
}
|
||||
out[o.Key] = list
|
||||
// DIRECT proposals first, alias-routed ones after — and DeltaYAML takes the term's dst from a DIRECT
|
||||
// one only. Merge routes an alias's rendering to the cluster owner so the ranking sees all of the
|
||||
// entity's evidence; letting that rendering become the OWNER's dst is a different act entirely. It
|
||||
// would put «Малыш Фан» on 方源 with no model involved, on the $0 path, with the terminology gate OFF —
|
||||
// the exact harm Variant.Via was introduced to prevent, arriving through the fix that introduced Via.
|
||||
list := make([]miner.DstProposal, 0, len(c.Variants))
|
||||
for _, v := range c.Variants {
|
||||
if v.Via == "" {
|
||||
list = append(list, miner.DstProposal{Dst: v.Dst, Type: c.Type, Chunks: v.Chunks})
|
||||
}
|
||||
}
|
||||
for _, v := range c.Variants {
|
||||
if v.Via != "" {
|
||||
list = append(list, miner.DstProposal{Dst: v.Dst, Type: c.Type, Chunks: v.Chunks, Via: v.Via})
|
||||
}
|
||||
}
|
||||
out[c.Key] = list
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -448,7 +546,19 @@ func unsignedEngineSurfaces(rows []store.GlossaryEntry) []store.GlossaryEntry {
|
|||
// writeAutoBank persists the unsigned rows the auto mode decided to carry forward, as the same seed-YAML
|
||||
// schema everything else in this pipeline speaks (so `tmctl seed-lint` reads it, and a row can be moved
|
||||
// into the owner's delta by copy-paste). Deterministic: the mined list is already sorted by src.
|
||||
func (r *Runner) writeAutoBank(mined []miner.Term, proposals map[string][]miner.DstProposal) error {
|
||||
//
|
||||
// It DIFFS the file it is about to replace, and that is the $0 minimum of backlog row 130. The file is
|
||||
// rewritten WHOLE from this run's delta, and the delta is capped at the miner's top-200 (emitRankCap,
|
||||
// applied BEFORE the emission filters, so seed and declined terms do not free their slots). The two
|
||||
// mechanisms were ratified separately and their INTERACTION never was: as a book grows, a term that was in
|
||||
// the bank for twenty chapters silently vanishes from it mid-run, with nothing in the artifacts saying so.
|
||||
// Naming the losers costs nothing and moves no bytes.
|
||||
//
|
||||
// The BOUNDARY is deliberate and is the owner's STOP: this reports, it does not accumulate. Merging the old
|
||||
// file into the new one would change what the bank CONTAINS, which moves memory_version and re-prices the
|
||||
// edit wave — a decision, not a hygiene fix.
|
||||
func (r *Runner) writeAutoBank(ctx context.Context, mined []miner.Term, proposals map[string][]miner.DstProposal, signed map[string]bool) error {
|
||||
before := r.autoBankSurfaces()
|
||||
body, err := miner.DeltaYAML(mined, proposals)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline: marshal auto-bank: %w", err)
|
||||
|
|
@ -456,9 +566,72 @@ func (r *Runner) writeAutoBank(mined []miner.Term, proposals map[string][]miner.
|
|||
if err := os.WriteFile(r.autoBankPath(), []byte(body), 0o644); err != nil {
|
||||
return fmt.Errorf("pipeline: write auto-bank %s: %w", r.autoBankPath(), err)
|
||||
}
|
||||
if len(before) == 0 {
|
||||
return nil
|
||||
}
|
||||
now := make(map[string]bool, len(mined))
|
||||
for _, m := range mined {
|
||||
now[text.NormalizeSourceKey(m.Src)] = true
|
||||
}
|
||||
var gone []string
|
||||
for _, src := range before {
|
||||
key := text.NormalizeSourceKey(src)
|
||||
if now[key] || signed[key] {
|
||||
// signed[] is the owner's two verbs — PROMOTED into the mined-delta or DECLINED in mined-rejects.
|
||||
// Both remove the term from this run's delta on purpose, and reporting them as a silent loss would
|
||||
// fire a false alarm on the normal signature cycle — advising the owner to do what he just did.
|
||||
continue
|
||||
}
|
||||
gone = append(gone, src)
|
||||
}
|
||||
if len(gone) > 0 {
|
||||
sort.Strings(gone)
|
||||
r.Log.WarnContext(ctx, "bank-mining: terms that were in the auto-bank are NOT in the one this run just wrote — the file is rewritten whole from a top-N-capped delta, so a term the book still uses can drop out of the bank mid-run; if one of these matters, promote it into the owner's mined-delta file (it is then a seed surface and cannot be cut again)",
|
||||
"book", r.Book.BookID, "dropped", len(gone), "kept", len(mined), "rank_cap", miner.EmitRankCap(),
|
||||
"terms", strings.Join(gone, ", "), "auto_bank", r.autoBankPath())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownerHandled is the surface set the OWNER has already decided about: every signed seed surface (a promoted
|
||||
// term is one) plus every declined one. A term leaving the delta through either door is not a loss.
|
||||
//
|
||||
// ⚠ The caller MUST pass it through unsignedEngineSurfaces — risk 2, the self-exclusion trap this file
|
||||
// documents twice and works around in two other places. From the SECOND auto-mode run the stored glossary
|
||||
// also holds the engine's OWN unsigned rows (seedGlossary re-seeds the auto-bank at start), so a raw seed
|
||||
// makes every term the engine ever proposed look owner-decided: the diff empties, and the row-130 warning —
|
||||
// whose entire purpose is to name terms the rewrite dropped — can never fire again in production.
|
||||
func ownerHandled(seed []store.GlossaryEntry, rejects map[string]bool) map[string]bool {
|
||||
out := make(map[string]bool, len(seed)+len(rejects))
|
||||
for _, e := range seed {
|
||||
out[text.NormalizeSourceKey(e.Src)] = true
|
||||
for _, a := range e.Aliases {
|
||||
out[text.NormalizeSourceKey(a.Alias)] = true
|
||||
}
|
||||
}
|
||||
for k := range rejects {
|
||||
out[k] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// autoBankSurfaces reads the src surfaces of the auto-bank file as it stands BEFORE this run rewrites it,
|
||||
// in file order. Absent or unreadable → nil: the diff is observability, and failing a paid run because the
|
||||
// PREVIOUS artifact cannot be parsed would be the tail wagging the dog.
|
||||
func (r *Runner) autoBankSurfaces() []string {
|
||||
entries, err := membank.LoadGlossarySeed(r.autoBankPath())
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.Src != "" {
|
||||
out = append(out, e.Src)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// minedRejectFile is the owner's mined-term reject list (Book.MinedRejects, R1-FL-B): the src surfaces the
|
||||
// owner reviewed and DECLINED. It is a PROPOSAL filter only — rejects never enter the bank content, so this
|
||||
// file is deliberately NOT folded into the snapshot (a reject affects the next mining PROPOSAL, not any
|
||||
|
|
|
|||
|
|
@ -576,29 +576,40 @@ func TestTerminologistSpendIsInTheRunTotal(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestTerminologistEmptyReplyIsLoud: a paid batch that comes back unreadable (empty completion, prose,
|
||||
// truncation) would otherwise be indistinguishable from «the role declined these terms» — which is a
|
||||
// DECISION in §C2-7 — and the run would exit 0 having bought nothing.
|
||||
// TestTerminologistEmptyReplyIsLoud: a paid batch that comes back unreadable would otherwise be
|
||||
// indistinguishable from «the role declined these terms» — which is a DECISION in §C2-7 — and the run would
|
||||
// exit 0 having bought nothing.
|
||||
//
|
||||
// TWO branches, and until the fix-pack only the first was covered while the test's NAME claimed the second
|
||||
// (§G3): prose is a non-empty completion the parser rejects line by line, whereas an EMPTY completion never
|
||||
// reached the parser at all and took a silent `continue`. The fixture below is the honest one.
|
||||
func TestTerminologistEmptyReplyIsLoud(t *testing.T) {
|
||||
var logBuf bytes.Buffer
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
return "Извините, я не понял задание.", "stop" // prose: no line the parser can use
|
||||
}
|
||||
return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
|
||||
for _, tc := range []struct{ name, reply, want string }{
|
||||
{"prose", "Извините, я не понял задание.", "returned nothing the parser could use"},
|
||||
{"empty completion", "", "came back with an EMPTY completion"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var logBuf bytes.Buffer
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
return tc.reply, "stop"
|
||||
}
|
||||
return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(logBuf.String(), "returned nothing the parser could use") {
|
||||
t.Fatalf("a paid batch that bought no terminology must say so:\n%s", logBuf.String())
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(logBuf.String(), tc.want) {
|
||||
t.Fatalf("a paid batch that bought no terminology must say so (%s):\n%s", tc.name, logBuf.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import (
|
|||
|
||||
"textmachine/backend/internal/obs"
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
"textmachine/backend/internal/text"
|
||||
)
|
||||
|
||||
// minirun_fixes_test.go: the two seams the 25.07 mini-run proved were producing a fact and losing it.
|
||||
|
|
@ -165,7 +167,11 @@ func TestBanknoteProposalsArePersistedAndJoined(t *testing.T) {
|
|||
if len(props) != 2 {
|
||||
t.Fatalf("want 2 stored proposals, got %d: %s", len(props), rs.BanknoteDetail)
|
||||
}
|
||||
byKey := proposalsFromObserved(bankObservedByKey([]store.RetrievalState{*rs}, nil))
|
||||
// The signature-map join reads the MERGED candidates (fix-pack §G3), so the assertion follows the same
|
||||
// path production takes: a proposal must land on the miner's normalized surface after the merge, not
|
||||
// only in the raw draft-side view.
|
||||
observed := bankObservedByKey([]store.RetrievalState{*rs}, nil)
|
||||
byKey := proposalsFromCandidates(terminology.Merge(nil, observed, text.NormalizeTargetForm))
|
||||
if got := byKey[props[0].SrcKey]; len(got) == 0 || got[0].Dst != props[0].Dst {
|
||||
t.Fatalf("the fold must key proposals by the miner's normalized surface, got %+v", byKey)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import (
|
|||
"textmachine/backend/internal/llm"
|
||||
"textmachine/backend/internal/membank"
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
)
|
||||
|
||||
// runner.go: runner setup — Runner and its wiring (config stack, store in the right
|
||||
|
|
@ -78,10 +79,21 @@ type Runner struct {
|
|||
// .target_script (config validates the name). nil when the book declares none — the answer-language
|
||||
// screen is then inert, which loadTargetScript says out loud if any channel could have used it.
|
||||
targetScript *unicode.RangeTable
|
||||
// familyParams is the §G1 family channel resolved from the source's declared morphology, once, at load
|
||||
// time (loadFamilyParams). The zero value is a disabled channel — the batcher then co-batches series
|
||||
// only, exactly as before the channel existed. Like the rest of the terminology axis it is NOT
|
||||
// snapshot-folded: it changes which candidates share a CALL, never a wave byte.
|
||||
familyParams terminology.FamilyParams
|
||||
// rateGuards is the per-model wave-concurrency guard set (WS1 §1б), built once in the precompute pass and
|
||||
// read-only in the waves — a transport axis, never snapshot-folded. nil until buildRateGuards.
|
||||
rateGuards map[string]*rateGuard
|
||||
|
||||
// evictedRows accumulates WHICH bank rows the injection budget dropped (src → how many units), so the
|
||||
// wave can NAME them once rather than leave n_evicted as an unactionable count. Written from N draft
|
||||
// workers, hence the mutex; drained by reportEvicted.
|
||||
evictMu sync.Mutex
|
||||
evictedRows map[string]int
|
||||
|
||||
// bankSrc is the banknote parser's "is this surface in the book?" index (backlog 19), built once from
|
||||
// the chunk manifest in the precompute pass. nil outside a book run → the parser then judges a line by
|
||||
// the chunk text it was handed, and accepts nothing without one.
|
||||
|
|
|
|||
|
|
@ -117,6 +117,9 @@ type projectOpts struct {
|
|||
// rebillConsentUSD writes book.yaml's `rebill_consent_usd` override (D20.2-Q2). 0 = omit the key, so
|
||||
// the book takes the ratified default threshold and every pre-existing fixture is byte-unchanged.
|
||||
rebillConsentUSD float64
|
||||
// glossaryTokenBudget overrides context.glossary_token_budget. 0 keeps the fixture's historical 800, so
|
||||
// every pre-existing project is byte-identical; a tiny value is how the EVICTION path is reachable at all.
|
||||
glossaryTokenBudget int
|
||||
}
|
||||
|
||||
func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string {
|
||||
|
|
@ -130,6 +133,9 @@ func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string {
|
|||
if o.bookUSD == 0 {
|
||||
o.bookUSD = 1.0
|
||||
}
|
||||
if o.glossaryTokenBudget == 0 {
|
||||
o.glossaryTokenBudget = 800
|
||||
}
|
||||
dir := t.TempDir()
|
||||
|
||||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||||
|
|
@ -163,12 +169,12 @@ core: C1
|
|||
version: 1
|
||||
defaults: { max_output_ratio: 2.0, min_max_tokens: %d }
|
||||
retries: { regenerate_before_escalate: %d }
|
||||
context: { glossary_injection: selective, glossary_token_budget: 800 }
|
||||
context: { glossary_injection: selective, glossary_token_budget: %d }
|
||||
waves: { workers: %d }
|
||||
stages:
|
||||
- { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
|
||||
- { name: edit, role: editor, model: fake-model, prompt_override: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||||
%s`, o.minMaxTokens, o.regenerate, o.waveWorkers, gatesBlock))
|
||||
%s`, o.minMaxTokens, o.regenerate, o.glossaryTokenBudget, o.waveWorkers, gatesBlock))
|
||||
|
||||
sourceName := "source.txt"
|
||||
if len(o.epub) > 0 {
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ const terminologyStageName = "terminology"
|
|||
// terminologyVersion versions the ASSEMBLY algorithm (merge → KWIC → §C2-3 ranking → batching → parse). It
|
||||
// is deliberately NOT snapshot-folded — see config.TerminologyGate — but it is logged with the run so a
|
||||
// signature map can be attributed to the algorithm that produced it.
|
||||
const terminologyVersion = "terminology-v2-merge+kwic+c2-3+series"
|
||||
const terminologyVersion = "terminology-v3-merge+kwic+c2-3+series+families+formfold"
|
||||
|
||||
// Engine defaults for the block sizing. They bound ONE call's input; the whole book is covered by batching.
|
||||
const (
|
||||
|
|
@ -83,14 +83,29 @@ type terminologyResult struct {
|
|||
// OBSERVABILITY, never a gate: the rows stay unverified either way, and this is what tells the owner
|
||||
// which of them to look at first.
|
||||
CanonConflicts int
|
||||
CostUSD float64 // what THIS run's RENDER phase paid
|
||||
CumUSD float64 // what the render calls cost in total (a replayed checkpoint is $0 now, not then)
|
||||
Fresh bool // at least one call actually reached the provider this run
|
||||
EstimateUSD float64 // the pre-call projection, logged before any money moves
|
||||
// SelfConflicts counts consolidations that contradict ANOTHER consolidation of the same run (§G2) — the
|
||||
// majority class on a live bank, and the one nothing looked at before this pack. Contradictions carries
|
||||
// the same finding per key, so the stop table can mark the rows instead of printing a bare total.
|
||||
SelfConflicts int
|
||||
Contradictions map[string][]string
|
||||
// Conf is the role's own stated confidence per key. It sorts the review list «least sure first» and does
|
||||
// nothing else — never a weight, a threshold or a cross-model comparison (D39.102).
|
||||
Conf map[string]int
|
||||
CostUSD float64 // what THIS run's RENDER phase paid
|
||||
CumUSD float64 // what the render calls cost in total (a replayed checkpoint is $0 now, not then)
|
||||
Fresh bool // at least one call actually reached the provider this run
|
||||
EstimateUSD float64 // the pre-call projection, logged before any money moves
|
||||
// Reclassified is how many candidate types the §2 classifier phase actually changed; ClassifyCostUSD is
|
||||
// what that phase paid this run. Both zero when classify_types is off.
|
||||
Reclassified int
|
||||
ClassifyCostUSD float64
|
||||
// Families is how many family GROUPS §G1 detected; FamiliesRefused how many of their merges the member
|
||||
// cap turned down, and FamiliesHeld how many a series held part of back. Both of the latter mean the same
|
||||
// thing to the owner — a family met split across two calls — and both are zero when the source declares
|
||||
// no family data.
|
||||
Families int
|
||||
FamiliesRefused int
|
||||
FamiliesHeld int
|
||||
}
|
||||
|
||||
// loadTerminologyTemplate loads the pair's terminologist prompt when the gate is on. Mirrors
|
||||
|
|
@ -104,9 +119,42 @@ func (r *Runner) loadTerminologyTemplate() error {
|
|||
return err
|
||||
}
|
||||
r.terminologyTemplate = tpl
|
||||
if err := r.loadFamilyParams(); err != nil {
|
||||
return err
|
||||
}
|
||||
return r.loadClassifierTemplate()
|
||||
}
|
||||
|
||||
// loadFamilyParams resolves the §G1 family channel from the SOURCE language's declared morphology, once, at
|
||||
// LOAD time — before any billing, like every other gate precondition. A source with no family data yields a
|
||||
// disabled channel and the batcher behaves exactly as it did before the channel existed.
|
||||
//
|
||||
// It is also where a data typo dies: the file names engine TYPES, and lang cannot check them against the
|
||||
// engine's closed vocabulary without depending on this layer. A rule for a type nothing emits would parse
|
||||
// fine and leave the channel quietly half-off — the same silent-empty-table class the pack loader refuses.
|
||||
func (r *Runner) loadFamilyParams() error {
|
||||
fm := lang.FamilyMorphology(r.Book.SourceLang)
|
||||
if !fm.Enabled() {
|
||||
r.familyParams = terminology.FamilyParams{}
|
||||
return nil
|
||||
}
|
||||
sEnabled, headFinal := lang.SeriesMorphology(r.Book.SourceLang)
|
||||
p := terminology.FamilyParams{
|
||||
Enabled: sEnabled, HeadFinal: headFinal,
|
||||
Affix: make(map[string]terminology.FamilyAffix, len(fm.Affix)),
|
||||
MinMembers: fm.MinMembers, MaxMembers: fm.MaxMembers, ContainmentRunes: fm.ContainmentRunes,
|
||||
}
|
||||
for typ, a := range fm.Affix {
|
||||
if !terminology.CandidateTypes[typ] {
|
||||
return fmt.Errorf("pipeline: the family morphology of source %q names type %q, which no candidate can carry (accepted: %s) — a typo here would parse fine and leave the family channel silently half-off",
|
||||
r.Book.SourceLang, typ, strings.Join(terminology.TypeNames(terminology.CandidateTypes), "|"))
|
||||
}
|
||||
p.Affix[typ] = terminology.FamilyAffix{Suffix: a.Suffix, MinRunes: a.MinRunes}
|
||||
}
|
||||
r.familyParams = p
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadClassifierTemplate loads the pair's §2 classifier prompt when classify_types is on. Off → nil, and the
|
||||
// classifier phase is inert. Called from loadTerminologyTemplate: the classifier only exists as a phase of
|
||||
// the terminology gate.
|
||||
|
|
@ -179,7 +227,11 @@ func (r *Runner) buildBankCandidates(mined []miner.Term, observed []terminology.
|
|||
Freq: m.Freq, SinceCh: m.SinceCh, Aliases: m.Aliases, Evidence: m.Evidence,
|
||||
})
|
||||
}
|
||||
cands := terminology.Merge(ms, observed)
|
||||
// The vote is counted per target-form CONVENTION, not per byte string (§G5): «Море истинной ци» and
|
||||
// «море истинной ци» are one decision, and splitting their evidence hands the §C2-3 frequency factor to
|
||||
// whichever spelling a chunk happened to repeat. The normalizer is the SAME one the post-check matches
|
||||
// against, so the fold cannot disagree with the check that reads the result.
|
||||
cands := terminology.Merge(ms, observed, text.NormalizeTargetForm)
|
||||
_, kwicPer, kwicWidth := r.terminologyOpts()
|
||||
cands = terminology.AttachKWIC(cands, chunks, kwicPer, kwicWidth)
|
||||
|
||||
|
|
@ -262,20 +314,31 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
}
|
||||
}
|
||||
|
||||
// §1: co-batch grade/rank series so the role picks ONE generic head for the whole set. The pair-data —
|
||||
// whether the source forms rune-morpheme series and where the head sits — comes from the language layer,
|
||||
// so the batcher stays pair-agnostic and an alphabetic source takes a byte-identical, series-free path.
|
||||
// §1 + §G1: co-batch each grade/rank SERIES and each term FAMILY so the role picks ONE generic head, and
|
||||
// ONE shared root, for the whole set. The pair-data — whether the source forms rune-morpheme series,
|
||||
// where the head sits, and which side of a surface carries a family's root — comes from the language
|
||||
// layer, so the batcher stays pair-agnostic and a source with neither takes a byte-identical path.
|
||||
sEnabled, headFinal := lang.SeriesMorphology(r.Book.SourceLang)
|
||||
seriesID := terminology.DetectSeries(cands, terminology.SeriesParams{Enabled: sEnabled, HeadFinal: headFinal})
|
||||
batches := terminology.Batch(cands, batchRunes, seriesID)
|
||||
fp := r.familyParams
|
||||
fams := terminology.DetectFamilies(cands, fp)
|
||||
unitID, ustats := terminology.MergeUnits(seriesID, fams, fp)
|
||||
batches := terminology.Batch(cands, batchRunes, unitID)
|
||||
res.Batches = len(batches)
|
||||
// §1 keeps a series (or a lone candidate) WHOLE even past the budget — splitting a series would defeat the
|
||||
// one-head co-batching it exists for — so a co-batched grade set or an evidence-heavy single term can render
|
||||
res.Families, res.FamiliesRefused, res.FamiliesHeld = len(fams), ustats.Refused, ustats.Held
|
||||
if ustats.Refused > 0 || ustats.Held > 0 {
|
||||
// Either guard leaves a family split across calls — the very defect this channel exists to close — so
|
||||
// both are named rather than left to be inferred from a bank that disagrees with itself.
|
||||
r.Log.WarnContext(ctx, "terminology: some families were NOT co-batched whole — the member cap refused the merge, or a series with an unrelated root kept its members; those families can still disagree with themselves across calls",
|
||||
"book", r.Book.BookID, "refused_by_cap", ustats.Refused, "held_by_series", ustats.Held, "max_members", fp.MaxMembers)
|
||||
}
|
||||
// §1/§G1 keep a unit (or a lone candidate) WHOLE even past the budget — splitting one would defeat the
|
||||
// co-batching it exists for — so a co-batched grade set or an evidence-heavy single term can render
|
||||
// over the cap. That is deliberate but NOT silent: an over-cap unit strains the model's output ceiling (the
|
||||
// cap-8000 mine), so name it while the run can still be watched.
|
||||
for i, b := range batches {
|
||||
if br := terminology.BatchRunes(b); br > batchRunes {
|
||||
r.Log.WarnContext(ctx, "terminology: a batch renders OVER the size budget and is sent WHOLE (a series is co-batched by design, §1; a lone candidate cannot be split) — watch the model's output cap on this call",
|
||||
r.Log.WarnContext(ctx, "terminology: a batch renders OVER the size budget and is sent WHOLE (a series/family is co-batched by design, §1/§G1; a lone candidate cannot be split) — watch the model's output cap on this call",
|
||||
"book", r.Book.BookID, "batch", i, "terms", len(b), "batch_runes", br, "budget_runes", batchRunes)
|
||||
}
|
||||
}
|
||||
|
|
@ -293,13 +356,26 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
res.EstimateUSD, res.CostUSD, res.CumUSD, res.Fresh = run.estimateUSD, run.costUSD, run.cumUSD, run.fresh
|
||||
|
||||
out := map[string]string{}
|
||||
res.Conf = map[string]int{}
|
||||
for i, b := range batches {
|
||||
if i >= run.attempted {
|
||||
break // the budget or a ceiling stopped the pass here; runBankRoleBatches already said so
|
||||
}
|
||||
if run.texts[i] == "" {
|
||||
// An EMPTY completion on a call that was actually made. It used to `continue` in silence, which is
|
||||
// how a paid batch that returned nothing became indistinguishable from «the role declined these
|
||||
// terms» — a DECISION in §C2-7 — with the run exiting 0.
|
||||
r.Log.WarnContext(ctx, "terminology: a paid batch came back with an EMPTY completion; its terms stay unconsolidated",
|
||||
"book", r.Book.BookID, "batch", i, "asked", len(b), "answered", 0)
|
||||
continue
|
||||
}
|
||||
got, st := terminology.ParseReply(run.texts[i], candKeys(b), text.NormalizeSourceKey, r.targetScript)
|
||||
got, conf, st := terminology.ParseReply(run.texts[i], candKeys(b), text.NormalizeSourceKey, r.targetScript)
|
||||
res.BadLines += st.Bad
|
||||
res.OffLanguage += st.OffLanguage
|
||||
// Asked-vs-answered per batch: the one number that separates «the model skipped half the block» from
|
||||
// «the parser refused half the lines», and neither is visible in a total.
|
||||
r.Log.InfoContext(ctx, "terminology render batch", "book", r.Book.BookID,
|
||||
"batch", i, "asked", len(b), "answered", len(got), "bad_lines", st.Bad, "reply_chars", len(run.texts[i]))
|
||||
// A batch answered largely in another language is the measured cold-start failure, not noise: say it
|
||||
// while the run is happening, naming the lines, because the terms themselves just stay auto.
|
||||
if st.OffLanguage > 0 {
|
||||
|
|
@ -317,6 +393,9 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
for k, v := range got {
|
||||
out[k] = v
|
||||
}
|
||||
for k, v := range conf {
|
||||
res.Conf[k] = v
|
||||
}
|
||||
}
|
||||
for _, c := range cands {
|
||||
v, answered := out[c.Key]
|
||||
|
|
@ -341,6 +420,21 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
r.Log.WarnContext(ctx, "terminology: consolidated renderings contradict the SIGNED bank; they stay unverified (⟨проверить⟩) and are the first rows to review at the stop",
|
||||
"book", r.Book.BookID, "conflicts", len(conflicts), "terms", strings.Join(named, "; "))
|
||||
}
|
||||
// §G2: the same check with the run's OWN consolidations on the right-hand side. On a live bank this is
|
||||
// the MAJORITY of the contradictions (18 of 149, research/24 §A4) and until now nobody looked: the canon
|
||||
// check can only see rows the owner already signed, and a book being consolidated for the first time has
|
||||
// almost none. $0, evidence-side, never a gate.
|
||||
if self := terminology.ConsolidationConflicts(cands, out); len(self) > 0 {
|
||||
res.SelfConflicts = len(self)
|
||||
res.Contradictions = map[string][]string{}
|
||||
named := make([]string, 0, len(self))
|
||||
for _, cf := range self {
|
||||
named = append(named, fmt.Sprintf("%s→%q drops %s→%q", cf.Src, cf.Dst, cf.PartSrc, cf.PartDst))
|
||||
res.Contradictions[cf.Src] = append(res.Contradictions[cf.Src], fmt.Sprintf("%s→%q", cf.PartSrc, cf.PartDst))
|
||||
}
|
||||
r.Log.WarnContext(ctx, "terminology: consolidations of THIS run contradict each other — a compound's rendering drops the rendering the same reply gave its own part; they stay unverified and are review rows at the stop",
|
||||
"book", r.Book.BookID, "conflicts", len(self), "terms", strings.Join(named, "; "))
|
||||
}
|
||||
// The $0 label screen (§2 warm-run hygiene): a name/place row whose rendering was clearly TRANSLATED is a
|
||||
// label/rendering disagreement worth a human's eye. It is a review FLAG, never a gate, and explicitly not
|
||||
// a safety net for the transliteration harm — the classifier phase is what prevents that (see
|
||||
|
|
@ -356,7 +450,9 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
r.Log.InfoContext(ctx, "terminology finished", "book", r.Book.BookID,
|
||||
"consolidated", res.Consolidated, "declined", res.Declined, "unanswered", res.Unanswered,
|
||||
"reclassified", res.Reclassified, "bad_lines", res.BadLines, "off_language", res.OffLanguage,
|
||||
"canon_conflicts", res.CanonConflicts, "cost_usd", fmt.Sprintf("%.6f", res.CostUSD),
|
||||
"canon_conflicts", res.CanonConflicts, "self_conflicts", res.SelfConflicts,
|
||||
"families", res.Families, "families_refused", res.FamiliesRefused, "families_held", res.FamiliesHeld,
|
||||
"cost_usd", fmt.Sprintf("%.6f", res.CostUSD),
|
||||
"classify_cost_usd", fmt.Sprintf("%.6f", res.ClassifyCostUSD))
|
||||
return out, classified, res, nil
|
||||
}
|
||||
|
|
@ -400,11 +496,21 @@ func (r *Runner) runClassifier(ctx context.Context, snapID string, cands []termi
|
|||
out := map[string]string{}
|
||||
bad := 0
|
||||
for i, b := range batches {
|
||||
if i >= run.attempted {
|
||||
break // the budget stopped the pass here, and it already said so
|
||||
}
|
||||
if run.texts[i] == "" {
|
||||
// The same silence the render phase carried: a paid classify batch that returned NOTHING left every
|
||||
// one of its terms on the draft heuristic type — the mistyping this phase exists to remove — and the
|
||||
// only trace was that `bad` stayed 0, which reads as a clean pass.
|
||||
r.Log.WarnContext(ctx, "terminology classify: a paid batch came back with an EMPTY completion; its terms keep the draft heuristic type",
|
||||
"book", r.Book.BookID, "batch", i, "asked", len(b), "answered", 0)
|
||||
continue
|
||||
}
|
||||
got, st := terminology.ParseTypes(run.texts[i], candKeys(b), text.NormalizeSourceKey)
|
||||
bad += st.Bad
|
||||
r.Log.InfoContext(ctx, "terminology classify batch", "book", r.Book.BookID,
|
||||
"batch", i, "asked", len(b), "answered", len(got), "bad_lines", st.Bad)
|
||||
for k, v := range got {
|
||||
out[k] = v
|
||||
}
|
||||
|
|
@ -571,7 +677,11 @@ type bankRolePlan struct {
|
|||
// bankRoleRun is what one role's pass produced: each batch's reply text (in batch order, "" for a batch the
|
||||
// budget cut or that failed soft), plus the cost accounting for the report.
|
||||
type bankRoleRun struct {
|
||||
texts []string
|
||||
texts []string
|
||||
// attempted is how many batches the pass actually reached before a budget ceiling or a soft denial stopped
|
||||
// it. Without it "" is ambiguous — an EMPTY completion the run paid for and a batch never called look the
|
||||
// same — and the caller cannot warn about the first without crying wolf about the second.
|
||||
attempted int
|
||||
estimateUSD float64
|
||||
costUSD float64
|
||||
cumUSD float64
|
||||
|
|
@ -597,7 +707,8 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban
|
|||
}
|
||||
r.Log.InfoContext(ctx, "terminology "+logKind+": estimate before any call",
|
||||
"book", r.Book.BookID, "role", plan.role, "batches", len(batches), "model", st.Model, "reasoning", st.Reasoning,
|
||||
"estimate_usd", fmt.Sprintf("%.6f", run.estimateUSD), "budget_usd", plan.budgetUSD, "version", terminologyVersion)
|
||||
"estimate_usd", fmt.Sprintf("%.6f", run.estimateUSD), "budget_usd", plan.budgetUSD,
|
||||
"version", terminologyVersion, "bank_data", lang.BankDataVersion())
|
||||
|
||||
spent, err := r.Store.RoleSpentUSD(r.Book.BookID, plan.role)
|
||||
if err != nil {
|
||||
|
|
@ -638,6 +749,7 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban
|
|||
run.fresh = run.fresh || att.freshCall
|
||||
spent += att.runCost
|
||||
run.texts[i] = att.text
|
||||
run.attempted = i + 1
|
||||
}
|
||||
return run, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -127,6 +127,8 @@ func (r *Runner) translateBookWaves(ctx context.Context, chunks []chunk.Chunk, s
|
|||
return nil, err
|
||||
}
|
||||
|
||||
r.reportEvicted(ctx, "draft")
|
||||
|
||||
// --- the bank-mining stop: bank-mining stop boundary (auto-continues when mining is not configured) ---
|
||||
if stopped, err := r.runBankMiningStop(ctx, chunks, draftSnapshot, editWave); err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
215
backend/internal/terminology/arbitration_test.go
Normal file
215
backend/internal/terminology/arbitration_test.go
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
package terminology
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// arbitration_test.go: the §G2/§G3/§G5 half of the fix-pack — the run reading its OWN output (self
|
||||
// contradictions), the confidence field, and the vote counted per convention instead of per byte string.
|
||||
|
||||
// lowerFold is a stand-in for the target-form normalizer the pipeline injects: enough of it (case and
|
||||
// whitespace) to exercise the fold without importing a target's morphology into this package.
|
||||
func lowerFold(s string) string { return strings.Join(strings.Fields(strings.ToLower(s)), " ") }
|
||||
|
||||
// TestConsolidationConflictsCatchesTheRunContradictingItself is the §G2 case measured on the live bank: the
|
||||
// same reply consolidated a part and then rendered the compound without it. The canon check cannot see this
|
||||
// — nothing here is signed — and until this pack nothing else looked either.
|
||||
func TestConsolidationConflictsCatchesTheRunContradictingItself(t *testing.T) {
|
||||
cands := []Candidate{
|
||||
{Key: "元海", Src: "元海"},
|
||||
{Key: "元海空窍", Src: "元海空窍"},
|
||||
{Key: "空窍", Src: "空窍"},
|
||||
{Key: "青茅山", Src: "青茅山"},
|
||||
}
|
||||
out := map[string]string{
|
||||
"元海": "море истинной ци",
|
||||
"空窍": "апертура",
|
||||
"元海空窍": "апертура Первозданного моря", // carries «апертура», DROPS «истинной ци»
|
||||
"青茅山": "гора Цинмао",
|
||||
}
|
||||
got := ConsolidationConflicts(cands, out)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("exactly one contradiction (the compound against 元海), got %+v", got)
|
||||
}
|
||||
if got[0].Src != "元海空窍" || got[0].PartSrc != "元海" {
|
||||
t.Fatalf("the conflict must name the compound and the part it dropped: %+v", got[0])
|
||||
}
|
||||
// A compound that DOES carry both parts is not a conflict, case endings and all.
|
||||
out["元海空窍"] = "апертура моря истинной ци"
|
||||
if got := ConsolidationConflicts(cands, out); len(got) != 0 {
|
||||
t.Fatalf("a rendering carrying every part is consistent, got %+v", got)
|
||||
}
|
||||
// And a term nobody consolidated cannot contradict anything (silence is not a decision).
|
||||
delete(out, "元海")
|
||||
out["元海空窍"] = "апертура Первозданного моря"
|
||||
if got := ConsolidationConflicts(cands, out); len(got) != 0 {
|
||||
t.Fatalf("an unconsolidated part is not a contradiction, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseReplyTakesTheConfidenceFieldBeforeRejoining is the §G3 parser trap, both directions: a THIRD
|
||||
// field of digits is the role's confidence and must come off before the rendering is re-joined, while a
|
||||
// rendering that merely ENDS in a number is one field and must survive whole.
|
||||
func TestParseReplyTakesTheConfidenceFieldBeforeRejoining(t *testing.T) {
|
||||
id := func(s string) string { return s }
|
||||
keys := []string{"方源", "花家", "一零八峰", "忘却"}
|
||||
reply := strings.Join([]string{
|
||||
"方源\tФан Юань\t95",
|
||||
"花家\tДом Хуа\t40", // a stray double space AND a confidence: both handled, or the name is halved
|
||||
"一零八峰\tПик 108", // a rendering ending in a number, single space → one field, untouched
|
||||
"忘却\tзабвение\t900", // a declared column that is not a confidence: dropped and counted, never glued
|
||||
}, "\n")
|
||||
got, conf, st := ParseReply(reply, keys, id, nil)
|
||||
if got["方源"] != "Фан Юань" || conf["方源"] != 95 {
|
||||
t.Fatalf("want the rendering without the confidence and the confidence beside it: %q / %d", got["方源"], conf["方源"])
|
||||
}
|
||||
if got["花家"] != "Дом Хуа" || conf["花家"] != 40 {
|
||||
t.Fatalf("a split rendering must be re-joined AFTER the confidence comes off: %q / %d", got["花家"], conf["花家"])
|
||||
}
|
||||
if got["一零八峰"] != "Пик 108" {
|
||||
t.Fatalf("a rendering that legitimately ends in a number must survive whole, got %q", got["一零八峰"])
|
||||
}
|
||||
if _, has := conf["一零八峰"]; has {
|
||||
t.Fatalf("there was no confidence field on that line: %v", conf)
|
||||
}
|
||||
if got["忘却"] != "забвение" || st.BadConfidence != 1 {
|
||||
t.Fatalf("an unreadable confidence column is counted, never glued onto the rendering: %q / %d", got["忘却"], st.BadConfidence)
|
||||
}
|
||||
if st.Bad != 0 {
|
||||
t.Fatalf("a bad confidence does not refuse the rendering, got %d bad", st.Bad)
|
||||
}
|
||||
// The whole class the third column opened: anything the model writes there that is not a bare 0..100 —
|
||||
// «95%», «0.9», «высокая» — used to ride into the rendering, pass wellFormedLemma, pass the answer-language
|
||||
// screen, and enter the bank as this book's canon with bad_lines reading 0.
|
||||
for _, tail := range []string{"95%", "95 %", "0.9", "~90", "высокая"} {
|
||||
g, c, s2 := ParseReply("师父\tнаставник\t"+tail, []string{"师父"}, id, nil)
|
||||
if g["师父"] != "наставник" {
|
||||
t.Fatalf("tail %q was glued onto the rendering: %q", tail, g["师父"])
|
||||
}
|
||||
if _, has := c["师父"]; has {
|
||||
t.Fatalf("tail %q must not be read as a confidence: %v", tail, c)
|
||||
}
|
||||
if s2.BadConfidence != 1 {
|
||||
t.Fatalf("tail %q must be COUNTED, or the prompt's third column can fail silently: %+v", tail, s2)
|
||||
}
|
||||
}
|
||||
// THE SPACE-RUN PATH, which every case above misses: all of them are TAB-delimited, so the confidence is
|
||||
// stripped by the positional column reader and the space-run branch is never executed. Its own stripping
|
||||
// could be removed without a single test noticing — and then a model that answers with spaces instead of
|
||||
// tabs (the reason that tolerance exists at all) banks «Фан Юань 85» as this book's canon.
|
||||
spaced, sconf, sst := ParseReply("方源 Фан Юань 85", []string{"方源"}, id, nil)
|
||||
if spaced["方源"] != "Фан Юань" {
|
||||
t.Fatalf("a space-delimited line must lose its confidence field, not bank it: %q", spaced["方源"])
|
||||
}
|
||||
if sconf["方源"] != 85 {
|
||||
t.Fatalf("and the confidence must be read: %v", sconf)
|
||||
}
|
||||
if sst.Bad != 0 {
|
||||
t.Fatalf("the line is usable, got %d bad", sst.Bad)
|
||||
}
|
||||
// A two-field reply — the shape every existing pair prompt asks for — is unchanged and carries no
|
||||
// confidence at all, so the field is additive rather than a new requirement.
|
||||
plain, conf2, _ := ParseReply("方源\tФан Юань", []string{"方源"}, id, nil)
|
||||
if plain["方源"] != "Фан Юань" || len(conf2) != 0 {
|
||||
t.Fatalf("a two-field reply must parse as before with no confidence: %q / %v", plain["方源"], conf2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseReplyAcceptsAMangledDeclineSentinel: declining is a DECISION (§C2-7), and a decline the parser
|
||||
// refuses is counted as a broken line instead — the term then reads as "the model produced garbage" rather
|
||||
// than "the model said it could not tell".
|
||||
func TestParseReplyAcceptsAMangledDeclineSentinel(t *testing.T) {
|
||||
id := func(s string) string { return s }
|
||||
keys := []string{"方源", "花家", "青茅山"}
|
||||
reply := "方源\t" + NoDst + "\n花家\t«" + NoDst + "»\n青茅山\t" + NoDst + "."
|
||||
got, _, st := ParseReply(reply, keys, id, nil)
|
||||
for _, k := range keys {
|
||||
v, answered := got[k]
|
||||
if !answered || v != "" {
|
||||
t.Fatalf("%s must read as an explicit decline, got %q (answered=%v)", k, v, answered)
|
||||
}
|
||||
}
|
||||
if st.Bad != 0 {
|
||||
t.Fatalf("a decline is not a parse failure, got %d bad lines", st.Bad)
|
||||
}
|
||||
// Narrow on purpose: a rendering that merely MENTIONS something is not a decline.
|
||||
other, _, _ := ParseReply("方源\tвариант "+NoDst, []string{"方源"}, id, nil)
|
||||
if other["方源"] == "" {
|
||||
t.Fatal("only a rendering that IS the sentinel declines; a prefix match would swallow real answers")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFoldVariantsCountsConventionsAndShowsRawForms is §G5. Before it, «Море истинной ци» and «море
|
||||
// истинной ци» were two variants: the §C2-3 frequency factor scored the consensus on half its evidence, and
|
||||
// the spread column reported a contest where there was a spelling difference.
|
||||
func TestFoldVariantsCountsConventionsAndShowsRawForms(t *testing.T) {
|
||||
observed := []Observed{{Key: "元海", Src: "元海", Proposals: []Proposal{
|
||||
{Dst: "море истинной ци", Chunks: 3},
|
||||
{Dst: "Море истинной ци", Chunks: 4}, // same convention, different case
|
||||
{Dst: "Первозданное море", Chunks: 5},
|
||||
}}}
|
||||
got := Merge(nil, observed, lowerFold)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("one surface, one candidate: %+v", got)
|
||||
}
|
||||
c := got[0]
|
||||
if c.Conventions() != 2 {
|
||||
t.Fatalf("two decisions were made, not three: %+v", c.Variants)
|
||||
}
|
||||
if c.Spread() != 3 {
|
||||
t.Fatalf("the drafts still wrote it three ways and the owner must see that: spread=%d", c.Spread())
|
||||
}
|
||||
var folded *Variant
|
||||
for i := range c.Variants {
|
||||
if strings.EqualFold(c.Variants[i].Dst, "море истинной ци") {
|
||||
folded = &c.Variants[i]
|
||||
}
|
||||
}
|
||||
if folded == nil {
|
||||
t.Fatalf("the folded class must be present as a RAW form, not a normalized one: %+v", c.Variants)
|
||||
}
|
||||
if folded.Chunks != 7 {
|
||||
t.Fatalf("the vote is the sum of the class, got %d", folded.Chunks)
|
||||
}
|
||||
if folded.Dst != "Море истинной ци" {
|
||||
t.Fatalf("the representative is the most-proposed RAW form, got %q", folded.Dst)
|
||||
}
|
||||
// And the fold decides the ranking: 7 folded chunks now outweigh the 5 that used to win on 4-vs-5.
|
||||
ScoreVariants(&c, ScoreOpts{})
|
||||
if !strings.EqualFold(c.Best(), "море истинной ци") {
|
||||
t.Fatalf("the consensus must stop losing to its own spelling variant, got %q", c.Best())
|
||||
}
|
||||
}
|
||||
|
||||
// TestFoldVariantsKeepsViaProvenanceHonest: a rendering that ALSO arrived on the candidate's own key is
|
||||
// direct. First-wins used to label such a consensus «proposed for <alias>» purely because the alias row was
|
||||
// seen first — presenting an agreed rendering as a mis-clustered alias's guess.
|
||||
func TestFoldVariantsKeepsViaProvenanceHonest(t *testing.T) {
|
||||
mined := []Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子", "小方"}}}
|
||||
observed := []Observed{
|
||||
{Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 1}}}, // through an alias, FIRST
|
||||
{Key: "方源", Src: "方源", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 6}}}, // and directly
|
||||
{Key: "小方", Src: "小方", Proposals: []Proposal{{Dst: "малыш Фан", Chunks: 2}}}, // alias only
|
||||
}
|
||||
for _, c := range Merge(mined, observed, lowerFold) {
|
||||
if c.Key != "方源" {
|
||||
continue
|
||||
}
|
||||
for _, v := range c.Variants {
|
||||
switch v.Dst {
|
||||
case "Фан Юань":
|
||||
if v.Via != "" {
|
||||
t.Fatalf("a rendering also proposed on the term's own key is DIRECT, got via=%q", v.Via)
|
||||
}
|
||||
if v.Chunks != 7 {
|
||||
t.Fatalf("the alias vote still counts toward the total, got %d", v.Chunks)
|
||||
}
|
||||
case "малыш Фан":
|
||||
if v.Via != "小方" {
|
||||
t.Fatalf("a rendering that ONLY ever came through an alias must keep saying so, got %q", v.Via)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
package terminology
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// classify.go: the TYPE re-derivation channel (bank-quality §2, D39.68). The draft type heuristic is wrong
|
||||
// 12–22% (D39.65 row 36a), and type ∈ {name,place} routes a candidate to transliteration — so a realia
|
||||
|
|
@ -9,11 +12,30 @@ import "strings"
|
|||
// WITH the rendering arrives too late to de-bias it). This file is the pure half — the reply parser and the
|
||||
// honest $0 label screen; the class DEFINITIONS live in the pair's classifier prompt, no pair literal here.
|
||||
|
||||
// Types is the engine's closed type vocabulary. The names are engine identifiers (the miner emits them, the
|
||||
// glossary stores them, emissionEligible gates on them), so they are Go constants here, not pair data; the
|
||||
// pair's prompt explains each class in its own language with its own examples.
|
||||
// Types is the engine's closed type vocabulary for the CLASSIFIER's answer. The names are engine identifiers
|
||||
// (the miner emits them, the glossary stores them, emissionEligible gates on them), so they are Go constants
|
||||
// here, not pair data; the pair's prompt explains each class in its own language with its own examples.
|
||||
var Types = map[string]bool{"name": true, "place": true, "title": true, "term": true}
|
||||
|
||||
// CandidateTypes is the wider set a CANDIDATE can actually carry, and it is not the same set — which is a
|
||||
// live seam, not pedantry. The banknote channel accepts `nickname` from a draft (pipeline.bankTypeOK), that
|
||||
// type rides into Observed and onto a draft-side-only candidate, and the classifier never overwrites a term
|
||||
// it was not asked about. So anything keyed on a candidate's type — the family rules, first — must be keyed
|
||||
// on THIS set: validating against Types alone rejects a legitimate pair rule for `nickname` while the type
|
||||
// keeps arriving, i.e. refuses the fix and keeps the defect.
|
||||
var CandidateTypes = map[string]bool{"name": true, "place": true, "title": true, "term": true, "nickname": true}
|
||||
|
||||
// TypeNames lists a type set in a stable order, so an error message can say what it actually accepts instead
|
||||
// of a hardcoded list that drifts from the map beside it.
|
||||
func TypeNames(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for t := range set {
|
||||
out = append(out, t)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ParseTypes turns a classifier reply into key → corrected type, keeping only the terms we asked about and
|
||||
// only the closed vocabulary. Same tolerant two-field line format and the same accounting discipline as
|
||||
// ParseReply: an unusable or off-vocabulary line is COUNTED (st.Bad), never silently dropped, so a paid call
|
||||
|
|
|
|||
348
backend/internal/terminology/family_test.go
Normal file
348
backend/internal/terminology/family_test.go
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
package terminology
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// family_test.go: the §G1 family channel. What it has to prove is not "groups form" but the four properties
|
||||
// the measurement asked for — a family stays whole across a batch boundary, a variable-length rank line
|
||||
// stays whole, an equal-length series does not regress, and neither of them turns into a blob.
|
||||
|
||||
// zhFamily mirrors the han rows the bank-data plane ships (internal/lang/bankdata/family-morphology.txt):
|
||||
// names lead with the clan morpheme, realia end with the generic head, both need two runes of root, a family
|
||||
// is two surfaces, a unit is at most 24.
|
||||
var zhFamily = FamilyParams{
|
||||
Enabled: true, HeadFinal: true,
|
||||
Affix: map[string]FamilyAffix{
|
||||
"name": {MinRunes: 2},
|
||||
"nickname": {MinRunes: 2},
|
||||
"place": {Suffix: true, MinRunes: 2},
|
||||
"title": {Suffix: true, MinRunes: 2},
|
||||
"term": {Suffix: true, MinRunes: 2},
|
||||
},
|
||||
MinMembers: 2, MaxMembers: 24, ContainmentRunes: 2,
|
||||
}
|
||||
|
||||
func typed(key, typ string, freq int) Candidate {
|
||||
return Candidate{Key: key, Src: key, Type: typ, Freq: freq}
|
||||
}
|
||||
|
||||
// unitsOf inverts a unit map into sorted member lists, for readable assertions.
|
||||
func unitsOf(unitID map[string]int) map[int][]string {
|
||||
out := map[int][]string{}
|
||||
for k, id := range unitID {
|
||||
out[id] = append(out[id], k)
|
||||
}
|
||||
for _, v := range out {
|
||||
sort.Strings(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sameUnit(t *testing.T, unitID map[string]int, keys ...string) int {
|
||||
t.Helper()
|
||||
id := unitID[keys[0]]
|
||||
if id == 0 {
|
||||
t.Fatalf("%s is in no unit: %v", keys[0], unitsOf(unitID))
|
||||
}
|
||||
for _, k := range keys[1:] {
|
||||
if unitID[k] != id {
|
||||
t.Fatalf("%s must share a unit with %s (%d vs %d): %v", k, keys[0], unitID[k], id, unitsOf(unitID))
|
||||
}
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// TestDetectFamiliesGroupsBySharedRootAsymmetrically is the measured case (research/24 §B4): the 古月 family
|
||||
// is eight surfaces of DIFFERENT lengths, which DetectSeries cannot see, and the drafts rendered it two ways
|
||||
// on either side of a batch boundary. The clan morpheme LEADS a name and the generic head ENDS a realia
|
||||
// term, and the asymmetry is data — so the same string in the other position forms nothing.
|
||||
func TestDetectFamiliesGroupsBySharedRootAsymmetrically(t *testing.T) {
|
||||
cands := []Candidate{
|
||||
typed("古月方源", "name", 40), typed("古月正", "name", 9), typed("古月赤练", "name", 5),
|
||||
typed("元海空窍", "term", 7), typed("天海空窍", "term", 4), // realia sharing the trailing 海空窍
|
||||
typed("方源古月", "name", 2), // the clan morpheme TRAILING a name: not a family under the name rule
|
||||
}
|
||||
fams := DetectFamilies(cands, zhFamily)
|
||||
byAnchor := map[string][]string{}
|
||||
for _, f := range fams {
|
||||
byAnchor[f.Anchor] = f.Keys
|
||||
}
|
||||
if got := byAnchor["古月"]; len(got) != 3 {
|
||||
t.Fatalf("the clan family must hold its three names, got %v (all: %v)", got, byAnchor)
|
||||
}
|
||||
if got := byAnchor["海空窍"]; len(got) != 2 {
|
||||
t.Fatalf("two realia sharing the trailing 海空窍 are one family, got %v", got)
|
||||
}
|
||||
// The asymmetry is real: 方源古月 shares 古月 as a SUFFIX, and the name rule reads prefixes.
|
||||
for _, f := range fams {
|
||||
if f.Anchor == "古月" {
|
||||
for _, k := range f.Keys {
|
||||
if k == "方源古月" {
|
||||
t.Fatal("a name sharing the clan morpheme as a SUFFIX must not join the prefix family")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamilyKeepsAVariableLengthRankLineWhole is the acceptance criterion of the fix-pack's own §2(а): the
|
||||
// 1..12 rank line must be ONE batch unit. 一转…九转 is an equal-length series; 十一转 and 十二转 are three
|
||||
// runes long and so fall outside it, and on the live corpus they landed in another batch — a rank scale
|
||||
// split in half is exactly the chimera the co-batching exists to prevent.
|
||||
func TestFamilyKeepsAVariableLengthRankLineWhole(t *testing.T) {
|
||||
var cands []Candidate
|
||||
for _, k := range []string{"一转", "二转", "三转", "四转", "五转", "六转", "七转", "八转", "九转", "十一转", "十二转"} {
|
||||
cands = append(cands, typed(k, "term", 5))
|
||||
}
|
||||
cands = append(cands, typed("中间", "term", 90)) // an unrelated term, so a whole-list unit would prove nothing
|
||||
sort.Slice(cands, func(i, j int) bool { return cands[i].Key < cands[j].Key })
|
||||
|
||||
seriesID := DetectSeries(cands, zhSeries)
|
||||
if seriesID["十一转"] != 0 {
|
||||
t.Fatal("test premise broken: a three-rune surface cannot be in an equal-length series")
|
||||
}
|
||||
unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
|
||||
if st.Refused != 0 || st.Held != 0 {
|
||||
t.Fatalf("nothing here should hit a guard: %+v", st)
|
||||
}
|
||||
id := sameUnit(t, unitID, "一转", "二转", "九转", "十一转", "十二转")
|
||||
if unitID["中间"] == id {
|
||||
t.Fatal("an unrelated term must not be swept into the rank unit")
|
||||
}
|
||||
// And the unit survives the batcher under a budget so tight every singleton is its own batch.
|
||||
batches := Batch(cands, 10, unitID)
|
||||
var rank []Candidate
|
||||
for _, b := range batches {
|
||||
for _, c := range b {
|
||||
if unitID[c.Key] == id {
|
||||
rank = b
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(rank) != 11 {
|
||||
t.Fatalf("the whole 1..12 line must ride ONE call, got %d: %v", len(rank), rank)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamilyDoesNotRegressAnEqualLengthSeries: the other half of the same criterion. 甲等/乙等/丙等 are two
|
||||
// runes, so no two-rune root is a PROPER affix of them and the family channel has nothing to say; the series
|
||||
// must come through untouched.
|
||||
func TestFamilyDoesNotRegressAnEqualLengthSeries(t *testing.T) {
|
||||
cands := []Candidate{typed("甲等", "term", 3), typed("乙等", "term", 3), typed("丙等", "term", 3), typed("丁等", "term", 3), typed("中间", "term", 50)}
|
||||
seriesID := DetectSeries(cands, zhSeries)
|
||||
unitID, _ := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
|
||||
id := sameUnit(t, unitID, "甲等", "乙等", "丙等", "丁等")
|
||||
for k, got := range unitID {
|
||||
if got == id && !strings.HasSuffix(k, "等") {
|
||||
t.Fatalf("%s joined the grade unit and shares no morpheme with it: %v", k, unitsOf(unitID))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestContainmentPairSharesAUnit is §2(б): two MINED surfaces where one contains the other (元海 ⊂ 元海空窍)
|
||||
// had neither a Related record (Merge computes those for banknote-only rows) nor any atomicity in the
|
||||
// batcher — so the compound could be consolidated in one call and its own part in another, which is how a
|
||||
// rendering stops carrying the element it is built from.
|
||||
func TestContainmentPairSharesAUnit(t *testing.T) {
|
||||
cands := []Candidate{typed("元海", "term", 30), typed("元海空窍", "term", 8), typed("青茅山", "place", 12)}
|
||||
unitID, _ := MergeUnits(nil, DetectFamilies(cands, zhFamily), zhFamily)
|
||||
sameUnit(t, unitID, "元海", "元海空窍")
|
||||
if unitID["青茅山"] != 0 {
|
||||
t.Fatalf("an unrelated surface must stay a singleton: %v", unitsOf(unitID))
|
||||
}
|
||||
// The anchor bound is real: a ONE-rune surface may not anchor the channel, or a generic morpheme sweeps
|
||||
// half the bank into one call.
|
||||
short := []Candidate{typed("蛊", "term", 900), typed("蛊虫", "term", 40), typed("蛊师", "term", 30), typed("月光蛊", "term", 10)}
|
||||
if unit, _ := MergeUnits(nil, DetectFamilies(short, zhFamily), zhFamily); unit["蛊"] != 0 {
|
||||
t.Fatalf("a one-rune anchor is below ContainmentRunes and must form no unit: %v", unitsOf(unit))
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamilyMergeRefusesAnOversizeUnit: the merge is capped and the refusal is REPORTED, not silent — a
|
||||
// family split across two calls is the defect this channel exists to close, so the caller has to be able to
|
||||
// say it happened.
|
||||
func TestFamilyMergeRefusesAnOversizeUnit(t *testing.T) {
|
||||
var cands []Candidate
|
||||
for _, k := range []string{"古月方源", "古月正", "古月赤练", "古月山寨"} {
|
||||
cands = append(cands, typed(k, "name", 5))
|
||||
}
|
||||
tight := zhFamily
|
||||
tight.MaxMembers = 3
|
||||
unitID, st := MergeUnits(nil, DetectFamilies(cands, tight), tight)
|
||||
if st.Refused == 0 {
|
||||
t.Fatalf("a four-member family under a three-member cap must be refused: %v", unitsOf(unitID))
|
||||
}
|
||||
for _, v := range unitsOf(unitID) {
|
||||
if len(v) > tight.MaxMembers {
|
||||
t.Fatalf("a unit past the cap was built anyway: %v", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamilyChannelOffIsTheSeriesMap guards the generality answer: a source that declares no family data
|
||||
// takes the byte-identical path it took before the channel existed — the SAME map object, so nothing
|
||||
// downstream can even observe a difference.
|
||||
func TestFamilyChannelOffIsTheSeriesMap(t *testing.T) {
|
||||
cands := []Candidate{typed("古月方源", "name", 5), typed("古月正", "name", 5), typed("甲等", "term", 3), typed("乙等", "term", 3), typed("丙等", "term", 3)}
|
||||
seriesID := DetectSeries(cands, zhSeries)
|
||||
off := FamilyParams{} // no data → inert
|
||||
if fams := DetectFamilies(cands, off); fams != nil {
|
||||
t.Fatalf("an undeclared family channel must detect nothing, got %v", fams)
|
||||
}
|
||||
unitID, st := MergeUnits(seriesID, DetectFamilies(cands, off), off)
|
||||
if st != (MergeStats{}) {
|
||||
t.Fatalf("an inert channel gives nothing up, got %+v", st)
|
||||
}
|
||||
// The SAME map object, not a copy: with the channel off nothing downstream can even observe a difference.
|
||||
if reflect.ValueOf(unitID).Pointer() != reflect.ValueOf(seriesID).Pointer() {
|
||||
t.Fatalf("the inert path must hand back the series map itself: %v vs %v", unitID, seriesID)
|
||||
}
|
||||
for k, v := range seriesID {
|
||||
if unitID[k] != v {
|
||||
t.Fatalf("series membership changed with the channel off: %v vs %v", unitID, seriesID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamiliesOnBankFullFixtureAreBoundedAndWhole runs the channel over the real corpus DetectSeries was
|
||||
// calibrated on. Two live properties, both of which a synthetic fixture cannot show: the 古月 family — the
|
||||
// one the drafts actually split across a batch boundary — comes out as ONE unit, and no unit grows past the
|
||||
// declared bound, so the co-batching cannot quietly turn into "the whole bank in one call".
|
||||
func TestFamiliesOnBankFullFixtureAreBoundedAndWhole(t *testing.T) {
|
||||
cands := loadBankFullSurfaces(t)
|
||||
seriesID := DetectSeries(cands, zhSeries)
|
||||
unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
|
||||
units := unitsOf(unitID)
|
||||
|
||||
clan := unitID["古月"]
|
||||
if clan == 0 {
|
||||
t.Fatalf("the clan surface must be in a unit with the names built on it: %v", units)
|
||||
}
|
||||
clanMembers := 0
|
||||
for _, k := range units[clan] {
|
||||
if strings.HasPrefix(k, "古月") {
|
||||
clanMembers++
|
||||
}
|
||||
}
|
||||
if clanMembers < 10 {
|
||||
t.Fatalf("the 古月 family is the measured chimera and must ride one call, got %d of %v", clanMembers, units[clan])
|
||||
}
|
||||
for id, keys := range units {
|
||||
if len(keys) > zhFamily.MaxMembers {
|
||||
t.Fatalf("unit %d grew past the declared bound (%d): %v", id, zhFamily.MaxMembers, keys)
|
||||
}
|
||||
}
|
||||
// EXACT numbers, not bounds. Two mutations survived a bounds-only version of this test: swapping the
|
||||
// relatedness test to substring containment (which splits 丙等 from 丙等资质 — the §B4 chimera) and
|
||||
// inverting the family ranking (which changes which candidates share a CALL, i.e. the money). Both leave
|
||||
// every inequality above satisfied, so only the composition can catch them.
|
||||
if len(units) != 14 || largestUnit(units) != 20 || st.Refused != 0 || st.Held != 2 {
|
||||
t.Fatalf("the corpus shape moved: %d units, largest %d, %+v — if this is intended, re-measure and update the numbers, do not relax the test",
|
||||
len(units), largestUnit(units), st)
|
||||
}
|
||||
// The rank line and the grade family, whole, by composition.
|
||||
sameUnit(t, unitID, "一转", "九转", "一转蛊师", "九转境界")
|
||||
sameUnit(t, unitID, "丙等", "丙等资质", "甲等", "甲等资质")
|
||||
// And the exact SHAPE of the partition. The aggregate counts above survive an inverted family ranking —
|
||||
// which silently re-cuts which candidates share a CALL, i.e. what the run buys — so the sizes are pinned.
|
||||
sizes := make([]int, 0, len(units))
|
||||
for _, keys := range units {
|
||||
sizes = append(sizes, len(keys))
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.IntSlice(sizes)))
|
||||
want := []int{20, 13, 10, 10, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2}
|
||||
if fmt.Sprint(sizes) != fmt.Sprint(want) {
|
||||
t.Fatalf("the partition SHAPE moved: %v want %v — re-measure before relaxing this", sizes, want)
|
||||
}
|
||||
t.Logf("bankfull: %d candidates → %d units, largest %d, gave up %+v",
|
||||
len(cands), len(units), largestUnit(units), st)
|
||||
}
|
||||
|
||||
// TestGradeFamilyMergesWithItsSeriesThroughASharedRune is the case that made the relatedness test wrong the
|
||||
// first time round, kept as its own pin: the 等资质 series and the {丙等, 丙等资, 丙等资质} family share no
|
||||
// substring in either direction, but they plainly share the morpheme 等 — and 丙等 landing in a different
|
||||
// call from the 丙等资质 built on it is the research/24 §B4 chimera, term for term.
|
||||
func TestGradeFamilyMergesWithItsSeriesThroughASharedRune(t *testing.T) {
|
||||
var cands []Candidate
|
||||
for _, k := range []string{"甲等", "乙等", "丙等", "甲等资质", "乙等资质", "丙等资质", "丙等资"} {
|
||||
cands = append(cands, typed(k, "term", 5))
|
||||
}
|
||||
unitID, st := MergeUnits(DetectSeries(cands, zhSeries), DetectFamilies(cands, zhFamily), zhFamily)
|
||||
if st.Held != 0 {
|
||||
t.Fatalf("a family sharing 等 with the series must not be held back: %+v (%v)", st, unitsOf(unitID))
|
||||
}
|
||||
sameUnit(t, unitID, "丙等", "丙等资质", "甲等", "甲等资质")
|
||||
}
|
||||
|
||||
func largestUnit(units map[int][]string) int {
|
||||
n := 0
|
||||
for _, keys := range units {
|
||||
if len(keys) > n {
|
||||
n = len(keys)
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestSeriesHoldsOnlyItsOwnMembers is the guard's honest boundary, and it exists because the first cut of it
|
||||
// was wrong in exactly the way that matters. «Series stronger than family» must hold back the SERIES — not
|
||||
// dissolve the family around it. Here the clan 古月 has five names, three of which happen to form a 雄-series;
|
||||
// the series' root (雄) has nothing to do with the clan morpheme, so the three keep their own unit, and the
|
||||
// other two must still be co-batched as the clan. Refusing the whole merge instead leaves the clan as five
|
||||
// singletons — the batch-boundary chimera of research/24 §B4, arriving through the guard meant to prevent it.
|
||||
func TestSeriesHoldsOnlyItsOwnMembers(t *testing.T) {
|
||||
cands := []Candidate{
|
||||
typed("古月方源", "name", 40), typed("古月正", "name", 9),
|
||||
typed("古月大雄", "name", 5), typed("古月二雄", "name", 5), typed("古月三雄", "name", 5),
|
||||
}
|
||||
seriesID := DetectSeries(cands, zhSeries)
|
||||
if seriesID["古月大雄"] == 0 {
|
||||
t.Fatal("test premise broken: the three *雄 names must form an equal-length series")
|
||||
}
|
||||
unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
|
||||
if st.Held == 0 {
|
||||
t.Fatalf("holding part of a family back is a split the owner must be told about: %+v", st)
|
||||
}
|
||||
// The series keeps its three members …
|
||||
sameUnit(t, unitID, "古月大雄", "古月二雄", "古月三雄")
|
||||
// … and the REST of the clan is still one unit, not two singletons.
|
||||
clan := sameUnit(t, unitID, "古月方源", "古月正")
|
||||
if clan == unitID["古月大雄"] {
|
||||
t.Fatalf("an unrelated series must not be swallowed by the family: %v", unitsOf(unitID))
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamilyMergesAcrossASeriesWhenTheRootIsShared is the other side of the same guard: when the series' own
|
||||
// root IS the family's anchor (the 转 case), nothing is held back and the whole line rides one call.
|
||||
func TestFamilyMergesAcrossASeriesWhenTheRootIsShared(t *testing.T) {
|
||||
var cands []Candidate
|
||||
for _, k := range []string{"一转", "二转", "三转", "四转", "五转", "十一转"} {
|
||||
cands = append(cands, typed(k, "term", 5))
|
||||
}
|
||||
_, st := MergeUnits(DetectSeries(cands, zhSeries), DetectFamilies(cands, zhFamily), zhFamily)
|
||||
if st.Held != 0 {
|
||||
t.Fatalf("a family sharing the series' own root must not be held back: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFamilyRulesCoverEveryTypeACandidateCanCarry: the family rules are keyed on a candidate's TYPE, and the
|
||||
// set a candidate can carry is wider than the classifier's answer vocabulary — the banknote channel accepts
|
||||
// `nickname` from a draft and that type rides onto a draft-side-only candidate. A type with no rule forms no
|
||||
// families, silently, which is a whole class of the bank going un-co-batched for no stated reason.
|
||||
func TestFamilyRulesCoverEveryTypeACandidateCanCarry(t *testing.T) {
|
||||
for _, typ := range TypeNames(CandidateTypes) {
|
||||
if _, has := zhFamily.Affix[typ]; !has {
|
||||
t.Fatalf("type %q can reach a candidate and has no family rule — declare one or say why in the data file", typ)
|
||||
}
|
||||
}
|
||||
// And it is live, not just declared: two nicknames sharing the clan morpheme are one family.
|
||||
cands := []Candidate{typed("方小子", "nickname", 5), typed("方小鬼", "nickname", 4)}
|
||||
if fams := DetectFamilies(cands, zhFamily); len(fams) == 0 {
|
||||
t.Fatalf("a nickname family must form: %v", fams)
|
||||
}
|
||||
}
|
||||
|
|
@ -68,7 +68,7 @@ func TestParseReplyRefusesForeignLanguageRenderings(t *testing.T) {
|
|||
keys := []string{"修行", "元海", "转", "是为", "资质"}
|
||||
reply := "修行\tcultivation\n元海\tPrimordial Sea\n转\trealm\n是为\tis\n资质\tталант"
|
||||
|
||||
got, st := ParseReply(reply, keys, id, unicode.Scripts["Cyrillic"])
|
||||
got, _, st := ParseReply(reply, keys, id, unicode.Scripts["Cyrillic"])
|
||||
if len(got) != 1 || got["资质"] != "талант" {
|
||||
t.Fatalf("only the target-language line may be banked, got %#v", got)
|
||||
}
|
||||
|
|
@ -80,7 +80,7 @@ func TestParseReplyRefusesForeignLanguageRenderings(t *testing.T) {
|
|||
}
|
||||
// Without a declared script the same reply is banked whole — the state the engine was in before, kept
|
||||
// visible so the cost of an undeclared script is a test, not a surprise.
|
||||
if inert, st := ParseReply(reply, keys, id, nil); len(inert) != 5 || st.OffLanguage != 0 {
|
||||
if inert, _, st := ParseReply(reply, keys, id, nil); len(inert) != 5 || st.OffLanguage != 0 {
|
||||
t.Fatalf("with no script the screen must be inert (that is what config refuses for an enabled gate), got %#v", inert)
|
||||
}
|
||||
}
|
||||
|
|
@ -95,7 +95,7 @@ func TestOffLanguageSamplesAreBounded(t *testing.T) {
|
|||
keys = append(keys, k)
|
||||
lines = append(lines, k+"\tforeign")
|
||||
}
|
||||
_, st := ParseReply(strings.Join(lines, "\n"), keys, id, unicode.Scripts["Cyrillic"])
|
||||
_, _, st := ParseReply(strings.Join(lines, "\n"), keys, id, unicode.Scripts["Cyrillic"])
|
||||
if st.OffLanguage != len(keys) {
|
||||
t.Fatalf("every foreign line must be counted, got %d of %d", st.OffLanguage, len(keys))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package terminology
|
||||
|
||||
import "sort"
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// series.go: the SERIES channel (bank-quality §1, D39.68). A grade/rank series — 甲等/乙等/丙等, 一转…九转 —
|
||||
// shares one generic HEAD and varies only on a modifier. Measured (D39.65 row 21): the drafts disagree on the
|
||||
|
|
@ -106,32 +109,456 @@ func blankAt(rs []rune, pos int) string {
|
|||
return string(out)
|
||||
}
|
||||
|
||||
// orderBySeries returns the candidates with every series' members made contiguous, anchored at the position
|
||||
// of the series' first member in the input order; non-series candidates keep their place. The input is
|
||||
// already key-sorted, so the result is deterministic and disturbs the key order minimally.
|
||||
func orderBySeries(cands []Candidate, seriesID map[string]int) []Candidate {
|
||||
if len(seriesID) == 0 {
|
||||
// orderByUnit returns the candidates with every batch unit's members made contiguous, anchored at the
|
||||
// position of the unit's first member in the input order; a candidate in no unit keeps its place. The input
|
||||
// is already key-sorted, so the result is deterministic and disturbs the key order minimally.
|
||||
func orderByUnit(cands []Candidate, unitID map[string]int) []Candidate {
|
||||
if len(unitID) == 0 {
|
||||
return cands
|
||||
}
|
||||
byID := map[int][]Candidate{}
|
||||
for _, c := range cands {
|
||||
if id := seriesID[c.Key]; id != 0 {
|
||||
if id := unitID[c.Key]; id != 0 {
|
||||
byID[id] = append(byID[id], c)
|
||||
}
|
||||
}
|
||||
out := make([]Candidate, 0, len(cands))
|
||||
emitted := map[int]bool{}
|
||||
for _, c := range cands {
|
||||
id := seriesID[c.Key]
|
||||
id := unitID[c.Key]
|
||||
if id == 0 {
|
||||
out = append(out, c)
|
||||
continue
|
||||
}
|
||||
if emitted[id] {
|
||||
continue // a later member of an already-emitted series
|
||||
continue // a later member of an already-emitted unit
|
||||
}
|
||||
out = append(out, byID[id]...)
|
||||
emitted[id] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- the FAMILY channel (fix-pack §G1) ---------------------------------------------------------------
|
||||
//
|
||||
// A SERIES is the narrow case: equal-length surfaces differing in ONE non-head rune. It leaves the wider
|
||||
// one open, and the wider one is where the measured damage is (research/24 §B4): the 古月-family — eight
|
||||
// surfaces of DIFFERENT lengths sharing the clan morpheme — came back «Гу Юэ» in batch 0 and «Гуюэ» in
|
||||
// batch 2 from ONE model, a split that correlates with the batch boundary perfectly. Nothing in the
|
||||
// terminologist can fix a disagreement it is never shown; only co-batching can.
|
||||
//
|
||||
// A family is anchored on a shared ROOT MORPHEME, and the side that root sits on is ASYMMETRIC by class —
|
||||
// a name leads with its clan surname, a realia term ends with its generic head. Both the side and the
|
||||
// required root length are pair/script DATA (lang.FamilyMorphology), so a source with no rows leaves the
|
||||
// channel inert and takes the byte-identical, family-free path.
|
||||
|
||||
// FamilyAffix is one type's family rule: which side carries the shared root and how long it must be.
|
||||
type FamilyAffix struct {
|
||||
Suffix bool
|
||||
MinRunes int
|
||||
}
|
||||
|
||||
// FamilyParams is the pair-data that governs the family channel. Like SeriesParams it arrives as a VALUE:
|
||||
// the package stays pure and pair-agnostic, and the pipeline resolves it from the source's declared
|
||||
// morphology.
|
||||
type FamilyParams struct {
|
||||
// Enabled gates the whole channel — true only for DENSE scripts, for the same reason the series channel
|
||||
// is: one rune ≈ one morpheme, so a shared affix is a shared MORPHEME and not a coincidence of spelling.
|
||||
Enabled bool
|
||||
// HeadFinal says the generic head is the trailing morpheme. It is used to read a SERIES' own root when
|
||||
// deciding whether a family may join it (see MergeUnits).
|
||||
HeadFinal bool
|
||||
// Affix is the per-type rule (name|place|title|term); a type absent here forms no families.
|
||||
Affix map[string]FamilyAffix
|
||||
// MinMembers is the smallest set that counts as a family (<2 → 2).
|
||||
MinMembers int
|
||||
// MaxMembers bounds the batch unit a family merge may produce; 0 = unbounded. A unit past it is a review
|
||||
// set nobody reads and an output cap nobody planned, so the merge is refused and counted.
|
||||
MaxMembers int
|
||||
// ContainmentRunes is the shortest candidate that may anchor the COMPOSITIONAL channel: a candidate that
|
||||
// is itself a substring of another candidate (元海 ⊂ 元海空窍 — both mined, so neither Related nor the
|
||||
// batcher held them together before this pack). Bounding the anchor is what keeps one generic morpheme
|
||||
// from sweeping half the bank into a single call. 0 → the channel is off.
|
||||
ContainmentRunes int
|
||||
}
|
||||
|
||||
// Family is one detected family: the shared ANCHOR morpheme and the candidate keys carrying it (key-sorted).
|
||||
type Family struct {
|
||||
Anchor string
|
||||
Keys []string
|
||||
}
|
||||
|
||||
// DetectFamilies returns the families of a candidate list, strongest first (larger set, then longer — i.e.
|
||||
// more specific — anchor, then first-seen). It does NOT assign membership: a candidate legitimately shows
|
||||
// up in several families, and which of them becomes a batch unit is decided in MergeUnits, where the
|
||||
// existing series are already on the table. Deterministic; nothing here iterates a map for output.
|
||||
//
|
||||
// There is no transitive closure over runes: an anchor is a WHOLE root morpheme of at least MinRunes (or a
|
||||
// whole candidate, for the compositional channel), never "these two differ in one position", which is the
|
||||
// rule that produced an 11-surface blob mixing three families and the protagonist's name.
|
||||
func DetectFamilies(cands []Candidate, p FamilyParams) []Family {
|
||||
if !p.Enabled || len(cands) < 2 {
|
||||
return nil
|
||||
}
|
||||
min := p.MinMembers
|
||||
if min < 2 {
|
||||
min = 2
|
||||
}
|
||||
type gkey struct{ side, anchor string }
|
||||
members := map[gkey][]string{}
|
||||
seen := map[gkey]map[string]bool{}
|
||||
var order []gkey
|
||||
add := func(g gkey, key string) {
|
||||
m := seen[g]
|
||||
if m == nil {
|
||||
m = map[string]bool{}
|
||||
seen[g] = m
|
||||
order = append(order, g)
|
||||
}
|
||||
if !m[key] {
|
||||
m[key] = true
|
||||
members[g] = append(members[g], key)
|
||||
}
|
||||
}
|
||||
isCand := make(map[string]bool, len(cands))
|
||||
for _, c := range cands {
|
||||
if c.Key != "" {
|
||||
isCand[c.Key] = true
|
||||
}
|
||||
}
|
||||
for _, c := range cands {
|
||||
if c.Key == "" {
|
||||
continue
|
||||
}
|
||||
rule, ok := p.Affix[typeOr(c.Type)]
|
||||
if !ok || rule.MinRunes <= 0 {
|
||||
continue
|
||||
}
|
||||
side := "prefix"
|
||||
if rule.Suffix {
|
||||
side = "suffix"
|
||||
}
|
||||
rs := []rune(c.Key)
|
||||
// PROPER affixes only (l < len): a surface is not a member of a family whose root IS the surface —
|
||||
// it is that root, and it joins below, once some other surface actually carries it.
|
||||
for l := rule.MinRunes; l < len(rs); l++ {
|
||||
anchor := string(rs[:l])
|
||||
if rule.Suffix {
|
||||
anchor = string(rs[len(rs)-l:])
|
||||
}
|
||||
g := gkey{side, anchor}
|
||||
add(g, c.Key)
|
||||
if isCand[anchor] {
|
||||
// The surface that IS the root belongs to its own family. Leaving it out is how the head of a
|
||||
// family ends up in a different call from the family it heads.
|
||||
//
|
||||
// ⚠ Bounded: an anchor is at least MinRunes long, so a ONE-rune head that is itself a bank term
|
||||
// (蛊, 转, 窍) is NOT reachable this way and does stay in another call from its family. That is
|
||||
// a data decision, not a Go one — family_containment_runes 1 turns it on — and it is left off
|
||||
// because a single generic morpheme anchors half the bank; the cost is measured at the cold run.
|
||||
add(g, anchor)
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.ContainmentRunes > 0 {
|
||||
for _, c := range cands {
|
||||
if c.Key == "" || len([]rune(c.Key)) < p.ContainmentRunes {
|
||||
continue
|
||||
}
|
||||
g := gkey{"contains", c.Key}
|
||||
for _, d := range cands {
|
||||
if d.Key == "" || d.Key == c.Key {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(d.Key, c.Key) {
|
||||
add(g, c.Key)
|
||||
add(g, d.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
all := make([]Family, 0, len(order))
|
||||
for _, g := range order {
|
||||
if len(members[g]) < min {
|
||||
continue
|
||||
}
|
||||
keys := append([]string(nil), members[g]...)
|
||||
sort.Strings(keys)
|
||||
all = append(all, Family{Anchor: g.anchor, Keys: keys})
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool {
|
||||
if len(all[i].Keys) != len(all[j].Keys) {
|
||||
return len(all[i].Keys) > len(all[j].Keys)
|
||||
}
|
||||
ai, aj := len([]rune(all[i].Anchor)), len([]rune(all[j].Anchor))
|
||||
if ai != aj {
|
||||
return ai > aj // the more SPECIFIC root wins the earlier merge
|
||||
}
|
||||
return all[i].Anchor < all[j].Anchor
|
||||
})
|
||||
// Dedupe AFTER the ranking, so the survivor of a duplicated set is the strongest description of it. The
|
||||
// affix and containment channels legitimately describe the same set from two directions (元海空窍/天海空窍
|
||||
// share the suffix 空窍 AND the longer 海空窍), and one of them is enough: a duplicate merges nothing the
|
||||
// first did not, but it double-counts in the merge's give-up statistics — and a number an operator reads
|
||||
// has to mean what it says.
|
||||
out := make([]Family, 0, len(all))
|
||||
seenSet := map[string]bool{}
|
||||
for _, f := range all {
|
||||
set := strings.Join(f.Keys, "\x00")
|
||||
if seenSet[set] {
|
||||
continue
|
||||
}
|
||||
seenSet[set] = true
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// MergeUnits folds the series and the families into ONE batch-unit map — the map Batch packs against.
|
||||
//
|
||||
// Membership is a PARTITION (a candidate is in at most one unit), built by merging whole units rather than
|
||||
// by claiming candidates: an exclusive assignment cannot express the case the fix-pack exists for. The
|
||||
// rank line 一转…九转 is a series; 十一转 is three runes long and so is not, and under exclusive assignment
|
||||
// it stays outside forever — the batch boundary splits a rank scale in half. Merging the family that spans
|
||||
// both puts the whole 1..12 line in one call, which is the stated criterion.
|
||||
//
|
||||
// Two guards keep the merge from becoming a blob, and BOTH are counted rather than silent — a guard that
|
||||
// splits a family is doing the same damage the channel exists to prevent, so it has to be visible:
|
||||
// - SERIES PRECEDENCE: a series whose own root is unrelated to a family's anchor keeps its members; the
|
||||
// family then forms WITHOUT them (转-case: root 转, anchors 转 / 一转 — related, so they merge instead).
|
||||
// - MaxMembers: a merge that would produce a unit larger than the pair declares is refused.
|
||||
//
|
||||
// With no families (the channel off, or nothing detected) the series map is returned UNCHANGED, so a source
|
||||
// with no family data takes the byte-identical path.
|
||||
func MergeUnits(seriesID map[string]int, fams []Family, p FamilyParams) (unitID map[string]int, stats MergeStats) {
|
||||
if len(fams) == 0 {
|
||||
return seriesID, MergeStats{}
|
||||
}
|
||||
u := newUnitSet()
|
||||
cappedSet := map[string]bool{} // key sets whose merge the member cap turned down
|
||||
bySeries := map[int][]string{}
|
||||
for k, id := range seriesID {
|
||||
if id != 0 {
|
||||
bySeries[id] = append(bySeries[id], k)
|
||||
}
|
||||
}
|
||||
ids := make([]int, 0, len(bySeries))
|
||||
for id := range bySeries {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Ints(ids)
|
||||
for _, id := range ids {
|
||||
mem := bySeries[id]
|
||||
sort.Strings(mem)
|
||||
for _, k := range mem[1:] {
|
||||
u.union(mem[0], k)
|
||||
}
|
||||
if root := commonAffix(mem, p.HeadFinal); root != "" {
|
||||
r := u.find(mem[0])
|
||||
u.roots[r] = append(u.roots[r], root)
|
||||
}
|
||||
}
|
||||
for _, f := range fams {
|
||||
var targets []string
|
||||
seen := map[string]bool{}
|
||||
for _, k := range f.Keys {
|
||||
r := u.find(k)
|
||||
if seen[r] {
|
||||
continue
|
||||
}
|
||||
seen[r] = true
|
||||
targets = append(targets, r)
|
||||
}
|
||||
if len(targets) < 2 {
|
||||
continue // already one unit
|
||||
}
|
||||
// SERIES PRECEDENCE, applied per TARGET rather than to the whole family. A series whose root is
|
||||
// unrelated to this anchor is HELD BACK — and only it. Refusing the whole merge instead would
|
||||
// dissolve the family around it: the clan 古月, minus three of its names that happen to form a
|
||||
// 雄-series, would go back to being five singletons, which is the batch-boundary chimera this
|
||||
// channel exists to close, arriving through the guard that was supposed to protect a series.
|
||||
var join []string
|
||||
size := 0
|
||||
for _, r := range targets {
|
||||
blocked := false
|
||||
for _, sr := range u.roots[r] {
|
||||
if !sharesMorpheme(f.Anchor, sr) {
|
||||
blocked = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if blocked {
|
||||
continue
|
||||
}
|
||||
join = append(join, r)
|
||||
size += u.size[r]
|
||||
}
|
||||
if len(join) < 2 {
|
||||
continue
|
||||
}
|
||||
if p.MaxMembers > 0 && size > p.MaxMembers {
|
||||
cappedSet[strings.Join(f.Keys, "\x00")] = true
|
||||
continue
|
||||
}
|
||||
for _, r := range join[1:] {
|
||||
u.union(join[0], r)
|
||||
}
|
||||
}
|
||||
// The give-up statistics are read off the FINAL partition, never off the attempts. A guard that fired
|
||||
// mid-way is not a family the owner meets split: a later merge routinely puts those keys back together,
|
||||
// and counting the attempt reports a split that does not exist. Measured on the coldrun-a distillation:
|
||||
// counting attempts claimed three, of which one (蛊师 against 一转蛊师) had ended up in ONE unit anyway.
|
||||
// A number an operator reads has to mean what it says — the standard this file sets for itself.
|
||||
final := u.ids()
|
||||
for _, f := range fams {
|
||||
if !splitAcrossUnits(f, final) {
|
||||
continue
|
||||
}
|
||||
if cappedSet[strings.Join(f.Keys, "\x00")] {
|
||||
stats.Refused++
|
||||
continue
|
||||
}
|
||||
stats.Held++
|
||||
}
|
||||
return final, stats
|
||||
}
|
||||
|
||||
// splitAcrossUnits reports whether a family's keys ended up in more than one batch unit (a key in no unit
|
||||
// counts as its own).
|
||||
func splitAcrossUnits(f Family, unitID map[string]int) bool {
|
||||
first, seen := 0, false
|
||||
for _, k := range f.Keys {
|
||||
id := unitID[k]
|
||||
if !seen {
|
||||
first, seen = id, true
|
||||
continue
|
||||
}
|
||||
if id != first || id == 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MergeStats is what the unit merge had to give up. Both numbers mean "a family the owner will meet split
|
||||
// across two calls", which is the defect the channel exists to close — so neither may be silent.
|
||||
type MergeStats struct {
|
||||
// Refused counts merges the member cap turned down.
|
||||
Refused int
|
||||
// Held counts families a SERIES held part of back: the series' own root is unrelated to the family's
|
||||
// anchor, so the series keeps its members and the family forms without them.
|
||||
Held int
|
||||
}
|
||||
|
||||
// sharesMorpheme reports whether two anchors share a root MORPHEME — the "делят одну корневую морфему" test
|
||||
// behind the series-vs-family precedence. In a dense script a morpheme is a rune, so that is the test.
|
||||
//
|
||||
// Substring containment is NOT it, and the difference was measured, not reasoned: on the coldrun-a bank the
|
||||
// containment version held back five families, every one of them wrongly. The 等资质 series (甲等资质/乙等资质/
|
||||
// 丙等资质, root 等资质) sits beside the family {丙等, 丙等资, 丙等资质}, anchored 丙等. Neither string contains
|
||||
// the other, so containment called them unrelated and split the grade 丙等 away from the graded aptitude
|
||||
// built on it — while they plainly share 等, which is the very morpheme whose rendering has to agree. The
|
||||
// blob these guards protect against is bounded by MaxMembers, not by making relatedness narrow.
|
||||
func sharesMorpheme(a, b string) bool {
|
||||
if a == "" || b == "" {
|
||||
return true
|
||||
}
|
||||
return sharedRunes(a, b) > 0
|
||||
}
|
||||
|
||||
// commonAffix returns the longest common trailing (headFinal) or leading morpheme of the keys — a series'
|
||||
// own root, read off its members so DetectSeries keeps its signature.
|
||||
func commonAffix(keys []string, headFinal bool) string {
|
||||
if len(keys) == 0 {
|
||||
return ""
|
||||
}
|
||||
best := []rune(keys[0])
|
||||
for _, k := range keys[1:] {
|
||||
rs := []rune(k)
|
||||
n := 0
|
||||
for n < len(best) && n < len(rs) {
|
||||
if headFinal {
|
||||
if best[len(best)-1-n] != rs[len(rs)-1-n] {
|
||||
break
|
||||
}
|
||||
} else if best[n] != rs[n] {
|
||||
break
|
||||
}
|
||||
n++
|
||||
}
|
||||
if headFinal {
|
||||
best = best[len(best)-n:]
|
||||
} else {
|
||||
best = best[:n]
|
||||
}
|
||||
if len(best) == 0 {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return string(best)
|
||||
}
|
||||
|
||||
// unitSet is the disjoint-set the merge runs on: parent/size by key, plus the series roots a unit carries
|
||||
// (the precedence guard reads them). Roots are the smallest member key, so the structure is deterministic
|
||||
// without any map iteration reaching the output.
|
||||
type unitSet struct {
|
||||
parent map[string]string
|
||||
size map[string]int
|
||||
roots map[string][]string
|
||||
}
|
||||
|
||||
func newUnitSet() *unitSet {
|
||||
return &unitSet{parent: map[string]string{}, size: map[string]int{}, roots: map[string][]string{}}
|
||||
}
|
||||
|
||||
func (u *unitSet) find(k string) string {
|
||||
p, ok := u.parent[k]
|
||||
if !ok {
|
||||
u.parent[k], u.size[k] = k, 1
|
||||
return k
|
||||
}
|
||||
if p == k {
|
||||
return k
|
||||
}
|
||||
r := u.find(p)
|
||||
u.parent[k] = r
|
||||
return r
|
||||
}
|
||||
|
||||
func (u *unitSet) union(a, b string) {
|
||||
ra, rb := u.find(a), u.find(b)
|
||||
if ra == rb {
|
||||
return
|
||||
}
|
||||
if rb < ra { // the smaller key is always the root → the same input yields the same structure
|
||||
ra, rb = rb, ra
|
||||
}
|
||||
u.parent[rb] = ra
|
||||
u.size[ra] += u.size[rb]
|
||||
u.roots[ra] = append(u.roots[ra], u.roots[rb]...)
|
||||
delete(u.size, rb)
|
||||
delete(u.roots, rb)
|
||||
}
|
||||
|
||||
// ids numbers the units of ≥2 members, in the order of their smallest key, so the map is a pure function of
|
||||
// the input.
|
||||
func (u *unitSet) ids() map[string]int {
|
||||
roots := make([]string, 0, len(u.size))
|
||||
for r, n := range u.size {
|
||||
if n > 1 {
|
||||
roots = append(roots, r)
|
||||
}
|
||||
}
|
||||
sort.Strings(roots)
|
||||
num := make(map[string]int, len(roots))
|
||||
for i, r := range roots {
|
||||
num[r] = i + 1
|
||||
}
|
||||
out := make(map[string]int, len(u.parent))
|
||||
for k := range u.parent {
|
||||
if id := num[u.find(k)]; id != 0 {
|
||||
out[k] = id
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,8 +80,12 @@ type Neighbour struct {
|
|||
|
||||
// Variant is one observed rendering with its consolidation score.
|
||||
type Variant struct {
|
||||
Dst string
|
||||
Chunks int
|
||||
Dst string
|
||||
Chunks int
|
||||
// Forms is how many DISTINCT raw renderings folded into this one (see foldVariants). 1 for an unfolded
|
||||
// variant; it is what keeps Spread() meaning "how many ways did the drafts write it" once the vote is
|
||||
// counted per CONVENTION rather than per byte string.
|
||||
Forms int
|
||||
Score float64
|
||||
Signals []string // which factors fired, in fixed order — the score's audit trail
|
||||
// Via names the surface that actually proposed this rendering when it is NOT the candidate's own —
|
||||
|
|
@ -125,7 +129,12 @@ type Candidate struct {
|
|||
//
|
||||
// Output order is by Key (never map order). Every mined candidate appears exactly once; every observed key
|
||||
// either merges into one or appears once in the reverse section.
|
||||
func Merge(mined []Mined, observed []Observed) []Candidate {
|
||||
//
|
||||
// foldDst is the TARGET-form normalizer the vote is counted under (see foldVariants); nil folds only
|
||||
// byte-identical renderings, which is what this did before the fix-pack. It is injected rather than
|
||||
// imported so the package keeps no target knowledge — the ё/case/whitespace folds a Russian target needs
|
||||
// are facts about that target, not about this algorithm.
|
||||
func Merge(mined []Mined, observed []Observed, foldDst func(string) string) []Candidate {
|
||||
byKey := make(map[string]*Candidate, len(mined))
|
||||
order := make([]string, 0, len(mined)+len(observed))
|
||||
// aliasOwner maps every alias surface to its owning mined key, so an aliased proposal merges.
|
||||
|
|
@ -206,31 +215,89 @@ func Merge(mined []Mined, observed []Observed) []Candidate {
|
|||
out := make([]Candidate, 0, len(order))
|
||||
for _, k := range order {
|
||||
c := *byKey[k]
|
||||
c.Variants = foldVariants(c.Variants)
|
||||
c.Variants = foldVariants(c.Variants, foldDst)
|
||||
out = append(out, c)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
|
||||
return out
|
||||
}
|
||||
|
||||
// foldVariants sums the chunk counts of IDENTICAL renderings. It matters on the alias path: 方源 and its
|
||||
// alias 方小子 arrive as two Observed rows, and both may carry the same rendering — appended raw that is
|
||||
// two variants of one word, which lies twice. Spread() would report 2 (the disagreement column the owner
|
||||
// reads to decide whether a term is contested) for a term nobody disagreed about, and the §C2-3 frequency
|
||||
// factor would score the consensus rendering on HALF its evidence, which can hand the win to a genuine
|
||||
// competitor. Deterministic: first-seen order is preserved, so the fold introduces no new ordering.
|
||||
func foldVariants(vs []Variant) []Variant {
|
||||
if len(vs) < 2 {
|
||||
// foldVariants sums the chunk counts of renderings that are the SAME CONVENTION and shows the raw form.
|
||||
//
|
||||
// It matters on the alias path: 方源 and its alias 方小子 arrive as two Observed rows, and both may carry the
|
||||
// same rendering — appended raw that is two variants of one word, which lies twice. It matters just as much
|
||||
// on orthography (fix-pack §G5, research/24 §A5): «Море истинной ци» and «море истинной ци» are one
|
||||
// convention written two ways, and folding only byte-identical strings splits the §C2-3 frequency factor
|
||||
// across them — the consensus rendering is then scored on half its evidence and a genuine competitor can
|
||||
// take the win. So the VOTE is counted under foldDst (the target-form normalizer), while the form the owner
|
||||
// and the model see stays the RAW one the drafts actually wrote.
|
||||
//
|
||||
// Two counts survive the fold and mean different things: Spread() (Σ Forms) is how many ways the drafts
|
||||
// wrote the term, Conventions() is how many of those are genuinely different decisions.
|
||||
//
|
||||
// Via is aggregated honestly rather than by first-wins. A rendering that ALSO arrived on the candidate's own
|
||||
// key is direct — labelling it «proposed for <alias>» because the alias happened to be seen first would
|
||||
// present a consensus as a mis-clustered alias's guess. Only when EVERY contribution came through aliases
|
||||
// is the provenance kept, and then it names all of them.
|
||||
//
|
||||
// Deterministic: first-seen order of the folded classes is preserved, and the representative is the raw form
|
||||
// with the most chunks, ties going to the one seen first.
|
||||
func foldVariants(vs []Variant, foldDst func(string) string) []Variant {
|
||||
if len(vs) == 0 {
|
||||
return vs
|
||||
}
|
||||
if foldDst == nil {
|
||||
foldDst = func(s string) string { return s }
|
||||
}
|
||||
type form struct {
|
||||
dst string
|
||||
chunks int
|
||||
}
|
||||
type class struct {
|
||||
forms []form
|
||||
byForm map[string]int
|
||||
chunks int
|
||||
direct bool
|
||||
vias []string
|
||||
seenVia map[string]bool
|
||||
}
|
||||
at := make(map[string]int, len(vs))
|
||||
out := make([]Variant, 0, len(vs))
|
||||
classes := make([]*class, 0, len(vs))
|
||||
for _, v := range vs {
|
||||
if i, seen := at[v.Dst]; seen {
|
||||
out[i].Chunks += v.Chunks
|
||||
continue
|
||||
k := foldDst(v.Dst)
|
||||
i, seen := at[k]
|
||||
if !seen {
|
||||
i = len(classes)
|
||||
at[k] = i
|
||||
classes = append(classes, &class{byForm: map[string]int{}, seenVia: map[string]bool{}})
|
||||
}
|
||||
cl := classes[i]
|
||||
cl.chunks += v.Chunks
|
||||
if j, had := cl.byForm[v.Dst]; had {
|
||||
cl.forms[j].chunks += v.Chunks
|
||||
} else {
|
||||
cl.byForm[v.Dst] = len(cl.forms)
|
||||
cl.forms = append(cl.forms, form{dst: v.Dst, chunks: v.Chunks})
|
||||
}
|
||||
if v.Via == "" {
|
||||
cl.direct = true
|
||||
} else if !cl.seenVia[v.Via] {
|
||||
cl.seenVia[v.Via] = true
|
||||
cl.vias = append(cl.vias, v.Via)
|
||||
}
|
||||
}
|
||||
out := make([]Variant, 0, len(classes))
|
||||
for _, cl := range classes {
|
||||
best := 0
|
||||
for j := 1; j < len(cl.forms); j++ {
|
||||
if cl.forms[j].chunks > cl.forms[best].chunks {
|
||||
best = j
|
||||
}
|
||||
}
|
||||
v := Variant{Dst: cl.forms[best].dst, Chunks: cl.chunks, Forms: len(cl.forms)}
|
||||
if !cl.direct {
|
||||
v.Via = strings.Join(cl.vias, ", ")
|
||||
}
|
||||
at[v.Dst] = len(out)
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
|
|
@ -596,8 +663,27 @@ func (c Candidate) Best() string {
|
|||
}
|
||||
|
||||
// Spread is the disagreement signal: how many DISTINCT renderings the drafts produced for this surface.
|
||||
// One term coming back three ways is the drift a canon closes — the reason to sign it at all.
|
||||
func (c Candidate) Spread() int { return len(c.Variants) }
|
||||
// One term coming back three ways is the drift a canon closes — the reason to sign it at all. It counts RAW
|
||||
// forms (Forms per folded class), so the fix-pack's convention fold did not silently shrink the column the
|
||||
// owner reads to decide whether a term is contested. A variant built outside Merge carries no Forms and
|
||||
// counts as one.
|
||||
func (c Candidate) Spread() int {
|
||||
n := 0
|
||||
for _, v := range c.Variants {
|
||||
if v.Forms > 1 {
|
||||
n += v.Forms
|
||||
continue
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Conventions is the disagreement that actually matters: how many distinct DECISIONS the drafts made, after
|
||||
// renderings that differ only in target form (case, ё, spacing) are folded together. Spread 3 with
|
||||
// Conventions 1 is one canon written three ways — a normalization nit; Spread 3 with Conventions 3 is a real
|
||||
// contest, and the two must not read the same at the stop.
|
||||
func (c Candidate) Conventions() int { return len(c.Variants) }
|
||||
|
||||
// --- the wire: request table and reply parser ---------------------------------------------------------
|
||||
|
||||
|
|
@ -611,6 +697,62 @@ const NoDst = "⟦TM-NO-DST⟧"
|
|||
// tolerance the banknote parser applies, for the same reason: models substitute spaces for tabs.
|
||||
var fieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`)
|
||||
|
||||
// columnSplit is the UNAMBIGUOUS half of that tolerance — a tab or a pipe is a delimiter the model chose and
|
||||
// cannot occur inside a rendering, so a line carrying one has declared its own columns. spaceRun folds the
|
||||
// stray spacing inside such a column. Both exist because guessing where the columns are, on a line that says
|
||||
// where they are, is how a fix for one mis-split rendering corrupts the lines that were never mis-split.
|
||||
var (
|
||||
columnSplit = regexp.MustCompile(`\t|\s*\|\s*`)
|
||||
spaceRun = regexp.MustCompile(` {2,}`)
|
||||
)
|
||||
|
||||
// replyColumns splits one reply line into src, rendering and the CONFIDENCE column, and says whether a
|
||||
// third column was present but unreadable as a confidence.
|
||||
//
|
||||
// A tab/pipe line is positional: field three IS the confidence column the pair's prompt declared, so a value
|
||||
// that is not a bare 0…100 is a malformed CONFIDENCE — counted, and never glued onto the rendering. That
|
||||
// glue is the failure this shape exists to prevent: «наставник» + «95%» silently became the rendering
|
||||
// «наставник 95%», passed every downstream screen (it is well-formed, it is in the target script, it is not
|
||||
// an echo) and would enter the bank as this book's canon with every counter reading clean.
|
||||
//
|
||||
// A line delimited only by a run of ≥2 spaces is ambiguous — that tolerance is exactly what splits «Фан␣␣
|
||||
// Юань» — so there the rendering is re-joined and only a trailing bare number is taken as a confidence.
|
||||
func replyColumns(line string) (src, dst string, conf int, badConf, ok bool) {
|
||||
conf = -1
|
||||
if cols := splitOn(columnSplit, line); len(cols) >= 2 {
|
||||
src, dst = cols[0], strings.TrimSpace(spaceRun.ReplaceAllString(cols[1], " "))
|
||||
if len(cols) >= 3 {
|
||||
if v, good := confidenceField(cols[2]); good {
|
||||
conf = v
|
||||
} else {
|
||||
badConf = true
|
||||
}
|
||||
}
|
||||
return src, dst, conf, badConf, dst != ""
|
||||
}
|
||||
f := splitOn(fieldSplit, line)
|
||||
if len(f) < 2 {
|
||||
return "", "", -1, false, false
|
||||
}
|
||||
if len(f) >= 3 {
|
||||
if v, good := confidenceField(f[len(f)-1]); good {
|
||||
conf, f = v, f[:len(f)-1]
|
||||
}
|
||||
}
|
||||
return f[0], strings.TrimSpace(strings.Join(f[1:], " ")), conf, false, true
|
||||
}
|
||||
|
||||
// splitOn splits a line on re, dropping empty and whitespace-only fields.
|
||||
func splitOn(re *regexp.Regexp, line string) []string {
|
||||
var out []string
|
||||
for _, p := range re.Split(strings.TrimSpace(line), -1) {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// splitFields splits one reply line into its non-empty trimmed fields.
|
||||
func splitFields(line string) []string {
|
||||
var parts []string
|
||||
|
|
@ -811,6 +953,49 @@ func CanonConflicts(cands []Candidate, consolidated map[string]string, ns []Neig
|
|||
return out
|
||||
}
|
||||
|
||||
// ConsolidationConflict is one consolidated rendering that contradicts ANOTHER consolidation of the SAME
|
||||
// run — the compositional rule broken inside one reply rather than against a signed row.
|
||||
type ConsolidationConflict struct {
|
||||
Src string // the containing candidate's source surface
|
||||
Dst string // what the role consolidated it to
|
||||
PartSrc string // the source surface contained in it, also consolidated this run
|
||||
PartDst string // the rendering its own dst fails to carry
|
||||
}
|
||||
|
||||
// ConsolidationConflicts is CanonConflicts' thin brother, and it exists because the canon check can only see
|
||||
// what the owner already signed. On a live bank that is the minority: 18 of 149 contradictions the same run
|
||||
// produced were between its OWN consolidations (research/24 §A4) — 元海空窍 rendered without the 元海 this
|
||||
// very reply had just fixed — and nothing looked at them, because the role runs once and never reads itself.
|
||||
// A step whose entire purpose is ONE consistent canon must not be the step that quietly breaks it.
|
||||
//
|
||||
// Same test as the canon side (containment + lexemeSubset, so Russian case endings do not false-flag), same
|
||||
// discipline: it REPORTS, never rewrites and never fails a run. $0 — it is arithmetic over a map that
|
||||
// already exists. Deterministic: candidates are key-ordered on both sides and nothing here iterates a map.
|
||||
func ConsolidationConflicts(cands []Candidate, consolidated map[string]string) []ConsolidationConflict {
|
||||
if len(consolidated) < 2 {
|
||||
return nil
|
||||
}
|
||||
var out []ConsolidationConflict
|
||||
for _, c := range cands {
|
||||
dst := consolidated[c.Key]
|
||||
if dst == "" || c.Key == "" {
|
||||
continue
|
||||
}
|
||||
have := wordSet(dst)
|
||||
for _, p := range cands {
|
||||
pd := consolidated[p.Key]
|
||||
if pd == "" || p.Key == "" || p.Key == c.Key || !strings.Contains(c.Key, p.Key) {
|
||||
continue
|
||||
}
|
||||
if lexemeSubset(wordSet(pd), have) {
|
||||
continue
|
||||
}
|
||||
out = append(out, ConsolidationConflict{Src: c.Src, Dst: dst, PartSrc: p.Src, PartDst: pd})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sharedRunes counts the DISTINCT runes two sources have in common.
|
||||
func sharedRunes(a, b string) int {
|
||||
if a == "" || b == "" {
|
||||
|
|
@ -838,6 +1023,10 @@ type ReplyStats struct {
|
|||
Bad int
|
||||
OffLanguage int
|
||||
OffLanguageSamples []string // first few refused lines verbatim, so the warning names them
|
||||
// BadConfidence counts lines whose declared CONFIDENCE column was not a bare 0…100 («95%», «0.9»,
|
||||
// «высокая»). The rendering is kept — it is fine — but the value is dropped rather than glued onto it,
|
||||
// and the count is what tells an operator the prompt's third column is not landing.
|
||||
BadConfidence int
|
||||
}
|
||||
|
||||
const offLanguageSampleCap = 8
|
||||
|
|
@ -850,25 +1039,27 @@ const offLanguageSampleCap = 8
|
|||
//
|
||||
// `target` is the target language's script (nil → the answer-language check is inert; see OffLanguage).
|
||||
//
|
||||
// Returns the accepted map plus the tally of unusable lines, which the caller logs: silence about a reply
|
||||
// the parser could not read is how a paid call turns into an empty bank with no signal.
|
||||
func ParseReply(reply string, expected []string, normalize func(string) string, target *unicode.RangeTable) (map[string]string, ReplyStats) {
|
||||
// Returns the accepted map, the role's own stated CONFIDENCE per key (absent when the reply carried none),
|
||||
// and the tally of unusable lines, which the caller logs: silence about a reply the parser could not read is
|
||||
// how a paid call turns into an empty bank with no signal.
|
||||
func ParseReply(reply string, expected []string, normalize func(string) string, target *unicode.RangeTable) (map[string]string, map[string]int, ReplyStats) {
|
||||
want := make(map[string]bool, len(expected))
|
||||
for _, k := range expected {
|
||||
want[k] = true
|
||||
}
|
||||
out := map[string]string{}
|
||||
conf := map[string]int{}
|
||||
var st ReplyStats
|
||||
for _, ln := range strings.Split(reply, "\n") {
|
||||
if strings.TrimSpace(ln) == "" || strings.HasPrefix(strings.TrimSpace(ln), "#") {
|
||||
continue
|
||||
}
|
||||
f := splitFields(ln)
|
||||
if len(f) < 2 {
|
||||
rawSrc, dst, c, badConf, ok := replyColumns(ln)
|
||||
if !ok {
|
||||
st.Bad++
|
||||
continue
|
||||
}
|
||||
key := normalize(f[0])
|
||||
key := normalize(rawSrc)
|
||||
if !want[key] {
|
||||
st.Bad++
|
||||
continue
|
||||
|
|
@ -876,15 +1067,23 @@ func ParseReply(reply string, expected []string, normalize func(string) string,
|
|||
if _, dup := out[key]; dup {
|
||||
continue // first answer wins; a model repeating itself is not a second opinion
|
||||
}
|
||||
if badConf {
|
||||
// The rendering is usable; the confidence column is not. Counted rather than refused — and never
|
||||
// folded into the rendering, which is the whole point.
|
||||
st.BadConfidence++
|
||||
}
|
||||
// Everything after the FIRST field is the rendering, re-joined on single spaces. The splitter is
|
||||
// deliberately tolerant (a tab, a run of ≥2 spaces, a padded pipe) because models substitute one for
|
||||
// another — but that tolerance also splits INSIDE a rendering the moment a model writes «Фан Юань»
|
||||
// with a stray double space, and taking f[1] alone would then bank the half-name «Фан» silently:
|
||||
// it passes wellFormedLemma, so no counter would ever mention it. The role's reply is specified as
|
||||
// exactly two fields, so there is no third column to lose by re-joining.
|
||||
dst := strings.TrimSpace(strings.Join(f[1:], " "))
|
||||
if dst == NoDst {
|
||||
// exactly two fields, so there is no third column to lose by re-joining — see replyColumns for how the
|
||||
// third (confidence) column and a mis-split rendering are told apart.
|
||||
if isNoDst(dst) {
|
||||
out[key] = ""
|
||||
if c >= 0 {
|
||||
conf[key] = c
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !wellFormedLemma(dst) {
|
||||
|
|
@ -905,18 +1104,60 @@ func ParseReply(reply string, expected []string, normalize func(string) string,
|
|||
st.Bad++
|
||||
st.OffLanguage++
|
||||
if len(st.OffLanguageSamples) < offLanguageSampleCap {
|
||||
st.OffLanguageSamples = append(st.OffLanguageSamples, f[0]+" → "+dst)
|
||||
st.OffLanguageSamples = append(st.OffLanguageSamples, rawSrc+" → "+dst)
|
||||
}
|
||||
continue
|
||||
}
|
||||
out[key] = dst
|
||||
if c >= 0 {
|
||||
conf[key] = c
|
||||
}
|
||||
}
|
||||
return out, st
|
||||
return out, conf, st
|
||||
}
|
||||
|
||||
// Batch splits candidates into groups whose rendered size stays under maxRunes. A series (a key present in
|
||||
// seriesID; pass nil to disable the channel) is kept WHOLE in one batch so the terminologist picks one
|
||||
// generic head for it (§1); a series or a single candidate larger than the budget still gets its own batch
|
||||
// confidenceField reads the role's stated confidence off a reply FIELD: 1–3 digits in 0…100 and nothing
|
||||
// else. Out of that range it is not a confidence and stays part of the rendering.
|
||||
//
|
||||
// ⚠ What this number may and may not do is RATIFIED (D39.102): it orders the review list «least sure
|
||||
// first» — an ordinal INSIDE one model's reply — and nothing more. It is not a weight, not a threshold, not
|
||||
// comparable between models, and never decides which rendering wins; the measured AUC (0.77–0.85) supports
|
||||
// exactly the ordering claim and nothing stronger.
|
||||
func confidenceField(s string) (int, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) == 0 || len(s) > 3 {
|
||||
return 0, false
|
||||
}
|
||||
n := 0
|
||||
for _, r := range s {
|
||||
if r < '0' || r > '9' {
|
||||
return 0, false
|
||||
}
|
||||
n = n*10 + int(r-'0')
|
||||
}
|
||||
if n > 100 {
|
||||
return 0, false
|
||||
}
|
||||
return n, true
|
||||
}
|
||||
|
||||
// isNoDst recognises the decline sentinel, including the forms a model mangles it into — a trailing period,
|
||||
// surrounding quotes. Narrow on purpose: the bracketed engine token must still OPEN the rendering and only
|
||||
// punctuation may follow, so no natural rendering can match. Without it a decline written «⟦TM-NO-DST⟧.»
|
||||
// fails wellFormedLemma-then-bad-line and the term is counted as a PARSE failure rather than as the
|
||||
// decision §C2-7 says it is.
|
||||
func isNoDst(dst string) bool {
|
||||
t := strings.Trim(strings.TrimSpace(dst), `"'«»“”`)
|
||||
if !strings.HasPrefix(t, NoDst) {
|
||||
return false
|
||||
}
|
||||
return strings.Trim(strings.TrimSpace(t[len(NoDst):]), `.,;:!?"'«»“”`) == ""
|
||||
}
|
||||
|
||||
// Batch splits candidates into groups whose rendered size stays under maxRunes. A batch UNIT — a series or
|
||||
// a family, i.e. a key present in unitID; pass nil to disable both channels — is kept WHOLE in one batch so
|
||||
// the terminologist picks one generic head or one shared root for it (§1, §G1); a unit or a single
|
||||
// candidate larger than the budget still gets its own batch
|
||||
// rather than being split or silently dropped (the caller WARNs on such an over-cap unit — see BatchRunes —
|
||||
// because the whole unit still bills against one output cap). The batches are then ordered by descending
|
||||
// source frequency, so a budget ceiling drops the least-frequent terms instead of the lexicographic tail
|
||||
|
|
@ -928,14 +1169,14 @@ func ParseReply(reply string, expected []string, normalize func(string) string,
|
|||
// into the request hash, so a batch's POSITION is part of its address. If a later run's frequencies reorder
|
||||
// the batches their ordinals shift and the affected batches re-pay — the accepted price of value-ordering,
|
||||
// and no worse than the content edit that would already re-pay them.
|
||||
func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candidate {
|
||||
func Batch(cands []Candidate, maxRunes int, unitID map[string]int) [][]Candidate {
|
||||
if len(cands) == 0 {
|
||||
return nil
|
||||
}
|
||||
if maxRunes <= 0 {
|
||||
return [][]Candidate{cands}
|
||||
}
|
||||
ordered := orderBySeries(cands, seriesID)
|
||||
ordered := orderByUnit(cands, unitID)
|
||||
var out [][]Candidate
|
||||
var cur []Candidate
|
||||
size := 0
|
||||
|
|
@ -946,9 +1187,9 @@ func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candida
|
|||
}
|
||||
}
|
||||
for i := 0; i < len(ordered); {
|
||||
j := i + 1 // extend over a whole series block; a singleton is a unit of one
|
||||
if id := seriesID[ordered[i].Key]; id != 0 {
|
||||
for j < len(ordered) && seriesID[ordered[j].Key] == id {
|
||||
j := i + 1 // extend over a whole unit block; a singleton is a unit of one
|
||||
if id := unitID[ordered[i].Key]; id != 0 {
|
||||
for j < len(ordered) && unitID[ordered[j].Key] == id {
|
||||
j++
|
||||
}
|
||||
}
|
||||
|
|
@ -967,7 +1208,7 @@ func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candida
|
|||
}
|
||||
|
||||
// BatchRunes reports the rendered size RenderBatch produces for a whole batch — the same measure Batch packs
|
||||
// against. The terminologist uses it to WARN when a co-batched series or a lone candidate exceeds the budget:
|
||||
// against. The terminologist uses it to WARN when a co-batched unit or a lone candidate exceeds the budget:
|
||||
// §1 keeps such a unit WHOLE rather than split it, so it is a real over-cap call the operator must see coming
|
||||
// (the cap-8000 output mine, PROBES §1), not a silent overrun.
|
||||
func BatchRunes(batch []Candidate) int { return unitRunes(batch) }
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ func TestMergeExactAndAliasCollapseButContainmentDoesNot(t *testing.T) {
|
|||
{Key: "方小子", Src: "方小子", Type: "nickname", Proposals: []Proposal{{Dst: "малец Фан", Chunks: 1}}},
|
||||
{Key: "古月方源", Src: "古月方源", Type: "name", Proposals: []Proposal{{Dst: "Гуюэ Фан Юань", Chunks: 2}}},
|
||||
}
|
||||
got := Merge(mined, observed)
|
||||
got := Merge(mined, observed, nil)
|
||||
|
||||
byKey := map[string]Candidate{}
|
||||
for _, c := range got {
|
||||
|
|
@ -71,7 +71,7 @@ func TestMergeFoldsIdenticalRenderingsOfOneEntity(t *testing.T) {
|
|||
{Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 2}, {Dst: "малец Фан", Chunks: 1}}},
|
||||
}
|
||||
var c Candidate
|
||||
for _, got := range Merge(mined, observed) {
|
||||
for _, got := range Merge(mined, observed, nil) {
|
||||
if got.Key == "方源" { // select by key: the alias surface is now a candidate of its own too
|
||||
c = got
|
||||
}
|
||||
|
|
@ -98,7 +98,7 @@ func TestMergeFoldsIdenticalRenderingsOfOneEntity(t *testing.T) {
|
|||
func TestMergeKeepsAnAliasSwallowedSurface(t *testing.T) {
|
||||
mined := []Mined{{Key: "葛家", Src: "葛家", Type: "name", Freq: 40, Aliases: []string{"族长"}}}
|
||||
observed := []Observed{{Key: "族长", Src: "族长", Type: "title", Proposals: []Proposal{{Dst: "глава клана", Chunks: 3}}}}
|
||||
got := Merge(mined, observed)
|
||||
got := Merge(mined, observed, nil)
|
||||
|
||||
var title, cluster *Candidate
|
||||
for i := range got {
|
||||
|
|
@ -137,9 +137,9 @@ func TestMergeIsDeterministic(t *testing.T) {
|
|||
{Key: "z", Src: "z", Proposals: []Proposal{{Dst: "Z", Chunks: 1}}},
|
||||
{Key: "a", Src: "a", Proposals: []Proposal{{Dst: "A", Chunks: 1}}},
|
||||
}
|
||||
first := keysOf(Merge(mined, observed))
|
||||
first := keysOf(Merge(mined, observed, nil))
|
||||
for i := 0; i < 20; i++ { // map iteration would show up here, not on one run
|
||||
if got := keysOf(Merge(mined, observed)); !reflect.DeepEqual(got, first) {
|
||||
if got := keysOf(Merge(mined, observed, nil)); !reflect.DeepEqual(got, first) {
|
||||
t.Fatalf("merge is order-unstable: %v vs %v", got, first)
|
||||
}
|
||||
}
|
||||
|
|
@ -340,13 +340,13 @@ func TestCanonConflictsFlagsOnlyRealContradictions(t *testing.T) {
|
|||
// stray double space inside a rendering must not truncate it (the splitter is tolerant BY DESIGN, and that
|
||||
// tolerance cuts both ways), and a batch must actually render within the budget it was split for.
|
||||
func TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds(t *testing.T) {
|
||||
got, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
|
||||
got, _, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
|
||||
if st.Bad != 0 || got["方源"] != "Фан Юань" {
|
||||
t.Fatalf("a double space inside a rendering must not bank half a name: %v (bad=%d)", got, st.Bad)
|
||||
}
|
||||
// A rendering that IS the source is not a rendering: a model echoing the term back would otherwise be
|
||||
// banked as this book's canon and shown to the editor as «方源 ⟨проверить⟩».
|
||||
echo, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
|
||||
echo, _, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
|
||||
if _, banked := echo["方源"]; banked || st.Bad != 1 {
|
||||
t.Fatalf("an echoed source must be refused and counted: %v (bad=%d)", echo, st.Bad)
|
||||
}
|
||||
|
|
@ -380,7 +380,7 @@ func TestParseReplyAcceptsOnlyAskedTerms(t *testing.T) {
|
|||
"忘却\t" + NoDst,
|
||||
"мусор",
|
||||
}, "\n")
|
||||
got, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil)
|
||||
got, _, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil)
|
||||
want := map[string]string{"方源": "Фан Юань", "花家": "Дом Хуа", "青茅山": "гора Цинмао", "忘却": ""}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("parse = %#v, want %#v", got, want)
|
||||
|
|
@ -398,7 +398,7 @@ func TestParseReplyNormalizesTheKey(t *testing.T) {
|
|||
// The caller passes the engine's source-key normalizer; a reply written in another orthography must
|
||||
// still land on its term.
|
||||
lower := strings.ToLower
|
||||
got, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil)
|
||||
got, _, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil)
|
||||
if got["fang"] != "Фан" {
|
||||
t.Fatalf("the reply key must be normalized by the caller's function, got %#v", got)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
<!-- v2 (26.07): KWIC-контексты, узкое правило транскрипции, пример формата ⟦TM-CANON⟧ — обоснования
|
||||
в D39.46/47/52 и отчётах полигон-пакетов. Файл — ДАННЫЕ: в снапшот не фолдится, текст не пинят тесты. -->
|
||||
в D39.46/47/52 и отчётах полигон-пакетов. Файл — ДАННЫЕ: в снапшот не фолдится, текст не пинят тесты.
|
||||
v3 (08.08, фикс-пак банка §G3): третье поле — уверенность 0–100. Она НЕ арбитр: движок сортирует ею лист
|
||||
ревью «сначала неуверенное» (ordinal внутри одного ответа, AUC 0.77–0.85 research/24 §B3) и ничего больше —
|
||||
ни весов, ни порогов, ни сравнения между моделями (запрещённые формы зафиксированы D39.102). -->
|
||||
|
||||
Ты — терминолог издательского перевода с языка «{{source_lang}}» на язык «{{target_lang}}».
|
||||
Книга: «{{title}}». Жанр: {{genre}}. Аудитория: {{audience}}.
|
||||
|
|
@ -33,12 +36,18 @@
|
|||
- Если по данным контекстам ты не можешь выбрать перевод уверенно — верни ⟦TM-NO-DST⟧. Это нормальный
|
||||
ответ: непереведённый термин останется на ручную подпись, а выдуманный попадёт в книгу.
|
||||
|
||||
Формат ответа — по одной строке на термин, ровно два поля, разделённые СИМВОЛОМ ТАБУЛЯЦИИ:
|
||||
первое поле — термин исходника, второе — перевод.
|
||||
Формат ответа — по одной строке на термин, ровно ТРИ поля, разделённые СИМВОЛОМ ТАБУЛЯЦИИ:
|
||||
первое поле — термин исходника, второе — перевод, третье — твоя уверенность в этом переводе,
|
||||
целое число от 0 до 100.
|
||||
|
||||
Пример строки ответа (символ между полями — настоящая табуляция):
|
||||
Уверенность — это оценка ТВОЕГО решения по данным контекстам, а не оценка термина: 90 — контексты
|
||||
однозначны и вариант очевиден; 40 — контекстов мало или они противоречивы, и ты выбрал наименее плохой.
|
||||
Она ни на что не влияет, кроме порядка, в котором человек будет просматривать список: сначала он посмотрит
|
||||
то, в чём ты сам не уверен. Занижать её «на всякий случай» так же бесполезно, как завышать.
|
||||
|
||||
师父 наставник
|
||||
Пример строки ответа (символы между полями — настоящие табуляции):
|
||||
|
||||
师父 наставник 95
|
||||
|
||||
Никаких заголовков, нумерации, комментариев и markdown. Термины, которых нет в списке, не добавляй.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue