From 5092ae3f0edd77e2ab2e07d02f3fbb425e7f6c2d Mon Sep 17 00:00:00 2001 From: heaven Date: Mon, 7 Sep 2026 04:53:43 +0300 Subject: [PATCH] Land the engine zone's bank-law work as the zone built it: the postcheck names the blind spot it cannot see, and an unsigned pair no longer passes the merge unseen --- backend/cmd/tmctl/render.go | 13 +- backend/cmd/tmctl/render_test.go | 27 +++ backend/internal/membank/mempostcheck.go | 20 +- backend/internal/membank/mempostcheck_test.go | 35 +++ backend/internal/membank/memseed.go | 216 +++++++++++++++-- backend/internal/membank/memseed_test.go | 180 +++++++++++++- .../pipeline/bankconflict_corpus_test.go | 222 ++++++++++++++++++ backend/internal/pipeline/bankfixpack_test.go | 188 +++++++++++++++ backend/internal/pipeline/disposition.go | 4 +- backend/internal/pipeline/export.go | 6 +- backend/internal/pipeline/mining.go | 26 +- .../internal/pipeline/miningstop_join_test.go | 11 +- .../pipeline/resume_updatedat_test.go | 93 ++++++++ backend/internal/pipeline/seeding.go | 11 +- backend/internal/pipeline/terminologist.go | 82 ++++++- backend/internal/pipeline/waverun.go | 11 +- backend/internal/store/chunkstatus.go | 15 +- backend/internal/text/norm_test.go | 24 ++ 18 files changed, 1114 insertions(+), 70 deletions(-) create mode 100644 backend/internal/pipeline/bankconflict_corpus_test.go create mode 100644 backend/internal/pipeline/resume_updatedat_test.go diff --git a/backend/cmd/tmctl/render.go b/backend/cmd/tmctl/render.go index 00b69194..dc9853c6 100644 --- a/backend/cmd/tmctl/render.go +++ b/backend/cmd/tmctl/render.go @@ -228,6 +228,12 @@ func renderBankStopRows(w io.Writer, rows []pipeline.BankStopRow) { // same reply gave its part. Loudest line of the row — it is a canon breaking inside one call. fmt.Fprintf(w, " ⚠ contradicts: %s\n", trunc(strings.Join(r.Contradicts, "; "), 80)) } + if len(r.BankHolds) > 0 { + // The other disagreement, and a different decision: the book already renders this surface some + // other way. It rides the same screen as its sibling above, or the operator's first view of the + // stop shows one and hides the other. + fmt.Fprintf(w, " ⚠ bank holds: %s\n", trunc(strings.Join(r.BankHolds, "; "), 80)) + } if len(r.Variants) > 1 { // The disagreement is the reason to sign: one term coming back several ways is exactly what a // canon exists to close, and hiding it behind one winner throws that reason away. @@ -359,8 +365,11 @@ func renderReport(w io.Writer, // Memory section (step 4): the per-chunk retrieval-state, surfaced so silent glossary // degradation is LOUD. The aggregate line always prints when a glossary is in use; - // each chunk with a post-check miss is listed (the flagger-mode signal — the model - // ignored an approved term, or we injected the wrong dst) with the offending src→dst. + // each chunk with a post-check miss is listed with the offending src→dst. Two limits: a + // chunk is listed on its CONFIRMED miss count while the detail beside it carries every + // miss including the unsigned ones, so a listed line is not necessarily an approved + // term; and a miss means the rendering is ABSENT — a wrong dst the model OBEYED is + // present in the output and appears in no line here (mempostcheck.go). states, err := fetchStates() if err != nil { return err diff --git a/backend/cmd/tmctl/render_test.go b/backend/cmd/tmctl/render_test.go index cfef41ff..aaa6ed42 100644 --- a/backend/cmd/tmctl/render_test.go +++ b/backend/cmd/tmctl/render_test.go @@ -561,3 +561,30 @@ func TestTheHumanManifestCallsTheBookLevelFigureACeiling(t *testing.T) { t.Errorf("a book with no book-level spend must not be told about a ceiling on it:\n%s", b2.String()) } } + +// TestRenderSignatureStopShowsBothDisagreements pins the review screen's two markers together. They are +// different decisions for the owner — the run contradicting itself, and the book already rendering the +// surface another way — and a screen that prints one and hides the other sends him to the sidecar to find +// out that the term he is signing is already in the bank. +func TestRenderSignatureStopShowsBothDisagreements(t *testing.T) { + rows := []pipeline.BankStopRow{{ + Src: "方源", Dst: "Фан Юань", Origin: "banknote", Freq: 9, Conf: 10, + Contradicts: []string{`方源空窍→"Пустота Фан Юаня"`}, + BankHolds: []string{`unsigned draft "方源"→"Странник" [ch 1..20]`}, + }} + var b bytes.Buffer + renderSignatureStop(&b, &pipeline.WaveSignatureStop{Terms: len(rows), SignaturePath: "p", Rows: rows}) + out := b.String() + for _, want := range []string{"contradicts:", "Пустота Фан Юаня", "bank holds:", `unsigned draft "方源"→"Странник"`} { + if !strings.Contains(out, want) { + t.Errorf("the review screen must carry %q:\n%s", want, out) + } + } + // CONTROL: a row with neither marker prints neither, so the assertions above are the fields answering. + var quiet bytes.Buffer + renderSignatureStop(&quiet, &pipeline.WaveSignatureStop{Terms: 1, SignaturePath: "p", + Rows: []pipeline.BankStopRow{{Src: "方源", Dst: "Фан Юань", Origin: "banknote", Freq: 9, Conf: 10}}}) + if strings.Contains(quiet.String(), "contradicts:") || strings.Contains(quiet.String(), "bank holds:") { + t.Errorf("control: an unmarked row must print neither marker:\n%s", quiet.String()) + } +} diff --git a/backend/internal/membank/mempostcheck.go b/backend/internal/membank/mempostcheck.go index 99fb0597..7716bc29 100644 --- a/backend/internal/membank/mempostcheck.go +++ b/backend/internal/membank/mempostcheck.go @@ -10,10 +10,22 @@ import ( // mempostcheck.go: the post-check (registry E1/E2 / research/13 Q5) — the main, almost // free detector of silent degradation. For every injected record whose SRC actually -// fired in THIS chunk, it asks: did an accepted DST form appear in the output? A miss -// catches BOTH failure modes — "the model ignored the glossary" (soft-following loses -// 17–36% of terms, 2310.05824) AND "we injected the wrong dst and the model obeyed it" -// (the owner's fear, 2510.00829). It is a post-CHECK, never a blind post-REPLACE (E2): +// fired in THIS chunk, it asks one question: did an accepted DST form appear in the +// output? A miss says the injected rendering is ABSENT and nothing more — "the model +// ignored the glossary" (soft-following loses 17–36% of terms, 2310.05824), whether the +// injected dst was right or wrong. +// +// BLIND SPOT, by the shape of the check: a wrong dst the model OBEYED (the owner's fear, +// 2510.00829) leaves the injected form PRESENT, so `if present { continue }` below is taken +// and no miss is written — the more faithfully a model executes a wrong injection, the +// quieter this gets. A zero confirmed-miss count therefore says the signed rows that fired +// on a TRUSTED key were EXECUTED — not that they were executed CORRECTLY, and nothing about +// the rest: an unsigned row's miss and a signed one demoted by a single-Han key both go to +// Unverified and are not counted. Judging the VALUE of an injected rendering needs a +// source-side check that does not live here (backlog row 286). Pinned by +// TestWrongDstObeyedIsNotAMiss. +// +// It is a post-CHECK, never a blind post-REPLACE (E2): // forcing a dictionary form into an oblique Russian case breaks agreement (46% of // constrained-model errors in en-cs are agreement, 2106.12398), so v1 only FLAGS. // diff --git a/backend/internal/membank/mempostcheck_test.go b/backend/internal/membank/mempostcheck_test.go index c7979ff1..ef941064 100644 --- a/backend/internal/membank/mempostcheck_test.go +++ b/backend/internal/membank/mempostcheck_test.go @@ -96,3 +96,38 @@ func TestAmbiguousMissNotDemoted(t *testing.T) { t.Errorf("the ambiguous miss must keep disp:ambiguous, got %q", res.Unverified[0].Disp) } } + +// TestWrongDstObeyedIsNotAMiss pins the SCOPE of a miss: the injected rendering is ABSENT, nothing more. +// A wrong dst the model faithfully executed leaves that rendering present, so it produces no miss at all. +// The two halves share bank, chunk and firing key and differ only in whether the output carries the +// injected form — without the control, the zero would be indistinguishable from a fixture that never +// reached the checker. +func TestWrongDstObeyedIsNotAMiss(t *testing.T) { + // 元石 is signed to a WRONG rendering: the book's stone is «первокамень», the bank says «монета». + b := bankFrom([]store.GlossaryEntry{gl("元石", "монета", "", "approved")}) + sel := b.Select("他手里握着元石。", 1, nil, 0) + if len(sel.Injected) != 1 { + t.Fatalf("the firing key must inject for either half of this pin to mean anything, got %d", len(sel.Injected)) + } + + // OBEYED. Nominative, so the whole-word branch answers and the pin does not need a stemmer. + obeyed := b.Postcheck(sel.Injected, "В его руке лежала монета.") + if obeyed.ConfirmedCount() != 0 || !obeyed.Empty() { + t.Errorf("an OBEYED injection leaves its dst present, so the post-check has nothing to report: confirmed=%d all=%+v", + obeyed.ConfirmedCount(), obeyed.All()) + } + + // CONTROL: the rendering is ABSENT, which is the only thing a miss says. + refused := b.Postcheck(sel.Injected, "В его руке лежал первокамень.") + if refused.ConfirmedCount() != 1 { + t.Fatalf("control: an absent rendering must be exactly one CONFIRMED miss, got %d (%+v)", refused.ConfirmedCount(), refused.All()) + } + if refused.Confirmed[0].Dst != "монета" { + t.Errorf("control: the miss must name the INJECTED rendering, got %q", refused.Confirmed[0].Dst) + } + // The wrong-value case is silent and the reported case is the model REFUSING that same wrong value, so + // a zero cannot be read as "the bank was executed correctly". + if obeyed.ConfirmedCount() == refused.ConfirmedCount() { + t.Errorf("the two halves must differ, or this test pins nothing: %d vs %d", obeyed.ConfirmedCount(), refused.ConfirmedCount()) + } +} diff --git a/backend/internal/membank/memseed.go b/backend/internal/membank/memseed.go index 32c4419f..01b4e659 100644 --- a/backend/internal/membank/memseed.go +++ b/backend/internal/membank/memseed.go @@ -222,9 +222,13 @@ func parseBankSeed(name string, raw []byte, engineWritten bool) (BankSeed, error // Scope to a MATCHABLE src: an A3-banned single-char src (道 — text.SignificantLen below // the min-key floor, never in the automaton) is inert and cannot livelock, so its // polysemy stays allowed. Both rows must be matchable for the contradiction to fire. + // + // Status-blind on purpose: under D39.104 п.2 every row on the wire is law whatever its + // signature (memory.go, "no fork by status at all"), so demoting one of the pair to + // auto/draft still shows the model both renderings. Hence no status among the remedies. if srcKeyFires(a.Src, a.AllowShort) && srcKeyFires(b.Src, b.AllowShort) { problems.addKeyed(subjectOf("term-polysemy/"+b.Sense, a.Src, a.Sense, a.SinceCh, a.UntilCh), - fmt.Sprintf("term %q: senses %q→%q and %q→%q have OVERLAPPING spoiler windows [%d,%d] and [%d,%d] — the deterministic matcher keys only on src and cannot pick a sense, so both inject as authoritative and the post-check is guaranteed to miss one (give them non-overlapping windows, merge the sense, or mark one status: auto)", + fmt.Sprintf("term %q: senses %q→%q and %q→%q have OVERLAPPING spoiler windows [%d,%d] and [%d,%d] — the deterministic matcher keys only on src and cannot pick a sense, so both inject as authoritative and the post-check is guaranteed to miss one (give them non-overlapping windows, merge the senses into one rendering, or drop one; a status will not separate them — every row on the wire is law whatever its signature)", a.Src, b.Sense, b.Dst, a.Sense, a.Dst, b.SinceCh, b.UntilCh, a.SinceCh, a.UntilCh)) } } @@ -676,48 +680,210 @@ func runesAllKana(s string) bool { return true } -// UnverifiedKeyConflicts reports the pairs S14 named: an APPROVED term and an UNSIGNED one that share a -// firing surface with DIFFERENT renderings and overlapping windows. Both are injected — the approved one -// as canon, the unsigned one as a working version — so the model receives two contradictory lines about -// the same surface. That is the likeliest source of a silent degradation in the auto mode, and until -// pack-20 no diagnostic could see it: ApprovedSharedKeyCollisions and InjectivityCollisions are both -// approved-only, by construction. +// UnverifiedKeyConflicts reports every pair of bank rows that share a firing surface, render it +// differently and have overlapping spoiler windows, except approved×approved — that pair belongs to +// ApprovedSharedKeyCollisions' fail-loud and reporting it here too would state one fault twice. // -// It is a WARNING, not a fail-loud, and deliberately so: the unsigned row is engine-produced, and -// aborting a paid run over the engine's own proposal would be a self-inflicted outage. The trust gate -// already protects the reader (the approved rendering keeps its precedence); this makes the collision -// VISIBLE so the owner can reconcile it. Deterministic (sorted keys, sorted pairs). +// Both rows of such a pair are injected, and a signature changes nothing about what the model is shown +// (D39.104 п.2: no fork by status), so an unsigned×unsigned pair is the same fault as a signed one. +// Nothing else catches it: ApprovedSharedKeyCollisions and InjectivityCollisions are approved-only, the +// seed loader's D16.1 guard compares only Src within one document, and the uniqueness checks key on the +// whole (src, sense, window) tuple, which two overlapping windows do not share. +// +// WARNING, not fail-loud: at least one row of every reported pair is engine-produced or unratified, and +// aborting a paid run over our own proposal would be a self-inflicted outage. +// +// Deterministic (sorted keys, then index order); a pair sharing two keys is reported once, under the +// first key. func UnverifiedKeyConflicts(entries []store.GlossaryEntry) []string { - approved := map[string][]int{} - unsigned := map[string][]int{} + owners := map[string][]int{} for i, e := range entries { if strings.TrimSpace(e.Dst) == "" { continue } - target := unsigned - if e.Status == "approved" { - target = approved - } for _, k := range entryFiringKeys(e) { - target[k] = append(target[k], i) + owners[k] = append(owners[k], i) } } var out []string - for _, k := range slices.Sorted(maps.Keys(approved)) { - for _, ai := range approved[k] { - for _, ui := range unsigned[k] { - ae, ue := entries[ai], entries[ui] - if ae.Dst == ue.Dst || !windowsOverlap(ae.SinceCh, ae.UntilCh, ue.SinceCh, ue.UntilCh) { + seenPair := map[[2]int]bool{} + for _, k := range slices.Sorted(maps.Keys(owners)) { + idxs := owners[k] + for a := 0; a < len(idxs); a++ { + for b := a + 1; b < len(idxs); b++ { + i, j := idxs[a], idxs[b] + ei, ej := entries[i], entries[j] + if ei.Status == "approved" && ej.Status == "approved" { + continue // ApprovedSharedKeyCollisions' fail-loud owns this pair + } + if ei.Dst == ej.Dst || !windowsOverlap(ei.SinceCh, ei.UntilCh, ej.SinceCh, ej.UntilCh) { continue } - out = append(out, fmt.Sprintf("firing key %q: approved %q→%q vs unsigned %s %q→%q (the model is shown two renderings of one surface)", - k, ae.Src, ae.Dst, ue.Status, ue.Src, ue.Dst)) + if seenPair[[2]int{i, j}] { + continue + } + seenPair[[2]int{i, j}] = true + out = append(out, fmt.Sprintf("firing key %q: %s %q→%q %s vs %s %q→%q %s (the model is shown two renderings of one surface)", + k, statusLabel(ei.Status), ei.Src, ei.Dst, windowLabel(ei.SinceCh, ei.UntilCh), + statusLabel(ej.Status), ej.Src, ej.Dst, windowLabel(ej.SinceCh, ej.UntilCh))) } } } return out } +// statusLabel renders a row's signature for an operator, keeping the specific status where there is one. +func statusLabel(status string) string { + if status == "approved" { + return "approved" + } + if status == "" { + return "unsigned" + } + return "unsigned " + status +} + +// windowLabel renders a spoiler window for a human. The raw zeros must not be printed: since_ch 0 means +// "from chapter 1" and until_ch 0 means "no end" (windowsOverlap), so "[0,0]" would read as an empty +// window rather than the whole book. +func windowLabel(since, until int) string { + lo := since + if lo == 0 { + lo = 1 + } + if until == 0 { + return fmt.Sprintf("[ch %d..end]", lo) + } + return fmt.Sprintf("[ch %d..%d]", lo, until) +} + +// ConsolidationKeyConflicts reports a rendering this run consolidated for a firing surface the bank +// already renders differently, in an overlapping window, whatever either side's status. +// +// Neither of the collapse stage's own checks can see this shape: terminology.CanonConflicts and +// terminology.ConsolidationConflicts both test CONTAINMENT and both skip the pair whose sources are +// equal, and CanonConflicts' right-hand side is approved-only. The consolidation reaches surfaces the +// bank already holds by two ordinary routes: the miner re-mines a surface held only by an engine-unsigned +// row (pipeline.unsignedEngineSurfaces hides those from it so the auto mode does not switch itself off), +// and the banknote channel merges every draft-proposed surface into the candidate list with no bank +// filter — that filter sits on emission, not on the role's input. +// +// It reports a DISAGREEMENT and does not predict that both renderings reach a wire. Three mechanisms +// downstream decide that, and only the first is grounds to stay silent: +// - the glossary UNIQUE key (book_id, src, sense, since_ch, until_ch) admits only one row per tuple, so +// a proposal landing on an existing row's tuple resolves against it rather than joining it. That pair +// is the one case skipped below; +// - the emission filter drops a proposal whose surface is already a seed surface +// (pipeline.reverseSectionTerms), leaving the disagreement on the review sheet only; +// - the auto-bank file is rewritten whole each run (pipeline.writeAutoBank), so a previous run's engine +// row does not survive beside this run's rendering of the same surface even when the windows differ. +// +// The last two are still reported: the sheet is where a term is decided, and "the book already calls it +// something else" is what the owner needs there. +// +// ⚠ WHAT THE SKIP HIDES, measured: a seed row with the DEFAULT window and no sense — the commonest shape a +// hand-written seed has — shares its tuple with a banknote proposal, which carries since_ch 0 likewise, +// so that pair is silent here whatever the seed row's status. The proposal is dropped rather than +// shipped, so no wire carries two renderings; what is lost is the sheet telling the owner that the role +// disagreed with a row he wrote. Closing it needs a rule for telling a REPLACED engine row from a DROPPED +// proposal, and the Source column cannot give one: the owner's delta and the engine's auto-bank both read +// "mined". +// +// In the auto mode a pair that DOES land is then reported a second time, by UnverifiedKeyConflicts on the +// re-seed. The two are different states of one term — "the role disagrees with the bank" before the row +// exists, "the bank holds both" after — and only the second means the model will be shown two lines. +// +// ⚠ WHO RESOLVES A SKIPPED PAIR, and it is not one mechanism. A proposal on a SEED surface never reaches +// the bank: pipeline.reverseSectionTerms drops it at emission, and an APPROVED delta row counts as a seed +// surface there too (pipeline.unsignedEngineSurfaces keeps it). A proposal on an UNSIGNED delta row's +// tuple does reach pipeline.loadAutoBank, which drops it whenever any earlier-gathered row holds the key — +// unsigned rows included, despite that function's wording. That drop is logged, but as "held by a signed +// term" whatever the holder's status, so the operator is told the wrong thing rather than nothing. +// +// Report-only and $0, like both siblings. Returns parts rather than finished sentences: the run log and +// the signature sheet each render them, and a caller free to re-phrase would state one finding two ways. +// +// Deterministic: proposal order, then each proposal's firing-key order, then bank index order. +func ConsolidationKeyConflicts(consolidated, bank []store.GlossaryEntry) []BankKeyConflict { + if len(consolidated) == 0 || len(bank) == 0 { + return nil + } + owners := map[string][]int{} + for i, e := range bank { + if strings.TrimSpace(e.Dst) == "" { + continue + } + for _, k := range entryFiringKeys(e) { + owners[k] = append(owners[k], i) + } + } + var out []BankKeyConflict + for _, c := range consolidated { + if strings.TrimSpace(c.Dst) == "" { + continue // the role declined or never answered: no rendering, nothing to contradict + } + reported := map[int]bool{} + for _, k := range entryFiringKeys(c) { + for _, i := range owners[k] { + if reported[i] { + continue + } + b := bank[i] + // Compared RAW, because the store's UNIQUE key is raw: 族長 and 族长 with one sense and + // window are two accepted rows, not one (migrate.go). Folding them here would skip a pair + // that really does coexist. Callers pass the source the row WILL carry — see + // pipeline.consolidatedRows. + if b.Src == c.Src && b.Sense == c.Sense && b.SinceCh == c.SinceCh && b.UntilCh == c.UntilCh { + continue // one uniqueness key — the store admits only one of the two + } + if b.Dst == c.Dst || !windowsOverlap(c.SinceCh, c.UntilCh, b.SinceCh, b.UntilCh) { + continue + } + reported[i] = true + out = append(out, BankKeyConflict{ + Key: k, Src: c.Src, Dst: c.Dst, SinceCh: c.SinceCh, UntilCh: c.UntilCh, + BankSrc: b.Src, BankDst: b.Dst, BankStatus: b.Status, + BankSinceCh: b.SinceCh, BankUntilCh: b.UntilCh, + }) + } + } + } + return out +} + +// BankKeyConflict is one consolidated rendering set against the bank row it contradicts. +type BankKeyConflict struct { + Key string // the firing surface the two share — which may be an ALIAS of either row + Src, Dst string // the proposal: the surface consolidated, and what this run rendered it as + SinceCh, UntilCh int // the window the proposal's row would carry + BankSrc string // the row already in the bank; its Src differs from the proposal's on an alias hit + BankDst string + BankStatus string + BankSinceCh int + BankUntilCh int +} + +// Message renders the whole finding as one sentence, for a log line. +func (c BankKeyConflict) Message() string { + return fmt.Sprintf("firing key %q: this run consolidated %q→%q %s while the bank already holds %s (one term, two renderings on the sheet the owner signs from)", + c.Key, c.Src, c.Dst, windowLabel(c.SinceCh, c.UntilCh), c.BankRowLabel()) +} + +// BankRowLabel names only the EXISTING row: the signature sheet already prints the proposal on its own +// line, so repeating it would bury the new fact. +func (c BankKeyConflict) BankRowLabel() string { + return fmt.Sprintf("%s %q→%q %s", statusLabel(c.BankStatus), c.BankSrc, c.BankDst, windowLabel(c.BankSinceCh, c.BankUntilCh)) +} + +// ConflictMessages renders a set of findings for one log line. +func ConflictMessages(cs []BankKeyConflict) []string { + out := make([]string, 0, len(cs)) + for _, c := range cs { + out = append(out, c.Message()) + } + return out +} + // MinedDeltaSeedCollisions returns a human message for every mined-delta entry whose store UNIQUE key // (src, sense, since_ch, until_ch) already exists among the seed/ruby entries — the D39.20 deviation-#1 // crash class. It keys on the FULL uniqueness tuple (not the firing surface, unlike diff --git a/backend/internal/membank/memseed_test.go b/backend/internal/membank/memseed_test.go index 8f8123bb..7fda8894 100644 --- a/backend/internal/membank/memseed_test.go +++ b/backend/internal/membank/memseed_test.go @@ -469,29 +469,49 @@ func TestSeedSurroundingWhitespaceTrimmed(t *testing.T) { } } -// TestUnverifiedKeyConflicts pins S14 — the collision class the approved-only diagnostics structurally -// cannot see. In the auto mode an approved term and an unsigned proposal can share a firing surface with -// different renderings, and both are injected (canon + working version), so the model receives two -// answers to one question. The trust gate keeps the approved one authoritative; this makes the collision -// VISIBLE, which is what neither ApprovedSharedKeyCollisions nor InjectivityCollisions can do. +// TestUnverifiedKeyConflicts pins the rule: every pair rendering one firing surface two ways in +// overlapping windows, except approved×approved (ApprovedSharedKeyCollisions' fail-loud). Both rows are +// injected and a signature changes nothing about that (D39.104 п.2), so an unsigned×unsigned pair is the +// same fault: «старейшина» (draft) against «глава клана» (auto) below. func TestUnverifiedKeyConflicts(t *testing.T) { rows := []store.GlossaryEntry{ {Src: "族長", Dst: "глава клана", Status: "approved"}, - {Src: "族長", Dst: "старейшина", Status: "draft"}, // ← the conflict - {Src: "族長", Dst: "глава клана", Status: "auto"}, // same rendering → no contradiction + {Src: "族長", Dst: "старейшина", Status: "draft"}, // ← contradicts BOTH rows that render «глава клана» + {Src: "族長", Dst: "глава клана", Status: "auto"}, // same rendering as the approved row → no contradiction there {Src: "鈴木", Dst: "Судзуки", Status: "approved"}, // untouched {Src: "田中", Dst: "Танака", Status: "draft"}, // unsigned but unopposed } got := UnverifiedKeyConflicts(rows) - if len(got) != 1 { - t.Fatalf("want exactly the approved-vs-unsigned contradiction, got %d: %v", len(got), got) + if len(got) != 2 { + t.Fatalf("want the approved×unsigned pair AND the unsigned×unsigned one, got %d: %v", len(got), got) } - for _, want := range []string{"族長", "глава клана", "старейшина", "draft"} { - if !strings.Contains(got[0], want) { - t.Fatalf("the report must name %q so the owner can reconcile it: %q", want, got[0]) + joined := strings.Join(got, " | ") + // The approved×unsigned pair (the original S14 guarantee). + if !strings.Contains(joined, `approved "族長"→"глава клана"`) || !strings.Contains(joined, `unsigned draft "族長"→"старейшина"`) { + t.Errorf("the approved×unsigned contradiction must still be named in full: %v", got) + } + // The unsigned×unsigned pair: neither side is signed, and an unsigned term owes one rendering too. + if !strings.Contains(joined, `unsigned draft "族長"→"старейшина" [ch 1..end] vs unsigned auto "族長"→"глава клана"`) { + t.Errorf("the unsigned×unsigned contradiction must be reported with both statuses: %v", got) + } + // Every message must locate the fault, or an operator cannot tell which two rows to reconcile. + for _, m := range got { + if !strings.Contains(m, `firing key "族长"`) || strings.Count(m, "[ch ") != 2 { + t.Errorf("a conflict must name its firing key and both windows: %q", m) } } - // The approved-only checks are blind to it — which is why this diagnostic exists at all. + // approved×approved belongs to the fail-loud check; reporting it here too would state one fault twice. + both := []store.GlossaryEntry{ + {Src: "族長", Dst: "глава клана", Status: "approved"}, + {Src: "族長", Dst: "старейшина", Status: "approved", Sense: "elder"}, + } + if got := UnverifiedKeyConflicts(both); len(got) != 0 { + t.Errorf("approved×approved belongs to the fail-loud check, not here: %v", got) + } + if cols := ApprovedSharedKeyCollisions(both); len(cols) != 1 { + t.Errorf("control: the fail-loud check must be the one that sees it, got %d", len(cols)) + } + // The approved-only checks are blind to the original fixture. if cols := ApprovedSharedKeyCollisions(rows); len(cols) != 0 { t.Fatalf("premise broken: the approved-only check should see nothing here, got %v", cols) } @@ -503,6 +523,23 @@ func TestUnverifiedKeyConflicts(t *testing.T) { if got := UnverifiedKeyConflicts(windowed); len(got) != 0 { t.Fatalf("disjoint windows are not a contradiction, got %v", got) } + // CONTROL: the same two rows with OVERLAPPING windows are reported, so the zero above is the window + // test answering and not a fixture that reaches nothing. + windowed[1].SinceCh = 3 + if got := UnverifiedKeyConflicts(windowed); len(got) != 1 { + t.Fatalf("control: overlapping windows must be reported, got %d: %v", len(got), got) + } + // A pair sharing TWO firing keys is one contradiction, reported once under the first key. + twice := []store.GlossaryEntry{ + {Src: "赵甲", Dst: "Чжао Цзя", Status: "draft", Aliases: []store.GlossaryAlias{alias("老赵"), alias("赵老")}}, + {Src: "赵乙", Dst: "Чжао И", Status: "auto", Aliases: []store.GlossaryAlias{alias("老赵"), alias("赵老")}}, + } + switch n := len(UnverifiedKeyConflicts(twice)); { + case n == 0: + t.Error("a pair sharing two firing keys must still be reported") + case n != 1: + t.Errorf("a pair sharing two keys is one fault, not %d", n) + } } // TestApprovedSharedKeyCollisionSeesAllowShort closes a mutation that SURVIVED the cold-run session's @@ -525,3 +562,120 @@ func TestApprovedSharedKeyCollisionSeesAllowShort(t *testing.T) { t.Fatalf("a single-key-BANNED shared src is inert and must not fail the load, got: %v", got) } } + +// TestPolysemyGuardIsStatusBlind pins three facts together: the D16.1 refusal fires for every status, an +// unsigned row really is injected beside the approved one (D39.104 п.2 — no fork by status on the wire), +// and the remedies the message names therefore do not include a status. Break any one and the message +// starts advising an operator to do something that cannot work. +func TestPolysemyGuardIsStatusBlind(t *testing.T) { + seedFor := func(status string) string { + return "terms:\n - { src: 神通, dst: сила, sense: power }\n - { src: 神通, dst: чудо, sense: miracle, status: " + status + " }\n" + } + for _, status := range []string{"approved", "draft", "auto"} { + _, err := LoadGlossarySeed(writeSeed(t, seedFor(status))) + if err == nil { + t.Fatalf("status %q: the polysemy livelock must be refused whatever the second row's signature", status) + } + if strings.Contains(err.Error(), "status: auto") || strings.Contains(err.Error(), "mark one status") { + t.Errorf("status %q: the refusal must not offer a STATUS as a remedy — the guard ignores it and so does the wire: %s", status, err) + } + } + // CONTROL for the loop above: the same two senses with NON-overlapping windows are a legitimate + // spoiler handoff and load, so the refusals above are the guard speaking and not a broken fixture. + if _, err := LoadGlossarySeed(writeSeed(t, "terms:\n - { src: 神通, dst: сила, sense: power, until_ch: 5 }\n - { src: 神通, dst: чудо, sense: miracle, since_ch: 6, status: auto }\n")); err != nil { + t.Fatalf("control: non-overlapping windows must still load: %v", err) + } + // WHY a status cannot be the remedy: an unsigned row is injected as law beside the approved one, so + // the model is handed both renderings exactly as it would be for two approved rows. + for _, status := range []string{"approved", "draft", "auto"} { + b := bankFrom([]store.GlossaryEntry{gl("神通", "сила", "power", "approved"), gl("神通", "чудо", "miracle", status)}) + if n := len(b.Select("他用神通打败了对手。", 1, nil, 0).Injected); n != 2 { + t.Errorf("status %q: both renderings must reach the wire (D39.104 п.2 — no fork by status), got %d injected", status, n) + } + } + // CONTROL for that loop: with the src absent from the chunk nothing fires, so the count of 2 above + // is the matcher answering and not a bank that injects unconditionally. + if n := len(bankFrom([]store.GlossaryEntry{gl("神通", "сила", "power", "approved")}).Select("他走了。", 1, nil, 0).Injected); n != 0 { + t.Errorf("control: a src absent from the chunk must inject nothing, got %d", n) + } +} + +// TestConsolidationKeyConflicts covers the four states the check has to tell apart: a disagreement, a +// legitimate agreement, a legitimate spoiler handoff, and a REPLACEMENT that only looks like a conflict. +// Neither side is asked for a signature. +func TestConsolidationKeyConflicts(t *testing.T) { + bank := []store.GlossaryEntry{ + {Src: "元始", Dst: "юаньши", Status: "draft", Source: "seed", SinceCh: 3, UntilCh: 9}, + {Src: "神通", Dst: "сила", Status: "approved", Source: "seed", UntilCh: 20}, + {Src: "赵甲", Dst: "Чжао Цзя", Status: "auto", Source: "mined", Aliases: []store.GlossaryAlias{alias("老赵")}}, + } + // A rendering that disagrees with an UNSIGNED bank row in an overlapping window. + got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "камень первоисточника"}}, bank) + if len(got) != 1 { + t.Fatalf("an unsigned row is owed one rendering too: got %d %v", len(got), got) + } + for _, want := range []string{`firing key "元始"`, "камень первоисточника", "юаньши", "unsigned draft", "[ch 3..9]", "[ch 1..end]"} { + if !strings.Contains(got[0].Message(), want) { + t.Errorf("the message must carry %q so the operator can reconcile the two rows: %q", want, got[0].Message()) + } + } + // The sheet prints only the existing row and the log prints the whole sentence, so the sheet's half + // must be a literal part of the log's — otherwise one fault reads as two facts on two surfaces. + if !strings.Contains(got[0].Message(), got[0].BankRowLabel()) { + t.Errorf("the sheet label must be the log sentence's own words: label=%q message=%q", got[0].BankRowLabel(), got[0].Message()) + } + if got[0].Src != "元始" || got[0].BankDst != "юаньши" || got[0].Key != "元始" { + t.Errorf("the parts must identify both rows for a caller that formats its own line: %+v", got[0]) + } + // The SIGNED side is not a special case. Neither of the collapse stage's own checks reports it (both + // skip the equal-source pair; CanonConflicts is approved-only). The windows differ, so both rows land. + if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "神通", Dst: "чудо"}}, bank); len(got) != 1 { + t.Errorf("a consolidation contradicting a SIGNED row of the same surface must be reported: %v", got) + } + // Reached through a shared ALIAS rather than the src — the key is the firing surface, not the source. + if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "赵乙", Dst: "Чжао И", + Aliases: []store.GlossaryAlias{alias("老赵")}}}, bank); len(got) != 1 { + t.Errorf("a contradiction reached through a shared alias must be reported: %v", got) + } + // AGREEMENT: the same rendering is the bank confirming itself, not a second rendering. + if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "юаньши"}}, bank); len(got) != 0 { + t.Errorf("an identical rendering is not a contradiction: %v", got) + } + // SPOILER HANDOFF: disjoint windows never inject together, so they are not two renderings at once. + if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "камень первоисточника", + SinceCh: 10}}, bank); len(got) != 0 { + t.Errorf("disjoint windows are a legitimate handoff: %v", got) + } + // REPLACEMENT: the proposal carries the bank row's whole UNIQUE tuple, so only one of the two exists. + if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "камень первоисточника", + SinceCh: 3, UntilCh: 9}}, bank); len(got) != 0 { + t.Errorf("a proposal on the row's own UNIQUE key replaces it and cannot contradict it: %v", got) + } + // A proposal with no rendering is a term the role declined; it becomes no row and contradicts nothing. + if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: ""}}, bank); len(got) != 0 { + t.Errorf("an unrendered proposal is not a contradiction: %v", got) + } + // CONTROL for the four zeros: the same bank and surfaces DO report when the rendering really differs + // in an overlapping window. + if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "первокамень", SinceCh: 5}}, bank); len(got) != 1 { + t.Fatalf("control: an overlapping window with a different rendering must report, got %d: %v", len(got), got) + } +} + +// TestConsolidationTupleSkipMirrorsTheStoreKey pins the skip against the store's ACTUAL uniqueness rule. +// The glossary UNIQUE key is raw (migrate.go), so 族長 and 族长 with one sense and window are two rows the +// store accepts and the matcher fires both on — a real contradiction. Folding the sources here would skip +// it, and the caller is what guarantees the proposal already carries the source its row will have. +func TestConsolidationTupleSkipMirrorsTheStoreKey(t *testing.T) { + bank := []store.GlossaryEntry{{Src: "族長", Dst: "глава клана", Status: "draft", Source: "seed"}} + // Different raw src, same sense and window: both rows can exist, so this is a contradiction. + folded := []store.GlossaryEntry{{Src: "族长", Dst: "старейшина"}} + if got := ConsolidationKeyConflicts(folded, bank); len(got) != 1 { + t.Fatalf("族长 against a stored 族長 is a pair the store admits, not a replacement: %v", got) + } + // Byte-identical raw src on the same tuple: the store admits one row, so it is a replacement. + same := []store.GlossaryEntry{{Src: "族長", Dst: "старейшина"}} + if got := ConsolidationKeyConflicts(same, bank); len(got) != 0 { + t.Errorf("an identical raw src on the same tuple is a replacement: %v", got) + } +} diff --git a/backend/internal/pipeline/bankconflict_corpus_test.go b/backend/internal/pipeline/bankconflict_corpus_test.go new file mode 100644 index 00000000..cb04308c --- /dev/null +++ b/backend/internal/pipeline/bankconflict_corpus_test.go @@ -0,0 +1,222 @@ +package pipeline + +import ( + "database/sql" + "fmt" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + _ "modernc.org/sqlite" + + "textmachine/backend/internal/membank" + "textmachine/backend/internal/store" +) + +// bankconflict_corpus_test.go: an opt-in corpus instrument, not a unit test. It measures how often the +// bank contradiction membank.UnverifiedKeyConflicts reports actually occurs on real books, because the +// open decision (report it, or refuse the run) turns on the rate rather than the shape. Run with +// `TM_CORPUS=/books go test ./internal/pipeline/ -run TestCorpusBankKeyConflicts -v`; skipped +// otherwise, so a bare clone stays green. +// +// Read-only by the connection, not by intent: every handle carries query_only(1), so a stray write fails +// loudly instead of touching a store another run owns. It goes to SQL rather than store.OpenReadOnly +// because that door refuses a schema this binary does not match, and older project files are part of the +// population worth counting. +// +// It prints its population beside its answer: "0 books" and "0 books that hold an unsigned row at all" +// are the same output and opposite facts. The rows read, the status histogram, and a planted +// contradiction on each book it just called clean are what separate them. +func TestCorpusBankKeyConflicts(t *testing.T) { + root := os.Getenv("TM_CORPUS") + if root == "" { + t.Skip("corpus instrument: set TM_CORPUS to the books tree") + } + var dbs []string + _ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error { + if err == nil && !fi.IsDir() && strings.HasSuffix(p, ".db") { + dbs = append(dbs, p) + } + return nil + }) + sort.Strings(dbs) + if len(dbs) == 0 { + t.Fatalf("TM_CORPUS=%s holds no .db file: a walk that found nothing reports the same zeros as a clean corpus", root) + } + + var withGlossary, noGlossary, failed int + var booksWithRows, totalRows int + var hitAny, hitOldOnly, hitUnsignedPair, spikeFired, booksWithUnsigned int + statuses := map[string]int{} + var lines []string + + for _, p := range dbs { + byBook, order, err := readGlossary(p) + if err != nil { + if strings.Contains(err.Error(), "no such table") { + noGlossary++ + } else { + failed++ + } + continue + } + withGlossary++ + for _, bid := range order { + g := byBook[bid] + booksWithRows++ + totalRows += len(g) + unsigned := 0 + for _, e := range g { + statuses[e.Status]++ + if e.Status != "approved" { + unsigned++ + } + } + if unsigned > 0 { + booksWithUnsigned++ + } + all := membank.UnverifiedKeyConflicts(g) + // A pair is "old rule" when EITHER side is approved. The label appears on both sides of the + // message, and row order decides which comes first, so testing only the first would + // misclassify every pair whose unsigned row sorts ahead of the approved one. + old, pair := 0, 0 + for _, m := range all { + if strings.Contains(m, `: approved "`) || strings.Contains(m, `vs approved "`) { + old++ + } else { + pair++ + } + } + switch { + case pair > 0: + hitAny++ + hitUnsignedPair++ + lines = append(lines, fmt.Sprintf("%s [%s] rows=%d: %d conflicts, %d unsigned×unsigned | %s", + filepath.Base(p), bid, len(g), len(all), pair, trimTo(all[0], 190))) + case old > 0: + hitAny++ + hitOldOnly++ + } + // The control that makes a zero readable: plant ONE contradicting unsigned row on a surface this + // book really has, and confirm the guard reports it. Every row is tried in turn, because not + // every row can carry a plant — an empty dst, or a src below the single-key floor without + // allow_short, produces no firing key, and stopping at the first row would report the + // instrument as broken on a correctly measured book. + for _, seed := range g { + planted := append(append([]store.GlossaryEntry(nil), g...), store.GlossaryEntry{ + Src: seed.Src, Dst: seed.Dst + "-ПОДСАДКА", Status: "draft", Sense: seed.Sense, + SinceCh: seed.SinceCh, UntilCh: seed.UntilCh + 1000, AllowShort: seed.AllowShort, + }) + if len(membank.UnverifiedKeyConflicts(planted)) > len(all) { + spikeFired++ + break + } + } + } + } + + fmt.Printf("\nCORPUS root=%s\n", root) + fmt.Printf(" .db walked=%d with a glossary table=%d without one=%d read-failed=%d\n", len(dbs), withGlossary, noGlossary, failed) + fmt.Printf(" books with a glossary=%d rows read=%d\n", booksWithRows, totalRows) + fmt.Printf(" POPULATION: books holding at least one UNSIGNED row=%d; status histogram:", booksWithUnsigned) + for _, k := range []string{"approved", "draft", "auto"} { + fmt.Printf(" %s=%d", k, statuses[k]) + } + fmt.Println() + fmt.Printf(" books with ANY conflict=%d; only approved×unsigned=%d; carrying an UNSIGNED×UNSIGNED pair=%d\n", hitAny, hitOldOnly, hitUnsignedPair) + fmt.Printf(" CONTROL: a planted contradicting unsigned row is reported on %d of %d books\n", spikeFired, booksWithRows) + for _, l := range lines { + fmt.Printf(" + %s\n", l) + } + if booksWithRows > 0 && spikeFired != booksWithRows { + t.Fatalf("the instrument is not measuring: a planted contradiction went unreported on %d of %d books", booksWithRows-spikeFired, booksWithRows) + } +} + +// readGlossary reads one project file's whole glossary, aliases attached, over a query_only connection. +func readGlossary(path string) (map[string][]store.GlossaryEntry, []string, error) { + v := url.Values{} + v.Add("_pragma", "busy_timeout(3000)") + v.Add("_pragma", "query_only(1)") + db, err := sql.Open("sqlite", "file:"+path+"?"+v.Encode()) + if err != nil { + return nil, nil, err + } + defer db.Close() + byBook, order, at, err := readTerms(db) + if err != nil { + return nil, nil, err + } + // Aliases are firing keys too: without them this UNDER-counts rather than reads zero, which is the + // difference between "no contradiction" and "did not look at half the keys". An older file with no + // alias table still answers on src keys, so its absence degrades rather than fails. + if err := attachAliases(db, byBook, at); err != nil && !strings.Contains(err.Error(), "no such table") { + return nil, nil, err + } + return byBook, order, nil +} + +// termLoc is where one glossary row landed, so an alias row can be attached to it by term_id. +type termLoc struct { + book string + i int +} + +func readTerms(db *sql.DB) (map[string][]store.GlossaryEntry, []string, map[int64]termLoc, error) { + rows, err := db.Query(`SELECT id, book_id, src, dst, sense, since_ch, until_ch, status, allow_short, source FROM glossary ORDER BY book_id, src, sense, since_ch, until_ch, status, dst`) + if err != nil { + return nil, nil, nil, err + } + defer rows.Close() + byBook := map[string][]store.GlossaryEntry{} + at := map[int64]termLoc{} + var order []string + for rows.Next() { + var id int64 + var e store.GlossaryEntry + var short int + if err := rows.Scan(&id, &e.BookID, &e.Src, &e.Dst, &e.Sense, &e.SinceCh, &e.UntilCh, &e.Status, &short, &e.Source); err != nil { + return nil, nil, nil, err + } + e.AllowShort = short != 0 + if _, ok := byBook[e.BookID]; !ok { + order = append(order, e.BookID) + } + byBook[e.BookID] = append(byBook[e.BookID], e) + at[id] = termLoc{e.BookID, len(byBook[e.BookID]) - 1} + } + // A truncated iteration is the exact failure this instrument exists to avoid: it would report a + // smaller bank as a clean one, and the two are indistinguishable in the output. + return byBook, order, at, rows.Err() +} + +func attachAliases(db *sql.DB, byBook map[string][]store.GlossaryEntry, at map[int64]termLoc) error { + rows, err := db.Query(`SELECT term_id, alias, alias_type FROM glossary_aliases`) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var tid int64 + var alias, aliasType string + if err := rows.Scan(&tid, &alias, &aliasType); err != nil { + return err + } + if l, ok := at[tid]; ok { + byBook[l.book][l.i].Aliases = append(byBook[l.book][l.i].Aliases, store.GlossaryAlias{Alias: alias, AliasType: aliasType}) + } + } + return rows.Err() +} + +// trimTo cuts on runes: the messages are Han and Cyrillic, and a byte cut would split a character. +func trimTo(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/backend/internal/pipeline/bankfixpack_test.go b/backend/internal/pipeline/bankfixpack_test.go index 281f6e3f..09659a07 100644 --- a/backend/internal/pipeline/bankfixpack_test.go +++ b/backend/internal/pipeline/bankfixpack_test.go @@ -10,7 +10,9 @@ import ( "testing" "textmachine/backend/internal/config" + "textmachine/backend/internal/membank" "textmachine/backend/internal/miner" + "textmachine/backend/internal/store" "textmachine/backend/internal/terminology" "textmachine/backend/internal/text" ) @@ -397,3 +399,189 @@ func TestEvictedBankRowsAreNamed(t *testing.T) { t.Fatalf("the evicted rows must be NAMED, not counted:\n%s", out) } } + +// TestConsolidationAgainstTheBankIsReportedAtTheStop is the bank-side disagreement end to end: the seed +// already renders 方源 one way (UNSIGNED — the point of the case) and the run consolidates the same +// surface differently. Neither of the collapse stage's own checks sees this: both test containment and +// skip the equal-source pair, and CanonConflicts is approved-only. +func TestConsolidationAgainstTheBankIsReportedAtTheStop(t *testing.T) { + var logBuf bytes.Buffer + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isTerminologyBody(body) { + return "方源\tФан Юань", "stop" // the seed calls the same surface «Странник» + } + return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop" + }) + defer srv.Close() + // until_ch keeps the seed row off the proposal's UNIQUE key, so this is a disagreement and not a + // replacement. + seed := "terms:\n - { src: 方源, dst: Странник, status: draft, until_ch: 20 }\n" + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, glossarySeed: seed})) + defer r.Close() + r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) + _ = runToSignatureStop(t, r) + + if r.lastTerminology.BankConflicts == 0 { + t.Fatalf("a rendering that disagrees with an UNSIGNED row of the same surface must be counted: %+v", r.lastTerminology) + } + log := logBuf.String() + if !strings.Contains(log, "DISAGREES with a row the bank already holds") { + t.Fatalf("the run must say it out loud:\n%s", log) + } + for _, want := range []string{"方源", "Фан Юань", "Странник", "[ch 1..20]"} { + if !strings.Contains(log, want) { + t.Errorf("the warning must name %q, or an operator cannot tell which two rows collided:\n%s", want, log) + } + } + // The finding has to reach the signature sheet, not only the log: that sheet is where a term is + // decided. It must also stay DISTINCT from the §G2 self-contradiction marker — "the run disagreed with + // itself" and "the book already calls it something else" are different decisions. + var marked *BankStopRow + for i, row := range r.lastBankStopRows { + if row.Src == "方源" { + marked = &r.lastBankStopRows[i] + } + } + if marked == nil { + t.Fatalf("premise broken: 方源 is not in the stop table at all: %+v", r.lastBankStopRows) + } + if len(marked.BankHolds) == 0 { + t.Fatalf("the contradicting row must be marked on the signature sheet: %+v", *marked) + } + if len(marked.Contradicts) != 0 { + t.Errorf("this is not a self-contradiction and must not be filed as one: %+v", marked.Contradicts) + } + if !strings.Contains(marked.BankHolds[0], `unsigned draft "方源"→"Странник"`) || !strings.Contains(marked.BankHolds[0], "[ch 1..20]") { + t.Errorf("the mark must name the EXISTING row, its signature and its window: %q", marked.BankHolds[0]) + } + table := renderBankStopTable(r.lastBankStopRows) + if !strings.Contains(table, "THE BANK ALREADY HOLDS: unsigned draft") { + t.Errorf("the rendered table must carry the mark:\n%s", table) + } + if strings.Contains(table, "CONTRADICTS this run's own") { + t.Errorf("the two markers must stay distinguishable on the sheet:\n%s", table) + } + // And nowhere else: the signature map's bytes become bank rows, so a diagnostic leaking into it would + // turn an observation into content. + sigMap := readSignatureMap(t, r) + for _, forbidden := range []string{"THE BANK ALREADY HOLDS", "Странник", "[ch 1..20]"} { + if strings.Contains(sigMap, forbidden) { + t.Errorf("the diagnostic must not enter the signature map (%q):\n%s", forbidden, sigMap) + } + } + // CONTROL for those three zeros. 方源 is deliberately not the control: it is a seed surface, so the + // emission skips it and it never reaches the map — which is why the stop TABLE (every candidate), + // not the map (signable rows only), is the right address for this mark. + if !strings.Contains(sigMap, "青茅山") || !strings.Contains(sigMap, "terms:") { + t.Fatalf("control: the signature map must hold this run's emittable terms, or the checks above assert nothing:\n%s", sigMap) + } + + // CONTROL: with the seed AGREEING with what the role consolidates, nothing is reported. + var quiet bytes.Buffer + srv2 := newJSONProvider(&reqRec{}, func(body string) (string, string) { + if isTerminologyBody(body) { + return "方源\tСтранник", "stop" + } + return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop" + }) + defer srv2.Close() + r2 := newVerifyRunner(t, setupMiningStopProject(t, srv2.URL, miningStopOpts{terminology: true, glossarySeed: seed})) + defer r2.Close() + r2.Log = slog.New(slog.NewTextHandler(&quiet, &slog.HandlerOptions{Level: slog.LevelWarn})) + _ = runToSignatureStop(t, r2) + if r2.lastTerminology.BankConflicts != 0 { + t.Errorf("control: agreeing with the bank is not a contradiction, got %d", r2.lastTerminology.BankConflicts) + } + if strings.Contains(renderBankStopTable(r2.lastBankStopRows), "THE BANK ALREADY HOLDS") { + t.Errorf("control: an agreeing run must leave the sheet unmarked:\n%s", renderBankStopTable(r2.lastBankStopRows)) + } +} + +// TestConsolidatedRowsCarryTheWindowAndTheAliases pins the two fields that decide what +// membank.ConsolidationKeyConflicts may say. Dropping either leaves every existing test green while +// changing the verdict: without SinceCh a re-mined engine row looks like a fresh contradiction instead of +// a replacement, and without the aliases an alias-only collision is invisible. +func TestConsolidatedRowsCarryTheWindowAndTheAliases(t *testing.T) { + // A banknote candidate whose raw surface differs from its normalized key, so the row's source is not + // ambiguous between the two: the row that lands carries the KEY, and the conflict check compares the + // store's raw UNIQUE key against it. + cands := []terminology.Candidate{ + {Key: "赵甲", Src: "趙甲", SinceCh: 7, Aliases: []string{"老赵"}}, + {Key: "神通", Src: "神通"}, // unanswered → no row + } + rows := consolidatedRows(cands, map[string]string{"赵甲": "Чжао Цзя"}) + if len(rows) != 1 { + t.Fatalf("only an answered candidate becomes a row: %+v", rows) + } + if rows[0].Src != "赵甲" { + t.Errorf("the row must carry the source it will LAND with (the key), got %q", rows[0].Src) + } + if rows[0].SinceCh != 7 { + t.Errorf("the candidate's window must reach the check: since_ch=%d, want 7", rows[0].SinceCh) + } + if len(rows[0].Aliases) != 1 || rows[0].Aliases[0].Alias != "老赵" { + t.Errorf("the identity cluster must reach the check as alias keys: %+v", rows[0].Aliases) + } + + // Both fields change the verdict of the real check, so neither is decoration. + bank := []store.GlossaryEntry{ + {Src: "赵甲", Dst: "Чжао Цзя-старый", Status: "auto", Source: "mined", SinceCh: 7}, + {Src: "赵乙", Dst: "Чжао И", Status: "draft", Aliases: []store.GlossaryAlias{{Alias: "老赵"}}}, + } + got := membank.ConsolidationKeyConflicts(rows, bank) + if len(got) != 1 || got[0].BankSrc != "赵乙" { + t.Fatalf("with the window carried, the same-tuple row is a replacement and only the alias hit is a conflict: %+v", got) + } + windowless := []store.GlossaryEntry{{Src: rows[0].Src, Dst: rows[0].Dst, Aliases: rows[0].Aliases}} + if n := len(membank.ConsolidationKeyConflicts(windowless, bank)); n != 2 { + t.Errorf("control: without the window the replacement is reported as a contradiction too, want 2 got %d", n) + } + aliasless := []store.GlossaryEntry{{Src: rows[0].Src, Dst: rows[0].Dst, SinceCh: rows[0].SinceCh}} + if n := len(membank.ConsolidationKeyConflicts(aliasless, bank)); n != 0 { + t.Errorf("control: without the aliases the alias-only collision is invisible, want 0 got %d", n) + } + + // The findings have to survive the trip to the sheet. They are keyed by the candidate KEY on both + // sides; keying either side by the raw surface loses every banknote candidate whose draft spelling + // differs from its key — which is the whole population the key choice above exists for. + // Straight through the production path: the findings go to the sheet ungrouped, so the key they are + // matched on is decided in one place. The candidate is chosen so that the four strings that could + // plausibly match it — candidate key, raw surface, firing key (an alias here) and the bank row's own + // source — are all different. + res := terminologyResult{BankHoldRows: got} + sheet := bankStopRows(cands, map[string]string{"赵甲": "Чжао Цзя"}, res) + if len(sheet) == 0 || sheet[0].Src != "趙甲" { + t.Fatalf("premise broken: the sheet row must be the raw-surface candidate: %+v", sheet) + } + if len(sheet[0].BankHolds) != 1 { + t.Fatalf("the mark must reach the sheet for a candidate whose key differs from its surface: %+v", sheet[0]) + } +} + +// TestApprovedNeighboursAdmitsOnlySignedRows pins the anchor's own rule, which until now lived only in a +// comment. The neighbours are what the CANON block shows a paid role, so an unsigned row leaking in +// changes the bytes that reach the model: the run would be told the book has already decided a term it +// has only proposed. Both halves of the filter matter — a signed row with no rendering anchors nothing. +func TestApprovedNeighboursAdmitsOnlySignedRows(t *testing.T) { + rows := []store.GlossaryEntry{ + {Src: "方源", Dst: "Фан Юань", Status: "approved"}, + {Src: "花家", Dst: "клан Хуа", Status: "draft"}, + {Src: "青茅山", Dst: "гора Цинмао", Status: "auto"}, + {Src: "元始", Dst: "", Status: "approved"}, // signed but unrendered: nothing to agree with + } + got := approvedNeighbours(rows) + if len(got) != 1 || got[0].Src != "方源" || got[0].Dst != "Фан Юань" { + t.Fatalf("only a signed row WITH a rendering may anchor the role: %+v", got) + } + // CONTROL: the same four rows all signed and rendered give four, so the one above is the filter + // answering and not a fixture the function cannot read. + all := make([]store.GlossaryEntry, len(rows)) + for i, e := range rows { + e.Status, e.Dst = "approved", "д" + all[i] = e + } + if n := len(approvedNeighbours(all)); n != len(rows) { + t.Fatalf("control: every signed rendered row must anchor, want %d got %d", len(rows), n) + } +} diff --git a/backend/internal/pipeline/disposition.go b/backend/internal/pipeline/disposition.go index c980ade3..cb213171 100644 --- a/backend/internal/pipeline/disposition.go +++ b/backend/internal/pipeline/disposition.go @@ -89,7 +89,9 @@ const ( // FlagGlossaryMiss is the memory-bank post-check verdict (E1, step 4): an approved // term's src fired in the chunk but no accepted dst form appears in the output — the - // model ignored the glossary, or we injected the wrong dst and it obeyed. Emitted + // model did not render the term the way the bank told it to. It says nothing about + // whether the dst we injected was RIGHT: a wrong rendering the model OBEYED is present + // in the output, so it raises no miss and no flag (mempostcheck.go). Emitted // ONLY when the opt-in glossary post-check gate is enabled (config); in the default // flagger mode a miss is recorded in the retrieval-state, not a disposition. NOT // retryable (a same-model retry re-produces the same rendering) and NOT auto- diff --git a/backend/internal/pipeline/export.go b/backend/internal/pipeline/export.go index b231c5f5..f8d0e1b6 100644 --- a/backend/internal/pipeline/export.go +++ b/backend/internal/pipeline/export.go @@ -111,8 +111,10 @@ type BookExport struct { // that do not ship are not consulted, so a $0 draft-wave re-write moves nothing here. A fact of the // store rather than of any process's clock, which is why the book writer uses it as the file's // modification time (two builds of one store then agree byte-for-byte). "" when no unit has a final - // row (a book that has shipped nothing). Additive and omitempty. ⚠ «Last written», not «last - // changed»: a resume that re-writes an unchanged final row moves it too (store.ChunkStatus.UpdatedAt). + // row (a book that has shipped nothing). Additive and omitempty. ⚠ «Last written», not «last changed»: + // a unit whose final row is SKIPPED is re-written by every run that passes it, so a book with a flagged + // unit reports a new TextModified after a $0 resume that bought nothing. A cleanly shipped book does + // not (store.ChunkStatus.UpdatedAt). TextModified string `json:"text_modified,omitempty"` // Chunks is always non-nil (an empty book exports [] not null). Chunks []ChunkExport `json:"chunks"` diff --git a/backend/internal/pipeline/mining.go b/backend/internal/pipeline/mining.go index 7583b886..f203b591 100644 --- a/backend/internal/pipeline/mining.go +++ b/backend/internal/pipeline/mining.go @@ -395,12 +395,16 @@ type BankStopRow struct { // class to read first. Conf is the role's own stated confidence, which orders the review list and // nothing else (D39.102) — NEGATIVE when the reply carried none, because «the role said it was 0% sure» // is the most important row on the sheet and «the role said nothing» is not a row at all. - // Contradicts names THIS RUN's other consolidations the rendering breaks (§G2). + // Contradicts names THIS RUN's other consolidations the rendering breaks (§G2). BankHolds names rows + // the bank ALREADY carries for the same firing surface with a different rendering. Separate field and + // separate marker: "the run disagreed with itself" and "the book already calls it something else" are + // different decisions, and one shared line would make the sheet shorter but the decision harder. Conventions int Signals []string Invented bool Conf int Contradicts []string + BankHolds []string } // BankStopVariant is ONE rendering the drafts produced, kept in its PARTS rather than as the sentence a @@ -445,6 +449,7 @@ func bankStopRows(cands []terminology.Candidate, consolidated map[string]string, Freq: c.Freq, Spread: c.Spread(), Conventions: c.Conventions(), Contexts: c.KWIC, Evidence: c.Evidence, Conf: confOrAbsent(tres.Conf, c.Key), Contradicts: tres.Contradictions[c.Src], + BankHolds: bankHoldLabels(tres.BankHoldRows, c), } for i, v := range c.Variants { row.Variants = append(row.Variants, BankStopVariant{Dst: v.Dst, Chunks: v.Chunks, Via: v.Via}) @@ -458,6 +463,19 @@ func bankStopRows(cands []terminology.Candidate, consolidated map[string]string, return out } +// bankHoldLabels picks the findings belonging to one candidate. They are matched on the candidate KEY, +// because that is the source consolidatedRows gives the proposal and therefore the conflict's own Src; a +// banknote candidate's raw surface is a different string, and its firing key may be an alias. +func bankHoldLabels(cols []membank.BankKeyConflict, c terminology.Candidate) []string { + var out []string + for _, cf := range cols { + if cf.Src == c.Key { + out = append(out, cf.BankRowLabel()) + } + } + return out +} + // confOrAbsent reads the role's stated confidence for a key, or -1 when the reply carried none. A plain // zero would merge the two, and they are opposites: one is the first row to review, the other is silence. func confOrAbsent(conf map[string]int, key string) int { @@ -486,7 +504,8 @@ func renderBankStopTable(rows []BankStopRow) string { var b strings.Builder fmt.Fprintf(&b, "BANK VERIFICATION TABLE — %d term(s)\n", len(rows)) b.WriteString("src · proposed dst · origin · type · freq · variant spread · conventions · confidence ·\n") - b.WriteString("why (the ranking factors that won) · contradictions · drafts · evidence · source contexts\n\n") + b.WriteString("why (the ranking factors that won) · contradictions (this run's own, and the bank's\n") + b.WriteString("existing rows) · drafts · evidence · source contexts\n\n") for _, r := range rows { fmt.Fprintf(&b, "%s\t%s\n", r.Src, dashIfEmpty(r.Dst)) fmt.Fprintf(&b, " origin=%s type=%s freq=%d spread=%d conventions=%d", r.Origin, dashIfEmpty(r.Type), r.Freq, r.Spread, r.Conventions) @@ -503,6 +522,9 @@ func renderBankStopTable(rows []BankStopRow) string { if len(r.Contradicts) > 0 { fmt.Fprintf(&b, " CONTRADICTS this run's own: %s\n", strings.Join(r.Contradicts, "; ")) } + if len(r.BankHolds) > 0 { + fmt.Fprintf(&b, " THE BANK ALREADY HOLDS: %s\n", strings.Join(r.BankHolds, "; ")) + } if len(r.Variants) > 0 { fmt.Fprintf(&b, " drafts: %s\n", strings.Join(r.VariantLabels(), " | ")) } diff --git a/backend/internal/pipeline/miningstop_join_test.go b/backend/internal/pipeline/miningstop_join_test.go index b207de08..a9d490d9 100644 --- a/backend/internal/pipeline/miningstop_join_test.go +++ b/backend/internal/pipeline/miningstop_join_test.go @@ -1511,6 +1511,9 @@ func TestAutoBankKeyCollisionDoesNotCrashTheRun(t *testing.T) { // approved term AND an unsigned proposal contradicting it on the same firing surface must say so out // loud. The check is a WARNING by design — the unsigned row is engine-produced, and aborting a paid run // over the engine's own proposal would be a self-inflicted outage — so the assertion is on the log. +// +// The log line must LOCATE the fault as well as announce it: on a book with a thousand terms, a warning +// without the firing key and both windows does not say which two rows to reconcile. func TestUnsignedConflictIsReportedByTheRun(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) @@ -1530,11 +1533,13 @@ func TestUnsignedConflictIsReportedByTheRun(t *testing.T) { t.Fatal(err) } out := logs.String() - if !strings.Contains(out, "contradicts an approved term") { + if !strings.Contains(out, "render ONE firing surface differently and at least one is unsigned") { t.Fatalf("the approved-vs-unsigned contradiction must be reported (S14):\n%s", out) } - if !strings.Contains(out, "Источник Фана") { - t.Fatalf("the report must name the contradicting rendering:\n%s", out) + for _, want := range []string{`approved \"方源\"→\"Фан Юань\"`, `unsigned draft \"方源\"→\"Источник Фана\"`, `firing key \"方源\"`, "[ch 1..end]", "[ch 2..end]"} { + if !strings.Contains(out, want) { + t.Fatalf("the report must name %s so the operator can reconcile the two rows:\n%s", want, out) + } } } diff --git a/backend/internal/pipeline/resume_updatedat_test.go b/backend/internal/pipeline/resume_updatedat_test.go new file mode 100644 index 00000000..140eced7 --- /dev/null +++ b/backend/internal/pipeline/resume_updatedat_test.go @@ -0,0 +1,93 @@ +package pipeline + +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +// TestOrdinaryResumeMovesOnlyTheSkippedRow pins which rows a $0 resume writes. store.ChunkStatus.UpdatedAt +// is read as the book's modification time, and a resume that changes nothing still writes — but only +// where a stage was SKIPPED: a skip is not a stored verdict a resume can serve, so the flagged upstream +// row is served from its checkpoint and the skip is re-derived and re-upserted (waverun.go — +// recordSkippedStages for a flagged edit unit, the flagged branch of runStageSequence for a multi-stage +// wave). Both halves are asserted: a run where EVERY row moved would be a clock, not a write. +func TestOrdinaryResumeMovesOnlyTheSkippedRow(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) { + return "Извините, я не могу перевести это.", "stop" // soft refusal → the edit stage is skipped + } + + return draftEdit(body) + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА"}) + ctx := context.Background() + + r1 := newRunner(t, bookPath) + if _, err := r1.TranslateBook(ctx); err != nil { + t.Fatal(err) + } + before := statusStamps(t, r1) + callsAfterFirstRun := rec.count() + r1.Close() + + // datetime('now') has one-second resolution: a rewrite inside the same second is indistinguishable from + // no rewrite, and the test would pass for the wrong reason. + time.Sleep(1100 * time.Millisecond) + + r2 := newRunner(t, bookPath) + defer r2.Close() + if _, err := r2.TranslateBook(ctx); err != nil { + t.Fatal(err) + } + after := statusStamps(t, r2) + + if rec.count() != callsAfterFirstRun { + t.Fatalf("premise broken: the resume must buy nothing, calls went %d → %d", callsAfterFirstRun, rec.count()) + } + if len(before) != 4 { + t.Fatalf("premise broken: want 4 chunk_status rows (2 units × 2 stages), got %d: %v", len(before), before) + } + const skipped = "ch1/chunk0/edit" + if before[skipped].disposition != string(DispSkipped) { + t.Fatalf("premise broken: %s must be the skipped row, got %q", skipped, before[skipped].disposition) + } + if after[skipped].updatedAt == before[skipped].updatedAt { + t.Errorf("a SKIPPED row is re-derived and re-written by every run past it: %s stayed at %q", + skipped, before[skipped].updatedAt) + } + for key, b := range before { + if key == skipped { + continue + } + if after[key].updatedAt != b.updatedAt { + t.Errorf("an ordinary resume must not write an %s row: %s moved %q → %q", + b.disposition, key, b.updatedAt, after[key].updatedAt) + } + } +} + +type statusStamp struct{ disposition, updatedAt string } + +// statusStamps maps "chapter/chunk/stage" to each row's disposition and write time. +func statusStamps(t *testing.T, r *Runner) map[string]statusStamp { + t.Helper() + rows, err := r.Store.ChunkStatusesForBook(r.Book.BookID) + if err != nil { + t.Fatal(err) + } + out := make(map[string]statusStamp, len(rows)) + for _, cs := range rows { + key := chunkStageKey(cs.Chapter, cs.ChunkIdx, cs.Stage) + out[key] = statusStamp{disposition: cs.Disposition, updatedAt: cs.UpdatedAt} + } + return out +} + +func chunkStageKey(chapter, chunkIdx int, stage string) string { + return fmt.Sprintf("ch%d/chunk%d/%s", chapter, chunkIdx, stage) +} diff --git a/backend/internal/pipeline/seeding.go b/backend/internal/pipeline/seeding.go index 72db933c..d5e99e75 100644 --- a/backend/internal/pipeline/seeding.go +++ b/backend/internal/pipeline/seeding.go @@ -65,13 +65,12 @@ func (r *Runner) seedGlossary(ctx context.Context) error { r.Log.WarnContext(ctx, "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)", "collisions", strings.Join(cols, "; ")) } - // S14: the diagnostic the approved-only checks structurally cannot make — an approved term and an - // unsigned one contradicting each other on the SAME firing surface. In the auto mode both are injected - // (canon + working version), so the model is handed two answers to one question; the trust gate keeps - // the approved one authoritative, but the collision is exactly the silent-degradation source the auto - // mode needs visible. + // S14: the diagnostic the approved-only checks cannot make — two rows contradicting each other on the + // SAME firing surface, at least one of them unsigned. Both are injected, and which is signed changes + // nothing about that (D39.104 п.2: no fork by status). Where a signature exists the trust gate still + // decides precedence; an unsigned×unsigned pair has none, so it is simply two renderings of one term. if cols := membank.UnverifiedKeyConflicts(rows); len(cols) > 0 { - r.Log.WarnContext(ctx, "bank: an UNSIGNED row contradicts an approved term on the same firing surface (the approved rendering keeps precedence; reconcile the seed or decline the proposal)", + r.Log.WarnContext(ctx, "bank: two rows render ONE firing surface differently and at least one is unsigned (the deterministic matcher cannot pick between them; reconcile the seed, the delta or the proposal)", "book", r.Book.BookID, "conflicts", strings.Join(cols, "; ")) } r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID, diff --git a/backend/internal/pipeline/terminologist.go b/backend/internal/pipeline/terminologist.go index 3b7986d7..4e7c0b14 100644 --- a/backend/internal/pipeline/terminologist.go +++ b/backend/internal/pipeline/terminologist.go @@ -11,6 +11,7 @@ import ( "textmachine/backend/internal/config" "textmachine/backend/internal/lang" "textmachine/backend/internal/llm" + "textmachine/backend/internal/membank" "textmachine/backend/internal/miner" "textmachine/backend/internal/obs" "textmachine/backend/internal/store" @@ -103,6 +104,13 @@ type terminologyResult struct { // the same finding per key, so the stop table can mark the rows instead of printing a bare total. SelfConflicts int Contradictions map[string][]string + // BankConflicts counts consolidated renderings that disagree with a row the bank already holds for the + // same firing surface — the shape neither check above can see (membank.ConsolidationKeyConflicts). + // BankHoldRows carries the findings themselves, ungrouped: bankStopRows matches them to its rows, so + // the key they are looked up by is decided in one place instead of once on each side of a map. Kept + // apart from Contradictions because the two are different decisions for the owner. + BankConflicts int + BankHoldRows []membank.BankKeyConflict // Conf is the role's own stated confidence per key. It sorts the review list «least sure first» and does // nothing else — never a weight, a threshold or a cross-model comparison (D39.102). Conf map[string]int @@ -262,7 +270,7 @@ func (r *Runner) buildBankCandidates(mined []miner.Term, observed []terminology. // the initial candidate build and the §2 post-classify re-score share ONE definition and can never disagree // about how a variant is scored. func (r *Runner) scoreOpts() terminology.ScoreOpts { - opts := terminology.ScoreOpts{Neighbours: r.approvedNeighbours()} + opts := terminology.ScoreOpts{Neighbours: approvedNeighbours(r.glossaryRows())} if r.pack != nil { pack := r.pack opts.Conformance = func(dst, typ string) float64 { @@ -275,13 +283,26 @@ func (r *Runner) scoreOpts() terminology.ScoreOpts { return opts } -// approvedNeighbours is the already-signed bank, as the §C2-3 "agreement with approved siblings" anchor. -// Only APPROVED rows qualify: an unverified row agreeing with an unverified row is not evidence. -func (r *Runner) approvedNeighbours() []terminology.Neighbour { +// glossaryRows is the book's stored bank, every row with its status and window. The anchor below narrows +// it to the signed rows; ConsolidationKeyConflicts needs it whole, since an unsigned row owes one +// rendering too. A read failure degrades rather than aborts — everything built on it improves a +// consolidation, none of it is a precondition — but it is LOGGED: a degraded read and a clean bank +// produce the same zero in every counter downstream. +func (r *Runner) glossaryRows() []store.GlossaryEntry { rows, err := r.Store.GlossaryForBook(r.Book.BookID) if err != nil { - return nil // an anchor is an improvement, never a precondition — a read failure degrades, not aborts + // Called from the scoring pass as well as the role, so it can repeat within one run; a repeated + // line is cheaper than a zero that reads as a clean bank. + r.Log.Warn("terminology: could not read the bank — whatever this call feeds goes silent (the canon anchor, the bank conflict check, or both), and its zero then means «not asked» rather than «nothing found»", + "book", r.Book.BookID, "err", err) + return nil } + return rows +} + +// approvedNeighbours is the already-signed bank, as the §C2-3 "agreement with approved siblings" anchor. +// Only APPROVED rows qualify: an unverified row agreeing with an unverified row is not evidence. +func approvedNeighbours(rows []store.GlossaryEntry) []terminology.Neighbour { out := make([]terminology.Neighbour, 0, len(rows)) for _, e := range rows { if e.Status == "approved" && e.Dst != "" { @@ -361,8 +382,10 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te } } - // The signed bank is read ONCE for the whole role, not per batch: it is the same law for every batch. - canon := r.approvedNeighbours() + // The bank is read ONCE for the whole role, not per batch: it is the same law for every batch. The + // signed rows are the anchor the model is shown; the whole set is what the conflict checks below read. + bank := r.glossaryRows() + canon := approvedNeighbours(bank) plan := bankRolePlan{ role: roleTerminologist, budgetUSD: r.Pipeline.Gates.Terminology.BudgetUSD, messages: func(b []terminology.Candidate) ([]llm.Message, error) { return r.terminologyMessages(b, canon) }, @@ -456,6 +479,15 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te r.Log.WarnContext(ctx, "terminology: consolidations of THIS run contradict each other — a compound's rendering drops the rendering the same reply gave its own part; they stay unverified and are review rows at the stop", "book", r.Book.BookID, "conflicts", len(self), "terms", strings.Join(named, "; ")) } + // The BANK side: a rendering this run consolidated for a surface the bank already renders differently, + // which neither check above reaches. Neither side is asked for a signature — one does not change what + // the model is shown (D39.104 п.2). $0, evidence-side, never a gate. + if cols := membank.ConsolidationKeyConflicts(consolidatedRows(cands, out), bank); len(cols) > 0 { + res.BankConflicts = len(cols) + res.BankHoldRows = cols + r.Log.WarnContext(ctx, "terminology: a consolidated rendering DISAGREES with a row the bank already holds for the same firing surface — the book calls this term something else, and the sheet at the stop is where that is decided (whether both renderings also reach a wire depends on the emission and the glossary UNIQUE key; this does not claim they will)", + "book", r.Book.BookID, "conflicts", len(cols), "terms", strings.Join(membank.ConflictMessages(cols), "; ")) + } // The $0 label screen (§2 warm-run hygiene): a name/place row whose rendering was clearly TRANSLATED is a // label/rendering disagreement worth a human's eye. It is a review FLAG, never a gate, and explicitly not // a safety net for the transliteration harm — the classifier phase is what prevents that (see @@ -477,6 +509,7 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te "classify_batches_dropped", res.ClassifyBatchesDropped) } r.Log.InfoContext(ctx, "terminology finished", "book", r.Book.BookID, + "bank_conflicts", res.BankConflicts, "batches_dropped", res.BatchesDropped, "classify_batches_dropped", res.ClassifyBatchesDropped, "consolidated", res.Consolidated, "declined", res.Declined, "unanswered", res.Unanswered, "reclassified", res.Reclassified, "bad_lines", res.BadLines, "off_language", res.OffLanguage, @@ -870,6 +903,41 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban return run, nil } +// consolidatedRows turns this run's consolidation into the rows those renderings would become, which is +// what ConsolidationKeyConflicts reads: the candidate's surface, its identity-cluster aliases and the +// window a mined row carries (first chapter of appearance, no end). An unanswered candidate carries no +// rendering, becomes no row, and contradicts nothing. +// +// Both extra fields are load-bearing for the check: the window decides whether an existing row is a +// replacement rather than a disagreement, and the aliases are the other surfaces a landed row fires on. +// Built for every answered candidate, not only those the emission keeps — a dropped candidate still +// reaches the review sheet (not the signature map, a different document), so its disagreement is read. +func consolidatedRows(cands []terminology.Candidate, consolidated map[string]string) []store.GlossaryEntry { + if len(consolidated) == 0 { + return nil + } + out := make([]store.GlossaryEntry, 0, len(consolidated)) + for _, c := range cands { + dst := consolidated[c.Key] + if dst == "" { + continue + } + // c.Key, not c.Src, and it is the landing source on BOTH emission paths — the check compares the + // store's raw UNIQUE key, so it must be given the string the row will actually hold. A banknote + // candidate carries the surface as the draft wrote it and lands as the key + // (mining.reverseSectionTerms). A mined candidate lands with the MINER's surface, which is a + // substring of already-normalized chunk text and therefore equals its own key — that equality is + // text.NormalizeSourceKey's idempotence, pinned by TestNormalizeSourceKeyIsIdempotent, because + // without it this line would compare a landing row against a source it does not have. + e := store.GlossaryEntry{Src: c.Key, Dst: dst, SinceCh: c.SinceCh} + for _, a := range c.Aliases { + e.Aliases = append(e.Aliases, store.GlossaryAlias{Alias: a}) + } + out = append(out, e) + } + return out +} + // attachConsolidatedDst stamps the terminologist's renderings onto the mined terms, which is what selects // the emission MODE in miner.DeltaYAML (§C2-7). Terms the role did not answer are left untouched. func attachConsolidatedDst(mined []miner.Term, consolidated map[string]string) []miner.Term { diff --git a/backend/internal/pipeline/waverun.go b/backend/internal/pipeline/waverun.go index 1e00b981..487cd360 100644 --- a/backend/internal/pipeline/waverun.go +++ b/backend/internal/pipeline/waverun.go @@ -691,9 +691,14 @@ func (r *Runner) runEditUnit(ctx context.Context, editSnapshot string, unit edit var editSel membank.Selection injectionByRole := map[string]string{} if r.memory != nil { - // FRESH Select over the WHOLE-unit source over the ENRICHED bank (§1(б)/F3): NOT the the precompute pass per-chunk - // memSel (that was over the base bank). Sticky is degenerate at unit scope (a unit == a chapter or a - // sub-chapter split, so the intra-chapter sticky window does not apply) → nil sticky_prev. + // FRESH Select over the WHOLE-unit source over the ENRICHED bank (§1(б)/F3): NOT the precompute + // pass's per-chunk memSel, which was taken over the BASE bank. + // + // sticky_prev is nil, and that is a LOSS, not a degenerate case: assignEditUnits closes a unit + // INSIDE a chapter (chunk/chunker.go) while the sticky window resets only at a CHAPTER boundary + // (precomputeSticky, wave.go), so every unit after the first in a chapter is selected with no + // memory of the rows its predecessors carried. On the stand book 1962 of its 2283 chapters are + // multi-unit (backlog row 321). Whether the loss produces drift in the text is not measured. editSel = r.memory.Select(leader.Text, unit.Chapter, nil, r.Pipeline.Context.GlossaryTokenBudget) // Render the per-role injection from the ENRICHED unit selection via the registry (D39 layer 7): the // editor gets its CONFIRMED-dst constraint block; other roles' blocks are rendered but consumed diff --git a/backend/internal/store/chunkstatus.go b/backend/internal/store/chunkstatus.go index 47accee6..0f78f128 100644 --- a/backend/internal/store/chunkstatus.go +++ b/backend/internal/store/chunkstatus.go @@ -53,8 +53,19 @@ type ChunkStatus struct { // `YYYY-MM-DD HH:MM:SS`, NOT NULL since the table's first schema — migrate.go). It is read, never // written from here: UpsertChunkStatus stamps it itself. The book writer takes the newest one as the // book's modification time — a fact of the store rather than of the clock of whichever process builds - // the file, which is what keeps two builds of one store byte-identical. A resume that re-writes an - // unchanged row moves it too, so it is «last written», not «last changed». + // the file, which is what keeps two builds of one store byte-identical. + // + // «Last written», not «last changed». Two ordinary $0 operations move it with nothing changed: + // - a SKIPPED row is re-written by every run that passes it. A skip is not a stored verdict a resume + // can serve: the flagged upstream row is served from its checkpoint, and the skip is re-derived + // from it and upserted again — by recordSkippedStages for a flagged edit unit, and by the flagged + // branch of runStageSequence where a wave holds two or more stages (both pipeline/waverun.go); + // - the --resnapshot RE-PIN rewrites an unchanged row to carry the new snapshot id + // (pipeline/stagerun.go). + // An `ok` or `flagged` row is not touched by an ordinary resume — it is read and its checkpoint served, + // with no write (pipeline/resume.go). So a cleanly shipped book keeps every timestamp across a re-run + // and a book with a flagged unit does not. Pinned by + // pipeline.TestOrdinaryResumeMovesOnlyTheSkippedRow. UpdatedAt string } diff --git a/backend/internal/text/norm_test.go b/backend/internal/text/norm_test.go index 07d20f81..73c24637 100644 --- a/backend/internal/text/norm_test.go +++ b/backend/internal/text/norm_test.go @@ -198,3 +198,27 @@ func TestNormalizeStripsDefaultIgnorable(t *testing.T) { t.Errorf("soft hyphen on the target side = %q, want %q (must fold to '-', not be stripped)", got, want) } } + +// TestNormalizeSourceKeyIsIdempotent pins a property the bank's conflict check now rests on. A MINED row +// lands with the miner's own surface, which is a substring of already-normalized chunk text, while the +// check compares against the candidate KEY — and the two are the same string only while re-normalizing +// changes nothing. A normalizer that stopped being idempotent would make the check compare a landing row +// against a source it does not have, silently, on the mined half of every bank. +func TestNormalizeSourceKeyIsIdempotent(t *testing.T) { + cases := []string{"趙甲", "赵甲", "強敵", "ゼロ", "ヴァン", "ＡＢ", "方源", "青茅山", "元始空窍", "老赵", "ＦＵＬＬ幅", "a b"} + changed := 0 + for _, s := range cases { + once := NormalizeSourceKey(s) + if twice := NormalizeSourceKey(once); twice != once { + t.Errorf("NormalizeSourceKey is not idempotent on %q: %q → %q", s, once, twice) + } + if once != s { + changed++ + } + } + // CONTROL: the normalizer really does work on this set, so the idempotence above is a property and + // not a table of inputs it happens to leave alone. + if changed == 0 { + t.Fatalf("control: no probe was changed by normalization, so idempotence proves nothing") + } +}