Land the engine zone's bank-law work as the zone built it: the postcheck names the blind spot it cannot see, and an unsigned pair no longer passes the merge unseen
This commit is contained in:
parent
3d90943421
commit
5092ae3f0e
18 changed files with 1114 additions and 70 deletions
|
|
@ -228,6 +228,12 @@ func renderBankStopRows(w io.Writer, rows []pipeline.BankStopRow) {
|
|||
// same reply gave its part. Loudest line of the row — it is a canon breaking inside one call.
|
||||
fmt.Fprintf(w, " ⚠ contradicts: %s\n", trunc(strings.Join(r.Contradicts, "; "), 80))
|
||||
}
|
||||
if len(r.BankHolds) > 0 {
|
||||
// The other disagreement, and a different decision: the book already renders this surface some
|
||||
// other way. It rides the same screen as its sibling above, or the operator's first view of the
|
||||
// stop shows one and hides the other.
|
||||
fmt.Fprintf(w, " ⚠ bank holds: %s\n", trunc(strings.Join(r.BankHolds, "; "), 80))
|
||||
}
|
||||
if len(r.Variants) > 1 {
|
||||
// The disagreement is the reason to sign: one term coming back several ways is exactly what a
|
||||
// canon exists to close, and hiding it behind one winner throws that reason away.
|
||||
|
|
@ -359,8 +365,11 @@ func renderReport(w io.Writer,
|
|||
|
||||
// Memory section (step 4): the per-chunk retrieval-state, surfaced so silent glossary
|
||||
// degradation is LOUD. The aggregate line always prints when a glossary is in use;
|
||||
// each chunk with a post-check miss is listed (the flagger-mode signal — the model
|
||||
// ignored an approved term, or we injected the wrong dst) with the offending src→dst.
|
||||
// each chunk with a post-check miss is listed with the offending src→dst. Two limits: a
|
||||
// chunk is listed on its CONFIRMED miss count while the detail beside it carries every
|
||||
// miss including the unsigned ones, so a listed line is not necessarily an approved
|
||||
// term; and a miss means the rendering is ABSENT — a wrong dst the model OBEYED is
|
||||
// present in the output and appears in no line here (mempostcheck.go).
|
||||
states, err := fetchStates()
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -561,3 +561,30 @@ func TestTheHumanManifestCallsTheBookLevelFigureACeiling(t *testing.T) {
|
|||
t.Errorf("a book with no book-level spend must not be told about a ceiling on it:\n%s", b2.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderSignatureStopShowsBothDisagreements pins the review screen's two markers together. They are
|
||||
// different decisions for the owner — the run contradicting itself, and the book already rendering the
|
||||
// surface another way — and a screen that prints one and hides the other sends him to the sidecar to find
|
||||
// out that the term he is signing is already in the bank.
|
||||
func TestRenderSignatureStopShowsBothDisagreements(t *testing.T) {
|
||||
rows := []pipeline.BankStopRow{{
|
||||
Src: "方源", Dst: "Фан Юань", Origin: "banknote", Freq: 9, Conf: 10,
|
||||
Contradicts: []string{`方源空窍→"Пустота Фан Юаня"`},
|
||||
BankHolds: []string{`unsigned draft "方源"→"Странник" [ch 1..20]`},
|
||||
}}
|
||||
var b bytes.Buffer
|
||||
renderSignatureStop(&b, &pipeline.WaveSignatureStop{Terms: len(rows), SignaturePath: "p", Rows: rows})
|
||||
out := b.String()
|
||||
for _, want := range []string{"contradicts:", "Пустота Фан Юаня", "bank holds:", `unsigned draft "方源"→"Странник"`} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("the review screen must carry %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
// CONTROL: a row with neither marker prints neither, so the assertions above are the fields answering.
|
||||
var quiet bytes.Buffer
|
||||
renderSignatureStop(&quiet, &pipeline.WaveSignatureStop{Terms: 1, SignaturePath: "p",
|
||||
Rows: []pipeline.BankStopRow{{Src: "方源", Dst: "Фан Юань", Origin: "banknote", Freq: 9, Conf: 10}}})
|
||||
if strings.Contains(quiet.String(), "contradicts:") || strings.Contains(quiet.String(), "bank holds:") {
|
||||
t.Errorf("control: an unmarked row must print neither marker:\n%s", quiet.String())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,10 +10,22 @@ import (
|
|||
|
||||
// mempostcheck.go: the post-check (registry E1/E2 / research/13 Q5) — the main, almost
|
||||
// free detector of silent degradation. For every injected record whose SRC actually
|
||||
// fired in THIS chunk, it asks: did an accepted DST form appear in the output? A miss
|
||||
// catches BOTH failure modes — "the model ignored the glossary" (soft-following loses
|
||||
// 17–36% of terms, 2310.05824) AND "we injected the wrong dst and the model obeyed it"
|
||||
// (the owner's fear, 2510.00829). It is a post-CHECK, never a blind post-REPLACE (E2):
|
||||
// fired in THIS chunk, it asks one question: did an accepted DST form appear in the
|
||||
// output? A miss says the injected rendering is ABSENT and nothing more — "the model
|
||||
// ignored the glossary" (soft-following loses 17–36% of terms, 2310.05824), whether the
|
||||
// injected dst was right or wrong.
|
||||
//
|
||||
// BLIND SPOT, by the shape of the check: a wrong dst the model OBEYED (the owner's fear,
|
||||
// 2510.00829) leaves the injected form PRESENT, so `if present { continue }` below is taken
|
||||
// and no miss is written — the more faithfully a model executes a wrong injection, the
|
||||
// quieter this gets. A zero confirmed-miss count therefore says the signed rows that fired
|
||||
// on a TRUSTED key were EXECUTED — not that they were executed CORRECTLY, and nothing about
|
||||
// the rest: an unsigned row's miss and a signed one demoted by a single-Han key both go to
|
||||
// Unverified and are not counted. Judging the VALUE of an injected rendering needs a
|
||||
// source-side check that does not live here (backlog row 286). Pinned by
|
||||
// TestWrongDstObeyedIsNotAMiss.
|
||||
//
|
||||
// It is a post-CHECK, never a blind post-REPLACE (E2):
|
||||
// forcing a dictionary form into an oblique Russian case breaks agreement (46% of
|
||||
// constrained-model errors in en-cs are agreement, 2106.12398), so v1 only FLAGS.
|
||||
//
|
||||
|
|
|
|||
|
|
@ -96,3 +96,38 @@ func TestAmbiguousMissNotDemoted(t *testing.T) {
|
|||
t.Errorf("the ambiguous miss must keep disp:ambiguous, got %q", res.Unverified[0].Disp)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWrongDstObeyedIsNotAMiss pins the SCOPE of a miss: the injected rendering is ABSENT, nothing more.
|
||||
// A wrong dst the model faithfully executed leaves that rendering present, so it produces no miss at all.
|
||||
// The two halves share bank, chunk and firing key and differ only in whether the output carries the
|
||||
// injected form — without the control, the zero would be indistinguishable from a fixture that never
|
||||
// reached the checker.
|
||||
func TestWrongDstObeyedIsNotAMiss(t *testing.T) {
|
||||
// 元石 is signed to a WRONG rendering: the book's stone is «первокамень», the bank says «монета».
|
||||
b := bankFrom([]store.GlossaryEntry{gl("元石", "монета", "", "approved")})
|
||||
sel := b.Select("他手里握着元石。", 1, nil, 0)
|
||||
if len(sel.Injected) != 1 {
|
||||
t.Fatalf("the firing key must inject for either half of this pin to mean anything, got %d", len(sel.Injected))
|
||||
}
|
||||
|
||||
// OBEYED. Nominative, so the whole-word branch answers and the pin does not need a stemmer.
|
||||
obeyed := b.Postcheck(sel.Injected, "В его руке лежала монета.")
|
||||
if obeyed.ConfirmedCount() != 0 || !obeyed.Empty() {
|
||||
t.Errorf("an OBEYED injection leaves its dst present, so the post-check has nothing to report: confirmed=%d all=%+v",
|
||||
obeyed.ConfirmedCount(), obeyed.All())
|
||||
}
|
||||
|
||||
// CONTROL: the rendering is ABSENT, which is the only thing a miss says.
|
||||
refused := b.Postcheck(sel.Injected, "В его руке лежал первокамень.")
|
||||
if refused.ConfirmedCount() != 1 {
|
||||
t.Fatalf("control: an absent rendering must be exactly one CONFIRMED miss, got %d (%+v)", refused.ConfirmedCount(), refused.All())
|
||||
}
|
||||
if refused.Confirmed[0].Dst != "монета" {
|
||||
t.Errorf("control: the miss must name the INJECTED rendering, got %q", refused.Confirmed[0].Dst)
|
||||
}
|
||||
// The wrong-value case is silent and the reported case is the model REFUSING that same wrong value, so
|
||||
// a zero cannot be read as "the bank was executed correctly".
|
||||
if obeyed.ConfirmedCount() == refused.ConfirmedCount() {
|
||||
t.Errorf("the two halves must differ, or this test pins nothing: %d vs %d", obeyed.ConfirmedCount(), refused.ConfirmedCount())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -222,9 +222,13 @@ func parseBankSeed(name string, raw []byte, engineWritten bool) (BankSeed, error
|
|||
// Scope to a MATCHABLE src: an A3-banned single-char src (道 — text.SignificantLen below
|
||||
// the min-key floor, never in the automaton) is inert and cannot livelock, so its
|
||||
// polysemy stays allowed. Both rows must be matchable for the contradiction to fire.
|
||||
//
|
||||
// Status-blind on purpose: under D39.104 п.2 every row on the wire is law whatever its
|
||||
// signature (memory.go, "no fork by status at all"), so demoting one of the pair to
|
||||
// auto/draft still shows the model both renderings. Hence no status among the remedies.
|
||||
if srcKeyFires(a.Src, a.AllowShort) && srcKeyFires(b.Src, b.AllowShort) {
|
||||
problems.addKeyed(subjectOf("term-polysemy/"+b.Sense, a.Src, a.Sense, a.SinceCh, a.UntilCh),
|
||||
fmt.Sprintf("term %q: senses %q→%q and %q→%q have OVERLAPPING spoiler windows [%d,%d] and [%d,%d] — the deterministic matcher keys only on src and cannot pick a sense, so both inject as authoritative and the post-check is guaranteed to miss one (give them non-overlapping windows, merge the sense, or mark one status: auto)",
|
||||
fmt.Sprintf("term %q: senses %q→%q and %q→%q have OVERLAPPING spoiler windows [%d,%d] and [%d,%d] — the deterministic matcher keys only on src and cannot pick a sense, so both inject as authoritative and the post-check is guaranteed to miss one (give them non-overlapping windows, merge the senses into one rendering, or drop one; a status will not separate them — every row on the wire is law whatever its signature)",
|
||||
a.Src, b.Sense, b.Dst, a.Sense, a.Dst, b.SinceCh, b.UntilCh, a.SinceCh, a.UntilCh))
|
||||
}
|
||||
}
|
||||
|
|
@ -676,48 +680,210 @@ func runesAllKana(s string) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
// UnverifiedKeyConflicts reports the pairs S14 named: an APPROVED term and an UNSIGNED one that share a
|
||||
// firing surface with DIFFERENT renderings and overlapping windows. Both are injected — the approved one
|
||||
// as canon, the unsigned one as a working version — so the model receives two contradictory lines about
|
||||
// the same surface. That is the likeliest source of a silent degradation in the auto mode, and until
|
||||
// pack-20 no diagnostic could see it: ApprovedSharedKeyCollisions and InjectivityCollisions are both
|
||||
// approved-only, by construction.
|
||||
// UnverifiedKeyConflicts reports every pair of bank rows that share a firing surface, render it
|
||||
// differently and have overlapping spoiler windows, except approved×approved — that pair belongs to
|
||||
// ApprovedSharedKeyCollisions' fail-loud and reporting it here too would state one fault twice.
|
||||
//
|
||||
// It is a WARNING, not a fail-loud, and deliberately so: the unsigned row is engine-produced, and
|
||||
// aborting a paid run over the engine's own proposal would be a self-inflicted outage. The trust gate
|
||||
// already protects the reader (the approved rendering keeps its precedence); this makes the collision
|
||||
// VISIBLE so the owner can reconcile it. Deterministic (sorted keys, sorted pairs).
|
||||
// Both rows of such a pair are injected, and a signature changes nothing about what the model is shown
|
||||
// (D39.104 п.2: no fork by status), so an unsigned×unsigned pair is the same fault as a signed one.
|
||||
// Nothing else catches it: ApprovedSharedKeyCollisions and InjectivityCollisions are approved-only, the
|
||||
// seed loader's D16.1 guard compares only Src within one document, and the uniqueness checks key on the
|
||||
// whole (src, sense, window) tuple, which two overlapping windows do not share.
|
||||
//
|
||||
// WARNING, not fail-loud: at least one row of every reported pair is engine-produced or unratified, and
|
||||
// aborting a paid run over our own proposal would be a self-inflicted outage.
|
||||
//
|
||||
// Deterministic (sorted keys, then index order); a pair sharing two keys is reported once, under the
|
||||
// first key.
|
||||
func UnverifiedKeyConflicts(entries []store.GlossaryEntry) []string {
|
||||
approved := map[string][]int{}
|
||||
unsigned := map[string][]int{}
|
||||
owners := map[string][]int{}
|
||||
for i, e := range entries {
|
||||
if strings.TrimSpace(e.Dst) == "" {
|
||||
continue
|
||||
}
|
||||
target := unsigned
|
||||
if e.Status == "approved" {
|
||||
target = approved
|
||||
}
|
||||
for _, k := range entryFiringKeys(e) {
|
||||
target[k] = append(target[k], i)
|
||||
owners[k] = append(owners[k], i)
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, k := range slices.Sorted(maps.Keys(approved)) {
|
||||
for _, ai := range approved[k] {
|
||||
for _, ui := range unsigned[k] {
|
||||
ae, ue := entries[ai], entries[ui]
|
||||
if ae.Dst == ue.Dst || !windowsOverlap(ae.SinceCh, ae.UntilCh, ue.SinceCh, ue.UntilCh) {
|
||||
seenPair := map[[2]int]bool{}
|
||||
for _, k := range slices.Sorted(maps.Keys(owners)) {
|
||||
idxs := owners[k]
|
||||
for a := 0; a < len(idxs); a++ {
|
||||
for b := a + 1; b < len(idxs); b++ {
|
||||
i, j := idxs[a], idxs[b]
|
||||
ei, ej := entries[i], entries[j]
|
||||
if ei.Status == "approved" && ej.Status == "approved" {
|
||||
continue // ApprovedSharedKeyCollisions' fail-loud owns this pair
|
||||
}
|
||||
if ei.Dst == ej.Dst || !windowsOverlap(ei.SinceCh, ei.UntilCh, ej.SinceCh, ej.UntilCh) {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("firing key %q: approved %q→%q vs unsigned %s %q→%q (the model is shown two renderings of one surface)",
|
||||
k, ae.Src, ae.Dst, ue.Status, ue.Src, ue.Dst))
|
||||
if seenPair[[2]int{i, j}] {
|
||||
continue
|
||||
}
|
||||
seenPair[[2]int{i, j}] = true
|
||||
out = append(out, fmt.Sprintf("firing key %q: %s %q→%q %s vs %s %q→%q %s (the model is shown two renderings of one surface)",
|
||||
k, statusLabel(ei.Status), ei.Src, ei.Dst, windowLabel(ei.SinceCh, ei.UntilCh),
|
||||
statusLabel(ej.Status), ej.Src, ej.Dst, windowLabel(ej.SinceCh, ej.UntilCh)))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// statusLabel renders a row's signature for an operator, keeping the specific status where there is one.
|
||||
func statusLabel(status string) string {
|
||||
if status == "approved" {
|
||||
return "approved"
|
||||
}
|
||||
if status == "" {
|
||||
return "unsigned"
|
||||
}
|
||||
return "unsigned " + status
|
||||
}
|
||||
|
||||
// windowLabel renders a spoiler window for a human. The raw zeros must not be printed: since_ch 0 means
|
||||
// "from chapter 1" and until_ch 0 means "no end" (windowsOverlap), so "[0,0]" would read as an empty
|
||||
// window rather than the whole book.
|
||||
func windowLabel(since, until int) string {
|
||||
lo := since
|
||||
if lo == 0 {
|
||||
lo = 1
|
||||
}
|
||||
if until == 0 {
|
||||
return fmt.Sprintf("[ch %d..end]", lo)
|
||||
}
|
||||
return fmt.Sprintf("[ch %d..%d]", lo, until)
|
||||
}
|
||||
|
||||
// ConsolidationKeyConflicts reports a rendering this run consolidated for a firing surface the bank
|
||||
// already renders differently, in an overlapping window, whatever either side's status.
|
||||
//
|
||||
// Neither of the collapse stage's own checks can see this shape: terminology.CanonConflicts and
|
||||
// terminology.ConsolidationConflicts both test CONTAINMENT and both skip the pair whose sources are
|
||||
// equal, and CanonConflicts' right-hand side is approved-only. The consolidation reaches surfaces the
|
||||
// bank already holds by two ordinary routes: the miner re-mines a surface held only by an engine-unsigned
|
||||
// row (pipeline.unsignedEngineSurfaces hides those from it so the auto mode does not switch itself off),
|
||||
// and the banknote channel merges every draft-proposed surface into the candidate list with no bank
|
||||
// filter — that filter sits on emission, not on the role's input.
|
||||
//
|
||||
// It reports a DISAGREEMENT and does not predict that both renderings reach a wire. Three mechanisms
|
||||
// downstream decide that, and only the first is grounds to stay silent:
|
||||
// - the glossary UNIQUE key (book_id, src, sense, since_ch, until_ch) admits only one row per tuple, so
|
||||
// a proposal landing on an existing row's tuple resolves against it rather than joining it. That pair
|
||||
// is the one case skipped below;
|
||||
// - the emission filter drops a proposal whose surface is already a seed surface
|
||||
// (pipeline.reverseSectionTerms), leaving the disagreement on the review sheet only;
|
||||
// - the auto-bank file is rewritten whole each run (pipeline.writeAutoBank), so a previous run's engine
|
||||
// row does not survive beside this run's rendering of the same surface even when the windows differ.
|
||||
//
|
||||
// The last two are still reported: the sheet is where a term is decided, and "the book already calls it
|
||||
// something else" is what the owner needs there.
|
||||
//
|
||||
// ⚠ WHAT THE SKIP HIDES, measured: a seed row with the DEFAULT window and no sense — the commonest shape a
|
||||
// hand-written seed has — shares its tuple with a banknote proposal, which carries since_ch 0 likewise,
|
||||
// so that pair is silent here whatever the seed row's status. The proposal is dropped rather than
|
||||
// shipped, so no wire carries two renderings; what is lost is the sheet telling the owner that the role
|
||||
// disagreed with a row he wrote. Closing it needs a rule for telling a REPLACED engine row from a DROPPED
|
||||
// proposal, and the Source column cannot give one: the owner's delta and the engine's auto-bank both read
|
||||
// "mined".
|
||||
//
|
||||
// In the auto mode a pair that DOES land is then reported a second time, by UnverifiedKeyConflicts on the
|
||||
// re-seed. The two are different states of one term — "the role disagrees with the bank" before the row
|
||||
// exists, "the bank holds both" after — and only the second means the model will be shown two lines.
|
||||
//
|
||||
// ⚠ WHO RESOLVES A SKIPPED PAIR, and it is not one mechanism. A proposal on a SEED surface never reaches
|
||||
// the bank: pipeline.reverseSectionTerms drops it at emission, and an APPROVED delta row counts as a seed
|
||||
// surface there too (pipeline.unsignedEngineSurfaces keeps it). A proposal on an UNSIGNED delta row's
|
||||
// tuple does reach pipeline.loadAutoBank, which drops it whenever any earlier-gathered row holds the key —
|
||||
// unsigned rows included, despite that function's wording. That drop is logged, but as "held by a signed
|
||||
// term" whatever the holder's status, so the operator is told the wrong thing rather than nothing.
|
||||
//
|
||||
// Report-only and $0, like both siblings. Returns parts rather than finished sentences: the run log and
|
||||
// the signature sheet each render them, and a caller free to re-phrase would state one finding two ways.
|
||||
//
|
||||
// Deterministic: proposal order, then each proposal's firing-key order, then bank index order.
|
||||
func ConsolidationKeyConflicts(consolidated, bank []store.GlossaryEntry) []BankKeyConflict {
|
||||
if len(consolidated) == 0 || len(bank) == 0 {
|
||||
return nil
|
||||
}
|
||||
owners := map[string][]int{}
|
||||
for i, e := range bank {
|
||||
if strings.TrimSpace(e.Dst) == "" {
|
||||
continue
|
||||
}
|
||||
for _, k := range entryFiringKeys(e) {
|
||||
owners[k] = append(owners[k], i)
|
||||
}
|
||||
}
|
||||
var out []BankKeyConflict
|
||||
for _, c := range consolidated {
|
||||
if strings.TrimSpace(c.Dst) == "" {
|
||||
continue // the role declined or never answered: no rendering, nothing to contradict
|
||||
}
|
||||
reported := map[int]bool{}
|
||||
for _, k := range entryFiringKeys(c) {
|
||||
for _, i := range owners[k] {
|
||||
if reported[i] {
|
||||
continue
|
||||
}
|
||||
b := bank[i]
|
||||
// Compared RAW, because the store's UNIQUE key is raw: 族長 and 族长 with one sense and
|
||||
// window are two accepted rows, not one (migrate.go). Folding them here would skip a pair
|
||||
// that really does coexist. Callers pass the source the row WILL carry — see
|
||||
// pipeline.consolidatedRows.
|
||||
if b.Src == c.Src && b.Sense == c.Sense && b.SinceCh == c.SinceCh && b.UntilCh == c.UntilCh {
|
||||
continue // one uniqueness key — the store admits only one of the two
|
||||
}
|
||||
if b.Dst == c.Dst || !windowsOverlap(c.SinceCh, c.UntilCh, b.SinceCh, b.UntilCh) {
|
||||
continue
|
||||
}
|
||||
reported[i] = true
|
||||
out = append(out, BankKeyConflict{
|
||||
Key: k, Src: c.Src, Dst: c.Dst, SinceCh: c.SinceCh, UntilCh: c.UntilCh,
|
||||
BankSrc: b.Src, BankDst: b.Dst, BankStatus: b.Status,
|
||||
BankSinceCh: b.SinceCh, BankUntilCh: b.UntilCh,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BankKeyConflict is one consolidated rendering set against the bank row it contradicts.
|
||||
type BankKeyConflict struct {
|
||||
Key string // the firing surface the two share — which may be an ALIAS of either row
|
||||
Src, Dst string // the proposal: the surface consolidated, and what this run rendered it as
|
||||
SinceCh, UntilCh int // the window the proposal's row would carry
|
||||
BankSrc string // the row already in the bank; its Src differs from the proposal's on an alias hit
|
||||
BankDst string
|
||||
BankStatus string
|
||||
BankSinceCh int
|
||||
BankUntilCh int
|
||||
}
|
||||
|
||||
// Message renders the whole finding as one sentence, for a log line.
|
||||
func (c BankKeyConflict) Message() string {
|
||||
return fmt.Sprintf("firing key %q: this run consolidated %q→%q %s while the bank already holds %s (one term, two renderings on the sheet the owner signs from)",
|
||||
c.Key, c.Src, c.Dst, windowLabel(c.SinceCh, c.UntilCh), c.BankRowLabel())
|
||||
}
|
||||
|
||||
// BankRowLabel names only the EXISTING row: the signature sheet already prints the proposal on its own
|
||||
// line, so repeating it would bury the new fact.
|
||||
func (c BankKeyConflict) BankRowLabel() string {
|
||||
return fmt.Sprintf("%s %q→%q %s", statusLabel(c.BankStatus), c.BankSrc, c.BankDst, windowLabel(c.BankSinceCh, c.BankUntilCh))
|
||||
}
|
||||
|
||||
// ConflictMessages renders a set of findings for one log line.
|
||||
func ConflictMessages(cs []BankKeyConflict) []string {
|
||||
out := make([]string, 0, len(cs))
|
||||
for _, c := range cs {
|
||||
out = append(out, c.Message())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// MinedDeltaSeedCollisions returns a human message for every mined-delta entry whose store UNIQUE key
|
||||
// (src, sense, since_ch, until_ch) already exists among the seed/ruby entries — the D39.20 deviation-#1
|
||||
// crash class. It keys on the FULL uniqueness tuple (not the firing surface, unlike
|
||||
|
|
|
|||
|
|
@ -469,29 +469,49 @@ func TestSeedSurroundingWhitespaceTrimmed(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestUnverifiedKeyConflicts pins S14 — the collision class the approved-only diagnostics structurally
|
||||
// cannot see. In the auto mode an approved term and an unsigned proposal can share a firing surface with
|
||||
// different renderings, and both are injected (canon + working version), so the model receives two
|
||||
// answers to one question. The trust gate keeps the approved one authoritative; this makes the collision
|
||||
// VISIBLE, which is what neither ApprovedSharedKeyCollisions nor InjectivityCollisions can do.
|
||||
// TestUnverifiedKeyConflicts pins the rule: every pair rendering one firing surface two ways in
|
||||
// overlapping windows, except approved×approved (ApprovedSharedKeyCollisions' fail-loud). Both rows are
|
||||
// injected and a signature changes nothing about that (D39.104 п.2), so an unsigned×unsigned pair is the
|
||||
// same fault: «старейшина» (draft) against «глава клана» (auto) below.
|
||||
func TestUnverifiedKeyConflicts(t *testing.T) {
|
||||
rows := []store.GlossaryEntry{
|
||||
{Src: "族長", Dst: "глава клана", Status: "approved"},
|
||||
{Src: "族長", Dst: "старейшина", Status: "draft"}, // ← the conflict
|
||||
{Src: "族長", Dst: "глава клана", Status: "auto"}, // same rendering → no contradiction
|
||||
{Src: "族長", Dst: "старейшина", Status: "draft"}, // ← contradicts BOTH rows that render «глава клана»
|
||||
{Src: "族長", Dst: "глава клана", Status: "auto"}, // same rendering as the approved row → no contradiction there
|
||||
{Src: "鈴木", Dst: "Судзуки", Status: "approved"}, // untouched
|
||||
{Src: "田中", Dst: "Танака", Status: "draft"}, // unsigned but unopposed
|
||||
}
|
||||
got := UnverifiedKeyConflicts(rows)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want exactly the approved-vs-unsigned contradiction, got %d: %v", len(got), got)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want the approved×unsigned pair AND the unsigned×unsigned one, got %d: %v", len(got), got)
|
||||
}
|
||||
for _, want := range []string{"族長", "глава клана", "старейшина", "draft"} {
|
||||
if !strings.Contains(got[0], want) {
|
||||
t.Fatalf("the report must name %q so the owner can reconcile it: %q", want, got[0])
|
||||
joined := strings.Join(got, " | ")
|
||||
// The approved×unsigned pair (the original S14 guarantee).
|
||||
if !strings.Contains(joined, `approved "族長"→"глава клана"`) || !strings.Contains(joined, `unsigned draft "族長"→"старейшина"`) {
|
||||
t.Errorf("the approved×unsigned contradiction must still be named in full: %v", got)
|
||||
}
|
||||
// The unsigned×unsigned pair: neither side is signed, and an unsigned term owes one rendering too.
|
||||
if !strings.Contains(joined, `unsigned draft "族長"→"старейшина" [ch 1..end] vs unsigned auto "族長"→"глава клана"`) {
|
||||
t.Errorf("the unsigned×unsigned contradiction must be reported with both statuses: %v", got)
|
||||
}
|
||||
// Every message must locate the fault, or an operator cannot tell which two rows to reconcile.
|
||||
for _, m := range got {
|
||||
if !strings.Contains(m, `firing key "族长"`) || strings.Count(m, "[ch ") != 2 {
|
||||
t.Errorf("a conflict must name its firing key and both windows: %q", m)
|
||||
}
|
||||
}
|
||||
// The approved-only checks are blind to it — which is why this diagnostic exists at all.
|
||||
// approved×approved belongs to the fail-loud check; reporting it here too would state one fault twice.
|
||||
both := []store.GlossaryEntry{
|
||||
{Src: "族長", Dst: "глава клана", Status: "approved"},
|
||||
{Src: "族長", Dst: "старейшина", Status: "approved", Sense: "elder"},
|
||||
}
|
||||
if got := UnverifiedKeyConflicts(both); len(got) != 0 {
|
||||
t.Errorf("approved×approved belongs to the fail-loud check, not here: %v", got)
|
||||
}
|
||||
if cols := ApprovedSharedKeyCollisions(both); len(cols) != 1 {
|
||||
t.Errorf("control: the fail-loud check must be the one that sees it, got %d", len(cols))
|
||||
}
|
||||
// The approved-only checks are blind to the original fixture.
|
||||
if cols := ApprovedSharedKeyCollisions(rows); len(cols) != 0 {
|
||||
t.Fatalf("premise broken: the approved-only check should see nothing here, got %v", cols)
|
||||
}
|
||||
|
|
@ -503,6 +523,23 @@ func TestUnverifiedKeyConflicts(t *testing.T) {
|
|||
if got := UnverifiedKeyConflicts(windowed); len(got) != 0 {
|
||||
t.Fatalf("disjoint windows are not a contradiction, got %v", got)
|
||||
}
|
||||
// CONTROL: the same two rows with OVERLAPPING windows are reported, so the zero above is the window
|
||||
// test answering and not a fixture that reaches nothing.
|
||||
windowed[1].SinceCh = 3
|
||||
if got := UnverifiedKeyConflicts(windowed); len(got) != 1 {
|
||||
t.Fatalf("control: overlapping windows must be reported, got %d: %v", len(got), got)
|
||||
}
|
||||
// A pair sharing TWO firing keys is one contradiction, reported once under the first key.
|
||||
twice := []store.GlossaryEntry{
|
||||
{Src: "赵甲", Dst: "Чжао Цзя", Status: "draft", Aliases: []store.GlossaryAlias{alias("老赵"), alias("赵老")}},
|
||||
{Src: "赵乙", Dst: "Чжао И", Status: "auto", Aliases: []store.GlossaryAlias{alias("老赵"), alias("赵老")}},
|
||||
}
|
||||
switch n := len(UnverifiedKeyConflicts(twice)); {
|
||||
case n == 0:
|
||||
t.Error("a pair sharing two firing keys must still be reported")
|
||||
case n != 1:
|
||||
t.Errorf("a pair sharing two keys is one fault, not %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApprovedSharedKeyCollisionSeesAllowShort closes a mutation that SURVIVED the cold-run session's
|
||||
|
|
@ -525,3 +562,120 @@ func TestApprovedSharedKeyCollisionSeesAllowShort(t *testing.T) {
|
|||
t.Fatalf("a single-key-BANNED shared src is inert and must not fail the load, got: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPolysemyGuardIsStatusBlind pins three facts together: the D16.1 refusal fires for every status, an
|
||||
// unsigned row really is injected beside the approved one (D39.104 п.2 — no fork by status on the wire),
|
||||
// and the remedies the message names therefore do not include a status. Break any one and the message
|
||||
// starts advising an operator to do something that cannot work.
|
||||
func TestPolysemyGuardIsStatusBlind(t *testing.T) {
|
||||
seedFor := func(status string) string {
|
||||
return "terms:\n - { src: 神通, dst: сила, sense: power }\n - { src: 神通, dst: чудо, sense: miracle, status: " + status + " }\n"
|
||||
}
|
||||
for _, status := range []string{"approved", "draft", "auto"} {
|
||||
_, err := LoadGlossarySeed(writeSeed(t, seedFor(status)))
|
||||
if err == nil {
|
||||
t.Fatalf("status %q: the polysemy livelock must be refused whatever the second row's signature", status)
|
||||
}
|
||||
if strings.Contains(err.Error(), "status: auto") || strings.Contains(err.Error(), "mark one status") {
|
||||
t.Errorf("status %q: the refusal must not offer a STATUS as a remedy — the guard ignores it and so does the wire: %s", status, err)
|
||||
}
|
||||
}
|
||||
// CONTROL for the loop above: the same two senses with NON-overlapping windows are a legitimate
|
||||
// spoiler handoff and load, so the refusals above are the guard speaking and not a broken fixture.
|
||||
if _, err := LoadGlossarySeed(writeSeed(t, "terms:\n - { src: 神通, dst: сила, sense: power, until_ch: 5 }\n - { src: 神通, dst: чудо, sense: miracle, since_ch: 6, status: auto }\n")); err != nil {
|
||||
t.Fatalf("control: non-overlapping windows must still load: %v", err)
|
||||
}
|
||||
// WHY a status cannot be the remedy: an unsigned row is injected as law beside the approved one, so
|
||||
// the model is handed both renderings exactly as it would be for two approved rows.
|
||||
for _, status := range []string{"approved", "draft", "auto"} {
|
||||
b := bankFrom([]store.GlossaryEntry{gl("神通", "сила", "power", "approved"), gl("神通", "чудо", "miracle", status)})
|
||||
if n := len(b.Select("他用神通打败了对手。", 1, nil, 0).Injected); n != 2 {
|
||||
t.Errorf("status %q: both renderings must reach the wire (D39.104 п.2 — no fork by status), got %d injected", status, n)
|
||||
}
|
||||
}
|
||||
// CONTROL for that loop: with the src absent from the chunk nothing fires, so the count of 2 above
|
||||
// is the matcher answering and not a bank that injects unconditionally.
|
||||
if n := len(bankFrom([]store.GlossaryEntry{gl("神通", "сила", "power", "approved")}).Select("他走了。", 1, nil, 0).Injected); n != 0 {
|
||||
t.Errorf("control: a src absent from the chunk must inject nothing, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConsolidationKeyConflicts covers the four states the check has to tell apart: a disagreement, a
|
||||
// legitimate agreement, a legitimate spoiler handoff, and a REPLACEMENT that only looks like a conflict.
|
||||
// Neither side is asked for a signature.
|
||||
func TestConsolidationKeyConflicts(t *testing.T) {
|
||||
bank := []store.GlossaryEntry{
|
||||
{Src: "元始", Dst: "юаньши", Status: "draft", Source: "seed", SinceCh: 3, UntilCh: 9},
|
||||
{Src: "神通", Dst: "сила", Status: "approved", Source: "seed", UntilCh: 20},
|
||||
{Src: "赵甲", Dst: "Чжао Цзя", Status: "auto", Source: "mined", Aliases: []store.GlossaryAlias{alias("老赵")}},
|
||||
}
|
||||
// A rendering that disagrees with an UNSIGNED bank row in an overlapping window.
|
||||
got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "камень первоисточника"}}, bank)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("an unsigned row is owed one rendering too: got %d %v", len(got), got)
|
||||
}
|
||||
for _, want := range []string{`firing key "元始"`, "камень первоисточника", "юаньши", "unsigned draft", "[ch 3..9]", "[ch 1..end]"} {
|
||||
if !strings.Contains(got[0].Message(), want) {
|
||||
t.Errorf("the message must carry %q so the operator can reconcile the two rows: %q", want, got[0].Message())
|
||||
}
|
||||
}
|
||||
// The sheet prints only the existing row and the log prints the whole sentence, so the sheet's half
|
||||
// must be a literal part of the log's — otherwise one fault reads as two facts on two surfaces.
|
||||
if !strings.Contains(got[0].Message(), got[0].BankRowLabel()) {
|
||||
t.Errorf("the sheet label must be the log sentence's own words: label=%q message=%q", got[0].BankRowLabel(), got[0].Message())
|
||||
}
|
||||
if got[0].Src != "元始" || got[0].BankDst != "юаньши" || got[0].Key != "元始" {
|
||||
t.Errorf("the parts must identify both rows for a caller that formats its own line: %+v", got[0])
|
||||
}
|
||||
// The SIGNED side is not a special case. Neither of the collapse stage's own checks reports it (both
|
||||
// skip the equal-source pair; CanonConflicts is approved-only). The windows differ, so both rows land.
|
||||
if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "神通", Dst: "чудо"}}, bank); len(got) != 1 {
|
||||
t.Errorf("a consolidation contradicting a SIGNED row of the same surface must be reported: %v", got)
|
||||
}
|
||||
// Reached through a shared ALIAS rather than the src — the key is the firing surface, not the source.
|
||||
if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "赵乙", Dst: "Чжао И",
|
||||
Aliases: []store.GlossaryAlias{alias("老赵")}}}, bank); len(got) != 1 {
|
||||
t.Errorf("a contradiction reached through a shared alias must be reported: %v", got)
|
||||
}
|
||||
// AGREEMENT: the same rendering is the bank confirming itself, not a second rendering.
|
||||
if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "юаньши"}}, bank); len(got) != 0 {
|
||||
t.Errorf("an identical rendering is not a contradiction: %v", got)
|
||||
}
|
||||
// SPOILER HANDOFF: disjoint windows never inject together, so they are not two renderings at once.
|
||||
if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "камень первоисточника",
|
||||
SinceCh: 10}}, bank); len(got) != 0 {
|
||||
t.Errorf("disjoint windows are a legitimate handoff: %v", got)
|
||||
}
|
||||
// REPLACEMENT: the proposal carries the bank row's whole UNIQUE tuple, so only one of the two exists.
|
||||
if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "камень первоисточника",
|
||||
SinceCh: 3, UntilCh: 9}}, bank); len(got) != 0 {
|
||||
t.Errorf("a proposal on the row's own UNIQUE key replaces it and cannot contradict it: %v", got)
|
||||
}
|
||||
// A proposal with no rendering is a term the role declined; it becomes no row and contradicts nothing.
|
||||
if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: ""}}, bank); len(got) != 0 {
|
||||
t.Errorf("an unrendered proposal is not a contradiction: %v", got)
|
||||
}
|
||||
// CONTROL for the four zeros: the same bank and surfaces DO report when the rendering really differs
|
||||
// in an overlapping window.
|
||||
if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "первокамень", SinceCh: 5}}, bank); len(got) != 1 {
|
||||
t.Fatalf("control: an overlapping window with a different rendering must report, got %d: %v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConsolidationTupleSkipMirrorsTheStoreKey pins the skip against the store's ACTUAL uniqueness rule.
|
||||
// The glossary UNIQUE key is raw (migrate.go), so 族長 and 族长 with one sense and window are two rows the
|
||||
// store accepts and the matcher fires both on — a real contradiction. Folding the sources here would skip
|
||||
// it, and the caller is what guarantees the proposal already carries the source its row will have.
|
||||
func TestConsolidationTupleSkipMirrorsTheStoreKey(t *testing.T) {
|
||||
bank := []store.GlossaryEntry{{Src: "族長", Dst: "глава клана", Status: "draft", Source: "seed"}}
|
||||
// Different raw src, same sense and window: both rows can exist, so this is a contradiction.
|
||||
folded := []store.GlossaryEntry{{Src: "族长", Dst: "старейшина"}}
|
||||
if got := ConsolidationKeyConflicts(folded, bank); len(got) != 1 {
|
||||
t.Fatalf("族长 against a stored 族長 is a pair the store admits, not a replacement: %v", got)
|
||||
}
|
||||
// Byte-identical raw src on the same tuple: the store admits one row, so it is a replacement.
|
||||
same := []store.GlossaryEntry{{Src: "族長", Dst: "старейшина"}}
|
||||
if got := ConsolidationKeyConflicts(same, bank); len(got) != 0 {
|
||||
t.Errorf("an identical raw src on the same tuple is a replacement: %v", got)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
222
backend/internal/pipeline/bankconflict_corpus_test.go
Normal file
222
backend/internal/pipeline/bankconflict_corpus_test.go
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"textmachine/backend/internal/membank"
|
||||
"textmachine/backend/internal/store"
|
||||
)
|
||||
|
||||
// bankconflict_corpus_test.go: an opt-in corpus instrument, not a unit test. It measures how often the
|
||||
// bank contradiction membank.UnverifiedKeyConflicts reports actually occurs on real books, because the
|
||||
// open decision (report it, or refuse the run) turns on the rate rather than the shape. Run with
|
||||
// `TM_CORPUS=<repo>/books go test ./internal/pipeline/ -run TestCorpusBankKeyConflicts -v`; skipped
|
||||
// otherwise, so a bare clone stays green.
|
||||
//
|
||||
// Read-only by the connection, not by intent: every handle carries query_only(1), so a stray write fails
|
||||
// loudly instead of touching a store another run owns. It goes to SQL rather than store.OpenReadOnly
|
||||
// because that door refuses a schema this binary does not match, and older project files are part of the
|
||||
// population worth counting.
|
||||
//
|
||||
// It prints its population beside its answer: "0 books" and "0 books that hold an unsigned row at all"
|
||||
// are the same output and opposite facts. The rows read, the status histogram, and a planted
|
||||
// contradiction on each book it just called clean are what separate them.
|
||||
func TestCorpusBankKeyConflicts(t *testing.T) {
|
||||
root := os.Getenv("TM_CORPUS")
|
||||
if root == "" {
|
||||
t.Skip("corpus instrument: set TM_CORPUS to the books tree")
|
||||
}
|
||||
var dbs []string
|
||||
_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
|
||||
if err == nil && !fi.IsDir() && strings.HasSuffix(p, ".db") {
|
||||
dbs = append(dbs, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(dbs)
|
||||
if len(dbs) == 0 {
|
||||
t.Fatalf("TM_CORPUS=%s holds no .db file: a walk that found nothing reports the same zeros as a clean corpus", root)
|
||||
}
|
||||
|
||||
var withGlossary, noGlossary, failed int
|
||||
var booksWithRows, totalRows int
|
||||
var hitAny, hitOldOnly, hitUnsignedPair, spikeFired, booksWithUnsigned int
|
||||
statuses := map[string]int{}
|
||||
var lines []string
|
||||
|
||||
for _, p := range dbs {
|
||||
byBook, order, err := readGlossary(p)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "no such table") {
|
||||
noGlossary++
|
||||
} else {
|
||||
failed++
|
||||
}
|
||||
continue
|
||||
}
|
||||
withGlossary++
|
||||
for _, bid := range order {
|
||||
g := byBook[bid]
|
||||
booksWithRows++
|
||||
totalRows += len(g)
|
||||
unsigned := 0
|
||||
for _, e := range g {
|
||||
statuses[e.Status]++
|
||||
if e.Status != "approved" {
|
||||
unsigned++
|
||||
}
|
||||
}
|
||||
if unsigned > 0 {
|
||||
booksWithUnsigned++
|
||||
}
|
||||
all := membank.UnverifiedKeyConflicts(g)
|
||||
// A pair is "old rule" when EITHER side is approved. The label appears on both sides of the
|
||||
// message, and row order decides which comes first, so testing only the first would
|
||||
// misclassify every pair whose unsigned row sorts ahead of the approved one.
|
||||
old, pair := 0, 0
|
||||
for _, m := range all {
|
||||
if strings.Contains(m, `: approved "`) || strings.Contains(m, `vs approved "`) {
|
||||
old++
|
||||
} else {
|
||||
pair++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case pair > 0:
|
||||
hitAny++
|
||||
hitUnsignedPair++
|
||||
lines = append(lines, fmt.Sprintf("%s [%s] rows=%d: %d conflicts, %d unsigned×unsigned | %s",
|
||||
filepath.Base(p), bid, len(g), len(all), pair, trimTo(all[0], 190)))
|
||||
case old > 0:
|
||||
hitAny++
|
||||
hitOldOnly++
|
||||
}
|
||||
// The control that makes a zero readable: plant ONE contradicting unsigned row on a surface this
|
||||
// book really has, and confirm the guard reports it. Every row is tried in turn, because not
|
||||
// every row can carry a plant — an empty dst, or a src below the single-key floor without
|
||||
// allow_short, produces no firing key, and stopping at the first row would report the
|
||||
// instrument as broken on a correctly measured book.
|
||||
for _, seed := range g {
|
||||
planted := append(append([]store.GlossaryEntry(nil), g...), store.GlossaryEntry{
|
||||
Src: seed.Src, Dst: seed.Dst + "-ПОДСАДКА", Status: "draft", Sense: seed.Sense,
|
||||
SinceCh: seed.SinceCh, UntilCh: seed.UntilCh + 1000, AllowShort: seed.AllowShort,
|
||||
})
|
||||
if len(membank.UnverifiedKeyConflicts(planted)) > len(all) {
|
||||
spikeFired++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\nCORPUS root=%s\n", root)
|
||||
fmt.Printf(" .db walked=%d with a glossary table=%d without one=%d read-failed=%d\n", len(dbs), withGlossary, noGlossary, failed)
|
||||
fmt.Printf(" books with a glossary=%d rows read=%d\n", booksWithRows, totalRows)
|
||||
fmt.Printf(" POPULATION: books holding at least one UNSIGNED row=%d; status histogram:", booksWithUnsigned)
|
||||
for _, k := range []string{"approved", "draft", "auto"} {
|
||||
fmt.Printf(" %s=%d", k, statuses[k])
|
||||
}
|
||||
fmt.Println()
|
||||
fmt.Printf(" books with ANY conflict=%d; only approved×unsigned=%d; carrying an UNSIGNED×UNSIGNED pair=%d\n", hitAny, hitOldOnly, hitUnsignedPair)
|
||||
fmt.Printf(" CONTROL: a planted contradicting unsigned row is reported on %d of %d books\n", spikeFired, booksWithRows)
|
||||
for _, l := range lines {
|
||||
fmt.Printf(" + %s\n", l)
|
||||
}
|
||||
if booksWithRows > 0 && spikeFired != booksWithRows {
|
||||
t.Fatalf("the instrument is not measuring: a planted contradiction went unreported on %d of %d books", booksWithRows-spikeFired, booksWithRows)
|
||||
}
|
||||
}
|
||||
|
||||
// readGlossary reads one project file's whole glossary, aliases attached, over a query_only connection.
|
||||
func readGlossary(path string) (map[string][]store.GlossaryEntry, []string, error) {
|
||||
v := url.Values{}
|
||||
v.Add("_pragma", "busy_timeout(3000)")
|
||||
v.Add("_pragma", "query_only(1)")
|
||||
db, err := sql.Open("sqlite", "file:"+path+"?"+v.Encode())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer db.Close()
|
||||
byBook, order, at, err := readTerms(db)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Aliases are firing keys too: without them this UNDER-counts rather than reads zero, which is the
|
||||
// difference between "no contradiction" and "did not look at half the keys". An older file with no
|
||||
// alias table still answers on src keys, so its absence degrades rather than fails.
|
||||
if err := attachAliases(db, byBook, at); err != nil && !strings.Contains(err.Error(), "no such table") {
|
||||
return nil, nil, err
|
||||
}
|
||||
return byBook, order, nil
|
||||
}
|
||||
|
||||
// termLoc is where one glossary row landed, so an alias row can be attached to it by term_id.
|
||||
type termLoc struct {
|
||||
book string
|
||||
i int
|
||||
}
|
||||
|
||||
func readTerms(db *sql.DB) (map[string][]store.GlossaryEntry, []string, map[int64]termLoc, error) {
|
||||
rows, err := db.Query(`SELECT id, book_id, src, dst, sense, since_ch, until_ch, status, allow_short, source FROM glossary ORDER BY book_id, src, sense, since_ch, until_ch, status, dst`)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
byBook := map[string][]store.GlossaryEntry{}
|
||||
at := map[int64]termLoc{}
|
||||
var order []string
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var e store.GlossaryEntry
|
||||
var short int
|
||||
if err := rows.Scan(&id, &e.BookID, &e.Src, &e.Dst, &e.Sense, &e.SinceCh, &e.UntilCh, &e.Status, &short, &e.Source); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
e.AllowShort = short != 0
|
||||
if _, ok := byBook[e.BookID]; !ok {
|
||||
order = append(order, e.BookID)
|
||||
}
|
||||
byBook[e.BookID] = append(byBook[e.BookID], e)
|
||||
at[id] = termLoc{e.BookID, len(byBook[e.BookID]) - 1}
|
||||
}
|
||||
// A truncated iteration is the exact failure this instrument exists to avoid: it would report a
|
||||
// smaller bank as a clean one, and the two are indistinguishable in the output.
|
||||
return byBook, order, at, rows.Err()
|
||||
}
|
||||
|
||||
func attachAliases(db *sql.DB, byBook map[string][]store.GlossaryEntry, at map[int64]termLoc) error {
|
||||
rows, err := db.Query(`SELECT term_id, alias, alias_type FROM glossary_aliases`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var tid int64
|
||||
var alias, aliasType string
|
||||
if err := rows.Scan(&tid, &alias, &aliasType); err != nil {
|
||||
return err
|
||||
}
|
||||
if l, ok := at[tid]; ok {
|
||||
byBook[l.book][l.i].Aliases = append(byBook[l.book][l.i].Aliases, store.GlossaryAlias{Alias: alias, AliasType: aliasType})
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
// trimTo cuts on runes: the messages are Han and Cyrillic, and a byte cut would split a character.
|
||||
func trimTo(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
|
|
@ -10,7 +10,9 @@ import (
|
|||
"testing"
|
||||
|
||||
"textmachine/backend/internal/config"
|
||||
"textmachine/backend/internal/membank"
|
||||
"textmachine/backend/internal/miner"
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
"textmachine/backend/internal/text"
|
||||
)
|
||||
|
|
@ -397,3 +399,189 @@ func TestEvictedBankRowsAreNamed(t *testing.T) {
|
|||
t.Fatalf("the evicted rows must be NAMED, not counted:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConsolidationAgainstTheBankIsReportedAtTheStop is the bank-side disagreement end to end: the seed
|
||||
// already renders 方源 one way (UNSIGNED — the point of the case) and the run consolidates the same
|
||||
// surface differently. Neither of the collapse stage's own checks sees this: both test containment and
|
||||
// skip the equal-source pair, and CanonConflicts is approved-only.
|
||||
func TestConsolidationAgainstTheBankIsReportedAtTheStop(t *testing.T) {
|
||||
var logBuf bytes.Buffer
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
return "方源\tФан Юань", "stop" // the seed calls the same surface «Странник»
|
||||
}
|
||||
return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
// until_ch keeps the seed row off the proposal's UNIQUE key, so this is a disagreement and not a
|
||||
// replacement.
|
||||
seed := "terms:\n - { src: 方源, dst: Странник, status: draft, until_ch: 20 }\n"
|
||||
r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, glossarySeed: seed}))
|
||||
defer r.Close()
|
||||
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
_ = runToSignatureStop(t, r)
|
||||
|
||||
if r.lastTerminology.BankConflicts == 0 {
|
||||
t.Fatalf("a rendering that disagrees with an UNSIGNED row of the same surface must be counted: %+v", r.lastTerminology)
|
||||
}
|
||||
log := logBuf.String()
|
||||
if !strings.Contains(log, "DISAGREES with a row the bank already holds") {
|
||||
t.Fatalf("the run must say it out loud:\n%s", log)
|
||||
}
|
||||
for _, want := range []string{"方源", "Фан Юань", "Странник", "[ch 1..20]"} {
|
||||
if !strings.Contains(log, want) {
|
||||
t.Errorf("the warning must name %q, or an operator cannot tell which two rows collided:\n%s", want, log)
|
||||
}
|
||||
}
|
||||
// The finding has to reach the signature sheet, not only the log: that sheet is where a term is
|
||||
// decided. It must also stay DISTINCT from the §G2 self-contradiction marker — "the run disagreed with
|
||||
// itself" and "the book already calls it something else" are different decisions.
|
||||
var marked *BankStopRow
|
||||
for i, row := range r.lastBankStopRows {
|
||||
if row.Src == "方源" {
|
||||
marked = &r.lastBankStopRows[i]
|
||||
}
|
||||
}
|
||||
if marked == nil {
|
||||
t.Fatalf("premise broken: 方源 is not in the stop table at all: %+v", r.lastBankStopRows)
|
||||
}
|
||||
if len(marked.BankHolds) == 0 {
|
||||
t.Fatalf("the contradicting row must be marked on the signature sheet: %+v", *marked)
|
||||
}
|
||||
if len(marked.Contradicts) != 0 {
|
||||
t.Errorf("this is not a self-contradiction and must not be filed as one: %+v", marked.Contradicts)
|
||||
}
|
||||
if !strings.Contains(marked.BankHolds[0], `unsigned draft "方源"→"Странник"`) || !strings.Contains(marked.BankHolds[0], "[ch 1..20]") {
|
||||
t.Errorf("the mark must name the EXISTING row, its signature and its window: %q", marked.BankHolds[0])
|
||||
}
|
||||
table := renderBankStopTable(r.lastBankStopRows)
|
||||
if !strings.Contains(table, "THE BANK ALREADY HOLDS: unsigned draft") {
|
||||
t.Errorf("the rendered table must carry the mark:\n%s", table)
|
||||
}
|
||||
if strings.Contains(table, "CONTRADICTS this run's own") {
|
||||
t.Errorf("the two markers must stay distinguishable on the sheet:\n%s", table)
|
||||
}
|
||||
// And nowhere else: the signature map's bytes become bank rows, so a diagnostic leaking into it would
|
||||
// turn an observation into content.
|
||||
sigMap := readSignatureMap(t, r)
|
||||
for _, forbidden := range []string{"THE BANK ALREADY HOLDS", "Странник", "[ch 1..20]"} {
|
||||
if strings.Contains(sigMap, forbidden) {
|
||||
t.Errorf("the diagnostic must not enter the signature map (%q):\n%s", forbidden, sigMap)
|
||||
}
|
||||
}
|
||||
// CONTROL for those three zeros. 方源 is deliberately not the control: it is a seed surface, so the
|
||||
// emission skips it and it never reaches the map — which is why the stop TABLE (every candidate),
|
||||
// not the map (signable rows only), is the right address for this mark.
|
||||
if !strings.Contains(sigMap, "青茅山") || !strings.Contains(sigMap, "terms:") {
|
||||
t.Fatalf("control: the signature map must hold this run's emittable terms, or the checks above assert nothing:\n%s", sigMap)
|
||||
}
|
||||
|
||||
// CONTROL: with the seed AGREEING with what the role consolidates, nothing is reported.
|
||||
var quiet bytes.Buffer
|
||||
srv2 := newJSONProvider(&reqRec{}, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
return "方源\tСтранник", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv2.Close()
|
||||
r2 := newVerifyRunner(t, setupMiningStopProject(t, srv2.URL, miningStopOpts{terminology: true, glossarySeed: seed}))
|
||||
defer r2.Close()
|
||||
r2.Log = slog.New(slog.NewTextHandler(&quiet, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
_ = runToSignatureStop(t, r2)
|
||||
if r2.lastTerminology.BankConflicts != 0 {
|
||||
t.Errorf("control: agreeing with the bank is not a contradiction, got %d", r2.lastTerminology.BankConflicts)
|
||||
}
|
||||
if strings.Contains(renderBankStopTable(r2.lastBankStopRows), "THE BANK ALREADY HOLDS") {
|
||||
t.Errorf("control: an agreeing run must leave the sheet unmarked:\n%s", renderBankStopTable(r2.lastBankStopRows))
|
||||
}
|
||||
}
|
||||
|
||||
// TestConsolidatedRowsCarryTheWindowAndTheAliases pins the two fields that decide what
|
||||
// membank.ConsolidationKeyConflicts may say. Dropping either leaves every existing test green while
|
||||
// changing the verdict: without SinceCh a re-mined engine row looks like a fresh contradiction instead of
|
||||
// a replacement, and without the aliases an alias-only collision is invisible.
|
||||
func TestConsolidatedRowsCarryTheWindowAndTheAliases(t *testing.T) {
|
||||
// A banknote candidate whose raw surface differs from its normalized key, so the row's source is not
|
||||
// ambiguous between the two: the row that lands carries the KEY, and the conflict check compares the
|
||||
// store's raw UNIQUE key against it.
|
||||
cands := []terminology.Candidate{
|
||||
{Key: "赵甲", Src: "趙甲", SinceCh: 7, Aliases: []string{"老赵"}},
|
||||
{Key: "神通", Src: "神通"}, // unanswered → no row
|
||||
}
|
||||
rows := consolidatedRows(cands, map[string]string{"赵甲": "Чжао Цзя"})
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("only an answered candidate becomes a row: %+v", rows)
|
||||
}
|
||||
if rows[0].Src != "赵甲" {
|
||||
t.Errorf("the row must carry the source it will LAND with (the key), got %q", rows[0].Src)
|
||||
}
|
||||
if rows[0].SinceCh != 7 {
|
||||
t.Errorf("the candidate's window must reach the check: since_ch=%d, want 7", rows[0].SinceCh)
|
||||
}
|
||||
if len(rows[0].Aliases) != 1 || rows[0].Aliases[0].Alias != "老赵" {
|
||||
t.Errorf("the identity cluster must reach the check as alias keys: %+v", rows[0].Aliases)
|
||||
}
|
||||
|
||||
// Both fields change the verdict of the real check, so neither is decoration.
|
||||
bank := []store.GlossaryEntry{
|
||||
{Src: "赵甲", Dst: "Чжао Цзя-старый", Status: "auto", Source: "mined", SinceCh: 7},
|
||||
{Src: "赵乙", Dst: "Чжао И", Status: "draft", Aliases: []store.GlossaryAlias{{Alias: "老赵"}}},
|
||||
}
|
||||
got := membank.ConsolidationKeyConflicts(rows, bank)
|
||||
if len(got) != 1 || got[0].BankSrc != "赵乙" {
|
||||
t.Fatalf("with the window carried, the same-tuple row is a replacement and only the alias hit is a conflict: %+v", got)
|
||||
}
|
||||
windowless := []store.GlossaryEntry{{Src: rows[0].Src, Dst: rows[0].Dst, Aliases: rows[0].Aliases}}
|
||||
if n := len(membank.ConsolidationKeyConflicts(windowless, bank)); n != 2 {
|
||||
t.Errorf("control: without the window the replacement is reported as a contradiction too, want 2 got %d", n)
|
||||
}
|
||||
aliasless := []store.GlossaryEntry{{Src: rows[0].Src, Dst: rows[0].Dst, SinceCh: rows[0].SinceCh}}
|
||||
if n := len(membank.ConsolidationKeyConflicts(aliasless, bank)); n != 0 {
|
||||
t.Errorf("control: without the aliases the alias-only collision is invisible, want 0 got %d", n)
|
||||
}
|
||||
|
||||
// The findings have to survive the trip to the sheet. They are keyed by the candidate KEY on both
|
||||
// sides; keying either side by the raw surface loses every banknote candidate whose draft spelling
|
||||
// differs from its key — which is the whole population the key choice above exists for.
|
||||
// Straight through the production path: the findings go to the sheet ungrouped, so the key they are
|
||||
// matched on is decided in one place. The candidate is chosen so that the four strings that could
|
||||
// plausibly match it — candidate key, raw surface, firing key (an alias here) and the bank row's own
|
||||
// source — are all different.
|
||||
res := terminologyResult{BankHoldRows: got}
|
||||
sheet := bankStopRows(cands, map[string]string{"赵甲": "Чжао Цзя"}, res)
|
||||
if len(sheet) == 0 || sheet[0].Src != "趙甲" {
|
||||
t.Fatalf("premise broken: the sheet row must be the raw-surface candidate: %+v", sheet)
|
||||
}
|
||||
if len(sheet[0].BankHolds) != 1 {
|
||||
t.Fatalf("the mark must reach the sheet for a candidate whose key differs from its surface: %+v", sheet[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestApprovedNeighboursAdmitsOnlySignedRows pins the anchor's own rule, which until now lived only in a
|
||||
// comment. The neighbours are what the CANON block shows a paid role, so an unsigned row leaking in
|
||||
// changes the bytes that reach the model: the run would be told the book has already decided a term it
|
||||
// has only proposed. Both halves of the filter matter — a signed row with no rendering anchors nothing.
|
||||
func TestApprovedNeighboursAdmitsOnlySignedRows(t *testing.T) {
|
||||
rows := []store.GlossaryEntry{
|
||||
{Src: "方源", Dst: "Фан Юань", Status: "approved"},
|
||||
{Src: "花家", Dst: "клан Хуа", Status: "draft"},
|
||||
{Src: "青茅山", Dst: "гора Цинмао", Status: "auto"},
|
||||
{Src: "元始", Dst: "", Status: "approved"}, // signed but unrendered: nothing to agree with
|
||||
}
|
||||
got := approvedNeighbours(rows)
|
||||
if len(got) != 1 || got[0].Src != "方源" || got[0].Dst != "Фан Юань" {
|
||||
t.Fatalf("only a signed row WITH a rendering may anchor the role: %+v", got)
|
||||
}
|
||||
// CONTROL: the same four rows all signed and rendered give four, so the one above is the filter
|
||||
// answering and not a fixture the function cannot read.
|
||||
all := make([]store.GlossaryEntry, len(rows))
|
||||
for i, e := range rows {
|
||||
e.Status, e.Dst = "approved", "д"
|
||||
all[i] = e
|
||||
}
|
||||
if n := len(approvedNeighbours(all)); n != len(rows) {
|
||||
t.Fatalf("control: every signed rendered row must anchor, want %d got %d", len(rows), n)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,7 +89,9 @@ const (
|
|||
|
||||
// FlagGlossaryMiss is the memory-bank post-check verdict (E1, step 4): an approved
|
||||
// term's src fired in the chunk but no accepted dst form appears in the output — the
|
||||
// model ignored the glossary, or we injected the wrong dst and it obeyed. Emitted
|
||||
// model did not render the term the way the bank told it to. It says nothing about
|
||||
// whether the dst we injected was RIGHT: a wrong rendering the model OBEYED is present
|
||||
// in the output, so it raises no miss and no flag (mempostcheck.go). Emitted
|
||||
// ONLY when the opt-in glossary post-check gate is enabled (config); in the default
|
||||
// flagger mode a miss is recorded in the retrieval-state, not a disposition. NOT
|
||||
// retryable (a same-model retry re-produces the same rendering) and NOT auto-
|
||||
|
|
|
|||
|
|
@ -111,8 +111,10 @@ type BookExport struct {
|
|||
// that do not ship are not consulted, so a $0 draft-wave re-write moves nothing here. A fact of the
|
||||
// store rather than of any process's clock, which is why the book writer uses it as the file's
|
||||
// modification time (two builds of one store then agree byte-for-byte). "" when no unit has a final
|
||||
// row (a book that has shipped nothing). Additive and omitempty. ⚠ «Last written», not «last
|
||||
// changed»: a resume that re-writes an unchanged final row moves it too (store.ChunkStatus.UpdatedAt).
|
||||
// row (a book that has shipped nothing). Additive and omitempty. ⚠ «Last written», not «last changed»:
|
||||
// a unit whose final row is SKIPPED is re-written by every run that passes it, so a book with a flagged
|
||||
// unit reports a new TextModified after a $0 resume that bought nothing. A cleanly shipped book does
|
||||
// not (store.ChunkStatus.UpdatedAt).
|
||||
TextModified string `json:"text_modified,omitempty"`
|
||||
// Chunks is always non-nil (an empty book exports [] not null).
|
||||
Chunks []ChunkExport `json:"chunks"`
|
||||
|
|
|
|||
|
|
@ -395,12 +395,16 @@ type BankStopRow struct {
|
|||
// class to read first. Conf is the role's own stated confidence, which orders the review list and
|
||||
// nothing else (D39.102) — NEGATIVE when the reply carried none, because «the role said it was 0% sure»
|
||||
// is the most important row on the sheet and «the role said nothing» is not a row at all.
|
||||
// Contradicts names THIS RUN's other consolidations the rendering breaks (§G2).
|
||||
// Contradicts names THIS RUN's other consolidations the rendering breaks (§G2). BankHolds names rows
|
||||
// the bank ALREADY carries for the same firing surface with a different rendering. Separate field and
|
||||
// separate marker: "the run disagreed with itself" and "the book already calls it something else" are
|
||||
// different decisions, and one shared line would make the sheet shorter but the decision harder.
|
||||
Conventions int
|
||||
Signals []string
|
||||
Invented bool
|
||||
Conf int
|
||||
Contradicts []string
|
||||
BankHolds []string
|
||||
}
|
||||
|
||||
// BankStopVariant is ONE rendering the drafts produced, kept in its PARTS rather than as the sentence a
|
||||
|
|
@ -445,6 +449,7 @@ func bankStopRows(cands []terminology.Candidate, consolidated map[string]string,
|
|||
Freq: c.Freq, Spread: c.Spread(), Conventions: c.Conventions(),
|
||||
Contexts: c.KWIC, Evidence: c.Evidence,
|
||||
Conf: confOrAbsent(tres.Conf, c.Key), Contradicts: tres.Contradictions[c.Src],
|
||||
BankHolds: bankHoldLabels(tres.BankHoldRows, c),
|
||||
}
|
||||
for i, v := range c.Variants {
|
||||
row.Variants = append(row.Variants, BankStopVariant{Dst: v.Dst, Chunks: v.Chunks, Via: v.Via})
|
||||
|
|
@ -458,6 +463,19 @@ func bankStopRows(cands []terminology.Candidate, consolidated map[string]string,
|
|||
return out
|
||||
}
|
||||
|
||||
// bankHoldLabels picks the findings belonging to one candidate. They are matched on the candidate KEY,
|
||||
// because that is the source consolidatedRows gives the proposal and therefore the conflict's own Src; a
|
||||
// banknote candidate's raw surface is a different string, and its firing key may be an alias.
|
||||
func bankHoldLabels(cols []membank.BankKeyConflict, c terminology.Candidate) []string {
|
||||
var out []string
|
||||
for _, cf := range cols {
|
||||
if cf.Src == c.Key {
|
||||
out = append(out, cf.BankRowLabel())
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// confOrAbsent reads the role's stated confidence for a key, or -1 when the reply carried none. A plain
|
||||
// zero would merge the two, and they are opposites: one is the first row to review, the other is silence.
|
||||
func confOrAbsent(conf map[string]int, key string) int {
|
||||
|
|
@ -486,7 +504,8 @@ func renderBankStopTable(rows []BankStopRow) string {
|
|||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "BANK VERIFICATION TABLE — %d term(s)\n", len(rows))
|
||||
b.WriteString("src · proposed dst · origin · type · freq · variant spread · conventions · confidence ·\n")
|
||||
b.WriteString("why (the ranking factors that won) · contradictions · drafts · evidence · source contexts\n\n")
|
||||
b.WriteString("why (the ranking factors that won) · contradictions (this run's own, and the bank's\n")
|
||||
b.WriteString("existing rows) · drafts · evidence · source contexts\n\n")
|
||||
for _, r := range rows {
|
||||
fmt.Fprintf(&b, "%s\t%s\n", r.Src, dashIfEmpty(r.Dst))
|
||||
fmt.Fprintf(&b, " origin=%s type=%s freq=%d spread=%d conventions=%d", r.Origin, dashIfEmpty(r.Type), r.Freq, r.Spread, r.Conventions)
|
||||
|
|
@ -503,6 +522,9 @@ func renderBankStopTable(rows []BankStopRow) string {
|
|||
if len(r.Contradicts) > 0 {
|
||||
fmt.Fprintf(&b, " CONTRADICTS this run's own: %s\n", strings.Join(r.Contradicts, "; "))
|
||||
}
|
||||
if len(r.BankHolds) > 0 {
|
||||
fmt.Fprintf(&b, " THE BANK ALREADY HOLDS: %s\n", strings.Join(r.BankHolds, "; "))
|
||||
}
|
||||
if len(r.Variants) > 0 {
|
||||
fmt.Fprintf(&b, " drafts: %s\n", strings.Join(r.VariantLabels(), " | "))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1511,6 +1511,9 @@ func TestAutoBankKeyCollisionDoesNotCrashTheRun(t *testing.T) {
|
|||
// approved term AND an unsigned proposal contradicting it on the same firing surface must say so out
|
||||
// loud. The check is a WARNING by design — the unsigned row is engine-produced, and aborting a paid run
|
||||
// over the engine's own proposal would be a self-inflicted outage — so the assertion is on the log.
|
||||
//
|
||||
// The log line must LOCATE the fault as well as announce it: on a book with a thousand terms, a warning
|
||||
// without the firing key and both windows does not say which two rows to reconcile.
|
||||
func TestUnsignedConflictIsReportedByTheRun(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, draftEdit)
|
||||
|
|
@ -1530,11 +1533,13 @@ func TestUnsignedConflictIsReportedByTheRun(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
out := logs.String()
|
||||
if !strings.Contains(out, "contradicts an approved term") {
|
||||
if !strings.Contains(out, "render ONE firing surface differently and at least one is unsigned") {
|
||||
t.Fatalf("the approved-vs-unsigned contradiction must be reported (S14):\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "Источник Фана") {
|
||||
t.Fatalf("the report must name the contradicting rendering:\n%s", out)
|
||||
for _, want := range []string{`approved \"方源\"→\"Фан Юань\"`, `unsigned draft \"方源\"→\"Источник Фана\"`, `firing key \"方源\"`, "[ch 1..end]", "[ch 2..end]"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("the report must name %s so the operator can reconcile the two rows:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
93
backend/internal/pipeline/resume_updatedat_test.go
Normal file
93
backend/internal/pipeline/resume_updatedat_test.go
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestOrdinaryResumeMovesOnlyTheSkippedRow pins which rows a $0 resume writes. store.ChunkStatus.UpdatedAt
|
||||
// is read as the book's modification time, and a resume that changes nothing still writes — but only
|
||||
// where a stage was SKIPPED: a skip is not a stored verdict a resume can serve, so the flagged upstream
|
||||
// row is served from its checkpoint and the skip is re-derived and re-upserted (waverun.go —
|
||||
// recordSkippedStages for a flagged edit unit, the flagged branch of runStageSequence for a multi-stage
|
||||
// wave). Both halves are asserted: a run where EVERY row moved would be a clock, not a write.
|
||||
func TestOrdinaryResumeMovesOnlyTheSkippedRow(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) {
|
||||
return "Извините, я не могу перевести это.", "stop" // soft refusal → the edit stage is skipped
|
||||
}
|
||||
|
||||
return draftEdit(body)
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА"})
|
||||
ctx := context.Background()
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := statusStamps(t, r1)
|
||||
callsAfterFirstRun := rec.count()
|
||||
r1.Close()
|
||||
|
||||
// datetime('now') has one-second resolution: a rewrite inside the same second is indistinguishable from
|
||||
// no rewrite, and the test would pass for the wrong reason.
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
|
||||
r2 := newRunner(t, bookPath)
|
||||
defer r2.Close()
|
||||
if _, err := r2.TranslateBook(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := statusStamps(t, r2)
|
||||
|
||||
if rec.count() != callsAfterFirstRun {
|
||||
t.Fatalf("premise broken: the resume must buy nothing, calls went %d → %d", callsAfterFirstRun, rec.count())
|
||||
}
|
||||
if len(before) != 4 {
|
||||
t.Fatalf("premise broken: want 4 chunk_status rows (2 units × 2 stages), got %d: %v", len(before), before)
|
||||
}
|
||||
const skipped = "ch1/chunk0/edit"
|
||||
if before[skipped].disposition != string(DispSkipped) {
|
||||
t.Fatalf("premise broken: %s must be the skipped row, got %q", skipped, before[skipped].disposition)
|
||||
}
|
||||
if after[skipped].updatedAt == before[skipped].updatedAt {
|
||||
t.Errorf("a SKIPPED row is re-derived and re-written by every run past it: %s stayed at %q",
|
||||
skipped, before[skipped].updatedAt)
|
||||
}
|
||||
for key, b := range before {
|
||||
if key == skipped {
|
||||
continue
|
||||
}
|
||||
if after[key].updatedAt != b.updatedAt {
|
||||
t.Errorf("an ordinary resume must not write an %s row: %s moved %q → %q",
|
||||
b.disposition, key, b.updatedAt, after[key].updatedAt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type statusStamp struct{ disposition, updatedAt string }
|
||||
|
||||
// statusStamps maps "chapter/chunk/stage" to each row's disposition and write time.
|
||||
func statusStamps(t *testing.T, r *Runner) map[string]statusStamp {
|
||||
t.Helper()
|
||||
rows, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := make(map[string]statusStamp, len(rows))
|
||||
for _, cs := range rows {
|
||||
key := chunkStageKey(cs.Chapter, cs.ChunkIdx, cs.Stage)
|
||||
out[key] = statusStamp{disposition: cs.Disposition, updatedAt: cs.UpdatedAt}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func chunkStageKey(chapter, chunkIdx int, stage string) string {
|
||||
return fmt.Sprintf("ch%d/chunk%d/%s", chapter, chunkIdx, stage)
|
||||
}
|
||||
|
|
@ -65,13 +65,12 @@ func (r *Runner) seedGlossary(ctx context.Context) error {
|
|||
r.Log.WarnContext(ctx, "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)",
|
||||
"collisions", strings.Join(cols, "; "))
|
||||
}
|
||||
// S14: the diagnostic the approved-only checks structurally cannot make — an approved term and an
|
||||
// unsigned one contradicting each other on the SAME firing surface. In the auto mode both are injected
|
||||
// (canon + working version), so the model is handed two answers to one question; the trust gate keeps
|
||||
// the approved one authoritative, but the collision is exactly the silent-degradation source the auto
|
||||
// mode needs visible.
|
||||
// S14: the diagnostic the approved-only checks cannot make — two rows contradicting each other on the
|
||||
// SAME firing surface, at least one of them unsigned. Both are injected, and which is signed changes
|
||||
// nothing about that (D39.104 п.2: no fork by status). Where a signature exists the trust gate still
|
||||
// decides precedence; an unsigned×unsigned pair has none, so it is simply two renderings of one term.
|
||||
if cols := membank.UnverifiedKeyConflicts(rows); len(cols) > 0 {
|
||||
r.Log.WarnContext(ctx, "bank: an UNSIGNED row contradicts an approved term on the same firing surface (the approved rendering keeps precedence; reconcile the seed or decline the proposal)",
|
||||
r.Log.WarnContext(ctx, "bank: two rows render ONE firing surface differently and at least one is unsigned (the deterministic matcher cannot pick between them; reconcile the seed, the delta or the proposal)",
|
||||
"book", r.Book.BookID, "conflicts", strings.Join(cols, "; "))
|
||||
}
|
||||
r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"textmachine/backend/internal/config"
|
||||
"textmachine/backend/internal/lang"
|
||||
"textmachine/backend/internal/llm"
|
||||
"textmachine/backend/internal/membank"
|
||||
"textmachine/backend/internal/miner"
|
||||
"textmachine/backend/internal/obs"
|
||||
"textmachine/backend/internal/store"
|
||||
|
|
@ -103,6 +104,13 @@ type terminologyResult struct {
|
|||
// the same finding per key, so the stop table can mark the rows instead of printing a bare total.
|
||||
SelfConflicts int
|
||||
Contradictions map[string][]string
|
||||
// BankConflicts counts consolidated renderings that disagree with a row the bank already holds for the
|
||||
// same firing surface — the shape neither check above can see (membank.ConsolidationKeyConflicts).
|
||||
// BankHoldRows carries the findings themselves, ungrouped: bankStopRows matches them to its rows, so
|
||||
// the key they are looked up by is decided in one place instead of once on each side of a map. Kept
|
||||
// apart from Contradictions because the two are different decisions for the owner.
|
||||
BankConflicts int
|
||||
BankHoldRows []membank.BankKeyConflict
|
||||
// Conf is the role's own stated confidence per key. It sorts the review list «least sure first» and does
|
||||
// nothing else — never a weight, a threshold or a cross-model comparison (D39.102).
|
||||
Conf map[string]int
|
||||
|
|
@ -262,7 +270,7 @@ func (r *Runner) buildBankCandidates(mined []miner.Term, observed []terminology.
|
|||
// the initial candidate build and the §2 post-classify re-score share ONE definition and can never disagree
|
||||
// about how a variant is scored.
|
||||
func (r *Runner) scoreOpts() terminology.ScoreOpts {
|
||||
opts := terminology.ScoreOpts{Neighbours: r.approvedNeighbours()}
|
||||
opts := terminology.ScoreOpts{Neighbours: approvedNeighbours(r.glossaryRows())}
|
||||
if r.pack != nil {
|
||||
pack := r.pack
|
||||
opts.Conformance = func(dst, typ string) float64 {
|
||||
|
|
@ -275,13 +283,26 @@ func (r *Runner) scoreOpts() terminology.ScoreOpts {
|
|||
return opts
|
||||
}
|
||||
|
||||
// approvedNeighbours is the already-signed bank, as the §C2-3 "agreement with approved siblings" anchor.
|
||||
// Only APPROVED rows qualify: an unverified row agreeing with an unverified row is not evidence.
|
||||
func (r *Runner) approvedNeighbours() []terminology.Neighbour {
|
||||
// glossaryRows is the book's stored bank, every row with its status and window. The anchor below narrows
|
||||
// it to the signed rows; ConsolidationKeyConflicts needs it whole, since an unsigned row owes one
|
||||
// rendering too. A read failure degrades rather than aborts — everything built on it improves a
|
||||
// consolidation, none of it is a precondition — but it is LOGGED: a degraded read and a clean bank
|
||||
// produce the same zero in every counter downstream.
|
||||
func (r *Runner) glossaryRows() []store.GlossaryEntry {
|
||||
rows, err := r.Store.GlossaryForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
return nil // an anchor is an improvement, never a precondition — a read failure degrades, not aborts
|
||||
// Called from the scoring pass as well as the role, so it can repeat within one run; a repeated
|
||||
// line is cheaper than a zero that reads as a clean bank.
|
||||
r.Log.Warn("terminology: could not read the bank — whatever this call feeds goes silent (the canon anchor, the bank conflict check, or both), and its zero then means «not asked» rather than «nothing found»",
|
||||
"book", r.Book.BookID, "err", err)
|
||||
return nil
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// approvedNeighbours is the already-signed bank, as the §C2-3 "agreement with approved siblings" anchor.
|
||||
// Only APPROVED rows qualify: an unverified row agreeing with an unverified row is not evidence.
|
||||
func approvedNeighbours(rows []store.GlossaryEntry) []terminology.Neighbour {
|
||||
out := make([]terminology.Neighbour, 0, len(rows))
|
||||
for _, e := range rows {
|
||||
if e.Status == "approved" && e.Dst != "" {
|
||||
|
|
@ -361,8 +382,10 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
}
|
||||
}
|
||||
|
||||
// The signed bank is read ONCE for the whole role, not per batch: it is the same law for every batch.
|
||||
canon := r.approvedNeighbours()
|
||||
// The bank is read ONCE for the whole role, not per batch: it is the same law for every batch. The
|
||||
// signed rows are the anchor the model is shown; the whole set is what the conflict checks below read.
|
||||
bank := r.glossaryRows()
|
||||
canon := approvedNeighbours(bank)
|
||||
plan := bankRolePlan{
|
||||
role: roleTerminologist, budgetUSD: r.Pipeline.Gates.Terminology.BudgetUSD,
|
||||
messages: func(b []terminology.Candidate) ([]llm.Message, error) { return r.terminologyMessages(b, canon) },
|
||||
|
|
@ -456,6 +479,15 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
r.Log.WarnContext(ctx, "terminology: consolidations of THIS run contradict each other — a compound's rendering drops the rendering the same reply gave its own part; they stay unverified and are review rows at the stop",
|
||||
"book", r.Book.BookID, "conflicts", len(self), "terms", strings.Join(named, "; "))
|
||||
}
|
||||
// The BANK side: a rendering this run consolidated for a surface the bank already renders differently,
|
||||
// which neither check above reaches. Neither side is asked for a signature — one does not change what
|
||||
// the model is shown (D39.104 п.2). $0, evidence-side, never a gate.
|
||||
if cols := membank.ConsolidationKeyConflicts(consolidatedRows(cands, out), bank); len(cols) > 0 {
|
||||
res.BankConflicts = len(cols)
|
||||
res.BankHoldRows = cols
|
||||
r.Log.WarnContext(ctx, "terminology: a consolidated rendering DISAGREES with a row the bank already holds for the same firing surface — the book calls this term something else, and the sheet at the stop is where that is decided (whether both renderings also reach a wire depends on the emission and the glossary UNIQUE key; this does not claim they will)",
|
||||
"book", r.Book.BookID, "conflicts", len(cols), "terms", strings.Join(membank.ConflictMessages(cols), "; "))
|
||||
}
|
||||
// The $0 label screen (§2 warm-run hygiene): a name/place row whose rendering was clearly TRANSLATED is a
|
||||
// label/rendering disagreement worth a human's eye. It is a review FLAG, never a gate, and explicitly not
|
||||
// a safety net for the transliteration harm — the classifier phase is what prevents that (see
|
||||
|
|
@ -477,6 +509,7 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
"classify_batches_dropped", res.ClassifyBatchesDropped)
|
||||
}
|
||||
r.Log.InfoContext(ctx, "terminology finished", "book", r.Book.BookID,
|
||||
"bank_conflicts", res.BankConflicts,
|
||||
"batches_dropped", res.BatchesDropped, "classify_batches_dropped", res.ClassifyBatchesDropped,
|
||||
"consolidated", res.Consolidated, "declined", res.Declined, "unanswered", res.Unanswered,
|
||||
"reclassified", res.Reclassified, "bad_lines", res.BadLines, "off_language", res.OffLanguage,
|
||||
|
|
@ -870,6 +903,41 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban
|
|||
return run, nil
|
||||
}
|
||||
|
||||
// consolidatedRows turns this run's consolidation into the rows those renderings would become, which is
|
||||
// what ConsolidationKeyConflicts reads: the candidate's surface, its identity-cluster aliases and the
|
||||
// window a mined row carries (first chapter of appearance, no end). An unanswered candidate carries no
|
||||
// rendering, becomes no row, and contradicts nothing.
|
||||
//
|
||||
// Both extra fields are load-bearing for the check: the window decides whether an existing row is a
|
||||
// replacement rather than a disagreement, and the aliases are the other surfaces a landed row fires on.
|
||||
// Built for every answered candidate, not only those the emission keeps — a dropped candidate still
|
||||
// reaches the review sheet (not the signature map, a different document), so its disagreement is read.
|
||||
func consolidatedRows(cands []terminology.Candidate, consolidated map[string]string) []store.GlossaryEntry {
|
||||
if len(consolidated) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]store.GlossaryEntry, 0, len(consolidated))
|
||||
for _, c := range cands {
|
||||
dst := consolidated[c.Key]
|
||||
if dst == "" {
|
||||
continue
|
||||
}
|
||||
// c.Key, not c.Src, and it is the landing source on BOTH emission paths — the check compares the
|
||||
// store's raw UNIQUE key, so it must be given the string the row will actually hold. A banknote
|
||||
// candidate carries the surface as the draft wrote it and lands as the key
|
||||
// (mining.reverseSectionTerms). A mined candidate lands with the MINER's surface, which is a
|
||||
// substring of already-normalized chunk text and therefore equals its own key — that equality is
|
||||
// text.NormalizeSourceKey's idempotence, pinned by TestNormalizeSourceKeyIsIdempotent, because
|
||||
// without it this line would compare a landing row against a source it does not have.
|
||||
e := store.GlossaryEntry{Src: c.Key, Dst: dst, SinceCh: c.SinceCh}
|
||||
for _, a := range c.Aliases {
|
||||
e.Aliases = append(e.Aliases, store.GlossaryAlias{Alias: a})
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// attachConsolidatedDst stamps the terminologist's renderings onto the mined terms, which is what selects
|
||||
// the emission MODE in miner.DeltaYAML (§C2-7). Terms the role did not answer are left untouched.
|
||||
func attachConsolidatedDst(mined []miner.Term, consolidated map[string]string) []miner.Term {
|
||||
|
|
|
|||
|
|
@ -691,9 +691,14 @@ func (r *Runner) runEditUnit(ctx context.Context, editSnapshot string, unit edit
|
|||
var editSel membank.Selection
|
||||
injectionByRole := map[string]string{}
|
||||
if r.memory != nil {
|
||||
// FRESH Select over the WHOLE-unit source over the ENRICHED bank (§1(б)/F3): NOT the the precompute pass per-chunk
|
||||
// memSel (that was over the base bank). Sticky is degenerate at unit scope (a unit == a chapter or a
|
||||
// sub-chapter split, so the intra-chapter sticky window does not apply) → nil sticky_prev.
|
||||
// FRESH Select over the WHOLE-unit source over the ENRICHED bank (§1(б)/F3): NOT the precompute
|
||||
// pass's per-chunk memSel, which was taken over the BASE bank.
|
||||
//
|
||||
// sticky_prev is nil, and that is a LOSS, not a degenerate case: assignEditUnits closes a unit
|
||||
// INSIDE a chapter (chunk/chunker.go) while the sticky window resets only at a CHAPTER boundary
|
||||
// (precomputeSticky, wave.go), so every unit after the first in a chapter is selected with no
|
||||
// memory of the rows its predecessors carried. On the stand book 1962 of its 2283 chapters are
|
||||
// multi-unit (backlog row 321). Whether the loss produces drift in the text is not measured.
|
||||
editSel = r.memory.Select(leader.Text, unit.Chapter, nil, r.Pipeline.Context.GlossaryTokenBudget)
|
||||
// Render the per-role injection from the ENRICHED unit selection via the registry (D39 layer 7): the
|
||||
// editor gets its CONFIRMED-dst constraint block; other roles' blocks are rendered but consumed
|
||||
|
|
|
|||
|
|
@ -53,8 +53,19 @@ type ChunkStatus struct {
|
|||
// `YYYY-MM-DD HH:MM:SS`, NOT NULL since the table's first schema — migrate.go). It is read, never
|
||||
// written from here: UpsertChunkStatus stamps it itself. The book writer takes the newest one as the
|
||||
// book's modification time — a fact of the store rather than of the clock of whichever process builds
|
||||
// the file, which is what keeps two builds of one store byte-identical. A resume that re-writes an
|
||||
// unchanged row moves it too, so it is «last written», not «last changed».
|
||||
// the file, which is what keeps two builds of one store byte-identical.
|
||||
//
|
||||
// «Last written», not «last changed». Two ordinary $0 operations move it with nothing changed:
|
||||
// - a SKIPPED row is re-written by every run that passes it. A skip is not a stored verdict a resume
|
||||
// can serve: the flagged upstream row is served from its checkpoint, and the skip is re-derived
|
||||
// from it and upserted again — by recordSkippedStages for a flagged edit unit, and by the flagged
|
||||
// branch of runStageSequence where a wave holds two or more stages (both pipeline/waverun.go);
|
||||
// - the --resnapshot RE-PIN rewrites an unchanged row to carry the new snapshot id
|
||||
// (pipeline/stagerun.go).
|
||||
// An `ok` or `flagged` row is not touched by an ordinary resume — it is read and its checkpoint served,
|
||||
// with no write (pipeline/resume.go). So a cleanly shipped book keeps every timestamp across a re-run
|
||||
// and a book with a flagged unit does not. Pinned by
|
||||
// pipeline.TestOrdinaryResumeMovesOnlyTheSkippedRow.
|
||||
UpdatedAt string
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -198,3 +198,27 @@ func TestNormalizeStripsDefaultIgnorable(t *testing.T) {
|
|||
t.Errorf("soft hyphen on the target side = %q, want %q (must fold to '-', not be stripped)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormalizeSourceKeyIsIdempotent pins a property the bank's conflict check now rests on. A MINED row
|
||||
// lands with the miner's own surface, which is a substring of already-normalized chunk text, while the
|
||||
// check compares against the candidate KEY — and the two are the same string only while re-normalizing
|
||||
// changes nothing. A normalizer that stopped being idempotent would make the check compare a landing row
|
||||
// against a source it does not have, silently, on the mined half of every bank.
|
||||
func TestNormalizeSourceKeyIsIdempotent(t *testing.T) {
|
||||
cases := []string{"趙甲", "赵甲", "強敵", "ゼロ", "ヴァン", "AB", "方源", "青茅山", "元始空窍", "老赵", "FULL幅", "a b"}
|
||||
changed := 0
|
||||
for _, s := range cases {
|
||||
once := NormalizeSourceKey(s)
|
||||
if twice := NormalizeSourceKey(once); twice != once {
|
||||
t.Errorf("NormalizeSourceKey is not idempotent on %q: %q → %q", s, once, twice)
|
||||
}
|
||||
if once != s {
|
||||
changed++
|
||||
}
|
||||
}
|
||||
// CONTROL: the normalizer really does work on this set, so the idempotence above is a property and
|
||||
// not a table of inputs it happens to leave alone.
|
||||
if changed == 0 {
|
||||
t.Fatalf("control: no probe was changed by normalization, so idempotence proves nothing")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue