From e48bca27df853df4e7e8ab40ad114eb0a9c8fe52 Mon Sep 17 00:00:00 2001 From: heaven Date: Tue, 8 Sep 2026 07:34:22 +0300 Subject: [PATCH] Take in the engine zone's work on operator truth and the surfaces the bank already renders, as the zone delivered it --- backend/cmd/tmctl/render.go | 10 +- backend/cmd/tmmutate/mutations.json | 383 ++++++++++++ backend/internal/membank/memory_e1_test.go | 8 +- backend/internal/membank/mempostcheck.go | 19 +- backend/internal/membank/memseed.go | 64 +- backend/internal/membank/memseed_test.go | 28 +- .../membank/postcheckdenominator_test.go | 99 ++++ backend/internal/pipeline/bankmaterialize.go | 10 +- .../pipeline/bankoperatortruth_test.go | 126 ++++ backend/internal/pipeline/banksettled_test.go | 561 ++++++++++++++++++ backend/internal/pipeline/mining.go | 88 ++- .../internal/pipeline/miningstop_join_test.go | 19 +- .../pipeline/operatormessages_test.go | 252 ++++++++ backend/internal/pipeline/quality.go | 4 +- .../pipeline/resume_updatedat_test.go | 92 +++ backend/internal/pipeline/runner_test.go | 8 + .../pipeline/stopsheetfindings_test.go | 181 ++++++ backend/internal/pipeline/terminologist.go | 237 +++++++- .../pipeline/testdata/operator-messages.txt | 121 ++++ backend/internal/store/chunkstatus.go | 8 + backend/internal/store/glossary.go | 7 +- backend/internal/store/ledger.go | 21 + backend/internal/store/migrate.go | 4 + backend/internal/terminology/terminology.go | 29 +- docs/PROGRESS.md | 27 + 25 files changed, 2317 insertions(+), 89 deletions(-) create mode 100644 backend/internal/membank/postcheckdenominator_test.go create mode 100644 backend/internal/pipeline/bankoperatortruth_test.go create mode 100644 backend/internal/pipeline/banksettled_test.go create mode 100644 backend/internal/pipeline/operatormessages_test.go create mode 100644 backend/internal/pipeline/stopsheetfindings_test.go create mode 100644 backend/internal/pipeline/testdata/operator-messages.txt diff --git a/backend/cmd/tmctl/render.go b/backend/cmd/tmctl/render.go index dc9853c6..e5cb14b0 100644 --- a/backend/cmd/tmctl/render.go +++ b/backend/cmd/tmctl/render.go @@ -500,9 +500,13 @@ func renderQuality(w io.Writer, q *pipeline.QualityReport) error { if q.UnverifiedShown > 0 { followed = fmt.Sprintf(" (%.0f%%)", 100*float64(q.UnverifiedFollowed)/float64(q.UnverifiedShown)) } - // "row-showings", not "units": the counter increments once per unsigned ROW put in front of the - // model, and one unit can show several — labelling it a unit count overstates the exposure. - fmt.Fprintf(w, "UNSIGNED BANK: terms=%d · row-showings=%d · model followed=%d%s — proposals, not canon; review the signature map beside the DB and decide via `tmctl bank-apply`\n", + // A ROW count, not a unit count: one unit can carry several, and labelling it a unit count would + // understate the exposure. "checked", not "shown": the counter increments once per unsigned row whose + // key FIRED in that unit, and a sticky carry is put in front of the model without being counted — + // its src is in the previous chunk, so the output is not expected to answer for it + // (membank.PostcheckResult). The follow percentage is a fraction OF THIS, so calling it a count of + // showings would have the operator read a ratio other than the one printed. + fmt.Fprintf(w, "UNSIGNED BANK: terms=%d · rows-checked=%d · model followed=%d%s — proposals, not canon; review the signature map beside the DB and decide via `tmctl bank-apply`\n", q.UnsignedBankTerms, q.UnverifiedShown, q.UnverifiedFollowed, followed) } // The pack-19 flaggers (D39.55). They were aggregated into the report struct from the start but no diff --git a/backend/cmd/tmmutate/mutations.json b/backend/cmd/tmmutate/mutations.json index dfd745b1..091340e4 100644 --- a/backend/cmd/tmmutate/mutations.json +++ b/backend/cmd/tmmutate/mutations.json @@ -2907,5 +2907,388 @@ "replace": "\t\tif p.Dst == \"\" {\n\t\t\tp.Dst = row.Src\n\t\t}\n\t\tout = append(out, p)" } ] + }, + { + "id": "BANKTRUTH-a-drop-names-its-holders-signature", + "why": "the drop an auto-bank key collision leaves calls its holder 'the signed' whatever the holder's status, and the operator signs a bank on the strength of that line. Every fixture in the package held the key with an APPROVED row, so the word was true of all of them and the lie was invisible", + "package": "./internal/pipeline/", + "run": "TestADropNamesItsHoldersSignature", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/mining.go", + "find": "membank.StatusLabel(prior.Status)", + "replace": "\"signed\"" + } + ] + }, + { + "id": "BANKTRUTH-the-drop-headline-promises-a-signature", + "why": "the headline over the dropped rows promises 'the signed term wins' over a set of drops whose holders may every one of them be unsigned. It is the line an operator reads first and often the only one, because the per-row detail is a semicolon-joined tail", + "package": "./internal/pipeline/", + "run": "TestTheDropHeadlineDoesNotPromiseASignature|TestEveryOperatorMessageIsCatalogued", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/bankmaterialize.go", + "find": "in.remark(\"auto-bank rows dropped: their key is already held by a row gathered earlier (the holder wins, whatever its status; each drop below names the holder's signature)\",", + "replace": "in.remark(\"auto-bank rows dropped: their key is already held by a signed term (the signed term wins)\"," + } + ] + }, + { + "id": "BANKTRUTH-a-sticky-carry-counted-as-judged", + "why": "the unsigned-wire denominator counts rows the chunk can be JUDGED on, which is narrower than the rows the model was shown: a sticky carry is in the injection block but its src is in the previous chunk. Counting it makes every pronominal chunk read as a channel nobody follows, and the follow RATIO then means something other than what four doc comments said it meant", + "package": "./internal/membank/", + "run": "TestTheUnverifiedDenominatorIsNotACountOfRowsShown", + "battery": true, + "edits": [ + { + "file": "internal/membank/mempostcheck.go", + "find": "if p.Sticky { // sticky context is not expected in the output", + "replace": "if false { // sticky context is not expected in the output" + } + ] + }, + { + "id": "OPMSG-an-operator-message-reworded-off-the-catalogue", + "why": "an operator warning is a sentence somebody acts on, and the ones that rot are the ones nothing else reads: a message that STOPS a run is held by the test asserting the stop, a warning by nobody. The catalogue is what makes a wording change a reviewed diff instead of a silent one; this planting is the ordinary way it is defeated", + "package": "./internal/pipeline/", + "run": "TestEveryOperatorMessageIsCatalogued", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/seeding.go", + "find": "r.Log.WarnContext(ctx, \"voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)\",", + "replace": "r.Log.WarnContext(ctx, \"voice profile windows leave chapters uncovered\"," + } + ] + }, + { + "id": "SEAM-confidence-keyed-by-the-raw-surface", + "why": "the role's stated confidence is looked up by the candidate KEY. Keyed by the raw surface it silently takes the confidence of whichever candidate happens to key as that surface, or none — and every fixture in the module used simplified spellings, where key and surface are the same string and the choice cannot be wrong", + "package": "./internal/pipeline/", + "run": "TestEveryStopSheetFindingReachesTheRowItBelongsTo|TestTheStopSheetJoinIsNotFooledByTheRawSurface", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/mining.go", + "find": "if v, ok := t.Conf[c.Key]; ok {", + "replace": "if v, ok := t.Conf[c.Src]; ok {" + } + ] + }, + { + "id": "SEAM-self-contradiction-matched-by-the-raw-surface", + "why": "the run's own self-contradiction is matched to its sheet row by the candidate KEY. Matched by the raw surface, every traditional or katakana spelling loses its mark — the population the fold exists for, and the one no fixture had", + "package": "./internal/pipeline/", + "run": "TestEveryStopSheetFindingReachesTheRowItBelongsTo", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/mining.go", + "find": "if cf.Key == c.Key {", + "replace": "if cf.Key == c.Src {" + } + ] + }, + { + "id": "SEAM-bank-hold-matched-by-the-raw-surface", + "why": "the bank's existing rendering is matched to its sheet row by the candidate KEY, because that is the source consolidatedRows gives the proposal. Matched by the raw surface, a banknote candidate's mark goes to another row or nowhere", + "package": "./internal/pipeline/", + "run": "TestEveryStopSheetFindingReachesTheRowItBelongsTo|TestTheStopSheetJoinIsNotFooledByTheRawSurface", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/mining.go", + "find": "if cf.Src == c.Key {", + "replace": "if cf.Src == c.Src {" + } + ] + }, + { + "id": "SEAM-the-conflict-carries-the-raw-surface-as-its-key", + "why": "a consolidation conflict is FOUND by comparing candidate keys, so the key is what it must carry back. Carrying the raw surface instead re-introduces the two-sided key choice the ungrouped carrier was built to end, one file away from where it is read", + "package": "./internal/pipeline/", + "run": "TestEveryStopSheetFindingReachesTheRowItBelongsTo", + "battery": true, + "edits": [ + { + "file": "internal/terminology/terminology.go", + "find": "out = append(out, ConsolidationConflict{Key: c.Key, Src: c.Src", + "replace": "out = append(out, ConsolidationConflict{Key: c.Src, Src: c.Src" + } + ] + }, + { + "id": "STATUSWRITE-the-pre-decided-skip-claims-the-other-writers-cause", + "why": "two different writers put a `skipped` row in chunk_status and which one fires is decided by the wave layout, not by anything the row records. The DETAIL each writes is the only durable trace of which path produced it — and the author of the pack that documented this named the wrong mechanism three times running", + "package": "./internal/pipeline/", + "run": "TestASkippedRowSaysWHICHWriterWroteIt", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/waverun.go", + "find": "detail := fmt.Sprintf(\"skipped: a member draft chunk of this edit unit was flagged (%s)\", flagReason)", + "replace": "detail := fmt.Sprintf(\"skipped: an upstream stage was flagged (%s)\", flagReason)" + } + ] + }, + { + "id": "STATUSWRITE-the-multi-stage-skip-claims-the-other-writers-cause", + "why": "the mirror of the entry above, on the writer that no fixture in the package reached until 08.09: breaking it survived the WHOLE package while breaking its sibling turned six tests red, so half of what the doc asserted had no witness at all", + "package": "./internal/pipeline/", + "run": "TestASkippedRowSaysWHICHWriterWroteIt", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/waverun.go", + "find": "detail := fmt.Sprintf(\"skipped: an upstream stage was flagged (%s)\", flagReason)", + "replace": "detail := fmt.Sprintf(\"skipped: a member draft chunk of this edit unit was flagged (%s)\", flagReason)" + } + ] + }, + { + "id": "BANKSHEET-the-tuple-skip-swallows-a-seed-rows-disagreement", + "why": "a proposal sharing a bank row's UNIQUE tuple is skipped only where it can LAND on that tuple. Over a row the owner wrote it never lands — the emission drops it — so the bank keeps its rendering and the disagreement is permanent. Unconditioned, the skip hides the commonest shape there is: a hand-written seed row takes the default window and no sense, and so does a banknote proposal", + "package": "./internal/pipeline/", + "run": "TestTheCommonestSeedShapeStillReachesTheSigningSheet", + "battery": true, + "edits": [ + { + "file": "internal/membank/memseed.go", + "find": "if IsEngineUnsigned(b) && b.Src == c.Src && b.Sense == c.Sense", + "replace": "if b.Src == c.Src && b.Sense == c.Sense" + } + ] + }, + { + "id": "BANKSHEET-the-one-carrier-of-engine-unsigned-inverted", + "why": "three places have to agree on what counts as the ENGINE's own unsigned row — the emission's seed-surface guard, the paid path's filter and the bank's tuple skip — and they agree by sharing one predicate. Inverting it is what a second spelling of the rule would eventually amount to", + "package": "./internal/membank/", + "run": "TestConsolidationKeyConflicts|TestConsolidationTupleSkipMirrorsTheStoreKey", + "battery": true, + "edits": [ + { + "file": "internal/membank/memseed.go", + "find": "return e.Source == \"mined\" && e.Status != \"approved\"", + "replace": "return e.Source == \"mined\" && e.Status == \"approved\"" + } + ] + }, + { + "id": "MONEY-a-settled-candidate-is-paid-for-anyway", + "why": "the role and the classifier were being paid for surfaces the bank already settles and the emission then throws away. The filter is the saving; without it every counter in the report still reads clean, because nothing downstream can tell a question that was asked from one that was not", + "package": "./internal/pipeline/", + "run": "TestABankSettledCandidateIsNotPaidFor", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\t\tif bankSettles(settled, c) {", + "replace": "\t\tif false && bankSettles(settled, c) {" + } + ] + }, + { + "id": "MONEY-the-filter-skips-a-candidate-whose-drafts-disagree", + "why": "the role is asked about banked surfaces ON PURPOSE: its answer is the mark that tells the owner the book already calls the term something else. Measured on the corpus, 366 of the 439 candidates the owner's seed holds had a draft that disagreed — so a filter that drops the disagreement half buys its saving by silencing them, and the sheet still looks complete", + "package": "./internal/pipeline/", + "run": "TestADisagreeingBankedCandidateIsStillPaidFor", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\t\tif text.NormalizeTargetForm(v.Dst) != want {", + "replace": "\t\tif false && text.NormalizeTargetForm(v.Dst) != want {" + } + ] + }, + { + "id": "MONEY-the-filter-reads-a-bank-that-moves-between-runs", + "why": "the filter reads the SEED surfaces, not the whole bank. Reading the whole bank folds in the engine's own auto-bank rows, which this very run rewrites — so the candidate set differs between run one and run two and the bank moves with it. Measured consequence is bigger than the pass: the second run refuses at the edit wave with «what moved: memory_version», i.e. the already-paid EDIT checkpoints are invalidated and the run demands --resnapshot. ⚠ Only an AUTO-CONTINUING pair of runs can see it: a run that stops at the bank boundary writes no auto-bank, and the first version of this pin stopped and let the planting through", + "package": "./internal/pipeline/", + "run": "TestAResumeFindsItsCheckpointsAfterTheFilter", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "settled := bankSettledSurfaces(unsignedEngineSurfaces(r.glossaryRows()))", + "replace": "settled := bankSettledSurfaces(r.glossaryRows())" + } + ] + }, + { + "id": "MONEY-the-filter-copies-the-candidates-and-loses-their-mutations", + "why": "the classifier and the scorer MUTATE candidates in place, and the caller renders the sheet and the delta from the slice it passed in. Applying them to the filtered COPY instead leaves the sheet carrying heuristic types and stale rankings with every counter reading clean — the defect the filter's own first implementation had, invisible to every fixture in the package because they all run with the classifier off", + "package": "./internal/pipeline/", + "run": "TestTheClassifiersTypeStillReachesTheSheetThroughTheFilter", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\t\tres.Reclassified = applyTypes(cands, classified)", + "replace": "\t\tres.Reclassified = applyTypes(paid, classified)" + } + ] + }, + { + "id": "MONEY-the-rescoring-is-applied-to-the-filtered-copy", + "why": "applyTypes and ScoreVariants are TWO in-place mutations of the candidates, and the caller renders the sheet from the slice it passed in. Pinning one leaves the other free: applied to the filtered copy, the re-scoring never reaches the sheet, and the row keeps a `conform` ranking factor the classification has just retired — a reason printed to the operator that is no longer true of the term. Measured 09.09: correct code gives no signals on that row, the planting gives [conform], and the whole package stays green", + "package": "./internal/pipeline/", + "run": "TestTheClassifiersTypeStillReachesTheSheetThroughTheFilter", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\t\tfor i := range cands {\n\t\t\tterminology.ScoreVariants(&cands[i], opts)\n\t\t}", + "replace": "\t\tfor i := range paid {\n\t\t\tterminology.ScoreVariants(&paid[i], opts)\n\t\t}" + } + ] + }, + { + "id": "MONEY-the-one-time-cost-warning-reads-this-runs-own-checkpoints", + "why": "the warning about a one-time re-consolidation needs BOTH halves and each is knowable at only one moment: that the book had paid BEFORE this run, and that this run paid anyway. Asked after the passes, the probe sees the checkpoints THIS run just settled, so a fresh book paying for the first time is told it paid twice. That is the second viton of one bug — the version before it warned on every ordinary resume — and each shipped because its test asserted only silence, in a fixture where the firing branch was unreachable. ⚠ Two edits, and the second is not decoration: replacing the condition alone leaves paidBefore unused and the package does not compile, so the planting would report «nothing ran» — a mutation that cannot build is not a mutation that survived, but it is not a caught one either", + "package": "./internal/pipeline/", + "run": "TestTheOneTimeCostIsAnnouncedExactlyWhenItHappens", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "if res.BankSettled > 0 && run.fresh && paidBefore {", + "replace": "if paidNow, _ := r.Store.HasCheckpointForStage(r.Book.BookID, terminologyStageName); res.BankSettled > 0 && run.fresh && paidNow {" + }, + { + "file": "internal/pipeline/terminologist.go", + "find": "\tpaidBefore, perr := r.Store.HasCheckpointForStage(r.Book.BookID, terminologyStageName)", + "replace": "\tpaidBefore, perr := r.Store.HasCheckpointForStage(r.Book.BookID, terminologyStageName)\n\t_ = paidBefore" + } + ] + }, + { + "id": "MONEY-the-filter-swallows-a-candidate-the-emission-would-have-emitted", + "why": "the drop is output-equivalent for the delta only because the filtered population is EXACTLY the one reverseSectionTerms drops: banknote-only candidates on a seed surface. Widen it to a mined candidate and the filter removes a term that would have reached the artifact the owner signs — money saved by losing a row. Every fixture that reached this filter carried a banknote candidate, so the condition was true of them by accident", + "package": "./internal/pipeline/", + "run": "TestTheSettledTestAnswersOnEachOfItsConditions", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\tif c.Origin != terminology.OriginBanknote || len(c.Variants) == 0 {", + "replace": "\tif len(c.Variants) == 0 {" + } + ] + }, + { + "id": "MONEY-the-filter-does-not-read-the-seed-rows-aliases", + "why": "the emission's own guard indexes a seed row by its src AND its aliases, so a proposal named by an alias is dropped there. Indexing only the src here pays the role for exactly those proposals and then throws the answer away — the waste this filter exists to end, surviving in the population hardest to notice", + "package": "./internal/pipeline/", + "run": "TestTheSettledTestAnswersOnEachOfItsConditions", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\t\tfor _, a := range e.Aliases {\n\t\t\tout[text.NormalizeSourceKey(a.Alias)] = e\n\t\t}", + "replace": "\t\t_ = e.Aliases" + } + ] + }, + { + "id": "MONEY-the-corrected-type-never-reaches-the-role", + "why": "the classifier is BOUGHT for the type field and that field travels to the role for the whole batch. The paid subset is rebuilt from the candidates AFTER the in-place passes precisely so the request carries the corrected type; drop the rebuild and the money is spent, the run prints reclassified=N, and the wire still says what the heuristic guessed. Third form of one root — a subset held as its own array — and the first two were both found after their own tests went green", + "package": "./internal/pipeline/", + "run": "TestTheClassifiersTypeStillReachesTheSheetThroughTheFilter", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\t\tpaid = pickCandidates(cands, paidIdx)\n", + "replace": "" + } + ] + }, + { + "id": "MONEY-the-filter-does-not-fold-the-seed-surface", + "why": "the seed surfaces are indexed by their NORMALIZED key because that is what a candidate carries. Indexing the raw src instead makes the filter silently stop firing for every traditional or katakana spelling — and BankSettled then reads 0, which is indistinguishable from «there was nothing to skip»", + "package": "./internal/pipeline/", + "run": "TestTheFilterFoldsBothSidesTheWayProductionDoes", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\t\tout[text.NormalizeSourceKey(e.Src)] = e", + "replace": "\t\tout[e.Src] = e" + } + ] + }, + { + "id": "MONEY-the-agreement-is-compared-byte-wise", + "why": "whether a draft AGREES with the bank is the money decision itself, and it is judged under the same target-form fold the vote is counted with. Compared byte-wise, a rendering differing only by case or ё reads as a disagreement and is paid for — the waste this filter exists to end, restored in the population nobody looks at", + "package": "./internal/pipeline/", + "run": "TestTheFilterFoldsBothSidesTheWayProductionDoes", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\t\tif text.NormalizeTargetForm(v.Dst) != want {", + "replace": "\t\tif v.Dst != want {" + } + ] + }, + { + "id": "MONEY-the-saving-loses-its-denominator", + "why": "the saving is the one line a later shift greps to learn what the filter bought, and «12 skipped» is unreadable without «of 300». The keys are pinned directly rather than by the operator-message catalogue, which guards texts and says arguments are outside it — so without this planting the denominator could go to zero unnoticed", + "package": "./internal/pipeline/", + "run": "TestABankSettledCandidateIsNotPaidFor", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\"book\", r.Book.BookID, \"skipped\", dropped, \"of_candidates\", len(cands), \"still_paid\", len(keptIdx),", + "replace": "\"book\", r.Book.BookID, \"skipped\", dropped, \"of_candidates\", 0, \"still_paid\", 0," + } + ] + }, + { + "id": "SEAM-the-normalizer-stops-folding-and-the-fixture-does-not-notice", + "why": "the stop-sheet fixture claims its candidates are the population whose KEY differs from its SURFACE, and it asks the real normalizer whether they are. Its first version asserted that against two typed literals — and the fold it named did not exist — so a normalizer replaced by identity would have left it green while every traditional spelling stopped keying at all", + "package": "./internal/pipeline/", + "run": "TestEveryStopSheetFindingReachesTheRowItBelongsTo", + "battery": true, + "edits": [ + { + "file": "internal/text/norm.go", + "find": "\t\tif m, ok := trad2simp[r]; ok {\n\t\t\tr = m\n\t\t}", + "replace": "\t\t_ = trad2simp" + } + ] + }, + { + "id": "BANKSHEET-a-row-nobody-was-asked-about-reads-as-undecided", + "why": "an empty dst on the signing sheet is the same glyph for four different facts: the role declined, no reply line covered the term, the budget did not reach it, and — since the already-banked filter — nobody was asked because there is nothing to decide. Three of those mean UNDECIDED and one means SETTLED, and the review ranking files all four together, so without the mark the owner cannot tell the row he can skip from the rows he must read", + "package": "./internal/pipeline/", + "run": "TestABankSettledCandidateIsNotPaidFor", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/mining.go", + "find": "\t\tif r.SettledByBank {\n\t\t\tb.WriteString(\" NOT ASKED(the bank already renders this surface and every draft agreed — nothing to decide)\")\n\t\t}\n", + "replace": "" + } + ] + }, + { + "id": "MONEY-the-one-time-cost-warning-drops-its-fresh-half", + "why": "the warning needs BOTH halves — that the book had paid BEFORE this run and that this run paid anyway — and dropping the second turns it into a claim of re-purchase on every ordinary resume, printed beside cost_usd=0.000000. It survived once already: the guard against it grepped the message's PROSE, and the prose had been reworded, so the substring occurred zero times in the module and the assertion could not fire. Both sides now read one structured key instead", + "package": "./internal/pipeline/", + "run": "TestAResumeFindsItsCheckpointsAfterTheFilter|TestTheOneTimeCostIsAnnouncedExactlyWhenItHappens", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "if res.BankSettled > 0 && run.fresh && paidBefore {", + "replace": "if res.BankSettled > 0 && paidBefore {" + } + ] } ] diff --git a/backend/internal/membank/memory_e1_test.go b/backend/internal/membank/memory_e1_test.go index 5640565d..057d5cd3 100644 --- a/backend/internal/membank/memory_e1_test.go +++ b/backend/internal/membank/memory_e1_test.go @@ -131,8 +131,10 @@ func TestPostcheckWhitespaceAndDashRobust(t *testing.T) { } // TestPostcheckUnverifiedChannelCountsExactly pins the $0 observability channel of D39.42 п.4 with -// NUMBERS, not with "nonzero". Shown is how many UNSIGNED rows were actually put in front of the model -// here; Followed is how many of those it went along with. Without the denominator "3 deviations" is +// NUMBERS, not with "nonzero". Shown is how many UNSIGNED rows this chunk can be JUDGED on — the ones +// whose key fired here, which is fewer than the rows the model is shown, see +// TestTheUnverifiedDenominatorIsNotACountOfRowsShown; Followed is how many of those the model went along +// with. Without the denominator "3 deviations" is // unreadable — 3 of 3 is a channel nobody follows, 3 of 90 is a model exercising the judgement the // unverified section explicitly grants it. Any wrong-but-nonzero pair passed before this test existed. func TestPostcheckUnverifiedChannelCountsExactly(t *testing.T) { @@ -148,7 +150,7 @@ func TestPostcheckUnverifiedChannelCountsExactly(t *testing.T) { // The model uses the signed rendering and ONE of the two unsigned ones. res := b.Postcheck(sel.Injected, "Бородатый Ван и А-кью пришли, а женщина в рясе молчала.") if res.Shown != 2 { - t.Fatalf("Shown counts the UNSIGNED rows put in front of the model, want 2, got %d", res.Shown) + t.Fatalf("Shown counts the UNSIGNED rows whose key fired here, want 2, got %d", res.Shown) } if res.Followed != 1 { t.Fatalf("Followed counts the unsigned rows the model went along with, want 1, got %d", res.Followed) diff --git a/backend/internal/membank/mempostcheck.go b/backend/internal/membank/mempostcheck.go index 7716bc29..90cdcd81 100644 --- a/backend/internal/membank/mempostcheck.go +++ b/backend/internal/membank/mempostcheck.go @@ -78,10 +78,21 @@ type PostcheckMiss struct { type PostcheckResult struct { Confirmed []PostcheckMiss Unverified []PostcheckMiss - // Shown / Followed are the $0 observability channel of the unsigned wire: how many unsigned rows were - // actually PUT IN FRONT of the model here, and how many of those it went along with. Without the - // denominator "3 deviations" is unreadable — 3 out of 3 is a channel nobody follows, 3 out of 90 is - // ordinary variance. + // Shown / Followed are the $0 observability channel of the unsigned wire: how many unsigned rows this + // chunk could be JUDGED on, and how many of those the model went along with. Without the denominator + // "3 deviations" is unreadable — 3 out of 3 is a channel nobody follows, 3 out of 90 is ordinary + // variance. + // + // ⚠ "JUDGED ON" IS NARROWER THAN "PUT IN FRONT OF THE MODEL", and the difference is the STICKY skip + // above. A sticky carry is rendered into the injection block like any other record — the model does see + // it — but its src is not in this chunk, so its dst is legitimately absent from the output; counting it + // would make every pronominal chunk read as a channel nobody follows. What is left is exactly the + // population Followed is a fraction of, and it is smaller than the set of rows shown. + // + // The other two skips do not narrow it on any real selection: MaterializeBank gives a row with no dst + // no source surfaces at all, so it can neither fire nor be carried, and a row-less record only exists + // when a caller assembles PickedEntry by hand. Both are defensive, and naming them as reasons the count + // is small would misdescribe it as badly as the old wording did. // // ⚠ WHAT THEY MEASURE NARROWED WITH ROW 134, and the numbers did not change shape. They were born on a // wire that marked an unsigned row ⟨проверить⟩ and told the model it was free to reject it, so a low diff --git a/backend/internal/membank/memseed.go b/backend/internal/membank/memseed.go index 0840ab78..bacecbba 100644 --- a/backend/internal/membank/memseed.go +++ b/backend/internal/membank/memseed.go @@ -740,16 +740,37 @@ func UnverifiedKeyConflicts(entries []store.GlossaryEntry) []string { } seenPair[[2]int{i, j}] = true out = append(out, fmt.Sprintf("firing key %q: %s %q→%q %s vs %s %q→%q %s (the model is shown two renderings of one surface)", - k, statusLabel(ei.Status), ei.Src, ei.Dst, windowLabel(ei.SinceCh, ei.UntilCh), - statusLabel(ej.Status), ej.Src, ej.Dst, windowLabel(ej.SinceCh, ej.UntilCh))) + k, StatusLabel(ei.Status), ei.Src, ei.Dst, windowLabel(ei.SinceCh, ei.UntilCh), + StatusLabel(ej.Status), ej.Src, ej.Dst, windowLabel(ej.SinceCh, ej.UntilCh))) } } } return out } -// statusLabel renders a row's signature for an operator, keeping the specific status where there is one. -func statusLabel(status string) string { +// IsEngineUnsigned reports whether a bank row is the ENGINE's own unsigned proposal — what the auto mode +// consolidated and wrote to its own document, as against anything a person put in the bank. +// +// ⚠ THIS IS THE ONE CARRIER OF THAT RULE, and it is exported for that reason rather than for reuse. Three +// separate places have to agree on it — the emission's seed-surface guard (pipeline.unsignedEngineSurfaces), +// the paid path's filter, and the tuple skip in ConsolidationKeyConflicts below — and a second spelling of +// it would be a second answer to "is this the engine's own word", which is precisely the distinction the +// bank exists to keep (18-bank-ontology.md: the owner's word and the engine's word must not merge). +// +// ⚠ WHAT IT CANNOT SEE, named because a caller will otherwise assume it can: an owner's mined-delta row +// that carries an unsigned status answers YES here, exactly like an auto-bank row. Both read Source +// "mined" and neither carries its document of origin, so no predicate over a bank row can separate them. +// What this does answer, exactly, is the question the emission asks: whether the miner is allowed to treat +// the surface as already seeded. +func IsEngineUnsigned(e store.GlossaryEntry) bool { + return e.Source == "mined" && e.Status != "approved" +} + +// StatusLabel renders a row's signature for an operator, keeping the specific status where there is one. +// Exported because the drop the pipeline reports on a key collision (pipeline.loadAutoBank) has to word a +// holder's signature the same way the bank's own findings do: a second wording is how one of them starts +// calling an unsigned row signed. +func StatusLabel(status string) string { if status == "approved" { return "approved" } @@ -797,13 +818,18 @@ func windowLabel(since, until int) string { // The last two are still reported: the sheet is where a term is decided, and "the book already calls it // something else" is what the owner needs there. // -// ⚠ WHAT THE SKIP HIDES, measured: a seed row with the DEFAULT window and no sense — the commonest shape a -// hand-written seed has — shares its tuple with a banknote proposal, which carries since_ch 0 likewise, -// so that pair is silent here whatever the seed row's status. The proposal is dropped rather than -// shipped, so no wire carries two renderings; what is lost is the sheet telling the owner that the role -// disagreed with a row he wrote. Closing it needs a rule for telling a REPLACED engine row from a DROPPED -// proposal, and the Source column cannot give one: the owner's delta and the engine's auto-bank both read -// "mined". +// ⚠ WHAT THE SKIP USED TO HIDE, and what it still does. A seed row with the DEFAULT window and no sense — +// the commonest shape a hand-written seed has — shares its tuple with a banknote proposal, which carries +// since_ch 0 likewise, and that pair was silent here whatever the seed row's status: the owner never saw +// on his signing sheet that the role disagreed with a row he had written. The skip is now conditioned on +// the row being the engine's own unsigned one (IsEngineUnsigned), which is exactly the population the +// emission does NOT treat as a seed surface, so the report agrees with what the emission will do. +// +// The residue, named rather than left to be discovered: an owner's mined-delta row that carries an +// unsigned status answers IsEngineUnsigned yes, so a proposal on ITS tuple is still skipped here — and +// that proposal does not resolve against the row either, it reaches the auto-bank document and is dropped +// by pipeline.loadAutoBank on the next run. Separating the two needs a row to say which document it came +// from, which no row does: the owner's delta and the engine's auto-bank both read Source "mined". // // In the auto mode a pair that DOES land is then reported a second time, by UnverifiedKeyConflicts on the // re-seed. The two are different states of one term — "the role disagrees with the bank" before the row @@ -849,8 +875,18 @@ func ConsolidationKeyConflicts(consolidated, bank []store.GlossaryEntry) []BankK // window are two accepted rows, not one (migrate.go). Folding them here would skip a pair // that really does coexist. Callers pass the source the row WILL carry — see // pipeline.consolidatedRows. - if b.Src == c.Src && b.Sense == c.Sense && b.SinceCh == c.SinceCh && b.UntilCh == c.UntilCh { - continue // one uniqueness key — the store admits only one of the two + // + // ⚠ THE SKIP IS FOR A PROPOSAL THAT CAN ACTUALLY LAND ON THE TUPLE, which is narrower than + // sharing it. Sharing a tuple means the store admits only one of the two, so a landing + // proposal resolves against the row instead of joining it and there is no lasting + // disagreement to report. A proposal on a SEED surface never lands at all — + // pipeline.reverseSectionTerms drops it at emission — so the bank keeps its rendering, the + // role's stays on the sheet, and staying silent hides the disagreement instead of + // forecasting its resolution. That was the commonest shape there is: a hand-written seed row + // takes the DEFAULT window and no sense, and so does a banknote proposal, so the two shared + // a tuple by construction and the whole population went unreported. + if IsEngineUnsigned(b) && b.Src == c.Src && b.Sense == c.Sense && b.SinceCh == c.SinceCh && b.UntilCh == c.UntilCh { + continue // the engine's own unsigned row, on one uniqueness key: this run's rendering replaces it } if b.Dst == c.Dst || !windowsOverlap(c.SinceCh, c.UntilCh, b.SinceCh, b.UntilCh) { continue @@ -888,7 +924,7 @@ func (c BankKeyConflict) Message() string { // BankRowLabel names only the EXISTING row: the signature sheet already prints the proposal on its own // line, so repeating it would bury the new fact. func (c BankKeyConflict) BankRowLabel() string { - return fmt.Sprintf("%s %q→%q %s", statusLabel(c.BankStatus), c.BankSrc, c.BankDst, windowLabel(c.BankSinceCh, c.BankUntilCh)) + return fmt.Sprintf("%s %q→%q %s", StatusLabel(c.BankStatus), c.BankSrc, c.BankDst, windowLabel(c.BankSinceCh, c.BankUntilCh)) } // ConflictMessages renders a set of findings for one log line. diff --git a/backend/internal/membank/memseed_test.go b/backend/internal/membank/memseed_test.go index 7fda8894..0a604936 100644 --- a/backend/internal/membank/memseed_test.go +++ b/backend/internal/membank/memseed_test.go @@ -646,10 +646,19 @@ func TestConsolidationKeyConflicts(t *testing.T) { SinceCh: 10}}, bank); len(got) != 0 { t.Errorf("disjoint windows are a legitimate handoff: %v", got) } - // REPLACEMENT: the proposal carries the bank row's whole UNIQUE tuple, so only one of the two exists. + // REPLACEMENT: the proposal carries the bank row's whole UNIQUE tuple AND the row is the engine's own + // unsigned one, so this run's rendering takes its place and there is nothing lasting to report. + if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "赵甲", Dst: "Чжао Цзя-старый"}}, bank); len(got) != 0 { + t.Errorf("a proposal on the engine's own unsigned row's UNIQUE key replaces it and cannot contradict it: %v", got) + } + // ⚠ THE SAME TUPLE OVER A ROW THE OWNER WROTE IS NOT A REPLACEMENT and must be reported. A proposal on + // a seed surface never lands at all — the emission drops it — so the bank keeps its rendering and the + // disagreement is permanent. This is the commonest shape there is: a hand-written seed row takes the + // default window and no sense, and so does a banknote proposal, so silence here hid the whole + // population from the sheet the owner signs by. if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: "камень первоисточника", - SinceCh: 3, UntilCh: 9}}, bank); len(got) != 0 { - t.Errorf("a proposal on the row's own UNIQUE key replaces it and cannot contradict it: %v", got) + SinceCh: 3, UntilCh: 9}}, bank); len(got) != 1 { + t.Errorf("a proposal sharing a SEED row's tuple is dropped, not merged, and the disagreement stands: %v", got) } // A proposal with no rendering is a term the role declined; it becomes no row and contradicts nothing. if got := ConsolidationKeyConflicts([]store.GlossaryEntry{{Src: "元始", Dst: ""}}, bank); len(got) != 0 { @@ -673,9 +682,16 @@ func TestConsolidationTupleSkipMirrorsTheStoreKey(t *testing.T) { if got := ConsolidationKeyConflicts(folded, bank); len(got) != 1 { t.Fatalf("族长 against a stored 族長 is a pair the store admits, not a replacement: %v", got) } - // Byte-identical raw src on the same tuple: the store admits one row, so it is a replacement. + // Byte-identical raw src on the same tuple: the store admits one row — but over a SEED row that does + // not make it a replacement, because the proposal is dropped at emission and never reaches the tuple. same := []store.GlossaryEntry{{Src: "族長", Dst: "старейшина"}} - if got := ConsolidationKeyConflicts(same, bank); len(got) != 0 { - t.Errorf("an identical raw src on the same tuple is a replacement: %v", got) + if got := ConsolidationKeyConflicts(same, bank); len(got) != 1 { + t.Errorf("the store admitting one row does not merge a proposal the emission drops: %v", got) + } + // CONTROL: the same tuple over the ENGINE's own unsigned row IS a replacement, so the assertion above + // is the row's provenance answering and not a skip that has stopped working. + engine := []store.GlossaryEntry{{Src: "族長", Dst: "глава клана", Status: "auto", Source: "mined"}} + if got := ConsolidationKeyConflicts(same, engine); len(got) != 0 { + t.Errorf("this run's rendering replaces the engine's own unsigned row for that tuple: %v", got) } } diff --git a/backend/internal/membank/postcheckdenominator_test.go b/backend/internal/membank/postcheckdenominator_test.go new file mode 100644 index 00000000..1b95b6a3 --- /dev/null +++ b/backend/internal/membank/postcheckdenominator_test.go @@ -0,0 +1,99 @@ +package membank + +import ( + "strings" + "testing" + + "textmachine/backend/internal/store" +) + +// TestTheUnverifiedDenominatorIsNotACountOfRowsShown pins what Shown is a count OF, which is narrower than +// the sentence that described it in four places ("how many unsigned rows were put in front of the model"). +// A sticky carry IS put in front of the model and is deliberately not counted, so the two populations are +// different numbers on the same chunk — and Followed is a fraction of the smaller one. +// +// The wire the same selection renders is asserted beside the counts, because that is what makes this a +// measurement rather than a restatement: the row missing from the denominator is visibly present in the +// block that goes to the model. +func TestTheUnverifiedDenominatorIsNotACountOfRowsShown(t *testing.T) { + fired := gl("阿Q", "А-кью", "", "draft") + fired.Decl = declJSON(true, "А-кью") + // Carried by scene inertia from the previous chunk: its src is NOT in this one. + carried := gl("王胡", "Бородатый Ван", "", "draft") + carried.Decl = declJSON(true, "Бородатый Ван") + b := bankFrom([]store.GlossaryEntry{fired, carried}) + + carriedID := "王胡\x1f\x1f0\x1f0" // src, sense, since_ch, until_ch — the UNIQUE key, as MaterializeBank builds it + sel := b.Select("阿Q在场。", 2, map[string]InjectionTrust{carriedID: {Disp: Ambiguous, KeyTrusted: true}}, 0) + // Premise: both records are injected and exactly one of them is a sticky carry. Without this the count + // below could come out right on a selection that never held the exclusion. + if len(sel.Injected) != 2 { + t.Fatalf("premise broken: want both records injected, got %d: %+v", len(sel.Injected), sel.Injected) + } + var sticky int + for _, p := range sel.Injected { + if p.Sticky { + sticky++ + if p.entry.src != "王胡" { + t.Fatalf("premise broken: the sticky carry must be the row absent from this chunk, got %q", p.entry.src) + } + } + } + if sticky != 1 { + t.Fatalf("premise broken: want exactly one sticky carry, got %d", sticky) + } + + // The model used BOTH renderings it was given — so a count of "rows shown that the model followed" + // would be 2 of 2 here, and the channel actually reports 1 of 1. + res := b.Postcheck(sel.Injected, "А-кью пришёл, и Бородатый Ван с ним.") + if res.Shown != 1 { + t.Errorf("Shown counts only the unsigned rows whose key fired HERE, want 1, got %d", res.Shown) + } + if res.Followed != 1 { + t.Errorf("Followed is a fraction of Shown, want 1, got %d", res.Followed) + } + block := RenderGlossaryBlock(sel.Injected, ruTX()) + if !strings.Contains(block, "Бородатый Ван") { + t.Errorf("premise broken: the sticky carry must be IN the block — that it is shown and not counted is the whole point:\n%s", block) + } + + // CONTROL: the same two rows with both keys firing here give 2. It proves the 1 above is the sticky + // exclusion answering, and not a bank that could only ever count one row. + selBoth := b.Select("阿Q和王胡在场。", 2, nil, 0) + if n := len(selBoth.Injected); n != 2 { + t.Fatalf("control premise: want two injected, got %d", n) + } + if got := b.Postcheck(selBoth.Injected, "А-кью пришёл, и Бородатый Ван с ним.").Shown; got != 2 { + t.Errorf("control: two unsigned rows firing here are two, got %d", got) + } +} + +// TestARowWithNoRenderingNeverReachesTheDenominator pins the OTHER skip in Postcheck as what it is — +// defensive, not a narrowing of the count. MaterializeBank gives a dst-less row no source surfaces at all, +// so it can neither fire nor be carried, and no selection can put one in front of Postcheck. Saying it +// narrows the denominator would misdescribe the number as badly as "rows shown" did, and the day the +// materializer starts admitting such a row this test is where that shows up. +func TestARowWithNoRenderingNeverReachesTheDenominator(t *testing.T) { + dstless := gl("小尼姑", "", "", "auto") + rendered := gl("阿Q", "А-кью", "", "draft") + rendered.Decl = declJSON(true, "А-кью") + b := bankFrom([]store.GlossaryEntry{dstless, rendered}) + sel := b.Select("阿Q和小尼姑在场。", 1, nil, 0) + for _, p := range sel.Injected { + if p.entry.src == "小尼姑" { + t.Fatalf("a row with no rendering became matchable; the Postcheck skip for it is no longer defensive: %+v", sel.Injected) + } + } + // CONTROL: the same source surface with a rendering DOES fire on the same chunk, so the absence above + // is the empty dst and not a chunk the matcher cannot read. + dstless.Dst = "монашка" + dstless.Decl = declJSON(true, "монашка") + with := bankFrom([]store.GlossaryEntry{dstless, rendered}) + selWith := with.Select("阿Q和小尼姑在场。", 1, nil, 0) + if len(selWith.Injected) != 2 { + t.Fatalf("control: with a rendering both rows must fire, got %d: %+v", len(selWith.Injected), selWith.Injected) + } + if got := with.Postcheck(selWith.Injected, "А-кью и монашка пришли.").Shown; got != 2 { + t.Errorf("control: both unsigned rows fired and both carry renderings, want 2, got %d", got) + } +} diff --git a/backend/internal/pipeline/bankmaterialize.go b/backend/internal/pipeline/bankmaterialize.go index 5955c613..61ffe546 100644 --- a/backend/internal/pipeline/bankmaterialize.go +++ b/backend/internal/pipeline/bankmaterialize.go @@ -127,9 +127,13 @@ func (r *Runner) gatherBankInputs() (bankInputs, error) { return in, err } if len(dropped) > 0 { - // A collision with a SIGNED row cannot abort a paid run over an engine-written proposal — the signed - // row simply wins and the proposal is dropped, loudly. - in.remark("auto-bank rows dropped: their key is already held by a signed term (the signed term wins)", + // A key collision cannot abort a paid run over an engine-written proposal — the row already gathered + // wins and the proposal is dropped, loudly. ⚠ The holder is whatever the seed, the ruby aliases or the + // owner's delta put on that tuple, of ANY status: the store's UNIQUE key does not read signatures. So + // the headline says what actually happened and each dropped row names its own holder's signature + // (loadAutoBank), because an operator told "the signed term wins" over an unsigned holder signs a bank + // he has been shown wrong. + in.remark("auto-bank rows dropped: their key is already held by a row gathered earlier (the holder wins, whatever its status; each drop below names the holder's signature)", "book", r.Book.BookID, "dropped", joinSemi(dropped)) } in.entries = append(in.entries, autoBank...) diff --git a/backend/internal/pipeline/bankoperatortruth_test.go b/backend/internal/pipeline/bankoperatortruth_test.go new file mode 100644 index 00000000..d34b0a00 --- /dev/null +++ b/backend/internal/pipeline/bankoperatortruth_test.go @@ -0,0 +1,126 @@ +package pipeline + +import ( + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "textmachine/backend/internal/config" + "textmachine/backend/internal/store" +) + +// bankoperatortruth_test.go holds the pins on what the bank contour TELLS the operator, as distinct from +// what it does. The behaviour under each of them was already right; the sentence describing it was not, +// and the owner signs a bank on the strength of the sentence (D39.104 п.1) — every row he is shown is +// injected whether or not he signed it, so a row named "signed" that nobody signed is not a cosmetic +// defect, it is the one fact he is reading the artifact to learn. + +// TestADropNamesItsHoldersSignature pins the message loadAutoBank leaves when a key collision drops an +// engine proposal. It used to say "key held by the signed …" for every holder, and the holder is whatever +// the seed, the ruby aliases or the owner's delta put on that tuple — the store's UNIQUE constraint does +// not read statuses, so an unsigned row holds a key exactly as hard as a signed one. An operator reading +// the old line was told a signature existed where none did. +func TestADropNamesItsHoldersSignature(t *testing.T) { + // Both cases are the SAME collision with only the holder's status moved, so what the assertions read + // is the status reaching the message and not some other difference in the fixture. + for _, tc := range []struct { + name string + status string + wantLabel string + wantAbsent string + }{ + // The defect: a draft seed row wins the tuple and the drop used to call it signed. The forbidden + // string carries its article — a bare "signed" is a substring of "unsigned" and would fail on the + // very message that fixes the defect. + {name: "unsigned holder", status: "draft", wantLabel: `unsigned draft "方源"→"Фан Юань"`, wantAbsent: "the signed"}, + // CONTROL: with a genuine signature the line says so, which is what proves the case above is the + // status being read and not the word "signed" simply having been deleted. + {name: "signed holder", status: "approved", wantLabel: `approved "方源"→"Фан Юань"`, wantAbsent: "unsigned"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "book.db") + // status:auto is required of the file itself — loadAutoBank refuses an `approved` row in the + // engine's own document, so the proposal cannot be made to win by mislabelling it. + if err := os.WriteFile(db+".auto-bank.yaml", []byte("terms:\n - src: 方源\n dst: Странник\n type: name\n status: auto\n"), 0o644); err != nil { + t.Fatalf("write the auto-bank: %v", err) + } + r := &Runner{ + Book: &config.Book{BookID: "b1", ProjectDB: db}, + Log: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})), + } + // One row, on the auto-bank row's own UNIQUE tuple (src, sense, since_ch, until_ch). + held := []store.GlossaryEntry{{Src: "方源", Dst: "Фан Юань", Status: tc.status}} + rows, dropped, err := r.loadAutoBank(held) + if err != nil { + t.Fatalf("load the auto-bank: %v", err) + } + if len(rows) != 0 { + t.Fatalf("premise broken: the proposal must lose the tuple to the row already gathered, got %+v", rows) + } + if len(dropped) != 1 { + t.Fatalf("the drop has to be reported, got %d line(s): %v", len(dropped), dropped) + } + if !strings.Contains(dropped[0], tc.wantLabel) { + t.Errorf("the drop must name the holder's signature as %q:\n%s", tc.wantLabel, dropped[0]) + } + if strings.Contains(dropped[0], tc.wantAbsent) { + t.Errorf("the drop calls the holder %q, which it is not:\n%s", tc.wantAbsent, dropped[0]) + } + // The proposal's own rendering has to be in the line as well: told only which row won, the + // operator cannot tell whether the engine had agreed with him or contradicted him. + if !strings.Contains(dropped[0], `"方源"→"Странник"`) { + t.Errorf("the drop must name the rendering that was dropped, not only the holder:\n%s", dropped[0]) + } + }) + } +} + +// TestTheDropHeadlineDoesNotPromiseASignature pins the line ABOVE the drops — the one an operator sees +// first and often the only one he reads, because the per-row detail is a semicolon-joined tail. It used +// to promise "the signed term wins" over a set of drops whose holders may every one of them be unsigned. +// +// The sentence is read off the REAL gather rather than retyped here: a test that retypes the message it +// guards proves that the test compiles. Its exact bytes are held by the catalogue +// (TestEveryOperatorMessageIsCatalogued); what this asserts is the property that made the old wording +// false, so the two are different guards over one sentence and not one guard twice. +func TestTheDropHeadlineDoesNotPromiseASignature(t *testing.T) { + srv := newJSONProvider(&reqRec{}, draftEdit) + defer srv.Close() + // A DRAFT seed row: the owner wrote it, it is not signed, and it holds the tuple the engine's proposal + // wants. This is the shape the old headline described as a signature. + bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: suzukiSource, glossarySeed: ` +terms: + - src: 鈴木 + dst: Судзуки + type: name + status: draft +`}) + if err := os.WriteFile(filepath.Join(filepath.Dir(bookPath), "test-book.db.auto-bank.yaml"), + []byte("terms:\n - src: 鈴木\n dst: Сузуки\n type: name\n status: auto\n"), 0o644); err != nil { + t.Fatalf("write the auto-bank: %v", err) + } + r := newRunner(t, bookPath) + defer r.Close() + in, err := r.gatherBankInputs() + if err != nil { + t.Fatalf("gather: %v", err) + } + var headline string + for _, rem := range in.remarks { + if strings.Contains(rem.msg, "auto-bank rows dropped") { + headline = rem.msg + } + } + if headline == "" { + t.Fatalf("premise broken: the collision produced no drop remark; remarks were %+v", in.remarks) + } + if strings.Contains(headline, "signed term wins") || strings.Contains(headline, "held by a signed") { + t.Errorf("the headline states a signature the holder may not have:\n%s", headline) + } + if !strings.Contains(headline, "whatever its status") { + t.Errorf("the headline must say the holder's status does not decide the collision:\n%s", headline) + } +} diff --git a/backend/internal/pipeline/banksettled_test.go b/backend/internal/pipeline/banksettled_test.go new file mode 100644 index 00000000..792b147a --- /dev/null +++ b/backend/internal/pipeline/banksettled_test.go @@ -0,0 +1,561 @@ +package pipeline + +import ( + "bytes" + "context" + "errors" + "log/slog" + "os" + "reflect" + "strings" + "testing" + + "textmachine/backend/internal/store" + "textmachine/backend/internal/terminology" + "textmachine/backend/internal/text" +) + +// askedAsCandidate reports whether a surface was sent to the paid role as a CANDIDATE of the block, which +// is the only question about money. The candidate list is spelled as markdown headings, and every +// candidate carries the source sentences it occurs in — so a surface can be all over a request without +// anybody having been asked about it. +func askedAsCandidate(rec *reqRec, src string) bool { + for _, body := range rec.all() { + if isTerminologyBody(body) && strings.Contains(body, "### "+src) { + return true + } + } + return false +} + +// terminologyBodies is what the role was actually sent, for a failure that has to be readable. +func terminologyBodies(rec *reqRec) string { + var out []string + for _, body := range rec.all() { + if isTerminologyBody(body) { + out = append(out, body) + } + } + return strings.Join(out, "\n---\n") +} + +// banksettled_test.go pins the filter that stops the paid terminology role being asked about surfaces the +// bank has already settled. The two cases below are the same run with ONE byte of the draft's proposal +// changed, so what they measure is the agreement and not two different fixtures. + +// TestABankSettledCandidateIsNotPaidFor is case (i): the seed holds the surface, the draft proposed the +// rendering the seed holds, and the role is therefore never asked about it. +func TestABankSettledCandidateIsNotPaidFor(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isTerminologyBody(body) { + return "青茅山\tгора Цинмао", "stop" + } + // The draft proposes for 方源 exactly what the seed already holds. + return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop" + }) + defer srv.Close() + seed := "terms:\n - { src: 方源, dst: Фан Юань, status: approved }\n" + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, glossarySeed: seed})) + defer r.Close() + var runLog bytes.Buffer + r.Log = slog.New(slog.NewTextHandler(&runLog, &slog.HandlerOptions{Level: slog.LevelInfo})) + _ = runToSignatureStop(t, r) + + tres := r.lastTerminology + // The number is stated WITH its denominator: "1 skipped" says nothing without "of how many". + if tres.BankSettled != 1 { + t.Fatalf("the settled candidate must be skipped before payment: skipped=%d of candidates=%d", tres.BankSettled, tres.Candidates) + } + if tres.Candidates < 2 { + t.Fatalf("premise broken: with fewer than two candidates a skip is indistinguishable from an empty run: %d", tres.Candidates) + } + // What the money question actually asks: did 方源 reach a paid request as a CANDIDATE. Asked on the + // block heading and not on the bare surface — every candidate's source contexts quote the sentences it + // occurs in, so 方源 appears in the request whatever the filter does, and a substring assertion here + // would have reported the filter broken while it was working. + if askedAsCandidate(rec, "方源") { + t.Errorf("the settled surface was sent to the paid role anyway:\n%s", terminologyBodies(rec)) + } + // CONTROL: the OTHER candidate did reach it, so the absence above is the filter and not a role that + // was never called. + if !askedAsCandidate(rec, "青茅山") { + t.Fatalf("control: the unsettled candidate must still be paid for, or nothing was measured") + } + // ⚠ THE KEYS OF THE SAVING LINE ARE PINNED HERE and not by the operator-message catalogue, because that + // catalogue guards message TEXTS and says so — the structured arguments are outside it by declaration. + // This is the one line a later shift will grep to learn what the filter bought, and a number without + // its denominator is unreadable: «12 skipped» says nothing without «of 300». + // ⚠ Asserted on the ONE line that carries the saving, not on the whole buffer: three Contains over a + // shared buffer are satisfiable by three DIFFERENT lines, so the denominator could go to zero and the + // test stay green. Found by review after it was written that way. + var saving string + for _, line := range strings.Split(runLog.String(), "\n") { + if strings.Contains(line, "already settled") { + saving = line + } + } + if saving == "" { + t.Fatalf("the saving must be logged at all:\n%s", runLog.String()) + } + for _, want := range []string{"skipped=1", "of_candidates=3", "still_paid=2"} { + if !strings.Contains(saving, want) { + t.Errorf("the saving line must carry %q — a number without its denominator is unreadable:\n%s", want, saving) + } + } + // The sheet still lists the settled surface — the skip is about what was BOUGHT, not about what the + // owner is shown. Its dst column is empty, because no role answered for it and the sheet must not + // print a rendering nobody produced this run. + var settled *BankStopRow + for i, row := range r.lastBankStopRows { + if row.Src == "方源" { + settled = &r.lastBankStopRows[i] + } + } + if settled == nil { + t.Fatalf("the skipped candidate must still be on the sheet: %+v", r.lastBankStopRows) + } + if settled.Dst != "" { + t.Errorf("no role answered for this candidate, so the sheet must not carry a rendering as if one had: %q", settled.Dst) + } + // ⛔ AND THE SHEET MUST SAY WHY. An empty dst is the same glyph as «the role declined», «no reply line + // covered it» and «the budget did not reach it» — three facts that mean UNDECIDED beside one that means + // NOTHING TO DECIDE, and the review ranking files all four together. The report of this pack first + // claimed the owner could read the difference off the row; he could not, and this is that claim made + // true instead of withdrawn. + if !settled.SettledByBank { + t.Errorf("a row the role was never asked about must be marked as such: %+v", *settled) + } + if !strings.Contains(renderBankStopTable(r.lastBankStopRows), "NOT ASKED(the bank already renders this surface") { + t.Errorf("the rendered sheet must carry the mark, not only the struct:\n%s", renderBankStopTable(r.lastBankStopRows)) + } + // CONTROL: the candidate that WAS paid for is not marked, so the mark is the filter answering. + for _, row := range r.lastBankStopRows { + if row.Src == "青茅山" && row.SettledByBank { + t.Errorf("a candidate the role WAS asked about must not be marked as unasked: %+v", row) + } + } + if len(settled.Variants) == 0 { + t.Errorf("the drafts' own proposal must still be on the sheet — that is what the owner reads instead: %+v", *settled) + } +} + +// TestADisagreeingBankedCandidateIsStillPaidFor is case (ii), and it is the half that keeps the filter +// from paying for itself with the owner's information: the seed holds the surface, the draft proposed +// something ELSE, and that disagreement is the whole reason the role is asked about banked surfaces. +// Measured on the corpus, this is the majority of the population — 366 of the 439 candidates the owner's +// own seed holds — so a filter without this half would buy its saving by silencing them. +func TestADisagreeingBankedCandidateIsStillPaidFor(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isTerminologyBody(body) { + return "方源\tИсточник Фана\n青茅山\tгора Цинмао", "stop" + } + // One byte of difference from the case above: the draft calls 方源 something the seed does not. + return "Странник пришёл." + "\n" + bankSeparator + "\n方源\tСтранник\tname\n青茅山\tгора Цинмао\tplace", "stop" + }) + defer srv.Close() + seed := "terms:\n - { src: 方源, dst: Фан Юань, status: approved }\n" + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, glossarySeed: seed})) + defer r.Close() + _ = runToSignatureStop(t, r) + + if n := r.lastTerminology.BankSettled; n != 0 { + t.Fatalf("a draft that disagrees with the bank must be PAID for, not skipped: skipped=%d", n) + } + if !askedAsCandidate(rec, "方源") { + t.Fatalf("the disagreeing surface must reach the paid role as a candidate:\n%s", terminologyBodies(rec)) + } + // And the finding the payment bought has to be on the sheet, which is what the payment is FOR. + var row *BankStopRow + for i, cur := range r.lastBankStopRows { + if cur.Src == "方源" { + row = &r.lastBankStopRows[i] + } + } + if row == nil { + t.Fatalf("premise broken: 方源 is not on the sheet: %+v", r.lastBankStopRows) + } + if len(row.BankHolds) == 0 { + t.Errorf("the payment bought the mark that the book already calls this term something else, and it must reach the sheet: %+v", *row) + } +} + +// TestAResumeFindsItsCheckpointsAfterTheFilter pins the resume cure, and it is the assertion that caught +// the first cure being wrong. Batches are packed greedily, so dropping a candidate changes the composition +// of a surviving batch, and a bank-role batch is addressed by the hash of its request: a second run whose +// composition differs from the first finds no checkpoint and buys the whole pass again. The first cure +// written here made the filter stand down for a book that had already paid — which produced exactly the +// re-purchase it was meant to prevent, because run one had paid under the FILTERED composition. +// +// What is asserted is money, not a flag: the resumed run reaches the provider for the bank role zero +// times and its pass costs $0. +func TestAResumeFindsItsCheckpointsAfterTheFilter(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isTerminologyBody(body) { + return "青茅山\tгора Цинмао", "stop" + } + return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop" + }) + defer srv.Close() + seed := "terms:\n - { src: 方源, dst: Фан Юань, status: approved }\n" + bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, glossarySeed: seed}) + + // ⚠ BOTH runs AUTO-CONTINUE (no --verify-bank), and that is load-bearing rather than convenience: a + // run that STOPS at the bank boundary does not write the auto-bank (18-bank-ontology.md — the view is + // not changed on a stop boundary), so the bank the second run reads would be byte-identical to the + // first's whatever the filter looked at. The first version of this test stopped, and a planting that + // made the filter read the WHOLE bank — auto-bank rows this very run rewrites, and therefore a bank + // that MOVES between runs — survived it. Auto-continuing is what makes the second run see a bank that + // the first one changed. + r1 := newRunner(t, bookPath) + if _, err := r1.TranslateBook(context.Background()); err != nil { + t.Fatalf("the first run: %v", err) + } + if r1.lastTerminology.BankSettled != 1 { + t.Fatalf("premise broken: the first run must filter, or the resume proves nothing: %+v", r1.lastTerminology) + } + if r1.lastTerminology.CostUSD == 0 { + t.Fatalf("premise broken: the first run must actually BUY the pass, or a $0 resume is meaningless") + } + terminologyCallsAfterFirst := countTerminologyCalls(rec) + r1.Close() + + // PREMISE for the whole test: the first run really did leave an auto-bank behind, so the second run's + // bank is not the same object. Without this the assertions below hold for a reason unrelated to them. + if _, err := os.Stat(r1.autoBankPath()); err != nil { + t.Fatalf("premise broken: the first run wrote no auto-bank, so the bank cannot move between runs: %v", err) + } + r2 := newRunner(t, bookPath) + defer r2.Close() + var log bytes.Buffer + r2.Log = slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{Level: slog.LevelWarn})) + if _, err := r2.TranslateBook(context.Background()); err != nil { + t.Fatalf("the resumed run: %v", err) + } + // ⚠ THE ONE-TIME-COST WARNING MUST NOT FIRE HERE, and this assertion exists because the first version + // of it did. It was emitted before the pass, on "this book has paid before" — which is true of every + // ordinary resume of a filtered book — so on a live run it announced a re-purchase while the pass made + // zero calls. An operator message that cries money on a $0 resume is the same defect this pack closes. + if strings.Contains(log.String(), logKeyReconsolidated+"=true") { + t.Errorf("the migration warning fired on a resume that re-bought nothing:\n%s", log.String()) + } + if r2.lastTerminology.BankSettled != 1 { + t.Errorf("the filter must decide the same way on every run, or the composition moves: skipped=%d", r2.lastTerminology.BankSettled) + } + if r2.lastTerminology.CostUSD != 0 { + t.Errorf("the resumed pass must replay from its checkpoints for $0, got $%f", r2.lastTerminology.CostUSD) + } + // Counted on the BANK ROLE's own calls: the second run still runs the edit wave the stop held back, so + // a count of all provider calls would move for a reason that has nothing to do with this filter. + if n := countTerminologyCalls(rec); n != terminologyCallsAfterFirst { + t.Errorf("the resume must reach the provider zero times for the bank role: role calls went %d → %d", + terminologyCallsAfterFirst, n) + } +} + +// countTerminologyCalls is how many provider calls carried the terminology role's block. +func countTerminologyCalls(rec *reqRec) int { + n := 0 + for _, body := range rec.all() { + if isTerminologyBody(body) { + n++ + } + } + return n +} + +// TestTheFilterChangesNothingOnTheCANDIDATESITKEEPS is the output-equivalence half of the order: the run +// must buy less and produce the same artifact for everything it still buys. +// +// The comparison is against a run of the SAME corpus and the SAME replies whose seed names an unrelated +// surface, so nothing is settled and the role is paid for every candidate. What the two runs must agree on, +// byte for byte, is the sheet row of the candidate both of them paid for; what they may differ in is the +// skipped surface itself, which is exactly "equal to the original minus the skipped rows". +func TestTheFilterChangesNothingOnTheCANDIDATESITKEEPS(t *testing.T) { + reply := func(body string) (string, string) { + if isTerminologyBody(body) { + return "方源\tФан Юань\n青茅山\tгора Цинмао", "stop" + } + return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop" + } + run := func(seed string) (*Runner, []BankStopRow) { + rec := &reqRec{} + srv := newJSONProvider(rec, reply) + t.Cleanup(srv.Close) + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, glossarySeed: seed})) + t.Cleanup(func() { _ = r.Close() }) + _ = runToSignatureStop(t, r) + return r, r.lastBankStopRows + } + // A: the seed settles 方源. B: the seed names a surface this book does not contain, so nothing is + // settled and every candidate is paid for. + withFilter, rowsA := run("terms:\n - { src: 方源, dst: Фан Юань, status: approved }\n") + _, rowsB := run("terms:\n - { src: 張三, dst: Чжан Сань, status: approved }\n") + + if withFilter.lastTerminology.BankSettled != 1 { + t.Fatalf("premise broken: run A must have filtered exactly one candidate: %+v", withFilter.lastTerminology) + } + find := func(rows []BankStopRow, src string) *BankStopRow { + for i := range rows { + if rows[i].Src == src { + return &rows[i] + } + } + return nil + } + kept, kept2 := find(rowsA, "青茅山"), find(rowsB, "青茅山") + if kept == nil || kept2 == nil { + t.Fatalf("premise broken: the paid-for candidate must be on both sheets: A=%+v B=%+v", rowsA, rowsB) + } + if !reflect.DeepEqual(*kept, *kept2) { + t.Errorf("the filter changed the sheet row of a candidate it did not skip:\n with filter: %+v\n without: %+v", *kept, *kept2) + } + // CONTROL for that equality: the SKIPPED surface is where the two runs are allowed to differ, and they + // must — otherwise the DeepEqual above is comparing two runs that took the same path. + skipped, paidFor := find(rowsA, "方源"), find(rowsB, "方源") + if skipped == nil || paidFor == nil { + t.Fatalf("premise broken: the skipped surface must be on both sheets: A=%+v B=%+v", rowsA, rowsB) + } + if skipped.Dst != "" { + t.Errorf("the skipped candidate was never asked about, so its sheet row carries no rendering: %q", skipped.Dst) + } + if paidFor.Dst == "" { + t.Fatalf("control: without the filter the same candidate IS answered for, or the two runs did not differ") + } +} + +// TestTheClassifiersTypeStillReachesTheSheetThroughTheFilter is the regression the filter's own first +// implementation carried, and it is here because the tests written for that filter could not have seen it. +// +// The classifier and the scorer MUTATE candidates in place, and the caller renders the sheet and the delta +// from the slice it passed in. Filtering the caller's slice copied the survivors into a new array, so those +// mutations stopped reaching the caller: the sheet would have carried the heuristic type and the unscored +// ranking, with every counter reading clean. Every other fixture in this package runs with the classifier +// OFF, where neither mutation happens — so this test turns it ON, which is the only state in which the +// defect is observable at all. +func TestTheClassifiersTypeStillReachesTheSheetThroughTheFilter(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isClassifierBody(body) { + // The heuristic drafted 青茅山 as a place; the classifier says otherwise, and THAT is the byte + // that has to survive the filter and reach the sheet. + return "青茅山\tterm\tnone", "stop" + } + if isTerminologyBody(body) { + return "青茅山\tгора Цинмао", "stop" + } + return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop" + }) + defer srv.Close() + seed := "terms:\n - { src: 方源, dst: Фан Юань, status: approved }\n" + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, classify: true, glossarySeed: seed})) + defer r.Close() + _ = runToSignatureStop(t, r) + + if r.lastTerminology.BankSettled != 1 { + t.Fatalf("premise broken: the filter must have skipped a candidate, or nothing is being crossed: %+v", r.lastTerminology) + } + if r.lastTerminology.Reclassified != 1 { + t.Fatalf("premise broken: the classifier must have CHANGED a type, or there is no mutation to lose: %+v", r.lastTerminology) + } + var row *BankStopRow + for i, cur := range r.lastBankStopRows { + if cur.Src == "青茅山" { + row = &r.lastBankStopRows[i] + } + } + if row == nil { + t.Fatalf("premise broken: 青茅山 is not on the sheet: %+v", r.lastBankStopRows) + } + if row.Type != "term" { + t.Errorf("the classifier's type must reach the sheet through the filter, got %q — the filter copied the candidates and the mutation was lost", row.Type) + } + // ⛔ AND THE WIRE, which neither of the two assertions above reaches. The classifier is bought FOR the + // type field, and that field travels to the ROLE for the whole batch — so the money is wasted the + // moment the request carries the heuristic's guess while the run prints `reclassified=N`. Asserted on + // the request bytes: this is the third form of one defect (the subset held as its own array), and the + // first two were both found after their own tests went green. + var asked string + for _, body := range rec.all() { + if isTerminologyBody(body) && strings.Contains(body, "### 青茅山") { + asked = body + } + } + if asked == "" { + t.Fatalf("premise broken: the surface must reach the paid role at all:\n%s", terminologyBodies(rec)) + } + if !strings.Contains(asked, "type: term") || strings.Contains(asked, "type: place") { + t.Errorf("the classifier's corrected type must reach the ROLE's request, not only the sheet — the run paid for that field:\n%s", asked) + } + // THE SECOND HALF OF THE SAME GUARANTEE, and it needs its own assertion: applyTypes and ScoreVariants + // are two in-place mutations, and pinning one leaves the other free. Re-scoring is what makes the + // heuristic's `conform` signal go away here — transliteration conformance speaks only about name/place + // (scoreOpts), so once the classifier calls this surface a `term` the signal is no longer true of it. + // Applied to the filtered copy instead, the sheet keeps a ranking factor the classification retired, + // and the operator reads a reason that no longer holds. Measured both ways: correct code gives no + // signals, the planting gives [conform]. + for _, sig := range row.Signals { + if sig == "conform" { + t.Errorf("the re-scoring must reach the sheet too: %q is ranked by a conformance that only speaks about name/place, and the classifier made it a term — signals=%v", row.Src, row.Signals) + } + } +} + +// TestTheOneTimeCostIsAnnouncedExactlyWhenItHappens pins the migration warning in BOTH directions, and it +// exists because that warning has been wrong twice, each time in a way its own test could not see. +// +// - first it was emitted before the pass, on «this book has paid» — true of every ordinary resume of a +// filtered book, so a live resume that re-bought nothing was told it had paid again; +// - then the probe moved after the pass, where the pass's OWN checkpoints make «has paid» trivially true +// — so a fresh book's FIRST run, having paid once, was told it had paid twice. +// +// Both halves are therefore asserted: the warning must be SILENT on a fresh first run and must FIRE on the +// one case it describes. A test that only asserts silence is vacuous here — the firing branch is +// unreachable in it — and that is exactly how the second version shipped. +func TestTheOneTimeCostIsAnnouncedExactlyWhenItHappens(t *testing.T) { + reply := func(body string) (string, string) { + if isTerminologyBody(body) { + return "青茅山\tгора Цинмао", "stop" + } + return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop" + } + + t.Run("silent on a fresh book that paid once", func(t *testing.T) { + srv := newJSONProvider(&reqRec{}, reply) + defer srv.Close() + seed := "terms:\n - { src: 方源, dst: Фан Юань, status: approved }\n" + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, glossarySeed: seed})) + defer r.Close() + var log bytes.Buffer + r.Log = slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{Level: slog.LevelWarn})) + _ = runToSignatureStop(t, r) + if r.lastTerminology.BankSettled != 1 || r.lastTerminology.CostUSD == 0 { + t.Fatalf("premise broken: the run must both FILTER and PAY, or the branch is not reached: %+v", r.lastTerminology) + } + if strings.Contains(log.String(), logKeyReconsolidated+"=true") { + t.Errorf("a book paying for the FIRST time was told it had paid before — the probe is reading this run's own checkpoints:\n%s", log.String()) + } + }) + + t.Run("fires when an earlier composition really was paid for", func(t *testing.T) { + srv := newJSONProvider(&reqRec{}, reply) + defer srv.Close() + // Run one has NO seed surface for 方源, so nothing is filtered and the pass is paid for over the + // FULL candidate set. + bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}) + r1 := newVerifyRunner(t, bookPath) + _ = runToSignatureStop(t, r1) + if r1.lastTerminology.BankSettled != 0 || r1.lastTerminology.CostUSD == 0 { + t.Fatalf("premise broken: run one must filter NOTHING and pay: %+v", r1.lastTerminology) + } + minedDelta := r1.Book.MinedDelta + r1.Close() + + // The owner signs 方源 with the rendering the drafts proposed. Its surface is now a seed surface, + // so run two filters it — and the composition it pays under is not the one run one paid under. + writeFile(t, minedDelta, "terms:\n - { src: 方源, dst: Фан Юань, status: approved }\n") + + r2 := newVerifyRunner(t, bookPath) + defer r2.Close() + var log bytes.Buffer + r2.Log = slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{Level: slog.LevelWarn})) + if _, err := r2.TranslateBook(context.Background()); err != nil { + var stop *WaveSignatureStop + if !errors.As(err, &stop) { + t.Fatalf("run two: %v", err) + } + } + if r2.lastTerminology.BankSettled != 1 { + t.Fatalf("premise broken: run two must filter the newly-signed surface: %+v", r2.lastTerminology) + } + if r2.lastTerminology.CostUSD == 0 { + t.Fatalf("premise broken: the changed composition must make run two PAY again, or there is nothing to announce") + } + if !strings.Contains(log.String(), logKeyReconsolidated+"=true") { + t.Errorf("the one-time re-consolidation happened and was not announced:\n%s", log.String()) + } + }) +} + +// TestTheSettledTestAnswersOnEachOfItsConditions pins the three conditions bankSettles rests on, one at a +// time. Two of them had no witness at all: every fixture that reached the filter carried a banknote-origin +// candidate whose surface matched a seed row's own src, so «banknote-only» and «aliases count» were both +// true of the fixture by accident and could have been dropped without a test noticing. +func TestTheSettledTestAnswersOnEachOfItsConditions(t *testing.T) { + seed := []store.GlossaryEntry{ + {Src: "方源", Dst: "Фан Юань", Status: "approved", Source: "seed", + Aliases: []store.GlossaryAlias{{Alias: "小方"}}}, + {Src: "青茅山", Dst: "", Status: "approved", Source: "seed"}, // no rendering: nothing to agree with + } + settled := bankSettledSurfaces(seed) + // A row with no rendering is not in the index at all — there is nothing for a draft to agree with. + if _, ok := settled[text.NormalizeSourceKey("青茅山")]; ok { + t.Errorf("a seed row with no dst cannot settle anything: %+v", settled) + } + banknote := func(key, draft string) terminology.Candidate { + return terminology.Candidate{Key: key, Src: key, Origin: terminology.OriginBanknote, + Variants: []terminology.Variant{{Dst: draft}}} + } + // BASE: the shape the filter is for. + if !bankSettles(settled, banknote("方源", "Фан Юань")) { + t.Fatalf("premise broken: a banknote candidate on a seeded surface whose draft agrees must be settled") + } + // (1) ORIGIN. A MINER-origin candidate is not dropped by reverseSectionTerms, so it DOES reach the + // delta — filtering it would lose a term from the artifact, not just a paid question. + mined := banknote("方源", "Фан Юань") + mined.Origin = terminology.OriginMined + if bankSettles(settled, mined) { + t.Errorf("only the population the EMISSION drops may be filtered; a mined candidate reaches the delta") + } + // (2) ALIASES. The emission's own guard reads a seed row's aliases, so this one must too, or a + // proposal named by an alias is paid for and then thrown away. + if !bankSettles(settled, banknote("小方", "Фан Юань")) { + t.Errorf("a surface the seed holds as an ALIAS is settled just as its src is") + } + // (3) AGREEMENT, per variant and not per candidate: one disagreeing draft among several is enough. + two := banknote("方源", "Фан Юань") + two.Variants = append(two.Variants, terminology.Variant{Dst: "Источник Фана"}) + if bankSettles(settled, two) { + t.Errorf("one disagreeing draft is the finding; the candidate must be paid for") + } + // (4) NO EVIDENCE is not evidence of agreement. Unreachable from a banknote candidate by construction + // — they always carry a proposal — and asserted so the rule cannot be quietly inverted. + none := banknote("方源", "") + none.Variants = nil + if bankSettles(settled, none) { + t.Errorf("a candidate with no observed rendering has not agreed with anything") + } +} + +// TestTheFilterFoldsBothSidesTheWayProductionDoes pins the two normalizations inside the money filter, +// each of which decides whether a paid call happens and neither of which had a witness. Breaking either +// left the whole battery green: the source fold silently stops the filter firing for every traditional or +// katakana spelling — and BankSettled then reads 0, indistinguishable from «nothing to skip» — while the +// target fold is the money decision itself. +func TestTheFilterFoldsBothSidesTheWayProductionDoes(t *testing.T) { + // A seed row spelled TRADITIONALLY, and a candidate keyed the way production keys it. + seed := []store.GlossaryEntry{{Src: "長空", Dst: "Пустота Небес", Status: "approved", Source: "seed"}} + settled := bankSettledSurfaces(seed) + if _, ok := settled["长空"]; !ok { + t.Fatalf("the seed surface must be indexed by its NORMALIZED key, or the filter never fires for the traditional population: %+v", settled) + } + cand := func(draft string) terminology.Candidate { + return terminology.Candidate{Key: "长空", Src: "長空", Origin: terminology.OriginBanknote, + Variants: []terminology.Variant{{Dst: draft}}} + } + if !bankSettles(settled, cand("Пустота Небес")) { + t.Fatalf("premise broken: an agreeing draft on a seeded surface must settle") + } + // THE TARGET FOLD. A draft differing only by case and ё is the SAME rendering under the fold the vote + // is counted with, so it agrees — and paying for it would be paying to be told what the bank says. + if !bankSettles(settled, cand("пустота небес")) { + t.Errorf("the agreement must be judged under the same target-form fold the vote uses, not byte-wise") + } + // CONTROL: a genuinely different rendering still disagrees, so the fold above is a fold and not a + // comparison that has stopped comparing. + if bankSettles(settled, cand("Небесная пустота")) { + t.Errorf("a different rendering is the finding and must be paid for") + } +} diff --git a/backend/internal/pipeline/mining.go b/backend/internal/pipeline/mining.go index f203b591..873e599b 100644 --- a/backend/internal/pipeline/mining.go +++ b/backend/internal/pipeline/mining.go @@ -405,6 +405,13 @@ type BankStopRow struct { Conf int Contradicts []string BankHolds []string + // SettledByBank marks a row the paid role was NEVER ASKED about, because the bank already renders the + // surface and every draft proposed that same rendering. Without it the row prints an empty dst, which + // on this sheet is the same glyph as «the role declined», «no reply line covered it» and «the budget + // did not reach it» — four different facts, one of which is «nothing to decide» and three of which are + // «undecided». Rendered by the text table only: the sidecar's proposal section is contract-pinned + // (backlog row 353) and is deliberately not touched. + SettledByBank bool } // BankStopVariant is ONE rendering the drafts produced, kept in its PARTS rather than as the sentence a @@ -448,9 +455,8 @@ func bankStopRows(cands []terminology.Candidate, consolidated map[string]string, Src: c.Src, Dst: dst, Origin: string(c.Origin), Type: c.Type, Freq: c.Freq, Spread: c.Spread(), Conventions: c.Conventions(), Contexts: c.KWIC, Evidence: c.Evidence, - Conf: confOrAbsent(tres.Conf, c.Key), Contradicts: tres.Contradictions[c.Src], - BankHolds: bankHoldLabels(tres.BankHoldRows, c), } + row.Conf, row.Contradicts, row.BankHolds, row.SettledByBank = tres.findingsFor(c) for i, v := range c.Variants { row.Variants = append(row.Variants, BankStopVariant{Dst: v.Dst, Chunks: v.Chunks, Via: v.Via}) if i == 0 { @@ -463,26 +469,44 @@ func bankStopRows(cands []terminology.Candidate, consolidated map[string]string, return out } -// bankHoldLabels picks the findings belonging to one candidate. They are matched on the candidate KEY, -// because that is the source consolidatedRows gives the proposal and therefore the conflict's own Src; a -// banknote candidate's raw surface is a different string, and its firing key may be an alias. -func bankHoldLabels(cols []membank.BankKeyConflict, c terminology.Candidate) []string { - var out []string - for _, cf := range cols { - if cf.Src == c.Key { - out = append(out, cf.BankRowLabel()) +// findingsFor is THE ONE PLACE a finding of the terminology phase is matched to the candidate it belongs +// to, and the only place the matching key is chosen. +// +// ⚠ WHY IT IS ONE FUNCTION AND NOT THREE LOOKUPS AT THE CALL SITE. Every one of these three crossed the +// file boundary as a map keyed on the writer's side and read on the reader's, which means the key was +// chosen TWICE per finding, in two files, with nothing comparing the two choices. They agreed; that they +// agreed was luck, and no test could have said otherwise, because a candidate whose Key differs from its +// Src — every traditional or katakana spelling — appeared in no fixture. Mutating any of the three keys +// survived the whole of pipeline, terminology and tmctl. The same shape was found and removed once +// already on the BankHolds half; this is that fix finished rather than repeated a fourth time (D39.216). +// +// The key is the candidate KEY throughout: it is the surface consolidatedRows gives a proposal, the +// surface the role was asked about (candKeys), and the surface ConsolidationConflicts compares on. A +// banknote candidate's raw Src is a different string and its firing key may be an alias, so either of +// those would lose the whole banknote population silently. +func (t terminologyResult) findingsFor(c terminology.Candidate) (conf int, contradicts, bankHolds []string, settled bool) { + // The role's stated confidence, or -1 when the reply carried none. A plain zero would merge the two, + // and they are opposites: one is the first row to review, the other is silence. + conf = -1 + if v, ok := t.Conf[c.Key]; ok { + conf = v + } + for _, cf := range t.SelfConflictRows { + if cf.Key == c.Key { + contradicts = append(contradicts, cf.PartLabel()) } } - return out -} - -// confOrAbsent reads the role's stated confidence for a key, or -1 when the reply carried none. A plain -// zero would merge the two, and they are opposites: one is the first row to review, the other is silence. -func confOrAbsent(conf map[string]int, key string) int { - if v, ok := conf[key]; ok { - return v + for _, cf := range t.BankHoldRows { + if cf.Src == c.Key { + bankHolds = append(bankHolds, cf.BankRowLabel()) + } } - return -1 + for _, k := range t.BankSettledKeys { + if k == c.Key { + settled = true + } + } + return conf, contradicts, bankHolds, settled } // proposedByDrafts reports whether the consolidated rendering is one the drafts actually produced, compared @@ -515,6 +539,9 @@ func renderBankStopTable(rows []BankStopRow) string { if r.Invented { b.WriteString(" INVENTED(no draft proposed it)") } + if r.SettledByBank { + b.WriteString(" NOT ASKED(the bank already renders this surface and every draft agreed — nothing to decide)") + } b.WriteString("\n") if len(r.Signals) > 0 { fmt.Fprintf(&b, " why: %s\n", strings.Join(r.Signals, ", ")) @@ -645,13 +672,19 @@ func (r *Runner) autoBankPath() string { return r.Book.ProjectDB + ".auto-bank.y // - the REJECT SET applies here too (risk 3). Until pack-20 rejects were consulted only at EMISSION, so // a term the owner declined could survive in an accumulated file and re-enter the bank through the // back door. «reject-set works in both modes» has to mean on the way IN, not only on the way out. -// - a row whose UNIQUE key (src, sense, since_ch, until_ch) is already held by a SIGNED row is dropped -// (risk 4). The flat INSERT in ReplaceGlossary would otherwise crash on the constraint and abort a -// paid run — and the right resolution is never "the engine's guess replaces the signature". +// - a row whose UNIQUE key (src, sense, since_ch, until_ch) is already held by a row gathered EARLIER is +// dropped (risk 4). The flat INSERT in ReplaceGlossary would otherwise crash on the constraint and +// abort a paid run. ⚠ THE HOLDER IS OF ANY STATUS, not only a signed one: what is passed in is the +// whole gathered set — seed, ruby aliases and the owner's mined-delta — and an unsigned row holds its +// tuple exactly as hard, because the store's constraint does not read statuses. Where the holder IS +// signed the resolution is never "the engine's guess replaces the signature"; where it is not, the +// drop is the constraint talking and nothing more, so each dropped row is reported WITH its holder's +// signature. Saying "the signed term wins" over an unsigned holder is the one thing this must not do: +// the operator signs the bank on the strength of these lines (D39.104 п.1). // - nothing here can carry `approved`: the loader is the same seed loader, and the status it reads is // whatever the emission wrote (auto/draft). A file hand-edited to say `approved` is refused loudly, // because that would be a signature nobody gave. -func (r *Runner) loadAutoBank(signed []store.GlossaryEntry) (rows []store.GlossaryEntry, dropped []string, err error) { +func (r *Runner) loadAutoBank(gathered []store.GlossaryEntry) (rows []store.GlossaryEntry, dropped []string, err error) { // Only ABSENT means "auto mode has not run"; an unreadable file must not drop the mined rows. if _, serr := os.Stat(r.autoBankPath()); serr != nil { if errors.Is(serr, fs.ErrNotExist) { @@ -679,7 +712,7 @@ func (r *Runner) loadAutoBank(signed []store.GlossaryEntry) (rows []store.Glossa since, until int } held := map[ukey]store.GlossaryEntry{} - for _, e := range signed { + for _, e := range gathered { held[ukey{e.Src, e.Sense, e.SinceCh, e.UntilCh}] = e } for _, e := range entries { @@ -688,7 +721,7 @@ func (r *Runner) loadAutoBank(signed []store.GlossaryEntry) (rows []store.Glossa continue } if prior, clash := held[ukey{e.Src, e.Sense, e.SinceCh, e.UntilCh}]; clash { - dropped = append(dropped, fmt.Sprintf("%q→%q (key held by the signed %q→%q)", e.Src, e.Dst, prior.Src, prior.Dst)) + dropped = append(dropped, fmt.Sprintf("%q→%q (key held by the %s %q→%q)", e.Src, e.Dst, membank.StatusLabel(prior.Status), prior.Src, prior.Dst)) continue } e.Source = "mined" // base-excluded: an unsigned row never moves the draft wave's snapshot @@ -708,7 +741,10 @@ func (r *Runner) loadAutoBank(signed []store.GlossaryEntry) (rows []store.Glossa func unsignedEngineSurfaces(rows []store.GlossaryEntry) []store.GlossaryEntry { out := rows[:0:0] for _, e := range rows { - if e.Source == "mined" && e.Status != "approved" { + // membank.IsEngineUnsigned rather than the test spelled out here: the same question is asked by the + // bank's own tuple skip and by the paid path's filter, and three spellings of it would be three + // answers to "is this the engine's own word". + if membank.IsEngineUnsigned(e) { continue } out = append(out, e) diff --git a/backend/internal/pipeline/miningstop_join_test.go b/backend/internal/pipeline/miningstop_join_test.go index c66a0f35..c108a8c9 100644 --- a/backend/internal/pipeline/miningstop_join_test.go +++ b/backend/internal/pipeline/miningstop_join_test.go @@ -1623,8 +1623,8 @@ func TestLoadAutoBankFiltersRejectsAndCollisions(t *testing.T) { " - src: 花家\n dst: Дом Хуа\n status: draft\n"+ " - src: 青茅山\n dst: гора Цинмао\n status: draft\n") - signed := []store.GlossaryEntry{{Src: "方源", Dst: "Фан Юань", Status: "approved", Source: "seed"}} - rows, dropped, err := r.loadAutoBank(signed) + gathered := []store.GlossaryEntry{{Src: "方源", Dst: "Фан Юань", Status: "approved", Source: "seed"}} + rows, dropped, err := r.loadAutoBank(gathered) if err != nil { t.Fatal(err) } @@ -1638,15 +1638,20 @@ func TestLoadAutoBankFiltersRejectsAndCollisions(t *testing.T) { if !strings.Contains(joined, "花家") || !strings.Contains(joined, "declined") { t.Fatalf("the DECLINED row must be dropped and said out loud: %q", joined) } - if !strings.Contains(joined, "方源") || !strings.Contains(joined, "signed") { - t.Fatalf("the row colliding with a SIGNED term must be dropped and said out loud: %q", joined) + // The holder here IS signed, and the line has to say so with the holder's own status rather than with + // the word "signed" — which the message used to print over any holder at all, signed or not + // (TestADropNamesItsHoldersSignature is where that half is pinned). + if !strings.Contains(joined, `key held by the approved "方源"`) { + t.Fatalf("the row colliding with a signed term must be dropped and say WHOSE key it hit: %q", joined) } } // TestAutoBankKeyCollisionDoesNotCrashTheRun is risk 4 end-to-end: an auto row whose UNIQUE key -// (src, sense, since_ch, until_ch) is already held by a signed term would make the flat INSERT in -// ReplaceGlossary hit the constraint and abort a PAID run mid-flight. The engine's own proposal must -// never be able to do that — the signed term wins and the run goes on. +// (src, sense, since_ch, until_ch) is already held by a row gathered earlier would make the flat INSERT in +// ReplaceGlossary hit the constraint and abort a PAID run mid-flight. The engine's own proposal must never +// be able to do that — the holder wins and the run goes on. The holder here IS signed, and that is this +// fixture's case rather than the rule: the store's constraint does not read statuses, and an unsigned +// holder wins the tuple exactly as hard (TestADropNamesItsHoldersSignature). func TestAutoBankKeyCollisionDoesNotCrashTheRun(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) diff --git a/backend/internal/pipeline/operatormessages_test.go b/backend/internal/pipeline/operatormessages_test.go new file mode 100644 index 00000000..5ab3adea --- /dev/null +++ b/backend/internal/pipeline/operatormessages_test.go @@ -0,0 +1,252 @@ +package pipeline + +import ( + "fmt" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" +) + +// operatormessages_test.go pins the TEXT of every operator message this package emits without stopping +// the run. +// +// ⚠ WHY A CATALOGUE AND NOT MORE SUBSTRING ASSERTS. The messages that rot are exactly the ones nothing +// else reads. A message that STOPS the run is held in place by the test that asserts the stop — change +// its meaning and that test says so. A warning is read by a person, once, in a log, and by nothing else: +// the S14 conflict warning was guarded by one substring covering one phrase of it, so when a rule change +// made the sentence false the substring still matched, and what caught it was an unrelated battery test. +// The localizing half of that message — both rows, both windows, the firing key — was guarded by nothing +// at all. Adding another substring assert would have moved the boundary, not closed it: the next message +// somebody adds is outside it again, silently. This is the same shape as flagseverity_test.go, and for +// the same reason: a hand-written list of what to check rots exactly like the code it checks. +// +// WHAT THE CATALOGUE ASSERTS, and it is not truth. It asserts DELIBERATENESS. Nothing here can know +// whether "the signed term wins" is true of the code below it; what it can do is make the day somebody +// changes those words the day they read the sentence again. A wording change is then a diff line in +// testdata/operator-messages.txt with the function's name beside it, which a reviewer can judge. +// +// THE BOUNDARY, stated so the next reader does not have to infer it: +// +// - COVERED: every Warn/WarnContext/Error/ErrorContext call on this package's logger, and every +// bankInputs.remark, whose message is a string LITERAL. These run on and leave only a line behind. +// - COVERED as a count, not as text: the same calls whose message is BUILT at run time. They cannot be +// pinned as text, so the catalogue holds their site with in place of the message — which +// still makes a NEW one visible, and a message quietly converted from a literal into a built string +// (the way to leave this gate without touching it) shows up as a changed line. +// - NOT COVERED, deliberately: Info/InfoContext — progress, not a decision, and a reader who misreads +// one loses nothing; and fmt.Errorf — a message that ends the run already has a test that asserts the +// ending, so it has a second reader and does not rot alone. +// - NOT COVERED, and named so it is not mistaken for coverage: the ARGUMENTS. Renaming a key or +// dropping a value from the structured tail is invisible here. +func TestEveryOperatorMessageIsCatalogued(t *testing.T) { + got := operatorMessagesOfPackage(t) + // The walk finding nothing at all would report a green "the catalogue matches" over an empty set, which + // is the failure mode this whole file exists to prevent. 41 source files carry ~100 of these; a floor + // well under that still catches a walk that stopped seeing the package. + if len(got) < 60 { + t.Fatalf("only %d operator messages found — the walk stopped seeing the package", len(got)) + } + const catalogue = "testdata/operator-messages.txt" + raw, err := os.ReadFile(catalogue) + if err != nil { + t.Fatalf("read %s: %v", catalogue, err) + } + var want []string + for _, line := range strings.Split(string(raw), "\n") { + if line = strings.TrimRight(line, "\r"); line != "" && !strings.HasPrefix(line, "#") { + want = append(want, line) + } + } + sort.Strings(want) + inWant := map[string]int{} + for _, l := range want { + inWant[l]++ + } + inGot := map[string]int{} + for _, l := range got { + inGot[l]++ + } + var added, gone []string + for l, n := range inGot { + for i := inWant[l]; i < n; i++ { + added = append(added, l) + } + } + for l, n := range inWant { + for i := inGot[l]; i < n; i++ { + gone = append(gone, l) + } + } + sort.Strings(added) + sort.Strings(gone) + if len(added) == 0 && len(gone) == 0 { + return + } + var b strings.Builder + fmt.Fprintf(&b, "the operator messages this package emits no longer match %s.\n", catalogue) + b.WriteString("This is not a formatting gate: each line below is a sentence an operator reads and acts on,\n") + b.WriteString("and the catalogue is where a change to one is reviewed. Read the new wording against what the\n") + b.WriteString("code now does, then paste the line into the file (it is sorted; keep it that way).\n") + if len(added) > 0 { + fmt.Fprintf(&b, "\nIN THE CODE, NOT IN THE CATALOGUE (%d):\n", len(added)) + for _, l := range added { + b.WriteString(l + "\n") + } + } + if len(gone) > 0 { + fmt.Fprintf(&b, "\nIN THE CATALOGUE, NOT IN THE CODE (%d) — a message was reworded, moved or removed:\n", len(gone)) + for _, l := range gone { + b.WriteString(l + "\n") + } + } + t.Fatal(b.String()) +} + +// operatorMessagesOfPackage returns one catalogue line per operator message the package emits, sorted. +// A line is `filefunctionquoted message`, and the message is quoted so a newline or a tab inside +// one cannot forge a line boundary. +func operatorMessagesOfPackage(t *testing.T) []string { + t.Helper() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read the package directory: %v", err) + } + var out []string + fset := token.NewFileSet() + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + f, perr := parser.ParseFile(fset, name, nil, 0) + if perr != nil { + t.Fatalf("parse %s: %v", name, perr) + } + // The enclosing function is tracked by walking declarations rather than by asking the node, because + // a message inside a closure belongs, for a reader, to the function that spells the closure. + for _, d := range f.Decls { + fd, ok := d.(*ast.FuncDecl) + if !ok { + continue + } + ast.Inspect(fd, func(n ast.Node) bool { + msg, found := operatorMessageArg(fset, n) + if found { + out = append(out, strings.Join([]string{name, fd.Name.Name, msg}, "\t")) + } + return true + }) + } + } + sort.Strings(out) + return out +} + +// operatorMessageArg reports the message of one operator-facing call, quoted, or found=false when the node +// is not one. `` stands for a message built at run time — see the boundary above. +func operatorMessageArg(fset *token.FileSet, n ast.Node) (msg string, found bool) { + call, ok := n.(*ast.CallExpr) + if !ok { + return "", false + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return "", false + } + var at int + switch sel.Sel.Name { + case "WarnContext", "ErrorContext": + at = 1 + case "Warn", "Error": + at = 0 // no ctx in front of the message, unlike the *Context pair above + // err.Error() and friends are spelled the same and are not messages. The logger is what decides: + // every one in this package is reached through a field or variable named log/Log/logger, and a + // call on anything else is somebody's error being rendered. + if !isLoggerExpr(fset, sel.X) { + return "", false + } + case "remark": + at = 0 + default: + return "", false + } + if sel.Sel.Name == "WarnContext" || sel.Sel.Name == "ErrorContext" { + if !isLoggerExpr(fset, sel.X) { + return "", false + } + } + if len(call.Args) <= at { + return "", false + } + lit, ok := call.Args[at].(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return "", true + } + s, err := strconv.Unquote(lit.Value) + if err != nil { + return "", true + } + return strconv.Quote(s), true +} + +// isLoggerExpr reports whether an expression is this package's logger. Matched on the LAST identifier of +// the rendered expression, so r.Log, e.log, logger and log all answer yes and a bare err does not. +func isLoggerExpr(fset *token.FileSet, x ast.Expr) bool { + var b strings.Builder + if err := printer.Fprint(&b, fset, x); err != nil { + return false + } + s := b.String() + if i := strings.LastIndexAny(s, ".)"); i >= 0 { + s = s[i+1:] + } + return strings.EqualFold(s, "log") || strings.EqualFold(s, "logger") +} + +// TestTheCatalogueFileIsWellFormed keeps the catalogue reviewable and keeps the comparison above honest: +// an unsorted file makes a one-line wording change read as a large diff, and a line with the wrong number +// of fields compares something other than a message. +// +// ⚠ DUPLICATE LINES ARE LEGITIMATE and are deliberately not rejected — two call sites in one function do +// sometimes carry the same words (quality.go does). An earlier version of this test forbade them on the +// reasoning that one would be "guarded by the other"; running it showed the reasoning was wrong, because +// the catalogue is compared as a MULTISET, so two identical messages need two identical lines and +// rewording either one still goes red. +func TestTheCatalogueFileIsWellFormed(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("testdata", "operator-messages.txt")) + if err != nil { + t.Fatalf("read the catalogue: %v", err) + } + var lines []string + for _, l := range strings.Split(string(raw), "\n") { + if l = strings.TrimRight(l, "\r"); l != "" && !strings.HasPrefix(l, "#") { + lines = append(lines, l) + } + } + if !sort.StringsAreSorted(lines) { + t.Errorf("the catalogue is not sorted; sort it so a wording change is a one-line diff") + } + // Every line must have exactly the three fields the walker writes, and the first must be a source file + // of this package — otherwise the comparison above silently compares something that is not a message. + for _, l := range lines { + if n := strings.Count(l, "\t"); n != 2 { + t.Errorf("catalogue line has %d tabs, want 2 (filefuncquoted message):\n%s", n, l) + continue + } + file := l[:strings.IndexByte(l, '\t')] + if !strings.HasSuffix(file, ".go") || strings.Contains(file, "/") { + t.Errorf("catalogue line does not start with a source file of this package:\n%s", l) + continue + } + if _, err := os.Stat(file); err != nil { + t.Errorf("catalogue names %s, which this package does not have: %v", file, err) + } + } +} diff --git a/backend/internal/pipeline/quality.go b/backend/internal/pipeline/quality.go index d642ae20..8119e43b 100644 --- a/backend/internal/pipeline/quality.go +++ b/backend/internal/pipeline/quality.go @@ -104,7 +104,9 @@ type QualityReport struct { // The UNSIGNED-BANK channel (pack-20 / D39.42 п.4). In the auto mode the bank carries renderings nobody // signed, and the run needs to say so out loud rather than let them pass for canon: // • UnsignedBankTerms — how many rows of the bank are unsigned right now (the exposure); - // • UnverifiedShown — how many times such a row actually fired in a unit's source (the denominator); + // • UnverifiedShown — how many times such a row could be JUDGED in a unit, i.e. its key fired in + // that unit's source (the denominator). NOT the count of rows the model was shown: a sticky carry is + // shown and deliberately not judged, since its src is back in the previous chunk; // • UnverifiedFollowed— of those, how often the model went along with the proposed rendering. // None of them is a verdict: an unsigned row is a candidate the model is entitled to reject, so a low // follow rate is information about the CHANNEL, not a defect in the text. All omitempty, so a book with diff --git a/backend/internal/pipeline/resume_updatedat_test.go b/backend/internal/pipeline/resume_updatedat_test.go index 140eced7..f7c83d5f 100644 --- a/backend/internal/pipeline/resume_updatedat_test.go +++ b/backend/internal/pipeline/resume_updatedat_test.go @@ -6,6 +6,8 @@ import ( "strings" "testing" "time" + + "textmachine/backend/internal/store" ) // TestOrdinaryResumeMovesOnlyTheSkippedRow pins which rows a $0 resume writes. store.ChunkStatus.UpdatedAt @@ -91,3 +93,93 @@ func statusStamps(t *testing.T, r *Runner) map[string]statusStamp { func chunkStageKey(chapter, chunkIdx int, stage string) string { return fmt.Sprintf("ch%d/chunk%d/%s", chapter, chunkIdx, stage) } + +// TestASkippedRowSaysWHICHWriterWroteIt pins the half of store.ChunkStatus.UpdatedAt's doc that could +// only be READ until now. It names two writers of a `skipped` row and says which condition selects +// each — and reading was not enough: the author of the pack that wrote that sentence named the wrong +// mechanism three times running, and each time it was a mutation that caught him. +// +// ⚠ MEASURED BEFORE THIS TEST EXISTED, and it is why the second case is here at all: breaking +// recordSkippedStages reddened six tests, while breaking the flagged branch of runStageSequence survived +// the WHOLE package. One of the two writers the doc names was exercised by nothing, so no mutation could +// have told a reader which one fires — the question the doc answers had no witness on either side. +// +// The two writers are told apart by the DETAIL they write, which is the only durable trace saying which +// code path produced the row. Swap the two sentences and this goes red naming both. +func TestASkippedRowSaysWHICHWriterWroteIt(t *testing.T) { + t.Run("a flagged member draft skips the unit's edit", func(t *testing.T) { + srv := newJSONProvider(&reqRec{}, func(body string) (string, string) { + if !isEditBody(body) { + return "Извините, я не могу перевести это.", "stop" // the draft soft-refuses → the edit is pre-decided skipped + } + return draftEdit(body) + }) + defer srv.Close() + r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА"})) + defer r.Close() + if _, err := r.TranslateBook(context.Background()); err != nil { + t.Fatal(err) + } + got := statusDetail(t, r, "edit") + if !strings.Contains(got, "a member draft chunk of this edit unit was flagged") { + t.Errorf("a unit whose member draft flagged is skipped by recordSkippedStages, and the row must say so: %q", got) + } + if strings.Contains(got, "an upstream stage was flagged") { + t.Errorf("this row is not the multi-stage skip; the two writers' rows are indistinguishable: %q", got) + } + }) + + t.Run("a flagged stage skips its own wave's later stages", func(t *testing.T) { + srv := newJSONProvider(&reqRec{}, func(body string) (string, string) { + if isEditBody(body) { + return "Извините, я не могу продолжить.", "stop" // the FIRST edit stage flags on the refusal blacklist + } + return draftEdit(body) + }) + defer srv.Close() + r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ОБЫЧНАЯГЛАВА", secondEditStage: true})) + defer r.Close() + if _, err := r.TranslateBook(context.Background()); err != nil { + t.Fatal(err) + } + // PREMISE: the first edit stage really did flag, so what follows is the skip of a LATER stage in + // the same wave and not a unit that was never edited. + if d := statusDisposition(t, r, "edit"); d != string(DispFlagged) { + t.Fatalf("premise broken: the first edit stage must be flagged, got %q", d) + } + got := statusDetail(t, r, "polish") + if !strings.Contains(got, "an upstream stage was flagged") { + t.Errorf("a later stage of the same wave is skipped by runStageSequence, and the row must say so: %q", got) + } + if strings.Contains(got, "a member draft chunk") { + t.Errorf("this row is not the pre-decided edit skip; the two writers' rows are indistinguishable: %q", got) + } + }) +} + +// statusDetail returns the stored chunk_status detail of one stage of the book's first chunk. +func statusDetail(t *testing.T, r *Runner, stage string) string { + t.Helper() + return statusRow(t, r, stage).Detail +} + +// statusDisposition returns the stored disposition of one stage of the book's first chunk. +func statusDisposition(t *testing.T, r *Runner, stage string) string { + t.Helper() + return statusRow(t, r, stage).Disposition +} + +func statusRow(t *testing.T, r *Runner, stage string) store.ChunkStatus { + t.Helper() + rows, err := r.Store.ChunkStatusesForBook(r.Book.BookID) + if err != nil { + t.Fatalf("read chunk_status: %v", err) + } + for _, cs := range rows { + if cs.Stage == stage && cs.Chapter == 1 && cs.ChunkIdx == 0 { + return cs + } + } + t.Fatalf("no chunk_status row for stage %q; rows were %+v", stage, rows) + return store.ChunkStatus{} +} diff --git a/backend/internal/pipeline/runner_test.go b/backend/internal/pipeline/runner_test.go index c58e09e1..2aba521f 100644 --- a/backend/internal/pipeline/runner_test.go +++ b/backend/internal/pipeline/runner_test.go @@ -123,6 +123,11 @@ type projectOpts struct { // draftOnly drops the editor stage, so the DRAFT is the shipping output (the pipeline shape the wave // executor calls draft-only). false keeps the historical two-stage fixture byte-identical. draftOnly bool + // secondEditStage puts a SECOND non-translator stage in the edit wave, which is the only shape that + // reaches runStageSequence's flagged-skip branch: with one stage per wave a flagged stage has no later + // sibling to skip, and that branch was unexercised by every fixture in this package. false leaves each + // pre-existing project byte-identical. + secondEditStage bool // systemMessagesSingle declares the fake PROVIDER as an endpoint that carries exactly ONE system // message (the Gemini OpenAI-compat quirk). false omits the capabilities block entirely, so every // pre-existing fixture's models.yaml stays byte-identical. @@ -176,6 +181,9 @@ models: gatesBlock += "\ngates:\n banknote:\n enabled: true\n" } editStage := " - { name: edit, role: editor, model: fake-model, prompt_override: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: \"off\" }\n" + if o.secondEditStage { + editStage += " - { name: polish, role: editor, model: fake-model, prompt_override: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: \"off\" }\n" + } if o.draftOnly { editStage = "" } diff --git a/backend/internal/pipeline/stopsheetfindings_test.go b/backend/internal/pipeline/stopsheetfindings_test.go new file mode 100644 index 00000000..c7007c10 --- /dev/null +++ b/backend/internal/pipeline/stopsheetfindings_test.go @@ -0,0 +1,181 @@ +package pipeline + +import ( + "strings" + "testing" + + "textmachine/backend/internal/membank" + "textmachine/backend/internal/terminology" + "textmachine/backend/internal/text" +) + +// stopsheetfindings_test.go pins the join between what the terminology phase FOUND and the row of the +// signature sheet the owner reads it on. +// +// ⚠ WHY THE FIXTURE IS SPELLED THE WAY IT IS, and it is the whole test. Every finding here used to cross +// the file boundary as a map: keyed on the writer's side in terminologist.go, read on the reader's side in +// mining.go. Two independent choices of key per finding, three findings, nothing comparing the six. They +// agreed — and no test in the module could have said whether they did, because a candidate whose KEY +// differs from its raw SRC appeared in no fixture, and on equal strings every wrong choice is right. +// Mutating any of the three keys survived the full pipeline, terminology and tmctl packages. +// +// So the candidates below are TRADITIONAL spellings, and the fixture asks the REAL normalizer whether +// they are: 長空 folds to 长空 and 長 to 长 (internal/text/data/trad2simp.txt lines 434, 193). ⚠ The first +// version of this file asserted the same premise about 雲海空竅 → 云海空窍 and was WRONG — neither 雲 nor 竅 +// is in that table, so the two strings differed only because they were typed differently, and the guard +// below compared two hand-written literals and would have stayed green against a normalizer replaced by +// identity. A fixture whose premise is checked by a literal certifies the literal. + +func TestEveryStopSheetFindingReachesTheRowItBelongsTo(t *testing.T) { + compound := terminology.Candidate{Key: "长空", Src: "長空", Type: "term", Freq: 4} + part := terminology.Candidate{Key: "长", Src: "長", Type: "term", Freq: 9} + // The premise is asked of the NORMALIZER, not of the literals: the key must be what production would + // derive from the surface, and it must differ from it. Either half checked against a typed constant + // would certify the constant — which is how the previous fixture stood on a fold that does not exist. + for _, c := range []terminology.Candidate{compound, part} { + if got := text.NormalizeSourceKey(c.Src); got != c.Key { + t.Fatalf("fixture premise: production keys %q as %q, not as %q — the fixture is not the population it claims", c.Src, got, c.Key) + } + if c.Key == c.Src { + t.Fatalf("degenerate fixture: %q keys as itself, so a wrong key choice cannot be seen here", c.Src) + } + } + cands := []terminology.Candidate{compound, part} + consolidated := map[string]string{"长空": "Пустота Небес", "长": "Море Облаков"} + + // All three findings are produced the way production produces them — the self-conflict by the real + // check, so its Key is whatever ConsolidationConflicts decided to carry and not a value typed here. + self := terminology.ConsolidationConflicts(cands, consolidated) + if len(self) != 1 { + t.Fatalf("premise broken: the compound's rendering must drop its own part's, got %+v", self) + } + tres := terminologyResult{ + Conf: map[string]int{"长空": 2}, + SelfConflictRows: self, + BankHoldRows: []membank.BankKeyConflict{{ + Key: "长空", Src: "长空", Dst: "Пустота Небес", + BankSrc: "長空", BankDst: "Полость Небес", BankStatus: "draft", + }}, + } + + rows := bankStopRows(cands, consolidated, tres) + if len(rows) != 2 || rows[0].Src != "長空" { + t.Fatalf("premise broken: the sheet must carry the raw surfaces, got %+v", rows) + } + marked, plain := rows[0], rows[1] + + if marked.Conf != 2 { + t.Errorf("the role's confidence must reach the row it was stated for, want 2, got %d", marked.Conf) + } + if len(marked.Contradicts) != 1 || !strings.Contains(marked.Contradicts[0], "長") || !strings.Contains(marked.Contradicts[0], "Море Облаков") { + t.Errorf("the self-contradiction must reach the row and name the part it drops: %+v", marked.Contradicts) + } + if len(marked.BankHolds) != 1 || !strings.Contains(marked.BankHolds[0], "Полость Небес") { + t.Errorf("the bank's existing rendering must reach the row: %+v", marked.BankHolds) + } + + // CONTROL, and it is what separates "the join works" from "the join attaches everything to everybody": + // the second candidate was named by none of the three findings and must carry none of them. Conf is + // -1 rather than 0 because silence and a stated zero are opposite facts for a reviewer. + if plain.Conf != -1 { + t.Errorf("a candidate the reply gave no confidence for must read as absent (-1), got %d", plain.Conf) + } + if len(plain.Contradicts) != 0 || len(plain.BankHolds) != 0 { + t.Errorf("a candidate no finding named must carry none: contradicts=%+v bankHolds=%+v", plain.Contradicts, plain.BankHolds) + } +} + +// TestTheStopSheetJoinIsNotFooledByTheRawSurface is the same join asked the question the old map shape +// could not be asked: it puts the OTHER candidate's raw surface where a wrong key choice would look for +// one, so a lookup by Src finds a real string belonging to the wrong row instead of finding nothing. +// A join that merely returned empty on a wrong key would be caught by the test above; one that returned +// the NEIGHBOUR's finding would not, and that is the failure an operator cannot detect — the sheet is +// complete, plausible, and about a different term. +func TestTheStopSheetJoinIsNotFooledByTheRawSurface(t *testing.T) { + // The two candidates cross-reference each other's strings: each one's SURFACE is the other's KEY, so a + // lookup on the wrong field finds a real finding belonging to the wrong row. Deliberately mirrored, so + // unlike the fixture above these are not both what production would derive — that premise is checked + // there, and here what matters is only that the four strings collide the way a wrong key would. + first := terminology.Candidate{Key: "长空", Src: "長空"} + second := terminology.Candidate{Key: "長空", Src: "长空"} + if first.Key == first.Src || second.Key == second.Src || first.Key == second.Key { + t.Fatalf("degenerate fixture: the two candidates must have distinct keys and each differ from its own surface") + } + cands := []terminology.Candidate{first, second} + tres := terminologyResult{ + Conf: map[string]int{"长空": 1, "長空": 9}, + BankHoldRows: []membank.BankKeyConflict{ + {Src: "长空", BankSrc: "长空", BankDst: "Море Облаков", BankStatus: "approved"}, + {Src: "長空", BankSrc: "長空", BankDst: "Облачное Море", BankStatus: "draft"}, + }, + } + rows := bankStopRows(cands, map[string]string{"长空": "А", "長空": "Б"}, tres) + if len(rows) != 2 { + t.Fatalf("premise broken: %d row(s)", len(rows)) + } + // Keyed by Src instead of Key, each row would take the other's confidence and the other's bank row — + // and every assertion about "a finding reached a row" would still be satisfied. + if rows[0].Conf != 1 { + t.Errorf("row %q took a confidence that is not its own: %d", rows[0].Src, rows[0].Conf) + } + if len(rows[0].BankHolds) != 1 || !strings.Contains(rows[0].BankHolds[0], "Море Облаков") { + t.Errorf("row %q took the wrong bank row: %+v", rows[0].Src, rows[0].BankHolds) + } + if rows[1].Conf != 9 { + t.Errorf("row %q took a confidence that is not its own: %d", rows[1].Src, rows[1].Conf) + } + if len(rows[1].BankHolds) != 1 || !strings.Contains(rows[1].BankHolds[0], "Облачное Море") { + t.Errorf("row %q took the wrong bank row: %+v", rows[1].Src, rows[1].BankHolds) + } +} + +// TestTheCommonestSeedShapeStillReachesTheSigningSheet is the same disagreement as +// TestConsolidationAgainstTheBankIsReportedAtTheStop, on the seed row shape that actually occurs. +// +// ⚠ THE FIXTURE IS THE FINDING. That test's seed carries `until_ch: 20`, and its own comment says why: +// "until_ch keeps the seed row off the proposal's UNIQUE key". A hand-written seed row does not carry +// until_ch — three of the forty-nine rows in the owner's own seeds have a window at all — so it shares +// its UNIQUE tuple (src, sense, since_ch, until_ch) with a banknote proposal, which takes the default +// window likewise. The tuple skip fired first and the disagreement never reached the sheet: the shape +// that the sheet exists for was the one shape it could not show. The e2e that was supposed to cover this +// passed because its fixture had been given a window that stepped around the hole — the second-degeneracy +// class of D39.208 п.5, which is why this test's seed is spelled with NO window and no sense at all. +func TestTheCommonestSeedShapeStillReachesTheSigningSheet(t *testing.T) { + srv := newJSONProvider(&reqRec{}, func(body string) (string, string) { + if isTerminologyBody(body) { + return "方源\tФан Юань", "stop" // the seed calls the same surface «Странник» + } + return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop" + }) + defer srv.Close() + // The commonest shape there is: a rendering, a status, and nothing else. + seed := "terms:\n - { src: 方源, dst: Странник, status: draft }\n" + r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, glossarySeed: seed})) + defer r.Close() + _ = runToSignatureStop(t, r) + + var marked *BankStopRow + for i, row := range r.lastBankStopRows { + if row.Src == "方源" { + marked = &r.lastBankStopRows[i] + } + } + if marked == nil { + t.Fatalf("premise broken: 方源 is not on the sheet at all: %+v", r.lastBankStopRows) + } + if len(marked.BankHolds) != 1 { + t.Fatalf("the row the owner wrote must be named on the sheet he signs by: %+v", *marked) + } + if !strings.Contains(marked.BankHolds[0], `unsigned draft "方源"→"Странник"`) { + t.Errorf("the mark must name the EXISTING row and its signature: %q", marked.BankHolds[0]) + } + // PREMISE, asserted rather than assumed: the two really do share the whole UNIQUE tuple. If a later + // change gave a banknote proposal a window of its own, this test would keep passing for a reason that + // has nothing to do with what it guards — the way its predecessor did. + if marked.BankHolds[0] != `unsigned draft "方源"→"Странник" [ch 1..end]` { + t.Errorf("premise broken: the seed row must carry the DEFAULT window, or the tuple is not shared: %q", marked.BankHolds[0]) + } + if !strings.Contains(renderBankStopTable(r.lastBankStopRows), "THE BANK ALREADY HOLDS: unsigned draft") { + t.Errorf("the rendered sheet must carry the mark:\n%s", renderBankStopTable(r.lastBankStopRows)) + } +} diff --git a/backend/internal/pipeline/terminologist.go b/backend/internal/pipeline/terminologist.go index 4e7c0b14..b86d5c75 100644 --- a/backend/internal/pipeline/terminologist.go +++ b/backend/internal/pipeline/terminologist.go @@ -50,6 +50,16 @@ const roleTerminologist = "terminologist" // separate from the render phase — "what did classification spend" is answerable with no migration. const roleClassifier = "classifier" +// logKeyReconsolidated is the structured key the one-time re-consolidation warning carries, and it exists +// so that the warning and the test that guards it read ONE carrier. +// +// ⚠ WHY A KEY AND NOT THE SENTENCE. That guard has now been written twice against the message's PROSE, and +// the second time the prose had already been reworded: the test grepped a substring that occurs zero times +// in this module, so it could not fire at all and the defect it was written for went green. A key is short, +// is not rewritten when the sentence is improved, and — being a constant both sides share — cannot be +// renamed on one side only: the test would stop compiling rather than stop guarding. +const logKeyReconsolidated = "reconsolidated" + // terminologyStageName is the synthetic stage name the calls carry. It is NOT a pipeline stage (a stage // would join a wave, take a snapshot axis and a chunk_status row per unit); it is an addressing label. const terminologyStageName = "terminology" @@ -100,17 +110,28 @@ type terminologyResult struct { // which of them to look at first. CanonConflicts int // SelfConflicts counts consolidations that contradict ANOTHER consolidation of the same run (§G2) — the - // majority class on a live bank, and the one nothing looked at before this pack. Contradictions carries - // the same finding per key, so the stop table can mark the rows instead of printing a bare total. - SelfConflicts int - Contradictions map[string][]string + // majority class on a live bank, and the one nothing looked at before this pack. SelfConflictRows + // carries the findings themselves, UNGROUPED, so the stop table can mark the rows instead of printing a + // bare total — and so the key they are matched by is decided in one place (bankStopRows' findingsFor) + // instead of once here and once on the far side of a map. + SelfConflicts int + SelfConflictRows []terminology.ConsolidationConflict // BankConflicts counts consolidated renderings that disagree with a row the bank already holds for the // same firing surface — the shape neither check above can see (membank.ConsolidationKeyConflicts). // BankHoldRows carries the findings themselves, ungrouped: bankStopRows matches them to its rows, so // the key they are looked up by is decided in one place instead of once on each side of a map. Kept - // apart from Contradictions because the two are different decisions for the owner. + // apart from SelfConflictRows because the two are different decisions for the owner. BankConflicts int BankHoldRows []membank.BankKeyConflict + // BankSettled is how many candidates never reached the paid role because the bank had already settled + // them — the surface is seeded and every draft proposed the rendering the bank holds. It is the saving, + // and it is carried rather than only logged so the report can state it with its denominator + // (Candidates) instead of with a word. + BankSettled int + // BankSettledKeys are the surfaces behind that count, carried so the sheet can SAY the row was never + // asked about instead of printing an empty rendering — which reads as «undecided» beside three other + // facts that share the glyph. + BankSettledKeys []string // Conf is the role's own stated confidence per key. It sorts the review list «least sure first» and does // nothing else — never a weight, a threshold or a cross-model comparison (D39.102). Conf map[string]int @@ -330,13 +351,54 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te res.Reverse++ } } + // ⚠ `paid` is a SEPARATE slice and `cands` is left whole on purpose. The caller renders the signature + // sheet and the delta from the candidates it passed in, and the two passes below MUTATE candidates in + // place — applyTypes stamps the classifier's type, ScoreVariants re-ranks the renderings. Filtering the + // caller's own slice would have copied the survivors into a new array, so those mutations would have + // stopped reaching the sheet: the operator would read heuristic types and stale rankings, and nothing + // would say so. Every fixture in the package runs with the classifier OFF, where neither mutation + // happens at all, so the tests written for this filter could not have seen it. + // ⛔ INDICES, not a filtered slice, and that is the fix for a defect this file has now grown three + // times. The two passes below MUTATE candidates in place (applyTypes stamps the classifier's type, + // ScoreVariants re-ranks), the caller renders the sheet from `cands`, and the ROLE's request is built + // from the paid subset — so a subset held as its own array carries the stamps to neither, or to only + // one of the two. Keeping positions lets the subset be REBUILT from `cands` after the mutations, so the + // sheet and the wire read the same candidates. + paidIdx, settled := r.dropBankSettled(ctx, cands) + res.BankSettled = settled + if settled > 0 { + kept := map[int]bool{} + for _, i := range paidIdx { + kept[i] = true + } + for i, c := range cands { + if !kept[i] { + res.BankSettledKeys = append(res.BankSettledKeys, c.Key) + } + } + } + if len(paidIdx) == 0 { + return nil, nil, nil, res, nil + } + paid := pickCandidates(cands, paidIdx) + // ⛔ ASKED HERE, BEFORE THE FIRST PAID CALL, and the position is the whole point. The question is + // whether this book had bank-role checkpoints BEFORE this run — and after the passes below it is + // unanswerable, because they settle checkpoints of their own: a FRESH book that has just paid once + // answers "yes, it had paid" and the operator is told he paid twice. That is exactly what the previous + // version of this warning did, one viton after the version before it told every ordinary resume the + // same lie. A read failure leaves it false — a warning about money must not be invented from an error. + paidBefore, perr := r.Store.HasCheckpointForStage(r.Book.BookID, terminologyStageName) + if perr != nil { + r.Log.WarnContext(ctx, "terminology: could not read whether this book had paid for the bank role before this run, so a one-time re-consolidation would go unannounced; nothing else is affected", + "book", r.Book.BookID, "err", perr) + } batchRunes, _, _ := r.terminologyOpts() // §2 type re-derivation, BEFORE the render: a focused pass re-classifies every candidate, so a realia // surface mistyped as a name (元石) is no longer FORCED to a transliteration. The corrected type routes // conformance (re-scored here), primes the wire block, and is returned so the caller stamps it as the // banked type. Off (or unanswered) → the heuristic draft type stands. - classified, gendered, crun, cerr := r.runClassifier(ctx, snapID, cands) + classified, gendered, crun, cerr := r.runClassifier(ctx, snapID, paid) if cerr != nil { return nil, nil, nil, res, cerr } @@ -351,18 +413,24 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te for i := range cands { terminology.ScoreVariants(&cands[i], opts) } + // REBUILT so the batches below carry the corrected type. The classifier is bought FOR that field + // and it travels to the role for the whole batch; without this the money is spent, the run prints + // `reclassified=N`, and the wire still says what the heuristic guessed. + paid = pickCandidates(cands, paidIdx) } // §1 + §G1: co-batch each grade/rank SERIES and each term FAMILY so the role picks ONE generic head, and // ONE shared root, for the whole set. The pair-data — whether the source forms rune-morpheme series, // where the head sits, and which side of a surface carries a family's root — comes from the language // layer, so the batcher stays pair-agnostic and a source with neither takes a byte-identical path. + // Over `paid`: a series or a family the role is not asked about cannot be co-batched, and detecting it + // over the full set would only make the units disagree with the batches built from them. sEnabled, headFinal := lang.SeriesMorphology(r.Book.SourceLang) - seriesID := terminology.DetectSeries(cands, terminology.SeriesParams{Enabled: sEnabled, HeadFinal: headFinal}) + seriesID := terminology.DetectSeries(paid, terminology.SeriesParams{Enabled: sEnabled, HeadFinal: headFinal}) fp := r.familyParams - fams := terminology.DetectFamilies(cands, fp) + fams := terminology.DetectFamilies(paid, fp) unitID, ustats := terminology.MergeUnits(seriesID, fams, fp) - batches := terminology.Batch(cands, batchRunes, unitID) + batches := terminology.Batch(paid, batchRunes, unitID) res.Batches = len(batches) res.Families, res.FamiliesRefused, res.FamiliesHeld = len(fams), ustats.Refused, ustats.Held if ustats.Refused > 0 || ustats.Held > 0 { @@ -398,6 +466,16 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te return nil, nil, nil, res, rerr } res.EstimateUSD, res.CostUSD, res.CumUSD, res.Fresh = run.estimateUSD, run.costUSD, run.cumUSD, run.fresh + // ⚠ THE ONE-TIME COST OF THE ALREADY-BANKED FILTER. It needs BOTH halves and each is knowable at only + // one moment: that the book had paid BEFORE this run (paidBefore, taken above the passes) and that this + // run paid ANYWAY (run.fresh, knowable only now). Either half alone is a lie that has already been told + // here twice — on "the book has paid", which every ordinary resume of a filtered book satisfies, and on + // a probe taken after the passes, which a freshly-paid first run satisfies trivially. + if res.BankSettled > 0 && run.fresh && paidBefore { + r.Log.WarnContext(ctx, "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — those earlier calls were made under a batch composition the already-banked filter does not reproduce, so their checkpoints could not be found. This is a ONE-TIME cost; every later run replays for $0", + logKeyReconsolidated, true, + "book", r.Book.BookID, "skipped", res.BankSettled, "of_candidates", res.Candidates, "cost_usd", fmt.Sprintf("%.6f", run.costUSD)) + } out := map[string]string{} res.Conf = map[string]int{} @@ -441,7 +519,9 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te res.Conf[k] = v } } - for _, c := range cands { + // Over `paid`, not over every candidate: a candidate nobody was asked about has not gone UNANSWERED — + // that word means the role was given it and said nothing, which is a fact about the model. + for _, c := range paid { v, answered := out[c.Key] switch { case !answered: @@ -470,14 +550,9 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te // almost none. $0, evidence-side, never a gate. if self := terminology.ConsolidationConflicts(cands, out); len(self) > 0 { res.SelfConflicts = len(self) - res.Contradictions = map[string][]string{} - named := make([]string, 0, len(self)) - for _, cf := range self { - named = append(named, fmt.Sprintf("%s→%q drops %s→%q", cf.Src, cf.Dst, cf.PartSrc, cf.PartDst)) - res.Contradictions[cf.Src] = append(res.Contradictions[cf.Src], fmt.Sprintf("%s→%q", cf.PartSrc, cf.PartDst)) - } + res.SelfConflictRows = self r.Log.WarnContext(ctx, "terminology: consolidations of THIS run contradict each other — a compound's rendering drops the rendering the same reply gave its own part; they stay unverified and are review rows at the stop", - "book", r.Book.BookID, "conflicts", len(self), "terms", strings.Join(named, "; ")) + "book", r.Book.BookID, "conflicts", len(self), "terms", strings.Join(terminology.ConsolidationConflictMessages(self), "; ")) } // The BANK side: a rendering this run consolidated for a surface the bank already renders differently, // which neither check above reaches. Neither side is asked for a signature — one does not change what @@ -994,3 +1069,131 @@ func attachClassifiedType(mined []miner.Term, classified map[string]string) []mi } return out } + +// dropBankSettled removes, BEFORE any money moves, the candidates the bank has already settled: the role +// and the classifier were being paid for surfaces whose renderings the emission then threw away. +// +// ⛔ WHY THE TEST IS TWO CONDITIONS AND NOT ONE. "The bank already holds this surface" is NOT sufficient, +// and a filter written that way would take back with one hand what the sheet is for. The role is asked +// about banked surfaces ON PURPOSE: its answer is what CanonConflicts and ConsolidationKeyConflicts read, +// and those are the marks that tell the owner, on the sheet he signs by, that the book already calls this +// term something else. Measured on the corpus (13 stop tables, 1365 candidates, against the owner's own +// signed seed): 439 candidates sit on a surface the seed holds, and in 366 of them at least one draft +// disagreed with the seed's rendering. A one-condition filter would have bought a saving by silencing 366 +// disputes — which is the defect the neighbouring row of this same pack exists to close. +// +// So a candidate is dropped only when BOTH hold: +// +// - the bank holds its surface as a SEED SURFACE, decided by the emission's own predicate +// (unsignedEngineSurfaces → membank.IsEngineUnsigned) rather than by a second rule of this function's +// own, and it is banknote-ONLY, which is the exact population reverseSectionTerms drops. That pairing +// is what makes the drop output-equivalent FOR THE DELTA: the term was never going to reach it; +// - every rendering the drafts actually proposed for it equals the bank's, under the same target-form +// fold the vote is counted with. One disagreement and it is PAID for, because that disagreement is the +// finding. A candidate with no observed rendering at all is paid for too — banknote candidates always +// carry one, so this cannot fire, but "no evidence" must never read as "evidence of agreement". +// +// ⛔ IT IS NOT OUTPUT-EQUIVALENT FOR THE SIGNING SHEET, and saying otherwise would be the same class of +// false description this pack exists to remove. The second condition tests what the DRAFTS proposed, not +// what the ROLE would have answered. Where the drafts agree with the bank and the role would have +// disagreed, the candidate is dropped and the disagreement is never produced, so the sheet loses a +// bankHolds mark it would otherwise carry — the very thing the neighbouring row of this pack widens. +// The population is bounded and measured: on the corpus (13 stop tables, 1365 candidates, against the +// owner's signed seed) 439 candidates sit on a surface the seed holds and 56 of them have every observed +// draft agreeing, so 56 of 1365 is the UPPER bound of what this can silence — the subset of those whose +// role answer would in fact have differed, which is smaller and not measurable without paying for it. The +// remaining 17 of that 439 carry no observed draft at all and are paid for by the rule above. +// +// ⛔ THE RESUME TRAP, AND WHY THE CURE IS "ALWAYS", NOT "SOMETIMES". Batches are packed greedily by rune +// budget, so removing any candidate changes the composition of at least one surviving batch, and a +// bank-role batch is addressed by the HASH of its request (bankCheckpointExists): a run whose composition +// differs from the run that paid finds no checkpoint and buys the pass again. The first cure written here +// was «stand down for a book that has already paid», and running it showed the cure was the bug: run one +// filtered and paid under the FILTERED composition, run two then stood down, rebuilt the FULL composition +// and re-bought everything. A rule that decides differently on the second run is not a resume rule. +// +// So the filter is unconditional, which makes the composition a pure function of the candidates and the +// bank and therefore identical on every resume — checkpoints replay for $0, forever, which is the property +// that was actually wanted. The one book this cannot help is one consolidated BEFORE the filter existed: +// its checkpoints carry the old composition and cannot be found again. That is a single re-consolidation, +// bounded by gates.terminology.budget_usd, and it is announced rather than discovered — see the warning +// below. Making even that unnecessary would take a checkpoint identity that survives a change of batch +// composition, which is a change to the money path's own addressing and belongs in its own decision. +func (r *Runner) dropBankSettled(ctx context.Context, cands []terminology.Candidate) (keptIdx []int, dropped int) { + all := func() []int { + out := make([]int, len(cands)) + for i := range cands { + out[i] = i + } + return out + } + settled := bankSettledSurfaces(unsignedEngineSurfaces(r.glossaryRows())) + if len(settled) == 0 { + return all(), 0 + } + var names []string + for i, c := range cands { + if bankSettles(settled, c) { + dropped++ + if len(names) < 20 { + names = append(names, c.Src) + } + continue + } + keptIdx = append(keptIdx, i) + } + if dropped == 0 { + return all(), 0 + } + // The saving is a NUMBER with its denominator, not a claim: "12 skipped" is unreadable without "of + // 300", and the operator is the one who decides whether that is worth a look at his seed. + r.Log.InfoContext(ctx, "terminology: candidates the bank has already settled are NOT sent to the paid role — the bank holds the surface and every draft proposed the rendering it holds, so the emission would have dropped the answer", + "book", r.Book.BookID, "skipped", dropped, "of_candidates", len(cands), "still_paid", len(keptIdx), + "terms", strings.Join(names, ", "), "terms_listed", len(names)) + return keptIdx, dropped +} + +// pickCandidates rebuilds the paid subset from the CURRENT candidates. Called again after the passes that +// mutate them, so the request the role sees and the sheet the owner reads are built from one set of rows. +func pickCandidates(cands []terminology.Candidate, idx []int) []terminology.Candidate { + out := make([]terminology.Candidate, 0, len(idx)) + for _, i := range idx { + out = append(out, cands[i]) + } + return out +} + +// bankSettledSurfaces indexes the seed surfaces by their firing key, aliases included: a banknote proposal +// can name a term by an alias, and the emission's own guard reads aliases too. +func bankSettledSurfaces(seed []store.GlossaryEntry) map[string]store.GlossaryEntry { + out := make(map[string]store.GlossaryEntry, len(seed)) + for _, e := range seed { + if strings.TrimSpace(e.Dst) == "" { + continue // nothing to agree with + } + out[text.NormalizeSourceKey(e.Src)] = e + for _, a := range e.Aliases { + out[text.NormalizeSourceKey(a.Alias)] = e + } + } + return out +} + +// bankSettles reports whether one candidate is settled by the bank — see dropBankSettled for why both +// halves are required. +func bankSettles(settled map[string]store.GlossaryEntry, c terminology.Candidate) bool { + if c.Origin != terminology.OriginBanknote || len(c.Variants) == 0 { + return false + } + e, held := settled[c.Key] + if !held { + return false + } + want := text.NormalizeTargetForm(e.Dst) + for _, v := range c.Variants { + if text.NormalizeTargetForm(v.Dst) != want { + return false // the disagreement IS the finding: pay, and let it reach the sheet + } + } + return true +} diff --git a/backend/internal/pipeline/testdata/operator-messages.txt b/backend/internal/pipeline/testdata/operator-messages.txt new file mode 100644 index 00000000..96933459 --- /dev/null +++ b/backend/internal/pipeline/testdata/operator-messages.txt @@ -0,0 +1,121 @@ +# The operator messages internal/pipeline emits without stopping the run — one line per call site, +# `filefunctionquoted message`, sorted. Pinned by TestEveryOperatorMessageIsCatalogued, whose +# comment holds the boundary (what is covered, what is deliberately not, and why a catalogue rather than +# more substring asserts). +# +# ⚠ THIS FILE IS NOT REGENERATED. A wording change is meant to arrive here as a one-line diff a reviewer +# reads against the code it now describes: paste the line the failure prints, and keep the file sorted. +# stands for a message built at run time — the site is pinned, the words cannot be. +bankexport.go exportBank "bank export: could not marshal the bank; the export artifact was NOT refreshed and may be stale" +bankexport.go exportBank "bank export: could not read the bank; the export artifact was NOT refreshed and may be stale" +bankexport.go exportBank "bank export: could not write the export artifact; it was NOT refreshed and may be stale" +bankmaterialize.go gatherBankInputs "auto-bank rows dropped: their key is already held by a row gathered earlier (the holder wins, whatever its status; each drop below names the holder's signature)" +bankmaterialize.go gatherBankInputs "ruby kana-alias skipped as a homophone collision (kana form left unmatchable; disambiguate in the seed if needed)" +bookbuild.go buildFromExport "build: CONFIG-DRIFT — the file carries the text the run shipped; the current config would render a different snapshot (not written into the file)" +bookbuild.go buildFromExport "build: a previous copy of a format this build did not produce could NOT be removed; the book IS written and this report stands, but that file is STALE and does not belong to this build" +bookbuild.go buildFromExport "build: an EXISTING copy of a format this build did not produce was deleted — the set beside the database is always from ONE build" +bookbuild.go buildFromExport "build: the target language ships no reader words; holes and the notice are marked in the NON-VERBAL form (a sign and numbers)" +bookbuild.go staleUnits "build: the config-drift state is UNKNOWN, so whether the source moved under the shipped rows is UNKNOWN too (reported as unknown, not as none)" +bookbuild.go staleUnits "build: the stale check could not be made for every shipped unit (a row without a content hash, or a position the run cannot re-render); those units are reported as unknown" +bookbuild.go staleUnits "build: the stale check could not run; whether the source moved under the shipped rows is UNKNOWN (reported as unknown, not as none)" +bookrun.go translateBook "the run ended before its VOLUME grant was used up — the stop below is NOT the volume ceiling" +chunkrun.go reportEvicted "memory: the injection token budget DROPPED bank rows before the model saw them — these terms had no canon on the wire for those units" +escalation.go maybeEscalate "escalation hop denied by a USD ceiling; keeping the primary flag" +escalation.go maybeEscalate "stage escalated to a fallback model" +events.go beginRunEvents "could not read the stored dispositions for the run-event counters; this run publishes no progress (the run continues; resync channel: `tmctl status --json`)" +events.go degrade +events.go enqueue "run-event outbox write failed; this event will not reach the platform's stream (the run continues; the resync channel is `tmctl status --json`)" +events.go ingestSource "the source reading has notes: page breaks ignored, documents folded or excluded, or table-of-contents targets that did not become boundaries" +events.go markAnnounced "could not record which run events have been delivered; a later run may announce them again" +events.go openEmitter "this run was given an id that has already written run events for this book; a stream identity must be unique per process, so the event stream announces itself under a fresh one (fix the caller: TM_TRACE_ID must be unique per invocation)" +events.go openEvents "could not open the run-event journal; the platform's live stream will be silent for this run (the run continues; resync channel: `tmctl status --json`)" +events.go project "the run-event journal is writable again; the pending events were caught up" +events.go spendLine "could not render the spend event; the settle proceeds without it (the counter is cumulative — the next one carries the total again)" +events.go unitResolved "run-event outbox write failed; this unit will not reach the platform's stream (the run continues; the resync channel is `tmctl status --json`)" +export.go Export "export: dropped chunk_status rows outside the current source manifest (source shrunk since the run?)" +export.go exportConfigDrift "export: CONFIG-DRIFT — current config renders a different snapshot than the stored rows; the gate/stage re-derivation may not match the run (translate would require --resnapshot)" +export.go exportConfigDrift "export: CONFIG-DRIFT — stored rows carry a stage the current config does not run (renamed or removed since the run); the shipping rows are not the ones the run shipped" +export.go exportConfigDrift "export: config-drift check failed for a wave; drift state is UNKNOWN, not none" +export.go exportConfigDrift "export: config-drift check failed; drift state is UNKNOWN, not none (config_drift_basis=unknown)" +export.go exportConfigDrift "export: config-drift not checked: no stored row carries a snapshot id, so drift is UNKNOWN, not none" +manifest.go buildManifest "manifest: no ingested text for a chapter the split emitted — its id falls back to the (unstable) chapter number" +manifest.go loadManifest "manifest: could not hash the source to validate the manifest; falling back to re-chunking the source" +manifest.go loadManifest "manifest: could not read the manifest; falling back to re-chunking the source" +manifest.go loadManifest "manifest: the manifest is not readable JSON; falling back to re-chunking the source" +manifest.go loadManifest "manifest: the manifest was written in another document version; falling back to re-chunking the source" +manifest.go loadManifest "manifest: the manifest's own counters do not describe its own contents; falling back to re-chunking the source" +manifest.go persistManifest "manifest: could not persist the chapter/chunk manifest; read paths fall back to re-chunking the source and the chapter tree may be stale" +manifest.go sourceFingerprintBeforeIngest "manifest: could not fingerprint the source before reading it; the chapter/chunk manifest will not be written this run" +mining.go loadMinedDelta "mined-delta holds term(s) the owner has DECLINED; they are NOT entering the bank (the two decision documents disagree — a hand edit, or a write interrupted between the two files)" +mining.go runBankMiningStop "bank-mining: --verify-bank has nothing to stop before in a draft-only pipeline (no edit wave); the signature map and table are written and the run continues" +mining.go runBankMiningStop "bank-mining: --verify-bank is raised and the delta is non-empty, but every cluster in it was already presented by an earlier stop — continuing (the map is rewritten and stays decidable via `tmctl bank-apply` at any time)" +mining.go runBankMiningStop "bank-mining: could not read the banknote proposals; the signature map falls back to WHICH-only (bare terms)" +mining.go runBankMiningStop "bank-mining: could not record the presented memory; the NEXT --verify-bank run will stop on this same map again (one extra stop, nothing lost)" +mining.go runBankMiningStop "bank-mining: could not write the stop table sidecar (the signature map is unaffected)" +mining.go runBankMiningStop "bank-mining: draft-side proposals were written in another script and are NOT offered for signature" +mining.go runBankMiningStop "bank-mining: the delta holds terms no stop has shown before; run STOPPED before the edit wave (review the signature map, decide what you wish via `tmctl bank-apply` — one act over the whole bank, one term, or nothing — then resume: the run continues and will not re-stop on these terms, and whatever you leave undecided is used on the wire like any signed row — D39.104, the bank is law for every row; a wrong one is corrected by a bank edit and a re-edit)" +mining.go runBankMiningStop "bank-mining: the reverse section is capped like the miner's own emission; the tail is NOT in this signature map and re-proposes on the next run once these are signed or declined" +mining.go runBankMiningStop +mining.go warnDroppedRows "wire fence removed term(s) from an engine-written bank document; they are NOT entering the bank and NOT reaching any request (the model's answer carried a rune that can write into a system message)" +mining.go writeAutoBank "bank-mining: terms that were in the auto-bank are NOT in the one this run just wrote — the file is rewritten whole from a top-N-capped delta, so a term the book still uses can drop out of the bank mid-run; if one of these matters, promote it into the owner's mined-delta file (it is then a seed surface and cannot be cut again)" +quality.go QualityReport "quality: could not read the bank; the UNSIGNED BANK count is unknown, not zero" +quality.go QualityReport "report: the paid-tail decomposition could not be read; what the money BOUGHT is unknown, not zero" +quality.go QualityReport "report: the paid-tail decomposition could not be read; what the money BOUGHT is unknown, not zero" +rebill.go bankRoleCommittedUSD "the bank-role spend could not be read; the book projection is missing that whole class of money, and is therefore a LOWER bound with an unknown gap rather than with a named one" +rebill.go checkRebillConsent "accepting a projected re-payment of already-billed work (--accept-rebill)" +repair.go maybeRepair "repair call denied by a USD ceiling; leaving the defect flagged" +repair.go maybeRepair "repair discarded: the re-gate refused the assembled text (original kept)" +repair.go maybeRepair "repair reply rejected by a guard" +repin.go repinnable "re-pin check could not read the stored snapshot; this unit will be re-translated rather than re-pinned" +runner.go loadLangPack "langpack declares no source-morphology channel — the name-miner is inert (no candidates); this is expected for a source without the pinyin name-miner schema" +runner.go openRunner "a stage sends the SOURCE text to a model whose wire SUPPRESSES thinking — the ratified echo zone (D19.1 point 2: reasoning-off + dense CJK is a property of the model class, not a vendor quirk). Not a stop: the downstream echo gate (cjk_artifact) still catches an echo on every configuration (D19.2), but it costs a paid call per hit" +runner.go openRunner "content routing is not runnable for this book; read-only projections continue (translate/redrive would refuse)" +runner.go openRunner "the banknote channel is ON but this book cannot reach the bank-mining stop, which is the only reader of what it collects: the pair's prompt asks the translator for a term table on every draft call, those output tokens are billed, and nothing on this configuration will ever read them (add the mining inputs — book.yaml `langpack_root` for this pair and pipeline.yaml `mining.contrast_path` — or drop `gates.banknote` together with the banknote prompt override)" +seeding.go seedGlossary "bank: two rows render ONE firing surface differently and at least one is unsigned (the deterministic matcher cannot pick between them; reconcile the seed, the delta or the proposal)" +seeding.go seedGlossary "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)" +seeding.go seedGlossary "voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)" +seeding.go seedGlossary +snapshotdiff.go describeSnapshotMoveFor "snapshot guard: the current payload could not be rendered, so the moved field cannot be named" +snapshotdiff.go describeSnapshotMoveFor "snapshot guard: the stored payload could not be read, so the moved field cannot be named" +stagerun.go releaseReservation "reservation release failed (reserved_usd leaks and tightens ceilings until the next process restart)" +stagerun.go runAttempt "SpentUSD read failed while formatting the ceiling error; money detail omitted" +stagerun.go runAttempt "paid 2xx with zero usage; settling the reservation estimate to keep the ceiling honest" +stagerun.go runStage "echo flagged, regenerating before escalation (echo is stochastic per call, D39.61)" +stagerun.go runStage "job re-pinned to new snapshot (--resnapshot)" +stagerun.go runStage "stage flagged" +stagerun.go runStage "stage flagged, regenerating with a larger budget" +stagerun.go setJobStatus "job status update failed (non-fatal; jobs table may lag chunk_status)" +status.go Status "CONFIG-DRIFT — stored rows carry a stage the current config does not run (renamed or removed since the run); the shipping rows are not the ones the run shipped" +status.go Status "config-drift check failed for a wave; drift state is UNKNOWN, not none" +status.go Status "config-drift not checked: no stored row carries a snapshot id, so drift is UNKNOWN, not none" +status.go Status "config-drift not checked: the bank could not be folded, so drift is UNKNOWN, not none" +status.go Status "config-drift not checked: the rows carry more than one snapshot within a wave (snapshot_drift), so config drift is UNKNOWN, not none" +status.go Status "re-bill projection failed; the re-payment cost of the drift is unknown (reported as unknown, not as none)" +status.go Status "status: no bank could be materialized; the unsigned-term count is unknown, not zero" +status.go Status "the bank fold refused, so the projections below are computed against the glossary the LAST run stored — they are a fact about the past, not a projection of the next run" +terminologist.go glossaryRows "terminology: could not read the bank — whatever this call feeds goes silent (the canon anchor, the bank conflict check, or both), and its zero then means «not asked» rather than «nothing found»" +terminologist.go loadTargetScript "the banknote channel is on but no gates.terminology.target_script is declared: draft-side proposals are NOT screened for the answer language, so a rendering in another script can enter the signature map and the auto-bank" +terminologist.go runBankRoleBatches +terminologist.go runBankRoleBatches +terminologist.go runClassifier "terminology classify: a paid batch came back with an EMPTY completion; its terms keep the draft heuristic type and produce no gender" +terminologist.go runClassifier "terminology classify: some reply lines were off-vocabulary or malformed; those terms keep their draft type" +terminologist.go runClassifier "terminology classify: the GENDER column did not land on every line — those terms carry no gender datum, and a bank with no gender renders no gender directive (backlog row 210)" +terminologist.go runTerminologist "terminology: a batch renders OVER the size budget and is sent WHOLE (a series/family is co-batched by design, §1/§G1; a lone candidate cannot be split) — watch the model's output cap on this call" +terminologist.go runTerminologist "terminology: a consolidated rendering DISAGREES with a row the bank already holds for the same firing surface — the book calls this term something else, and the sheet at the stop is where that is decided (whether both renderings also reach a wire depends on the emission and the glossary UNIQUE key; this does not claim they will)" +terminologist.go runTerminologist "terminology: a paid batch came back with an EMPTY completion; its terms stay unconsolidated" +terminologist.go runTerminologist "terminology: a paid batch returned nothing the parser could use; its terms stay unconsolidated" +terminologist.go runTerminologist "terminology: consolidated renderings contradict the SIGNED bank; they stay UNSIGNED — which no longer holds them back from the wire (D39.104) — and are the first rows to review at the stop" +terminologist.go runTerminologist "terminology: consolidations of THIS run contradict each other — a compound's rendering drops the rendering the same reply gave its own part; they stay unverified and are review rows at the stop" +terminologist.go runTerminologist "terminology: could not read whether this book had paid for the bank role before this run, so a one-time re-consolidation would go unannounced; nothing else is affected" +terminologist.go runTerminologist "terminology: name/place rows carry a translated rendering — a label/rendering mismatch to review (hygiene flag, not a gate)" +terminologist.go runTerminologist "terminology: some families were NOT co-batched whole — the member cap refused the merge, or a series with an unrelated root kept its members; those families can still disagree with themselves across calls" +terminologist.go runTerminologist "terminology: the model answered in another script; those lines are REFUSED (the terms stay unconsolidated) — a book with few signed rows gives the model no target-language anchor" +terminologist.go runTerminologist "terminology: this bank is PARTIALLY consolidated — a budget cut a pass, so some terms were never offered to the role at all; `unanswered` below counts them together with terms the role saw and did not answer" +terminologist.go runTerminologist "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — those earlier calls were made under a batch composition the already-banked filter does not reproduce, so their checkpoints could not be found. This is a ONE-TIME cost; every later run replays for $0" +volume.go deliveredUnits +volume.go planVolume "a VOLUME ceiling on a book that MINES its bank: each purchase drafts more, so the bank-mining stop writes a larger auto-bank, and the next purchase moves the edit-wave snapshot the previous purchase's edit jobs are pinned to. Expect that run to need --resnapshot and to re-pay the units the new terms actually touch (the re-payment consent gate still bounds it)" +volume.go rescopeEditWave "the bank moved between planning and the edit wave, so units judged FREE are no longer free: they have been re-judged against the snapshot the edit wave actually uses" +waverun.go runDraftChunk "glossary post-check gate flagged the chunk" +waverun.go runDraftChunk "memory: lower-trust longer key refused from suppressing a higher-trust nested key (approved term preserved; reconcile the seed)" +waverun.go runEditUnit "glossary post-check gate flagged the edit unit" +waverun.go runEditUnit "spoiler leak: a rendering this chapter must not know yet reached the output" diff --git a/backend/internal/store/chunkstatus.go b/backend/internal/store/chunkstatus.go index 0f78f128..ff98be9f 100644 --- a/backend/internal/store/chunkstatus.go +++ b/backend/internal/store/chunkstatus.go @@ -66,6 +66,14 @@ type ChunkStatus struct { // with no write (pipeline/resume.go). So a cleanly shipped book keeps every timestamp across a re-run // and a book with a flagged unit does not. Pinned by // pipeline.TestOrdinaryResumeMovesOnlyTheSkippedRow. + // + // ⚠ WHICH of the two writers wrote a row is readable off the row itself — they write different Detail + // sentences — and that is the only way to tell, since the code path is decided by the wave layout + // rather than by anything the row records. Pinned in both directions by + // pipeline.TestASkippedRowSaysWHICHWriterWroteIt, which exists because reading this paragraph was NOT + // enough: measured 08.09, breaking recordSkippedStages turned six tests red while breaking the + // runStageSequence branch survived its whole package, so half of what this paragraph asserted had no + // witness at all. UpdatedAt string } diff --git a/backend/internal/store/glossary.go b/backend/internal/store/glossary.go index 756cf212..a000f9df 100644 --- a/backend/internal/store/glossary.go +++ b/backend/internal/store/glossary.go @@ -94,9 +94,10 @@ type RetrievalState struct { // map; the proposal stays status:auto, so delivering the WHAT grants no trust. BanknoteDetail string // JSON // NUnverifiedShown / NUnverifiedFollowed are the $0 observability channel of the unverified wire - // (pack-20): how many UNSIGNED bank rows actually fired in this chunk, and how many of those the - // model went along with. Measured on the wire that SHOWED the row, and never a gate — an unverified - // row is a candidate the model is entitled to reject. + // (pack-20): how many UNSIGNED bank rows this chunk could be judged on — the ones whose key fired HERE — + // and how many of those the model went along with. Never a gate: an unverified row is a candidate the + // model is entitled to reject. ⚠ Not a count of the rows the model was SHOWN: a sticky carry is shown + // and is deliberately not judged, because its src is in the previous chunk (membank.PostcheckResult). NUnverifiedShown int NUnverifiedFollowed int // NVoiceFlags / VoiceDetail are the deterministic voice flagger (pack-19, gates.voice): T/V diff --git a/backend/internal/store/ledger.go b/backend/internal/store/ledger.go index b20408a1..06d1d864 100644 --- a/backend/internal/store/ledger.go +++ b/backend/internal/store/ledger.go @@ -467,3 +467,24 @@ func (s *Store) EscalationSpentUSD(bookID string) (float64, error) { WHERE j.book_id = ? AND c.escalation = 1`, bookID).Scan(&sum) return sum, err } + +// HasCheckpointForStage reports whether a book has ever PAID for a call of one synthetic stage — asked as +// an EXISTS rather than by listing, because a translated book carries a checkpoint per chunk per stage and +// the caller wants one bit. +// +// It exists for a decision about money that has to be made before any call: a pass whose batch COMPOSITION +// would change cannot reuse the checkpoints of the old composition (they are addressed by the request's +// hash), so a change of composition on a book that has already paid is a second purchase of work already +// bought. The question "has this stage ever been paid for here" is what lets such a change stand down. +func (s *Store) HasCheckpointForStage(bookID, stage string) (bool, error) { + ctx, cancel := opContext() + defer cancel() + var one int + err := s.r.QueryRowContext(ctx, ` + SELECT 1 FROM checkpoints c JOIN jobs j ON j.id = c.job_id + WHERE j.book_id = ? AND c.stage = ? LIMIT 1`, bookID, stage).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} diff --git a/backend/internal/store/migrate.go b/backend/internal/store/migrate.go index be70f704..2bfe2e9d 100644 --- a/backend/internal/store/migrate.go +++ b/backend/internal/store/migrate.go @@ -339,6 +339,10 @@ var migrations = []string{ // • n_unverified_shown — unsigned rows whose src fired in THIS chunk (the denominator); // • n_unverified_followed — of those, the ones whose rendering appears in the output. // + // ⚠ The question above is narrower than the pair answers, and the bullets are the exact rule: a STICKY + // carry is put in front of the model and is NOT counted, because its src is in the previous chunk and + // the output is not expected to answer for it (membank.PostcheckResult). "Fired here", not "shown". + // // The pair is deliberately measured ON THE WIRE THAT SHOWED THE ROW: before pack-20 the ambiguous // counter was collected on the editor wire, whose block was CONFIRMED-only — it counted deviations // from something the editor was never shown. Neither number gates anything: an unverified row is a diff --git a/backend/internal/terminology/terminology.go b/backend/internal/terminology/terminology.go index f1da8f4a..7247bd45 100644 --- a/backend/internal/terminology/terminology.go +++ b/backend/internal/terminology/terminology.go @@ -956,12 +956,37 @@ func CanonConflicts(cands []Candidate, consolidated map[string]string, ns []Neig // ConsolidationConflict is one consolidated rendering that contradicts ANOTHER consolidation of the SAME // run — the compositional rule broken inside one reply rather than against a signed row. type ConsolidationConflict struct { - Src string // the containing candidate's source surface + // Key is the candidate KEY the finding belongs to — the normalized surface the check compared on, and + // the only string a consumer may match it back by. Src is the raw one and the two differ for every + // traditional or katakana spelling, so matching on Src loses exactly the candidates the fold exists + // for. Carried rather than re-derived: a consumer deriving it again would be the second place the key + // is chosen, which is how this pair drifted apart unnoticed for three packs. + Key string + Src string // the containing candidate's source surface, as a human reads it Dst string // what the role consolidated it to PartSrc string // the source surface contained in it, also consolidated this run PartDst string // the rendering its own dst fails to carry } +// PartLabel is the one rendering of the CONTRADICTED part every human-facing surface uses — the sheet +// column and the sidecar table — so the finding cannot be phrased two ways by two callers. Same +// discipline as membank.BankKeyConflict.BankRowLabel on the other half of the sheet. +func (c ConsolidationConflict) PartLabel() string { + return fmt.Sprintf("%s→%q", c.PartSrc, c.PartDst) +} + +// ConsolidationConflictMessages renders the findings for the run log, where the reader needs BOTH sides: +// which rendering dropped which. Returned as parts by ConsolidationConflicts and assembled here for the +// same reason the bank side does it (membank.ConflictMessages) — a caller free to re-phrase would state +// one finding two ways. +func ConsolidationConflictMessages(cs []ConsolidationConflict) []string { + out := make([]string, 0, len(cs)) + for _, c := range cs { + out = append(out, fmt.Sprintf("%s→%q drops %s", c.Src, c.Dst, c.PartLabel())) + } + return out +} + // ConsolidationConflicts is CanonConflicts' thin brother, and it exists because the canon check can only see // what the owner already signed. On a live bank that is the minority: 18 of 149 contradictions the same run // produced were between its OWN consolidations (research/24 §A4) — 元海空窍 rendered without the 元海 this @@ -990,7 +1015,7 @@ func ConsolidationConflicts(cands []Candidate, consolidated map[string]string) [ if lexemeSubset(wordSet(pd), have) { continue } - out = append(out, ConsolidationConflict{Src: c.Src, Dst: dst, PartSrc: p.Src, PartDst: pd}) + out = append(out, ConsolidationConflict{Key: c.Key, Src: c.Src, Dst: dst, PartSrc: p.Src, PartDst: pd}) } } return out diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index fff5a0fb..161d052e 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -498,6 +498,33 @@ terminology finished … bank_conflicts=2 batches_dropped=0 classify_batches_dro **Работа завершена, править не планирую.** +##### ДОФИКС ПО СЛЕПОЙ ЛИНЗЕ (09.09) — одна строка, и это ТРЕТИЙ раз, когда меня ловит один класс + +⛔ **Мой отрицательный страж грепал подстроку, которой в модуле НЕТ.** `banksettled_test.go` искал `"still bought the pass"`, а сообщение к тому моменту звучало «…bought the pass **again**…». Пере-снято: **0 вхождений** в боевом коде при 351 go-файле (контроль: строк с «bought the pass» всего 4 — одна боевая и три тестовых). То есть страж не мог сработать НИКОГДА, и линза доказала это исполнением: снятие `run.fresh` — ровно того блокера, ради которого делался предыдущий круг, — оставляло `internal/pipeline` ЗЕЛЁНЫМ. + +⭐ **Это тот самый класс, который я сформулировала утром и который уехал в ролевой промт оркестратора** («пин, утверждающий только молчание, вакуумен»). Здесь он вакуумен вдвойне: не просто ветвь недостижима — искомой строки не существует. Я сформулировала правило и в тот же день его нарушила, в том же файле. + +**Лечение — НЕ подставленная новая подстрока**, она протухла бы так же при следующей переформулировке. Обе стороны сведены к ОДНОМУ носителю: заведена константа `logKeyReconsolidated = "reconsolidated"`, сообщение несёт её структурным ключом, все ТРИ утверждения (два отрицательных и одно положительное) грепают `logKeyReconsolidated+"=true"`. Свойства формы: переформулировка сообщения страж не ломает; переименование ключа на одной стороне не компилируется; удаление константы не компилируется. Проза больше ни при чём — `grep` по тестам даёт 0 прежних подстрок, константа встречается 6 раз в 2 файлах. + +**Предъявлено ОБЕИМИ сторонами исполнением.** Посадка «снять `run.fresh`» теперь КРАСНАЯ, и текст падения выдаёт ложь дословно: `reconsolidated=true … cost_usd=0.000000` — предупреждение о повторной покупке при нулевой стоимости. Штатный первый прогон молчит (тот же пин, другой подслучай). Запись каталога `MONEY-the-one-time-cost-warning-drops-its-fresh-half` заведена. + +⚠ **Почему это не нарушило заморозку:** вторая линза судит КОПИЮ (`scratchpad/frozen-pack2`), а не рабочее дерево, и правка в дереве её вердикт не двигает. Правила ровно этот предмет, больше ничего. + +##### Числа, пере-снятые ПОСЛЕ дофикса по слепой линзе — командой, не памятью + +| Число | Значение | Чем получено | +|---|---|---| +| Батарея | **19 `ok` · 0 `FAIL` · 4 «no test files» · 4 скипа поимённо · `0 issues` · exit 0** | `make battery` | +| Мутации | **48 запусков · 48 RED · выживших 0 · `0 unexpected outcome(s)` · `anchors swept: 0 of 259 entr(ies) rotten` · exit 0** | `make mutations` | +| Каталог мутаций | **259 записей, 48 в батарее** (на входе 232 / 21) | `json.load` по `cmd/tmmutate/mutations.json` | +| Тестов в зоне | **1267 → 1285, +18, −0** | `git grep '^func Test' 060b13b` против дерева, сверка `comm` ПО ИМЕНАМ | +| Каталог операторских сообщений | **113 строк** | `grep -vc '^#'` | +| Мёртвых прозаических стражей | **0** (контроль: `logKeyReconsolidated` — 6 вхождений в 2 файлах) | `grep -rc` по `*_test.go` | +| Дерево | **24 позиции, все в `backend/`; вне `backend/` и `docs/PROGRESS.md` — 0** | `git status --short` | + +**Работа завершена, править не планирую.** + + #### Пак «гейт вместо прозы» (07.09, промт `archive/prompts/BACKEND_GATES_NOT_PROSE_SESSION_PROMPT.md`, вход HEAD `bd652fd`). ✅ ЗАЛЕНДЕН и ПРИНЯТ — акт `D39.225`, три дофикса, оба верификатора окнули закрытие **ЗАПИСКА-ПЛАН (§7), написана ДО первой правки.** Пять швов держатся прозой и вырожденными фикстурами; ставлю под каждый гейт, боевой код трогаю ровно в одном месте (§4.2а — единственная заказанная смена поведения).