Add a settled-basis table so a term the book already decided is served from memory instead of being bought again, and report bank rows whose chapter windows leave a gap
This commit is contained in:
parent
d443c12059
commit
d25e418acb
32 changed files with 4497 additions and 70 deletions
|
|
@ -44,12 +44,29 @@ func TestTheSigningScreenSaysHowCompleteTheBankIs(t *testing.T) {
|
|||
want: []string{whole, "41 consolidated", "2 declined", "3 unanswered"},
|
||||
mustNotSay: []string{partial, unmeasured, types},
|
||||
}, {
|
||||
name: "the paid role was asked for nothing",
|
||||
name: "the paid role was asked for nothing — because the BANK renders them",
|
||||
// Reachable state, not a hypothetical: the already-banked filter can empty the paid set, and then
|
||||
// zero batches were planned. «Every render batch was bought» is true of zero batches and reads as a
|
||||
// report on work that happened — the instrument answering its own question (D39.202).
|
||||
c: &pipeline.BankConsolidation{Complete: true, NeverAsked: 3},
|
||||
want: []string{whole, "asked for nothing", "No batch was planned"},
|
||||
want: []string{whole, "asked for nothing", "No batch was planned", "the bank already renders every candidate this run found"},
|
||||
mustNotSay: []string{partial, unmeasured, types, "every render batch was bought", "earlier purchase"},
|
||||
}, {
|
||||
// ⚠ MAINTENANCE DECLARED, not a quiet edit: the settled basis added a SECOND reason for «the role
|
||||
// was asked nothing», and the single sentence this pin used to require asserted the FIRST one for
|
||||
// both. For a book answered from the memory of decisions it was false in both halves — the bank
|
||||
// holds no seed surface for those terms and the drafts were never asked to agree with anything.
|
||||
name: "the paid role was asked for nothing — because the BOOK had already decided them",
|
||||
c: &pipeline.BankConsolidation{Complete: true, SettledEarlier: 4},
|
||||
want: []string{whole, "asked for nothing", "already decided every candidate", "4 term(s)", "has not moved"},
|
||||
mustNotSay: []string{partial, unmeasured, types, "every render batch was bought", "the bank already renders"},
|
||||
}, {
|
||||
// And the state that makes the two reasons a CHOICE rather than an either/or: both savings fired,
|
||||
// and the line must name both with their own numbers. Three different counts, so a line printing
|
||||
// one number in the other's slot cannot pass.
|
||||
name: "asked for nothing, and BOTH savings explain it",
|
||||
c: &pipeline.BankConsolidation{Complete: true, NeverAsked: 3, SettledEarlier: 7},
|
||||
want: []string{whole, "asked for nothing", "3 term(s)", "7 because", "this book decided them in an earlier purchase"},
|
||||
mustNotSay: []string{partial, unmeasured, types, "every render batch was bought"},
|
||||
}, {
|
||||
name: "a budget cut the render pass",
|
||||
|
|
|
|||
|
|
@ -35,14 +35,19 @@ import (
|
|||
// case backlog row 49а names (those steps are not re-appliable), and a fixture that ignored it would
|
||||
// re-apply the step onto its own effect and fail on a duplicate column instead of on the property the
|
||||
// test is about.
|
||||
var headOfSchemaAddedColumns = []string{"request_log.reasoning_in_completion"}
|
||||
// ⚠ EMPTY AT THIS HEAD, AND THAT IS A STATEMENT RATHER THAN AN OMISSION: migration 19 creates a TABLE
|
||||
// (bank_basis) and adds no column to any existing one. A CREATE TABLE step is re-appliable — it carries
|
||||
// IF NOT EXISTS — so rolling the version row back is the whole roll-back, and there is nothing for this
|
||||
// list to take with it. The assertion below is what keeps the emptiness honest: the day a head arrives
|
||||
// that DOES add a column, the version check fires and this list has to be written again.
|
||||
var headOfSchemaAddedColumns []string
|
||||
|
||||
// headOfSchemaVersion is the migration the list above DESCRIBES. It is asserted rather than assumed,
|
||||
// because a stale list does not fail where it is wrong: with a newer head the roll-back would drop this
|
||||
// column while deleting the NEWER version row, so the migrator would re-apply the new step and never
|
||||
// restore the old column — the "DDL missing while the version says present" state the fixture exists to
|
||||
// avoid, surfacing later inside some unrelated test. The assertion turns that into one sentence here.
|
||||
const headOfSchemaVersion = 18
|
||||
const headOfSchemaVersion = 19
|
||||
|
||||
// staleTheProjectSchema rolls the project database back one vintage — the recorded version AND the
|
||||
// newest step's own columns — which is what an older engine binary left behind.
|
||||
|
|
|
|||
|
|
@ -252,13 +252,30 @@ func renderBankConsolidation(w io.Writer, c *pipeline.BankConsolidation) {
|
|||
}
|
||||
switch {
|
||||
case c.Complete && c.Consolidated+c.Declined+c.Unanswered == 0:
|
||||
// ⛔ THE PAID ROLE WAS ASKED NOTHING — every candidate was already settled by the bank, so no batch
|
||||
// was ever planned. «Every render batch was bought» is technically true of zero batches and reads as
|
||||
// a report on work that happened, which is the exact shape this surface exists to remove: an
|
||||
// instrument answering its own question (D39.202). Reachable whenever the already-banked filter
|
||||
// empties the paid set.
|
||||
fmt.Fprintln(w, "Bank completeness: WHOLE — the paid role was asked for nothing, because the bank "+
|
||||
"already renders every candidate this run found. No batch was planned and none was bought.")
|
||||
// ⛔ THE PAID ROLE WAS ASKED NOTHING, AND THE LINE MUST SAY WHY — there are now TWO reasons and they
|
||||
// are different facts. «Every render batch was bought» is technically true of zero batches and reads
|
||||
// as a report on work that happened, which is the shape this surface exists to remove: an instrument
|
||||
// answering its own question (D39.202).
|
||||
//
|
||||
// ⚠ THE SECOND REASON ARRIVED WITH THE SETTLED BASIS, and until it was named this line asserted the
|
||||
// FIRST one for both: it said the bank already renders every candidate, which is false of a book
|
||||
// whose answers came from the memory of decisions — there the bank holds no seed surface for them
|
||||
// and the drafts were never asked to agree with anything. A message that states the wrong cause is
|
||||
// worse than one that states none, and D39.93 п.2 refuses it next door.
|
||||
switch {
|
||||
case c.SettledEarlier > 0 && c.NeverAsked > 0:
|
||||
fmt.Fprintf(w, "Bank completeness: WHOLE — the paid role was asked for nothing. %d term(s) because "+
|
||||
"the bank already renders them and every draft agreed, and %d because this book decided them in an "+
|
||||
"earlier purchase and the source around them has not moved. No batch was planned and none was bought.\n",
|
||||
c.NeverAsked, c.SettledEarlier)
|
||||
case c.SettledEarlier > 0:
|
||||
fmt.Fprintf(w, "Bank completeness: WHOLE — the paid role was asked for nothing, because this book had "+
|
||||
"already decided every candidate this run found (%d term(s)) and the source around them has not "+
|
||||
"moved. No batch was planned and none was bought.\n", c.SettledEarlier)
|
||||
default:
|
||||
fmt.Fprintln(w, "Bank completeness: WHOLE — the paid role was asked for nothing, because the bank "+
|
||||
"already renders every candidate this run found. No batch was planned and none was bought.")
|
||||
}
|
||||
case c.Complete:
|
||||
fmt.Fprintf(w, "Bank completeness: WHOLE — every render batch was bought (%d consolidated, %d declined, %d unanswered).\n",
|
||||
c.Consolidated, c.Declined, c.Unanswered)
|
||||
|
|
@ -282,6 +299,13 @@ func renderBankConsolidation(w io.Writer, c *pipeline.BankConsolidation) {
|
|||
fmt.Fprintf(w, " %d term(s) were NOT ASKED about: the bank already renders those surfaces and every "+
|
||||
"draft agreed — nothing to decide, and they are marked in the table and the full sidecar.\n", c.NeverAsked)
|
||||
}
|
||||
if c.SettledEarlier > 0 {
|
||||
// A SECOND saving with a different cause, and it gets its own sentence because the one above is
|
||||
// false of these rows in both halves: no seed holds their surface, and no draft was asked to agree.
|
||||
fmt.Fprintf(w, " %d term(s) were SETTLED EARLIER: this book decided them in a previous purchase and "+
|
||||
"the source around them has not moved, so the roles were not paid to decide them again. They are "+
|
||||
"unsigned like any other row and you may replace them.\n", c.SettledEarlier)
|
||||
}
|
||||
}
|
||||
|
||||
// bankStopStdoutCap bounds the stdout table. The emission cap is 200 terms (miner.emitRankCap), and 200
|
||||
|
|
@ -327,6 +351,12 @@ func renderBankStopRows(w io.Writer, rows []pipeline.BankStopRow) {
|
|||
// differently. The count above is what survives the cap; this is what a shown row says.
|
||||
fmt.Fprintln(w, " NOT ASKED: the bank already renders this surface and every draft agreed — nothing to decide")
|
||||
}
|
||||
if r.SettledByBasis {
|
||||
// A DIFFERENT sentence, because the one above is false of this row in both of its halves: the
|
||||
// bank holds no seed surface for it and the drafts were never asked to agree with anything.
|
||||
// Printing the other sentence here would tell the owner the bank renders a surface it does not.
|
||||
fmt.Fprintln(w, " SETTLED EARLIER: this book decided this rendering in an earlier purchase and the source around the term has not moved since — the roles were not paid to decide it again; it is unsigned like any other row and you may replace it")
|
||||
}
|
||||
if len(r.Contradicts) > 0 {
|
||||
// A contradiction against this run's OWN consolidations: the compound dropped the rendering the
|
||||
// same reply gave its part. Loudest line of the row — it is a canon breaking inside one call.
|
||||
|
|
|
|||
|
|
@ -3433,9 +3433,9 @@
|
|||
},
|
||||
{
|
||||
"id": "BANKMONEY-the-run-total-reports-the-contours-cumulative-spend",
|
||||
"why": "CostUSD is what the render phase paid THIS run; CumUSD is what the contour has cost since the book began, replayed checkpoints included at what they cost then. On a FIRST run the two are equal to the cent, so a total built from the wrong one is invisible there — and on a resume it reports money nobody spent, beside a ledger that did not move. The pin's second leg is a resume precisely because the first cannot tell them apart",
|
||||
"why": "CostUSD is what the render phase paid THIS run; CumUSD is what the contour has cost since the book began, replayed checkpoints included at what they cost then. On a FIRST run the two are equal to the cent, so a total built from the wrong one is invisible there — and on a resume it reports money nobody spent, beside a ledger that did not move. The pin's second leg is a resume precisely because the first cannot tell them apart. ⛔ THE CATCHER MOVED ON 17.09 AND THE ENTRY DID NOT NOTICE — which is why the filter now names BOTH tests. The settled-basis pack split the original pin in two: its resume leg needs a budget with HEADROOM (so a re-asked batch can be bought and then replayed) while its first leg needs a budget that CUTS a batch (so the render and classify phases cost different amounts and the sum cannot be satisfied by the wrong addend), and with the basis in the tree those two requirements became arithmetically incompatible on one fixture. The resume leg — the ONLY leg that can tell CostUSD from CumUSD — moved to the second test, and this entry went on naming the first. It survived a full catalogue run and nothing cheaper would have seen it: the test was green, the split was reviewed, and the entry still pointed at a file that no longer holds the assertion",
|
||||
"package": "./internal/pipeline/",
|
||||
"run": "TestTheRunTotalIsThisRunsSpendIncludingTheClassifier",
|
||||
"run": "TestTheRunTotalIsThisRunsSpendIncludingTheClassifier|TestAReplayedBankContourIsFreeWhileItsCumulativeFigureRemembers",
|
||||
"battery": true,
|
||||
"edits": [
|
||||
{
|
||||
|
|
@ -6513,5 +6513,481 @@
|
|||
"replace": " regenerate: 0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-fp-hashes-what-the-drafts-merely-guessed",
|
||||
"why": "the fingerprint's TRIGGER, attacked where money is. Draft variants, freq and since_ch all reach the role's prompt and all three move between purchases for reasons that are not new knowledge about the term — model noise, a cap, a re-cut. Hashing any of them turns ordinary variation into a re-purchase of the whole bank out of a LIFETIME per-book ceiling whose exhaustion is permanent (row 356). Nothing else goes red for it: the run still consolidates, the sheet still renders, only the bill grows",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestBasisFingerprintIgnoresWhatTheRoleMerelyGUESSED",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "fmt.Fprintf(h, \"width\\x00%d\\x1e\", width)",
|
||||
"replace": "fmt.Fprintf(h, \"width\\x00%d\\x1e\", width)\n\t\tfmt.Fprintf(h, \"freq\\x00%d\\x1e\", c.Freq)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-fp-reads-the-subset-shown-to-the-model",
|
||||
"why": "the property the whole fingerprint SHAPE exists for. KWIC is capped and takes the FIRST occurrences, and a stratified selection is already a backlog row (449): built over that subset, the fingerprint moves for every book on earth the day anyone turns kwic_per_term or lands the new selection, and each move re-buys a bank. Built over EVERY occurrence it moves only when the source around the surface moves",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestBasisFingerprintIgnoresWhatTheRoleMerelyGUESSED",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "for _, w := range terminology.EvidenceWindows(nsource, c.Key, width) {",
|
||||
"replace": "for _, w := range c.KWIC {"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-evidence-windows-capped-like-KWIC",
|
||||
"why": "the same property one layer down, in the pure function. A cap here makes an edit past the third occurrence invisible to the fingerprint, so the engine keeps serving a decision whose evidence has moved — the failure direction that costs quality rather than money, and the one no counter would show. ⚠ THE EDIT IS IN internal/terminology AND THE CATCHER IS IN internal/pipeline, so the entry names the CATCHER's package — the harness runs the test filter there. Pointed at the edited package instead, the filter matches no test and the entry asserts nothing: which is exactly how it was first written, and how tmmutate's own rotted-entry guard caught it",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestBasisFingerprintReadsEveryOccurrenceNotTheFirstFew",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/terminology/evidence.go",
|
||||
"find": "\t\tout = append(out, strings.TrimSpace(string(runes[lo:hi])))",
|
||||
"replace": "\t\tif len(out) < 3 {\n\t\t\tout = append(out, strings.TrimSpace(string(runes[lo:hi])))\n\t\t}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-fp-forgets-the-signed-rows-it-depends-on",
|
||||
"why": "the book's own law changing under a settled term. CanonConflicts fires on strict containment of a SIGNED source; with the canon out of the hash a signature on a relative leaves the fingerprint unmoved, the term is served from memory, and the rendering decided BEFORE the owner's act keeps shipping. The Canon half is hashed separately so the re-purchase can name its cause instead of arriving as «the source moved»",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestBasisFingerprintNamesTheSignedRowsItDependsOn",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "fmt.Fprintf(h, \"canon\\x00%s\\x1e\", canonPart)",
|
||||
"replace": "_ = canonPart"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-units-read-a-partial-map-as-unit-zero",
|
||||
"why": "a zero value that means «no unit» against one that means «unit number zero» — the same integer, and only one is a decision. MergeUnits returns a PARTIAL map and terminology.Batch reads a missing key as a singleton; indexed raw, every unit-less candidate shares unit 0, so ONE unsettled row re-buys all of them. ⚠ MEASURED, not imagined: the pack's own first measurement reported 22 of 69 candidates served where the truth is 65, and the number looked entirely plausible",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestBasisUnitsGiveEveryCandidateAUnitOfItsOwn",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "out := make(map[string]int, len(cands))\n\t// Singleton ids are handed out",
|
||||
"replace": "return merged\n\t//nolint\n\tout := make(map[string]int, len(cands))\n\t// Singleton ids are handed out"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-predicate-serves-a-row-that-is-not-draft",
|
||||
"why": "«non-empty dst» is NOT enough and the miner proves it: it produces status:auto WITH a rendering — the raw first-chunk banknote guess — a pair four engine comments declare impossible and no pin guards (row 487). A predicate reading the rendering alone serves an answer nobody consolidated, and the same hole lets an owner-SIGNED row be republished as an engine decision",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestBasisServesOnlyADecidedRow",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "if row.Status != \"draft\" || strings.TrimSpace(row.Dst) == \"\" {",
|
||||
"replace": "if strings.TrimSpace(row.Dst) == \"\" {"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-unit-rule-dropped",
|
||||
"why": "a family decided in halves is the chimera co-batching exists to prevent — «old members frozen, a new one asked alone, and the result correlates with the batch boundary perfectly». Without the rule the predicate serves half a series from memory while the rest is re-asked, and the two halves answer to different laws",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestBasisServesAUnitOnlyWhole",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "if v.Reason == basisServed && blocked[unitID[c.Key]] {",
|
||||
"replace": "if false {"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-signed-row-holds-its-family-open",
|
||||
"why": "the owner's signature re-buying the family it belongs to. A signed row is settled harder than any draft one, but it carries no basis record (the basis remembers ENGINE decisions), so counted as a hole it blocks its whole unit. Measured on both bought cold runs: the two signed rows were the ONLY blockers, and exempting them moves the saving from 65/69 and 62/66 to 67/69 and 64/66",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestASignedRowDoesNotHoldItsFamilyOpen",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "if r := own[c.Key].Reason; r != basisServed && r != basisSigned && r != basisOwnerRow {",
|
||||
"replace": "if r := own[c.Key].Reason; r != basisServed {"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-owner-canon-served-as-an-engine-answer",
|
||||
"why": "the two words the bank ontology keeps apart, merged. The basis is the memory of what the ENGINE decided; handing back the owner's signed rendering as one of its answers republishes his canon as a machine proposal, and «which translations did the human actually approve» stops being answerable — which is the question the whole bank exists for",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestASignedRowDoesNotHoldItsFamilyOpen",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "return bankBasisVerdict{Reason: basisSigned}",
|
||||
"replace": "return bankBasisVerdict{Answer: bankBasisAnswer{Dst: row.Dst}, Reason: basisSigned}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-is-a-filter-instead-of-a-source-of-answers",
|
||||
"why": "⛔ THE PACK'S QUIETEST REGRESSION. Dropping settled candidates from the paid set WITHOUT handing their answer back does not leave the book where it was: attachConsolidatedDst stamps nothing for a term the role did not answer, so the emission falls to the raw first-chunk banknote guess at status auto, or to no rendering at all. The run looks CHEAPER and the bank gets POORER, and nothing goes red — the saving is visible and the loss is silent",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheBasisCarriesTheBoughtAnswerRatherThanDroppingIt",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/terminologist.go",
|
||||
"find": "for k, a := range basisAnswers {\n\t\tout[k] = a.Dst\n\t}",
|
||||
"replace": "_ = basisAnswers"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-classifier-half-of-the-answer-dropped",
|
||||
"why": "the answer has TWO halves bought on the same money and reaching the row through different doors — the terminologist's rendering and the classifier's type and gender. A basis that carried only the first drops a wire directive the engine has PROMISED since D39.21, on rows nobody will ever be asked about again. ⚠ The pin's fixture runs the classifier ON deliberately: with the phase off this mutation survived, because the comparison was \"\" against \"\"",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheBasisCarriesTheBoughtAnswerRatherThanDroppingIt",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/terminologist.go",
|
||||
"find": "if a.Gender != \"\" {",
|
||||
"replace": "if false {"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-written-before-the-run-reaches-a-boundary",
|
||||
"why": "a run that dies after the bank roles are paid leaves their checkpoints, and a resume rebuilds the SAME batch composition and replays them for $0. Record the basis before the fork and the crashed run has already stored its decisions: the resume serves them, builds a NARROWER composition, misses every checkpoint addressed by request hash, and buys the whole pass again — money out of a lifetime ceiling for work already done",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheBasisIsNotWrittenBeforeTheRunReachesABoundary",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/terminologist.go",
|
||||
"find": "\tres.BasisRows = basisRowsToStore(cands, out, classified, gendered, basisFPs, basisRows, basisAmbiguous, basisShape(r.basisWidth(), r.bankRoleProducer()))",
|
||||
"replace": "\tres.BasisRows = basisRowsToStore(cands, out, classified, gendered, basisFPs, basisRows, basisAmbiguous, basisShape(r.basisWidth(), r.bankRoleProducer()))\n\tif err := r.Store.PutBankBasis(r.Book.BookID, res.BasisRows); err != nil {\n\t\t_ = err\n\t}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-store-accepts-an-empty-rendering",
|
||||
"why": "the guarantee at the WRITE, where a future caller cannot forget it. A basis row with no rendering served back is the same silent loss the mechanism exists to prevent, arriving through its own memory: the emission drops to the raw guess at status auto and the run reads as cheaper",
|
||||
"package": "./internal/store/",
|
||||
"battery": true,
|
||||
"run": "TestBasisStoreRefusesAnEmptyRendering",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/store/bankbasis.go",
|
||||
"find": "if r.SrcKey == \"\" || r.Dst == \"\" || r.FPWhole == \"\" || r.FPVersion == \"\" {",
|
||||
"replace": "if false {"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-store-accumulates-where-it-must-replace",
|
||||
"why": "the guarantee this table does NOT inherit from its neighbour. bank_stop_presented only grows and has no delete on purpose; the basis must OVERWRITE when a term is decided again, or the predicate keeps comparing against evidence the book has moved past and serves an answer that was right two purchases ago",
|
||||
"package": "./internal/store/",
|
||||
"battery": true,
|
||||
"run": "TestBasisStoreReplacesRatherThanAccumulates",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/store/bankbasis.go",
|
||||
"find": "ON CONFLICT(book_id, src_key) DO UPDATE SET",
|
||||
"replace": "ON CONFLICT(book_id, src_key) DO NOTHING WHERE 0 = (SELECT 0) --"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-term-window-gap-goes-unreported",
|
||||
"why": "backlog row 490: the loader refuses two bank rows that OVERLAP and says nothing about two that leave a chapter BETWEEN them — «until_ch 3» then «since_ch 5», and chapter 4 has no law for the term at all. It is not a schema violation and cannot be: the UNIQUE key admits both rows, windowsOverlap permits them, and the spoiler rule then picks neither. The term stops being law for those chapters, the editor is shown nothing, and their drafts render it however they like — inside a book whose whole point is one word per term. The guarantee is taken from the voice sibling and NOT improvised: a deliberate gap is legitimate, so the outcome is a loud line, never a refused load",
|
||||
"package": "./internal/membank/",
|
||||
"battery": true,
|
||||
"run": "TestTermWindowGapsSeesTheHoleTheLoaderCannot",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/membank/memvoice.go",
|
||||
"find": "if win[i].SinceCh > prevUntil+1 {\n\t\t\t\tout = append(out, fmt.Sprintf(\"term %q (sense %q): chapters %d–%d have no bank row",
|
||||
"replace": "if false {\n\t\t\t\tout = append(out, fmt.Sprintf(\"term %q (sense %q): chapters %d–%d have no bank row"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-reaches-a-book-whose-gate-is-off",
|
||||
"why": "the promise every new mechanism owes the books already in the field. The basis lives on the paid path and the paid path is GATED; a gate whose OFF branch grew a side effect is the quiet way a mechanism reaches books nobody enabled it for — rows in a table for a contour that never ran, and a later purchase serving answers nobody bought. The pin also asserts the run really reached the mining stop first, so its zeros are about a path that was walked",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestABookWithTheGateOffTouchesNoBasisAtAll",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "\tt := r.lastTerminology\n\tif t == nil || len(t.BasisRows) == 0 {\n\t\treturn\n\t}",
|
||||
"replace": "\tt := r.lastTerminology\n\tif t == nil {\n\t\tt = &terminologyResult{BasisRows: []store.BankBasisRow{{SrcKey: \"x\", FPVersion: basisFingerprintVersion, FPWhole: \"h\", Dst: \"d\"}}}\n\t}\n\tif len(t.BasisRows) == 0 {\n\t\treturn\n\t}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-banked-type-never-reaches-the-sheet",
|
||||
"why": "⛔ FOUND BY ADVERSARIAL REVIEW OF THIS PACK, ON THE TREE. The sheet, the stop table and the sidecar's `kind` read Candidate.Type, and applyTypes — the only writer — runs BEFORE the basis answers reach the maps it consults. So the banked type travelled to the auto-bank and NOT to the sheet: in the steady state the row the owner signs by said `place` while the bank row for the same term said `term`. The stamp is deliberately not applyTypes, because that one's return value is published as `reclassified` and a term the classifier never saw must not inflate it",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheBankedTypeReachesTheSheetWhenTheBasisAnswers",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "\t\tcands[i].Type = a.Type\n\t\tn++",
|
||||
"replace": "\t\tn++"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-records-the-owners-signed-rendering",
|
||||
"why": "⛔ FOUND BY ADVERSARIAL REVIEW. A signed surface still goes to the paid role on purpose (row 447), so its consolidation reaches the recorder — and recording it puts the ENGINE's word into the memory of decisions under a surface whose law is the OWNER's. Lift the signature and the predicate finds a record, sees an unsigned draft row, and serves the engine's rendering as «this book decided it earlier», over his own: exactly the merging of the two words the bank ontology keeps apart, arriving through the door this pack built",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheOwnersSignedRowIsNeverRecordedInTheBasis",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "if row, inBank := rows[c.Key]; inBank && !membank.IsEngineUnsigned(row) {\n\t\t\tcontinue",
|
||||
"replace": "if row, inBank := rows[c.Key]; false && inBank && !membank.IsEngineUnsigned(row) {\n\t\t\tcontinue"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-collapses-two-senses-of-one-surface",
|
||||
"why": "⛔ FOUND BY ADVERSARIAL REVIEW. The bank's uniqueness key is the FOUR-tuple (src, sense, since_ch, until_ch) — two senses are two laws by design, and a rendering that changes across a spoiler boundary is two rows with different windows — while a candidate carries only a Key. Indexed as out[key]=row, the surface keeps whichever row the store returned LAST (it orders by that tuple, so the latest window wins) and answers for the other: a surface the owner had signed on one window, holding an engine draft on another, served the draft",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestAnAmbiguousSurfaceIsReAskedRatherThanGuessed",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "\t\tif len(es) != 1 {\n\t\t\tambiguous[k] = true\n\t\t\tcontinue\n\t\t}",
|
||||
"replace": "\t\tif false {\n\t\t\tambiguous[k] = true\n\t\t\tcontinue\n\t\t}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-width-read-from-the-prompt-knob",
|
||||
"why": "⛔ FOUND BY ADVERSARIAL REVIEW: this mutation SURVIVED the whole tree before its pin existed. The pack's first pin for the pair datum built a Runner with no pack at all, so only the DEFAULT branch ran and reading kwic_width instead of basis_width went unnoticed — which would make a prompt-shape knob decide which stored decisions survive, and tuning translation quality would re-buy every book's bank out of a lifetime ceiling. No shipping langpack states either key, so nothing else in the tree exercises the pair branch",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestBasisWidthIsReadFromThePairWhenThePairStatesIt",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "if pair := r.packTerminology(); pair != nil && pair.BasisWidth > 0 {\n\t\treturn pair.BasisWidth",
|
||||
"replace": "if pair := r.packTerminology(); pair != nil && pair.KWICWidth > 0 {\n\t\treturn pair.KWICWidth"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-and-the-bank-filter-claim-the-same-row",
|
||||
"why": "⛔ FOUND BY ADVERSARIAL REVIEW, reproduced end to end. The two savings are different facts with different sentences on the sheet — «the bank already renders this surface and every draft agreed» against «this book decided it earlier» — and a candidate offered to both carries BOTH marks: the sheet prints two mutually exclusive sentences about one row, the read-out counts it twice, and SettledByBank lifts it out of the contested queue, which is precisely the exemption stopcontest.go refuses for a basis row. ⚠ THE PLANTING IS `&& false` AND NOT `if false`: the latter leaves the loop index unused, the package does not compile, and tmmutate reports «nothing ran» — a non-zero exit that is NOT a catch. Measured on this very entry. ⚠ WHAT THIS MUTATION STILL MEASURES CHANGED WHEN «whose row is this» WAS FIXED, and the entry says so rather than being retired: the two savings can no longer collide on one surface (the filter takes SEED surfaces, the recorder keeps ENGINE rows, and those are disjoint), so the damage is no longer a double MARK but a double COUNT — the predicate reports on candidates nobody offered it, and every re-ask number the run prints is inflated by the other saving. The pin follows the subject",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestASettledRowIsNotAlsoMarkedNeverAsked",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "\t\tif !consider[i] {\n\t\t\tcontinue\n\t\t}",
|
||||
"replace": "\t\tif !consider[i] && false {\n\t\t\tcontinue\n\t\t}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-row-accused-of-being-INVENTED",
|
||||
"why": "⛔ FOUND BY ADVERSARIAL REVIEW: this mutation SURVIVED the whole tree before its pin existed. `Invented` means «the engine produced a rendering no draft proposed» and is a finding about THIS run's drafts; a rendering carried from an earlier purchase was never offered to them, so the flag fires on every settled row and prints «INVENTED (no draft proposed it)» against the book's own established canon, on the sheet the owner signs by",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheThreeDeclaredDecisionsAboutABasisRowAreGuarded",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/mining.go",
|
||||
"find": "row.Invented = dst != \"\" && !row.SettledByBasis && !proposedByDrafts(dst, c.Variants)",
|
||||
"replace": "row.Invented = dst != \"\" && !proposedByDrafts(dst, c.Variants)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-row-exempted-from-the-contested-queue",
|
||||
"why": "⛔ FOUND BY ADVERSARIAL REVIEW: SURVIVED the whole tree before its pin existed, against a ⛔-comment written directly above the line refusing exactly this. SettledByBank means «the bank renders the surface and every draft agreed» — genuinely nothing to doubt. A basis row carries a real rendering chosen by a paid role and neither half of that sentence is true of it; exempting it by the resemblance of two field names silences the queue over most of a settled book",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheThreeDeclaredDecisionsAboutABasisRowAreGuarded",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/stopcontest.go",
|
||||
"find": "\tif r.SettledByBank {",
|
||||
"replace": "\tif r.SettledByBank || r.SettledByBasis {"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-stop-table-label-swallowed-on-read-back",
|
||||
"why": "⛔ FOUND BY ADVERSARIAL REVIEW: SURVIVED the whole tree before its pin existed. Both labels are trailing clauses of one line and the renderer prints NOT ASKED first, so a reader that cuts at the first label it finds swallows the other — and the two really do co-occur. The shipped table then reads back missing a state the sheet shows",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheThreeDeclaredDecisionsAboutABasisRowAreGuarded",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankstopparse.go",
|
||||
"find": "if i := strings.Index(rest, \" SETTLED EARLIER(\"); i >= 0 {",
|
||||
"replace": "if i := strings.Index(rest, \" NOT ASKED(\"); i >= 0 {\n\t\tr.SettledByBank = true\n\t\trest = rest[:i]\n\t}\n\tif i := strings.Index(rest, \" SETTLED EARLIER(\"); i >= 0 {"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-evidence-window-differs-from-the-KWIC-window",
|
||||
"why": "the fingerprint's meaning rests on this equality. EvidenceWindows stopped calling window() for COST — window() re-derives the whole rune view per call, which is quadratic over a book and measured at ×15 time for a ×4 source — so the two now compute the same shape by two routes. Let them drift by one rune and every stored fingerprint stops matching the one computed next purchase: a silent, book-wide re-purchase out of a lifetime budget",
|
||||
"package": "./internal/terminology/",
|
||||
"battery": true,
|
||||
"run": "TestEvidenceWindowsCutsTheSameWindowAsKWIC",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/terminology/evidence.go",
|
||||
"find": "lo := runeAt[at] - width",
|
||||
"replace": "lo := runeAt[at] - width + 1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-evidence-scan-goes-quadratic-again",
|
||||
"why": "the $0 pass that runs before EVERY purchase. window() declares in its own comment that its inputs are «chunk-sized»; called once per occurrence over the WHOLE book it is quadratic, and a source four times longer took FIFTEEN times as long — minutes on a real volume, plus tens of gigabytes of garbage, all before the first paid call and whether or not anything is saved. A later «simplification» back to window() restores it INVISIBLY: the result stays correct",
|
||||
"package": "./internal/terminology/",
|
||||
"battery": true,
|
||||
"run": "TestEvidenceWindowsDoesNotGrowQUADRATICALLYWithTheBook",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/terminology/evidence.go",
|
||||
"find": "out = append(out, strings.TrimSpace(string(runes[lo:hi])))",
|
||||
"replace": "out = append(out, window(nsource, at, at+len(key), width))"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-producer-change-reported-as-a-source-change",
|
||||
"why": "⛔ F1, FOUND BY ACCEPTANCE: a SILENT QUALITY REGRESSION, not a missing feature. Before the settled basis, improving the terminologist's prompt or moving it to another model missed every checkpoint (RequestHash folds model, effort and snapshot) and the book re-bought its bank from the new producer. With the producer out of the fingerprint a settled row is never asked again no matter who is asking now — a better prompt reaches the unsettled TAIL of every book and nothing else, permanently, and the only exit is bumping the shape, which zeroes every book at once. Improving a bank role's prompt is a direct lever on translationese, goal №1 of this project. ⚠ RE-AIMED: the producer moved OUT of the evidence hash and into the stored SHAPE, because hashed with the evidence it reported every prompt edit as «the source around this term changed» — the wrong cause, told to the operator on every row, about a book nobody had touched. The pin now asks the CLASS and not merely that something moved; its first version asserted movement alone and was green and empty on exactly this",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestAProducerChangeReportsItsOwnCause",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "return fmt.Sprintf(\"%s|w%d|p%s\", basisFingerprintVersion, width, producer)",
|
||||
"replace": "_ = producer\n\treturn fmt.Sprintf(\"%s|w%d\", basisFingerprintVersion, width)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-producer-identity-spelled-twice",
|
||||
"why": "a producer identity assembled at the call site is two identities the day the spellings disagree — the same rule membank.IsEngineUnsigned keeps for «whose row is this», and the rule this pack had already broken once. The classifier rides in it because it authors the TYPE and GENDER halves of every stored answer, and a role that is not configured contributes an empty FIELD rather than vanishing: «off» and «absent» must not hash the same, or turning a phase on would leave every decision it now co-authors untouched. ⚠ THE PLANTING TARGETS THE CLASSIFIER'S CONTRIBUTION rather than deleting a line of the Join: deleting one leaves its variable unused, the package does not compile, and tmmutate reports «nothing ran» — a non-zero exit that is NOT a catch. Measured on this very entry. ⚠ RE-AIMED at the RENDERED prompt: keying on the template's own SHA misses the BRIEF the pair's prompt reads ({{honorifics}}, {{transcription}}, {{venuti}}, {{audience}}, {{title}}), so changing the book's law re-buys both waves through BriefHash while the basis serves renderings decided under the old one. Rendering makes it pair-agnostic by SHAPE rather than by a list of fields in Go",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheProducerIdentityIsResolvedInOnePlace",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "\tout, err := Render(tpl.System, RenderVars{Book: r.Book})",
|
||||
"replace": "\tout, err := tpl.System, error(nil)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-asks-whose-row-by-a-second-spelling",
|
||||
"why": "⛔ F2, FOUND BY ACCEPTANCE. The engine has ONE definition of «whose row is this» — membank.IsEngineUnsigned, `Source == \"mined\" && Status != \"approved\"` — with other callers. A private `Status == \"approved\"` is a SECOND spelling, and two spellings are two answers the day they disagree. They already did: a term the OWNER wrote by hand and left unsigned (`source: seed`, `status: draft` — a shape the loader admits and even REQUIRES a rendering for) answered «not approved», so the engine recorded its own consolidation under that surface and served it back as «this book decided it earlier»",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestAnOwnersUNSIGNEDRowIsNotTheEnginesMemoryEither",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "if row, inBank := rows[c.Key]; inBank && !membank.IsEngineUnsigned(row) {\n\t\t\tcontinue",
|
||||
"replace": "if row, inBank := rows[c.Key]; inBank && row.Status == \"approved\" {\n\t\t\tcontinue"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-width-change-reported-as-a-source-change",
|
||||
"why": "⛔ D1, FOUND BY ACCEPTANCE against the pack's own order. Hashed only inside the evidence, a change of basis_width moves every fingerprint of every book — and the predicate reports it as `evidence-moved`, «the source around this term changed». That is the WRONG CAUSE for every one of those rows, and the owner's reaction to the two is opposite: one is nothing to do, the other is a knob somebody turned. Carried in the stored SHAPE, the same change is reported by the version check with both widths printed",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestAWidthChangeReportsItsOwnCauseAndNotTheSources",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "return fmt.Sprintf(\"%s|w%d|p%s\", basisFingerprintVersion, width, producer)",
|
||||
"replace": "_ = width\n\treturn fmt.Sprintf(\"%s|p%s\", basisFingerprintVersion, producer)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-signature-stop-records-nothing",
|
||||
"why": "⛔ D2, FOUND BY ACCEPTANCE WITH A CONTROL: removing the write at `run-finished` turned the package RED, removing it at the SIGNATURE STOP left it entirely `ok`. The two boundaries are not interchangeable and the stop is the one that cannot be reached from the other — it returns as an ERROR VALUE, so a run that stops for the owner's signature never arrives at `run-finished`. Lose the write there and the resume after the owner signs re-buys the ENTIRE bank pass out of a lifetime ceiling, while every fixture that only auto-continues stays green",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestTheSignatureStopWritesTheBasisToo",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/mining.go",
|
||||
"find": "\tr.writeBankBasis(ctx, \"bank-mining/signature-stop\")",
|
||||
"replace": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-serves-an-answer-the-bank-contradicts",
|
||||
"why": "⛔ D3. The predicate's sentence is «this book decided this AND the bank still holds that decision», and only the first half was checked: the stored answer was served and travelled into the delta, overwriting whatever the bank held, with the sheet reporting the row as settled earlier. ⚠ The population is NAMED rather than claimed — on the run that writes them the two agree by construction, so the reachable divergence is a hand edit of the auto-bank FILE, which the ontology lists among the SOURCES. The comparison folds target forms, so a case difference is one decision and not a re-purchase",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestABankRowThatDisagreesWithTheMemoryIsReAsked",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "if text.NormalizeTargetForm(row.Dst) != text.NormalizeTargetForm(rec.Answer.Dst) {\n\t\treturn bankBasisVerdict{Reason: basisDiverged}\n\t}",
|
||||
"replace": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-completeness-line-states-the-wrong-cause",
|
||||
"why": "⛔ D5, FOUND BY ACCEPTANCE. «The paid role was asked for nothing» now has TWO reasons and they are different facts; the single sentence asserted the FIRST for both. For a book answered from the memory of decisions it was false in BOTH halves — the bank holds no seed surface for those terms and the drafts were never asked to agree with anything. A message that states the wrong cause is worse than one that states none, and D39.93 п.2 refuses it next door. The pin was maintained together with the sentence, and the maintenance is declared",
|
||||
"package": "./cmd/tmctl/",
|
||||
"battery": true,
|
||||
"run": "TestTheSigningScreenSaysHowCompleteTheBankIs",
|
||||
"edits": [
|
||||
{
|
||||
"file": "cmd/tmctl/render.go",
|
||||
"find": "case c.SettledEarlier > 0 && c.NeverAsked > 0:",
|
||||
"replace": "case false:"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "BASIS-shape-mismatch-reported-as-never-decided",
|
||||
"why": "a record whose RULE moved must be kept, not dropped. Dropped, it leaves no record — and no record reads as `new`, «this surface has never been decided», which is false of every one of those rows. The summary line would state the right cause once while every row carried the wrong one, and the two would disagree in the same run. ⚠ RE-AIMED at the extracted loading rule: inside bankBasisPass it needs a store, a book and a runner, and a money rule that can only be exercised end to end is one nobody plants a mutation into — the first version of this entry SURVIVED for exactly that reason",
|
||||
"package": "./internal/pipeline/",
|
||||
"battery": true,
|
||||
"run": "TestARecordWhoseRuleMovedIsKeptAndNamed",
|
||||
"edits": [
|
||||
{
|
||||
"file": "internal/pipeline/bankbasis.go",
|
||||
"find": "\t\t\told[row.FPVersion]++\n\t\t\tchanged++\n\t\t}",
|
||||
"replace": "\t\t\told[row.FPVersion]++\n\t\t\tchanged++\n\t\t\tcontinue\n\t\t}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -135,6 +135,18 @@ type SpeechCue struct {
|
|||
type TerminologySizing struct {
|
||||
KWICPerTerm int
|
||||
KWICWidth int
|
||||
// BasisWidth is how much context around an occurrence DEFINES that occurrence, for the settled-basis
|
||||
// fingerprint. It is pair data for the same reason KWICWidth is (D39.50 п.4: a width counted in RUNES
|
||||
// is a fact about the source's information density, so an engine constant would be pair-blind), and it
|
||||
// is a SEPARATE key on purpose.
|
||||
//
|
||||
// ⛔ THE SEPARATION IS THE MONEY. KWICWidth shapes the PROMPT — how much context the model is shown —
|
||||
// and it is the kind of knob a pair tunes while looking at translation quality. BasisWidth decides
|
||||
// which stored decisions survive the next purchase. Folded into one key, tuning the prompt would
|
||||
// invalidate every fingerprint of every book and re-buy each bank out of a LIFETIME role budget, with
|
||||
// nothing in the run saying that is what happened. Two keys cost one line of parsing; one key costs a
|
||||
// consolidated bank per turn of a quality dial.
|
||||
BasisWidth int
|
||||
}
|
||||
|
||||
// Palladius is the pair transliteration table: the pinyin→Cyrillic maps and the syllable-generator's
|
||||
|
|
@ -741,8 +753,8 @@ func parseSpeechCue(b []byte) (*SpeechCue, error) {
|
|||
}
|
||||
|
||||
// parseTerminology reads terminology.txt into a TerminologySizing. Format: `key<TAB>value`, keys
|
||||
// kwic_per_term | kwic_width, each optional but positive when present. Fail-loud like parseHeading; an
|
||||
// empty file is refused too, because its bytes move Version() while changing nothing.
|
||||
// kwic_per_term | kwic_width | basis_width, each optional but positive when present. Fail-loud like
|
||||
// parseHeading; an empty file is refused too, because its bytes move Version() while changing nothing.
|
||||
func parseTerminology(b []byte) (*TerminologySizing, error) {
|
||||
ts := &TerminologySizing{}
|
||||
seen := false
|
||||
|
|
@ -765,13 +777,15 @@ func parseTerminology(b []byte) (*TerminologySizing, error) {
|
|||
ts.KWICPerTerm = n
|
||||
case "kwic_width":
|
||||
ts.KWICWidth = n
|
||||
case "basis_width":
|
||||
ts.BasisWidth = n
|
||||
default:
|
||||
return nil, fmt.Errorf("line %d: unknown key %q (want kwic_per_term|kwic_width)", i+1, key)
|
||||
return nil, fmt.Errorf("line %d: unknown key %q (want kwic_per_term|kwic_width|basis_width)", i+1, key)
|
||||
}
|
||||
seen = true
|
||||
}
|
||||
if !seen {
|
||||
return nil, fmt.Errorf("the file states nothing (want at least one of kwic_per_term|kwic_width); delete it instead — an empty table moves the pack version without changing behaviour")
|
||||
return nil, fmt.Errorf("the file states nothing (want at least one of kwic_per_term|kwic_width|basis_width); delete it instead — an empty table moves the pack version without changing behaviour")
|
||||
}
|
||||
return ts, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -400,3 +400,59 @@ func (b *Bank) VoiceProjection(chapter int) ([]CharacterView, []AddressView) {
|
|||
}
|
||||
return chars, addrs
|
||||
}
|
||||
|
||||
// TermWindowGaps reports a bank SURFACE whose rows leave a chapter uncovered between two windows — a term
|
||||
// signed «until chapter 3» whose next row starts at chapter 5, so chapter 4 has no law for it at all.
|
||||
//
|
||||
// ⛔ WHY IT EXISTS WHEN THE LOADER ALREADY CHECKS WINDOWS. The loader refuses an OVERLAP (two rows claiming
|
||||
// one chapter) and polysemy collisions; the opposite shape — a HOLE — passes every check there is. It is
|
||||
// not a schema violation and it cannot be: the UNIQUE key admits the two rows, membank.windowsOverlap
|
||||
// permits them, and the spoiler rule then picks neither. The term simply stops being law for those
|
||||
// chapters, the editor is shown nothing about it, and the drafts of chapter 4 render it however they like —
|
||||
// inside a book whose whole point is one word per term.
|
||||
//
|
||||
// ⚠ IT REPORTS AND NEVER REFUSES, and the guarantee is taken from the exemplar next to it rather than
|
||||
// improvised: VoiceWindowGaps says a deliberate gap is legitimate, and the same is true here — a term that
|
||||
// genuinely does not apply between two reveals is a real editorial shape. The difference is that a gap is
|
||||
// invisible in a seed file and is far more often a typo, so it is said out loud and left to the owner.
|
||||
//
|
||||
// ⚠ AND IT IS KEYED ON (src, sense), NOT ON src ALONE. Two senses of one surface are two laws by design
|
||||
// (the A3 disambiguator is part of the uniqueness key), so comparing their windows against each other would
|
||||
// report a gap for every polysemous term in the book — a warning that fires on correct data is one an
|
||||
// operator learns to ignore, which costs more than the check earns.
|
||||
//
|
||||
// Deterministic: surfaces are reported in first-seen order and the rows of one surface are sorted by the
|
||||
// chapter their window opens at; nothing here iterates a map for output.
|
||||
func TermWindowGaps(rows []store.GlossaryEntry) []string {
|
||||
byTerm := map[[2]string][]store.GlossaryEntry{}
|
||||
var order [][2]string
|
||||
for _, e := range rows {
|
||||
if strings.TrimSpace(e.Src) == "" {
|
||||
continue
|
||||
}
|
||||
k := [2]string{e.Src, e.Sense}
|
||||
if _, seen := byTerm[k]; !seen {
|
||||
order = append(order, k)
|
||||
}
|
||||
byTerm[k] = append(byTerm[k], e)
|
||||
}
|
||||
var out []string
|
||||
for _, k := range order {
|
||||
win := byTerm[k]
|
||||
if len(win) < 2 {
|
||||
continue
|
||||
}
|
||||
sort.Slice(win, func(i, j int) bool { return win[i].SinceCh < win[j].SinceCh })
|
||||
for i := 1; i < len(win); i++ {
|
||||
prevUntil := win[i-1].UntilCh
|
||||
if prevUntil == 0 {
|
||||
continue // open-ended: nothing after it can be a gap
|
||||
}
|
||||
if win[i].SinceCh > prevUntil+1 {
|
||||
out = append(out, fmt.Sprintf("term %q (sense %q): chapters %d–%d have no bank row (window ends at %d, next starts at %d)",
|
||||
k[0], k[1], prevUntil+1, win[i].SinceCh-1, prevUntil, win[i].SinceCh))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
|
|||
46
backend/internal/membank/termgaps_test.go
Normal file
46
backend/internal/membank/termgaps_test.go
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
package membank
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/store"
|
||||
)
|
||||
|
||||
// TestTermWindowGapsSeesTheHoleTheLoaderCannot pins backlog row 490: the loader refuses two rows that
|
||||
// OVERLAP and says nothing about two that leave a chapter between them.
|
||||
func TestTermWindowGapsSeesTheHoleTheLoaderCannot(t *testing.T) {
|
||||
rows := []store.GlossaryEntry{
|
||||
// THE HOLE: chapter 4 belongs to neither window.
|
||||
{Src: "古月", Dst: "Гуюэ", SinceCh: 1, UntilCh: 3, Status: "approved"},
|
||||
{Src: "古月", Dst: "клан Гу Юэ", SinceCh: 5, UntilCh: 0, Status: "approved"},
|
||||
// CONTIGUOUS: a handover, not a hole — the commonest legitimate shape and the one a noisy check
|
||||
// would report on every spoiler boundary in the book.
|
||||
{Src: "方源", Dst: "Фан Юань", SinceCh: 1, UntilCh: 10, Status: "approved"},
|
||||
{Src: "方源", Dst: "Демон", SinceCh: 11, UntilCh: 0, Status: "approved"},
|
||||
// ONE ROW, open-ended: nothing to compare.
|
||||
{Src: "元石", Dst: "юаньши", SinceCh: 0, UntilCh: 0, Status: "draft"},
|
||||
// TWO SENSES of one surface: two laws by design (the A3 disambiguator is part of the uniqueness
|
||||
// key), so their windows must NOT be measured against each other.
|
||||
{Src: "青茅", Sense: "гора", Dst: "Цинмао", SinceCh: 1, UntilCh: 3, Status: "approved"},
|
||||
{Src: "青茅", Sense: "трава", Dst: "синяя осока", SinceCh: 40, UntilCh: 0, Status: "approved"},
|
||||
}
|
||||
got := TermWindowGaps(rows)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("reported %d gap(s), want exactly the one real hole: %v", len(got), got)
|
||||
}
|
||||
for _, want := range []string{`"古月"`, "chapters 4–4", "window ends at 3", "next starts at 5"} {
|
||||
if !strings.Contains(got[0], want) {
|
||||
t.Errorf("the message must name the chapters the owner has to look at; %q is missing from:\n %s", want, got[0])
|
||||
}
|
||||
}
|
||||
// The control that makes the «exactly one» readable: this function can see MORE than one, so the count
|
||||
// above is a measurement of the fixture rather than a ceiling of the function.
|
||||
wider := append([]store.GlossaryEntry(nil), rows...)
|
||||
wider = append(wider,
|
||||
store.GlossaryEntry{Src: "蛊师", Dst: "гу-мастер", SinceCh: 1, UntilCh: 2, Status: "approved"},
|
||||
store.GlossaryEntry{Src: "蛊师", Dst: "мастер Гу", SinceCh: 9, UntilCh: 0, Status: "approved"})
|
||||
if n := len(TermWindowGaps(wider)); n != 2 {
|
||||
t.Fatalf("control: a second genuine hole produced %d report(s), want 2 — the function reports at most one and the assertion above measured that, not the data", n)
|
||||
}
|
||||
}
|
||||
819
backend/internal/pipeline/bankbasis.go
Normal file
819
backend/internal/pipeline/bankbasis.go
Normal file
|
|
@ -0,0 +1,819 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"textmachine/backend/internal/lang"
|
||||
"textmachine/backend/internal/membank"
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
"textmachine/backend/internal/text"
|
||||
)
|
||||
|
||||
// bankbasis.go: the SETTLED BASIS — the memory that a row the engine already decided is not bought again.
|
||||
//
|
||||
// WHAT PROBLEM IT IS FOR. Every purchase of chapters re-ran both bank roles over surfaces whose rendering
|
||||
// the book had already settled, and paid for them out of a LIFETIME per-book ceiling
|
||||
// (gates.terminology.budget_usd / classify_budget_usd). Exhausting that ceiling is irreversible: the book
|
||||
// then keeps whatever bank it has, forever. So this is not an optimisation — it is the difference between
|
||||
// a long book that can afford a consolidated bank and one that cannot.
|
||||
//
|
||||
// WHAT IS IN HERE. The two PURE halves: the fingerprint of what the role reads about one candidate, and
|
||||
// the predicate that reads a fingerprint back and says whether the stored answer still holds. Storage,
|
||||
// boundaries and wiring live elsewhere; nothing in this file touches a store, a clock or a provider.
|
||||
//
|
||||
// ⛔ THE BASIS IS A SOURCE OF ANSWERS, NOT A FILTER, and the distinction is the whole money risk of the
|
||||
// mechanism. Dropping a settled candidate from the paid set and stopping there does NOT leave the book
|
||||
// where it was: attachConsolidatedDst stamps nothing for a candidate the role did not answer, and the
|
||||
// emission then falls back to the raw first-chunk banknote guess at status `auto`
|
||||
// (miner_emit.go, `st.Dst = props[0].Dst` under `props[0].Via == ""`), or to no rendering at all. The book
|
||||
// would look CHEAPER and be POORER, and nothing in the run would say so. So a settled candidate must be
|
||||
// handed its stored answer, and the answer handed over is the CONSOLIDATED one — never a raw guess, for
|
||||
// exactly the reason the emission gives for its own two-mode rule.
|
||||
|
||||
// basisFingerprintVersion versions the SHAPE of the fingerprint, and it is load-bearing rather than
|
||||
// hygienic: every stored fingerprint was produced by one shape, so changing what goes into the hash
|
||||
// invalidates every book's basis at once. Bumping this is therefore a DECISION with a price — one
|
||||
// re-consolidation per book, out of the lifetime ceiling — and the version makes that decision explicit
|
||||
// and one-time instead of silent and repeated.
|
||||
const basisFingerprintVersion = "bank-basis-fp-v1"
|
||||
|
||||
// basisShape is the fingerprint's SHAPE as it is stored beside every row: the version above plus the
|
||||
// EFFECTIVE evidence width the row was decided under.
|
||||
//
|
||||
// ⛔ THE WIDTH BELONGS IN THE SHAPE, NOT ONLY INSIDE THE HASH, and the difference is which sentence the
|
||||
// operator reads when money moves. Hashed only inside the evidence, a change of `basis_width` moves every
|
||||
// fingerprint of every book — and the predicate reports that as `evidence-moved`, i.e. «the source around
|
||||
// this term changed». That is the WRONG CAUSE for every one of those rows: the source did not move, the
|
||||
// rule for reading it did, and the owner's reaction to the two is opposite (one is nothing to do, the other
|
||||
// is a knob somebody turned). Carried in the shape, the same change is reported by the version check as
|
||||
// «the rule that decided this is no longer the rule», with the old and the new width printed.
|
||||
//
|
||||
// ⚠ It is the EFFECTIVE width — what terminologyOpts actually resolved — and not the pair's declared value:
|
||||
// a pair that states nothing runs on the engine default, and the day that default moves those rows must
|
||||
// report a shape change too, not silence.
|
||||
func basisShape(width int, producer string) string {
|
||||
return fmt.Sprintf("%s|w%d|p%s", basisFingerprintVersion, width, producer)
|
||||
}
|
||||
|
||||
// basisShapeParts splits a stored shape back into its components, so a mismatch can say WHICH of them moved
|
||||
// instead of reporting «the shape changed» and leaving the operator to guess between a knob they turned and
|
||||
// a prompt somebody edited.
|
||||
func basisShapeParts(shape string) (version, width, producer string) {
|
||||
p := strings.SplitN(shape, "|", 3)
|
||||
for len(p) < 3 {
|
||||
p = append(p, "")
|
||||
}
|
||||
return p[0], p[1], p[2]
|
||||
}
|
||||
|
||||
// terminologyDefaultBasisWidth is the engine's fallback for the evidence width when the pair states none.
|
||||
// It equals today's KWIC default by coincidence of judgement, not by construction: the two are separate
|
||||
// knobs (lang.TerminologySizing.BasisWidth) precisely so that tuning the prompt cannot re-buy a bank.
|
||||
const terminologyDefaultBasisWidth = 40
|
||||
|
||||
// bankBasisFP is one candidate's evidence reduced to hashes.
|
||||
//
|
||||
// ⚠ TWO VALUES AND NOT ONE, and the second is not decoration. `Whole` is what the predicate compares;
|
||||
// `Canon` is the part contributed by the book's SIGNED rows alone. Without it the engine can say "this
|
||||
// term is being bought again" and not say WHY, and the two reasons call for opposite reactions from the
|
||||
// owner: «the source around this term changed» is nothing to do, «you signed a relative of this term» is
|
||||
// the consequence of an act they just performed. A message that cannot name its own cause is the class
|
||||
// this codebase already refuses elsewhere.
|
||||
type bankBasisFP struct {
|
||||
Whole string
|
||||
Canon string
|
||||
}
|
||||
|
||||
// bankBasisAnswer is what the paid roles decided about one surface — the thing the basis exists to hand
|
||||
// back. Gender rides with it because the classifier is the only automatic producer of that field, and a
|
||||
// settled row that lost its gender would silently drop a directive the wire already promised.
|
||||
type bankBasisAnswer struct {
|
||||
Dst string
|
||||
Type string
|
||||
Gender string
|
||||
}
|
||||
|
||||
// bankBasisRecord is one row of the memory of decisions: what was decided, and the fingerprint of the
|
||||
// evidence it was decided on.
|
||||
type bankBasisRecord struct {
|
||||
Key string
|
||||
FP bankBasisFP
|
||||
Answer bankBasisAnswer
|
||||
Shape string // the fingerprint SHAPE this row was decided under: version | evidence width | producer
|
||||
UnitMembers []string // the unit this key was decided as part of, so a later split is visible
|
||||
}
|
||||
|
||||
// basisReason is why a candidate was NOT served from the basis — the class the report counts by, and the
|
||||
// line the operator reads when a purchase costs money.
|
||||
type basisReason string
|
||||
|
||||
const (
|
||||
basisServed basisReason = "" // served from the basis: not bought
|
||||
basisNew basisReason = "new" // no record: this surface has never been decided
|
||||
basisKin basisReason = "kin-of-signed" // the canon anchors of THIS candidate moved (a relative was signed)
|
||||
basisMoved basisReason = "evidence-moved" // the source around this surface changed
|
||||
basisUnsettled basisReason = "unsettled" // the bank row is not a decided one (status != draft, or no rendering)
|
||||
basisAbsent basisReason = "no-bank-row" // decided once, but the bank no longer holds the row at all
|
||||
basisUnit basisReason = "unit-incomplete" // this member is settled, another member of its unit is not
|
||||
// basisSigned: the OWNER signed this row. Not served (his word is not the engine's memory to hand
|
||||
// back) and not a hole either (it is settled harder than any draft row) — see basisVerdictFor.
|
||||
basisSigned basisReason = "signed-by-owner"
|
||||
// basisOwnerRow: the bank row for this surface is not the ENGINE's at all — it came from the owner's
|
||||
// hand (a seed term) and he has not signed it. Not served and not a hole, exactly like a signed row:
|
||||
// see basisVerdictFor.
|
||||
basisOwnerRow basisReason = "owners-unsigned-row"
|
||||
// basisProducerMoved / basisWidthMoved / basisShapeMoved: the RULE that decided this row is not the rule
|
||||
// being applied now — a different prompt or model, a different evidence width, or a different
|
||||
// fingerprint shape. Three classes and not one, because the owner's reaction differs: somebody edited a
|
||||
// prompt, somebody turned a pair datum, or the engine's own shape was bumped.
|
||||
basisProducerMoved basisReason = "producer-changed"
|
||||
basisWidthMoved basisReason = "evidence-width-changed"
|
||||
basisShapeMoved basisReason = "fingerprint-shape-changed"
|
||||
// basisDiverged: the memory of decisions and the bank disagree about this surface's rendering. Serving
|
||||
// the stored answer would overwrite the bank's row and report the result as «decided earlier».
|
||||
basisDiverged basisReason = "bank-disagrees-with-the-memory"
|
||||
// basisAmbiguous: the bank holds MORE THAN ONE row for this surface — two senses, or two chapter
|
||||
// windows — and a candidate carries no sense and no window, so nothing here can say WHICH row is the
|
||||
// answer. Re-asked rather than guessed: the cheap direction of a doubt on this path is always to buy.
|
||||
basisAmbiguous basisReason = "ambiguous-surface"
|
||||
)
|
||||
|
||||
// basisWidth resolves the evidence width in ONE place: the pair's own datum, else the engine default.
|
||||
//
|
||||
// ⚠ TWO STEPS, WHERE terminologyOpts HAS THREE (explicit config → pair → engine default), and the missing
|
||||
// one is deliberate: there is no `gates.terminology.basis_width`. A per-book config override of what counts
|
||||
// as the same evidence would let one book's pipeline.yaml silently decide which of its stored decisions
|
||||
// survive, and that is a decision about the SOURCE LANGUAGE, not about a book. Said here because an earlier
|
||||
// revision of this comment claimed it "mirrors terminologyOpts", which the tree does not support — and a
|
||||
// comment describing a resolution order the code does not have is how the next edit adds the key without
|
||||
// anyone deciding to.
|
||||
//
|
||||
// ⛔ NOT A Go CONSTANT, and the reason is ratified rather than stylistic: D39.50 п.4 settled that a width
|
||||
// counted in RUNES is a fact about the source language, so freezing one in Go makes a PAIR-BLIND decision
|
||||
// — and here it would make it on the money path, where being pair-blind means buying the wrong bank for
|
||||
// every pair that is not the one the constant was chosen for. The review question this pack answers to:
|
||||
// a pair that is not in the repository states `basis_width` in its own terminology.txt and needs no Go edit.
|
||||
func (r *Runner) basisWidth() int {
|
||||
if pair := r.packTerminology(); pair != nil && pair.BasisWidth > 0 {
|
||||
return pair.BasisWidth
|
||||
}
|
||||
return terminologyDefaultBasisWidth
|
||||
}
|
||||
|
||||
// bankRoleProducer is WHO would answer a bank-role question on this configuration, as one comparable
|
||||
// string: each role's model, the effort they share, and the canonical SHA256 of the pair's prompt for it.
|
||||
//
|
||||
// ⚠ IT IS RESOLVED IN ONE PLACE AND HASHED WHOLE, rather than assembled field by field at the call site,
|
||||
// because a producer identity spelled twice is two identities the day the two spellings disagree — the same
|
||||
// rule membank.IsEngineUnsigned keeps for «whose row is this». The classifier rides here too: it produces
|
||||
// the TYPE and the GENDER halves of a stored answer, so a book whose classifier changed has decisions the
|
||||
// new one would take differently.
|
||||
//
|
||||
// A role that is not configured contributes an empty field rather than being skipped: «off» and «absent»
|
||||
// must not hash the same, or turning a phase on would leave every decision it now co-authors untouched.
|
||||
func (r *Runner) bankRoleProducer() string {
|
||||
g := r.Pipeline.Gates.Terminology
|
||||
return hashOf(func(h io.Writer) {
|
||||
fmt.Fprintf(h, "tm-bank-producer-v2\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1e",
|
||||
g.Model, r.roleSystemPrompt(r.terminologyTemplate),
|
||||
g.ClassifierModel(), r.roleSystemPrompt(r.classifierTemplate),
|
||||
g.Reasoning)
|
||||
})
|
||||
}
|
||||
|
||||
// roleSystemPrompt is what a bank role's system turn ACTUALLY says on this book — the template with the
|
||||
// book's brief rendered into it — reduced to a comparable string.
|
||||
//
|
||||
// ⛔ THE RENDERED TEXT, NOT THE TEMPLATE'S SHA, and the difference is a whole class of law. The pair's
|
||||
// terminologist prompt reads the BRIEF: `{{title}}`, `{{audience}}`, `{{honorifics}}`, `{{transcription}}`,
|
||||
// `{{venuti}}`. Change `honorifics: keep` to `adapt`, or the transcription convention, and both waves of the
|
||||
// book are re-bought through BriefHash — while a basis keyed on the template's SHA would go on serving
|
||||
// renderings decided under the OLD law into waves paid for under the new one. Neither the evidence nor the
|
||||
// model would have moved; the law the role was told to apply would have.
|
||||
//
|
||||
// ⚠ AND IT IS PAIR-AGNOSTIC BY CONSTRUCTION, which is why it is one line rather than a list of brief
|
||||
// fields in Go: whatever a pair's own prompt chooses to read enters the identity because it enters the
|
||||
// rendered text. A pair that is not in this repository needs no edit here — the canon's standing review
|
||||
// question, answered by shape instead of by enumeration.
|
||||
//
|
||||
// A template that fails to render contributes the error text rather than falling back to silence: «this
|
||||
// producer could not be described» must not hash the same as «this role is off», and a run in that state
|
||||
// re-buys rather than serves.
|
||||
func (r *Runner) roleSystemPrompt(tpl *PromptTemplate) string {
|
||||
if tpl == nil {
|
||||
return "" // the role is not configured; an empty FIELD, never an absent one (see bankRoleProducer)
|
||||
}
|
||||
out, err := Render(tpl.System, RenderVars{Book: r.Book})
|
||||
if err != nil {
|
||||
return "unrenderable:" + err.Error()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// candidateBasisFingerprint hashes what the role reads ABOUT ONE CANDIDATE, in a form invariant to how the
|
||||
// contexts shown to the role are selected.
|
||||
//
|
||||
// WHAT GOES IN, and why each:
|
||||
//
|
||||
// - the candidate's key, its MINER-heuristic type, aliases, related surfaces and evidence tags — the
|
||||
// identity and the detector's account of it. The type is the heuristic one on purpose: hashing the
|
||||
// CLASSIFIER's type would mean buying the classifier on every purchase merely to learn whether the
|
||||
// terminologist may be skipped, which is the cost this whole mechanism exists to avoid;
|
||||
// - the context around EVERY occurrence of the key in the source, at the pair's evidence width;
|
||||
// - the SIGNED bank rows related to this candidate, uncapped.
|
||||
//
|
||||
// ⚠ WHO WOULD ANSWER is NOT here — it lives in the stored SHAPE (basisShape), for the same reason the
|
||||
// evidence width does: a producer is part of the RULE that decided a row, not part of the evidence, and
|
||||
// hashing it in with the evidence would report every prompt edit as «the source around this term changed».
|
||||
//
|
||||
// ⛔ THE PRODUCER IS IN, AND LEAVING IT OUT WAS A SILENT QUALITY REGRESSION. The fingerprint summarises the
|
||||
// evidence a decision stands on, and a decision stands on WHO READ that evidence as much as on the evidence
|
||||
// itself. Before this mechanism, improving the terminologist's prompt or moving it to another model missed
|
||||
// every checkpoint (RequestHash folds model, effort and snapshot) and the book simply re-bought its bank
|
||||
// from the new producer. With the producer out of the fingerprint, a settled row would never be asked again
|
||||
// no matter who is asking now — so a better prompt would reach the unsettled TAIL of every book and nothing
|
||||
// else, permanently, and the only exit would be bumping basisFingerprintVersion, which zeroes every book at
|
||||
// once. Improving a bank role's prompt is a direct lever on translationese, which is goal №1 of this
|
||||
// project; a money mechanism must not take that lever away to save a purchase it was never asked to save.
|
||||
//
|
||||
// ⚠ THE PRODUCER IS NOT THE SNAPSHOT, and the narrowing is deliberate. SnapshotID folds the BANK among many
|
||||
// other things, so hashing it would move this fingerprint every time any row of the bank changed — the
|
||||
// mechanism would save nothing on exactly the books it exists for. What identifies a producer is: the model
|
||||
// each bank role will be called on, the effort they share, and the canonical SHA256 of the pair's prompt for
|
||||
// each of them. Change any of those and the answer can differ; change the bank's contents and it cannot.
|
||||
//
|
||||
// WHAT STAYS OUT, and why each:
|
||||
//
|
||||
// - the draft variants. They leave the TRIGGER and stay EVIDENCE: a rendering the drafts happened to
|
||||
// produce this time is not new information about the term, and hashing it would re-buy a bank on
|
||||
// ordinary model noise. They are still printed on the sheet and in the log;
|
||||
// - freq and since_ch — measured constant between purchases, and since_ch is a chapter number, which a
|
||||
// re-cut moves without changing a word of the book;
|
||||
// - the KWIC subset, its cap and its selection rule — see terminology.EvidenceWindows;
|
||||
// - the batch this candidate ends up in. The fingerprint is per CANDIDATE, so it cannot inherit a
|
||||
// dependency on who its batch-mates were — which is the dependency the pack exists to remove.
|
||||
//
|
||||
// ⚠ AND TWO THINGS NEITHER THE EVIDENCE NOR THE RULE CAN SEE, named here because a blind spot that is not
|
||||
// written down is one the next reader assumes away:
|
||||
//
|
||||
// - A MODEL SLUG IS NOT A MODEL. The producer identity carries the name the call will be addressed to,
|
||||
// and a vendor that moves the weights behind that name changes who answers without changing anything
|
||||
// this engine can hash. «Слаг живой» is not «модель та же» (docs/experiments/00-provider-quirks.md),
|
||||
// and there is no mechanism here that would notice.
|
||||
// - AN EMPTY `reasoning` MEANS «the vendor's default», which is a value this project has already watched
|
||||
// move. The shipping pipelines pin the effort explicitly, so the identity is stable for them; a book
|
||||
// that leaves it empty inherits a producer whose behaviour can change under a constant string.
|
||||
//
|
||||
// ⚠ THE NARROWING THIS ACCEPTS, said out loud because it is invisible and deliberate. The role is shown
|
||||
// the anchor of its whole BATCH, so a signed row that relates to a NEIGHBOUR changes what the role reads
|
||||
// about this candidate and does NOT change this fingerprint. That is the measured case — a batch where
|
||||
// the anchor related to none of the six terms whose rendering moved and was there because of three other
|
||||
// members — and re-asking a term because somebody else's relative was signed is exactly the re-purchase
|
||||
// this mechanism refuses.
|
||||
func candidateBasisFingerprint(c terminology.Candidate, nsource string, canon []terminology.Neighbour, width int) bankBasisFP {
|
||||
canonPart := hashOf(func(h io.Writer) {
|
||||
// CanonFor with no cap: terminologyCanonCap bounds what one PROMPT may carry, which is a wire
|
||||
// concern. Applied here it could drop the very signed row CanonConflicts reads, and the fingerprint
|
||||
// would then call a term settled whose canon had moved.
|
||||
for _, pair := range terminology.CanonFor([]terminology.Candidate{c}, canon, len(canon)) {
|
||||
fmt.Fprintf(h, "canon\x00%s\x00%s\x1e", pair[0], pair[1])
|
||||
}
|
||||
})
|
||||
whole := hashOf(func(h io.Writer) {
|
||||
fmt.Fprintf(h, "v\x00%s\x1e", basisFingerprintVersion)
|
||||
fmt.Fprintf(h, "key\x00%s\x1e", c.Key)
|
||||
fmt.Fprintf(h, "type\x00%s\x1e", c.Type)
|
||||
fmt.Fprintf(h, "width\x00%d\x1e", width)
|
||||
for _, a := range c.Aliases {
|
||||
fmt.Fprintf(h, "alias\x00%s\x1e", a)
|
||||
}
|
||||
for _, rel := range c.Related {
|
||||
fmt.Fprintf(h, "related\x00%s\x1e", rel)
|
||||
}
|
||||
for _, e := range c.Evidence {
|
||||
fmt.Fprintf(h, "evidence\x00%s\x1e", e)
|
||||
}
|
||||
for _, w := range terminology.EvidenceWindows(nsource, c.Key, width) {
|
||||
fmt.Fprintf(h, "ctx\x00%s\x1e", w)
|
||||
}
|
||||
fmt.Fprintf(h, "canon\x00%s\x1e", canonPart)
|
||||
})
|
||||
return bankBasisFP{Whole: whole, Canon: canonPart}
|
||||
}
|
||||
|
||||
// hashOf runs write against a hash and returns it hex-encoded. Full-width sha256 rather than a truncation:
|
||||
// a collision here does not read as a bug, it reads as «already settled» — the engine would hand back an
|
||||
// answer decided on OTHER evidence and never buy the term again. The cheap direction of a wrong guess is
|
||||
// to re-buy, never to keep.
|
||||
func hashOf(write func(io.Writer)) string {
|
||||
h := sha256.New()
|
||||
write(h)
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// bankBasisRecordsFrom turns the stored rows into records, KEEPING the ones whose shape has moved.
|
||||
//
|
||||
// ⛔ KEPT, NOT DROPPED, and this is the whole reason the loop is a function. A dropped record leaves the
|
||||
// predicate with nothing, and nothing reads as `new` — «this surface has never been decided» — which is
|
||||
// false of every one of those rows. The summary line would then state the right cause ONCE while every row
|
||||
// carried the wrong one, and the two would disagree inside one run. The record carries its own shape so the
|
||||
// verdict can name which part of the rule moved: the producer, the evidence width, or the engine's shape.
|
||||
//
|
||||
// It is also the only way this rule can be asked a question: inside bankBasisPass it needs a store, a book
|
||||
// and a runner, and a money rule that can only be exercised end to end is one nobody plants a mutation into.
|
||||
func bankBasisRecordsFrom(stored map[string]store.BankBasisRow, shape string) (map[string]bankBasisRecord, int, map[string]int) {
|
||||
basis := make(map[string]bankBasisRecord, len(stored))
|
||||
changed, old := 0, map[string]int{}
|
||||
for k, row := range stored {
|
||||
if row.FPVersion != shape {
|
||||
old[row.FPVersion]++
|
||||
changed++
|
||||
}
|
||||
basis[k] = bankBasisRecord{Key: k, Shape: row.FPVersion,
|
||||
FP: bankBasisFP{Whole: row.FPWhole, Canon: row.FPCanon},
|
||||
Answer: bankBasisAnswer{Dst: row.Dst, Type: row.Type, Gender: row.Gender}}
|
||||
}
|
||||
return basis, changed, old
|
||||
}
|
||||
|
||||
// bankBasisUnits assigns every candidate a unit id over the WHOLE candidate list — the series and families
|
||||
// a term must be decided together with.
|
||||
//
|
||||
// ⛔ OVER ALL CANDIDATES, NOT OVER THE PAID SUBSET, and this is a deliberate departure from the batcher's
|
||||
// own rule rather than an oversight. The batcher detects units over `paid` because co-batching has to
|
||||
// agree with the batches built from it — a statement about BATCHES. The predicate asks a different
|
||||
// question: «is this term's whole family settled». Answer it over `paid` and the unit is defined by the
|
||||
// very filter it feeds, so a family would count as whole precisely because its other members had been
|
||||
// filtered out — the circularity would make the guarantee vacuous exactly when it matters.
|
||||
//
|
||||
// The batcher is left untouched: it still detects over `paid` and its batches stay byte-identical.
|
||||
//
|
||||
// ⛔ THE RESULT IS TOTAL: EVERY candidate gets a unit of its own when it belongs to none, and that is a
|
||||
// correction of a real defect rather than tidiness. MergeUnits returns a PARTIAL map — a candidate in no
|
||||
// series and no family is simply absent from it — and the engine's own convention reads a missing key as
|
||||
// «a singleton» (terminology.Batch: `if id := unitID[...]; id != 0`). A predicate that indexed the raw map
|
||||
// instead would give every unit-less candidate the id 0, i.e. ONE shared unit, so a single unsettled
|
||||
// candidate anywhere in that crowd would re-buy all of them.
|
||||
//
|
||||
// ⚠ IT IS NOT A THEORETICAL HAZARD: the first run of this measurement did exactly that and reported 22 of
|
||||
// 69 candidates served where the truth is 65. The number looked entirely plausible — a strict-sounding
|
||||
// rule with a visible cost — and it was found only because a second view of the same rule, written with
|
||||
// the ids assigned explicitly, disagreed with it. A zero value that means «no unit» and a zero value that
|
||||
// means «unit number zero» are the same integer, and only one of them is a decision.
|
||||
func bankBasisUnits(cands []terminology.Candidate, sp terminology.SeriesParams, fp terminology.FamilyParams) map[string]int {
|
||||
seriesID := terminology.DetectSeries(cands, sp)
|
||||
fams := terminology.DetectFamilies(cands, fp)
|
||||
merged, _ := terminology.MergeUnits(seriesID, fams, fp)
|
||||
out := make(map[string]int, len(cands))
|
||||
// Singleton ids are handed out from a region the detectors never reach, so a real unit id and a
|
||||
// synthesised one can never collide however many units a long book grows.
|
||||
next := 1 << 30
|
||||
for _, c := range cands {
|
||||
if id, ok := merged[c.Key]; ok && id != 0 {
|
||||
out[c.Key] = id
|
||||
continue
|
||||
}
|
||||
next++
|
||||
out[c.Key] = next
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// bankBasisVerdict is one candidate's outcome of the predicate.
|
||||
type bankBasisVerdict struct {
|
||||
Answer bankBasisAnswer
|
||||
Reason basisReason // basisServed when the answer is served from memory
|
||||
}
|
||||
|
||||
// bankBasisSettle is THE PREDICATE: for each candidate, does the basis already answer it?
|
||||
//
|
||||
// A candidate is served only when ALL of these hold, and each condition is one that has already cost
|
||||
// somebody money to learn:
|
||||
//
|
||||
// - the basis holds a record for its key, and the record's fingerprint equals the one computed now;
|
||||
// - the bank holds an ENGINE row for it with status `draft` and a non-empty rendering. Status is asked
|
||||
// EXPLICITLY and «non-empty dst» is not enough: the miner produces `status:auto` WITH a rendering
|
||||
// (the raw first-chunk guess), a pair four engine comments declare impossible and no pin guards, so a
|
||||
// predicate reading the rendering alone would serve an answer nobody consolidated;
|
||||
// - every other member of its unit satisfies the same. A family decided in halves is the chimera the
|
||||
// co-batching exists to prevent, and half a family served from memory while the other half is re-asked
|
||||
// produces exactly it.
|
||||
//
|
||||
// Pure: no store, no clock, no map-order iteration in the output.
|
||||
func bankBasisSettle(
|
||||
cands []terminology.Candidate,
|
||||
fps map[string]bankBasisFP,
|
||||
basis map[string]bankBasisRecord,
|
||||
rows map[string]store.GlossaryEntry,
|
||||
ambiguous map[string]bool,
|
||||
unitID map[string]int,
|
||||
shape string,
|
||||
) map[string]bankBasisVerdict {
|
||||
|
||||
// Pass one: the per-candidate verdict, unit ignored.
|
||||
own := make(map[string]bankBasisVerdict, len(cands))
|
||||
for _, c := range cands {
|
||||
own[c.Key] = basisVerdictFor(c, fps[c.Key], basis, rows, ambiguous, shape)
|
||||
}
|
||||
// Pass two: a unit is served only whole. Computed over the units' members rather than by re-walking
|
||||
// candidates per member, so the cost is linear and the answer cannot depend on candidate order.
|
||||
blocked := map[int]bool{}
|
||||
for _, c := range cands {
|
||||
// A signed row is neither served nor a hole: it does not hold its family open. Written as an
|
||||
// explicit second case rather than folded into the test above, because «not served» and «blocks»
|
||||
// stopped being the same statement the moment the owner's signature acquired a class of its own.
|
||||
if r := own[c.Key].Reason; r != basisServed && r != basisSigned && r != basisOwnerRow {
|
||||
blocked[unitID[c.Key]] = true
|
||||
}
|
||||
}
|
||||
out := make(map[string]bankBasisVerdict, len(cands))
|
||||
for _, c := range cands {
|
||||
v := own[c.Key]
|
||||
if v.Reason == basisServed && blocked[unitID[c.Key]] {
|
||||
// The member itself is settled; its unit is not. Named as its own class rather than folded
|
||||
// into «evidence moved», because the two are different facts and only this one is a
|
||||
// consequence of a NEIGHBOUR — the operator who sees it should look at the family, not at
|
||||
// the source.
|
||||
v = bankBasisVerdict{Reason: basisUnit}
|
||||
}
|
||||
out[c.Key] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// basisVerdictFor is the per-candidate half of the predicate, split out so the unit rule above reads as
|
||||
// the one thing it is and so the class of a re-ask is decided in exactly one place.
|
||||
func basisVerdictFor(c terminology.Candidate, fp bankBasisFP, basis map[string]bankBasisRecord, rows map[string]store.GlossaryEntry, ambiguous map[string]bool, shape string) bankBasisVerdict {
|
||||
// A surface the bank answers with SEVERAL rows is refused before anything else is asked: a candidate
|
||||
// carries no sense and no window, so there is no honest way to choose one of them — see bankBasisRows.
|
||||
if ambiguous[c.Key] {
|
||||
return bankBasisVerdict{Reason: basisAmbiguous}
|
||||
}
|
||||
// THE OWNER'S SIGNATURE IS ANSWERED FIRST, before the memory of engine decisions is consulted at all,
|
||||
// and it is answered with a class of its own. Both halves of what follows are deliberate:
|
||||
//
|
||||
// - the basis NEVER hands back a rendering for a signed row. That rendering is the owner's canon, not
|
||||
// a decision this mechanism took, and a memory of engine decisions that starts serving the owner's
|
||||
// words has merged the two things the bank ontology keeps apart on purpose;
|
||||
// - the row is still sent to the paid role, because the two-condition filter pays for banked surfaces
|
||||
// ON PURPOSE — that is what keeps the dispute marks on the sheet the owner signs by. ⚠ PREMISE:
|
||||
// backlog row 447 is OPEN, and it is the row that would change this. Should it close, a signed
|
||||
// surface stops being bought — and that must be a decision somebody takes, not a behaviour that
|
||||
// shifts because a predicate here was written to assume one answer.
|
||||
//
|
||||
// What it does NOT do is hold its family open. «Bought» and «blocks its unit» are two consequences of
|
||||
// one cause, and only the second is free to change without touching product behaviour: a row the owner
|
||||
// has signed is settled harder than any draft one, so counting it as a hole in its own family made the
|
||||
// owner's signature re-buy the family it belongs to. Measured on both bought runs: the two signed rows
|
||||
// were the ONLY blockers, and exempting them serves 67 of 69 and 64 of 66 instead of 65 and 62.
|
||||
if row, inBank := rows[c.Key]; inBank && !membank.IsEngineUnsigned(row) && strings.TrimSpace(row.Dst) != "" {
|
||||
// ⛔ ASKED THROUGH membank.IsEngineUnsigned AND NOT BY A SECOND SPELLING. The engine has ONE
|
||||
// definition of «whose row is this» — `Source == "mined" && Status != "approved"` — and it has
|
||||
// other callers; a private `Status == "approved"` here is a SECOND spelling of the same question,
|
||||
// and two spellings are two answers the day they disagree. They already did: a term the OWNER
|
||||
// wrote by hand and left unsigned (`source: seed`, `status: draft` — a shape the loader admits,
|
||||
// and for which it even REQUIRES a rendering) answered «not approved», so the engine recorded its
|
||||
// own consolidation under that surface and later served it back as «this book decided it earlier».
|
||||
// Found by acceptance, on a population this pack's own fixture builds.
|
||||
if row.Status == "approved" {
|
||||
return bankBasisVerdict{Reason: basisSigned}
|
||||
}
|
||||
return bankBasisVerdict{Reason: basisOwnerRow}
|
||||
}
|
||||
rec, held := basis[c.Key]
|
||||
if !held {
|
||||
return bankBasisVerdict{Reason: basisNew}
|
||||
}
|
||||
// THE RULE IS COMPARED BEFORE THE EVIDENCE, and each part of it names itself. A row decided under
|
||||
// another width, or by another producer, has evidence that never moved — reporting it as «the source
|
||||
// around this term changed» would send the owner looking at the book for a change somebody made to a
|
||||
// knob or to a prompt, and those three call for three different reactions.
|
||||
if rec.Shape != "" && rec.Shape != shape {
|
||||
wantV, wantW, wantP := basisShapeParts(shape)
|
||||
gotV, gotW, gotP := basisShapeParts(rec.Shape)
|
||||
switch {
|
||||
case gotP != wantP:
|
||||
return bankBasisVerdict{Reason: basisProducerMoved}
|
||||
case gotW != wantW:
|
||||
return bankBasisVerdict{Reason: basisWidthMoved}
|
||||
case gotV != wantV:
|
||||
return bankBasisVerdict{Reason: basisShapeMoved}
|
||||
}
|
||||
}
|
||||
if rec.FP.Whole != fp.Whole {
|
||||
// WHICH PART moved, so the re-purchase can state its own cause. Canon first: a signed relative is
|
||||
// an act the owner performed and can recognise, while «the source moved» is not.
|
||||
if rec.FP.Canon != fp.Canon {
|
||||
return bankBasisVerdict{Reason: basisKin}
|
||||
}
|
||||
return bankBasisVerdict{Reason: basisMoved}
|
||||
}
|
||||
row, inBank := rows[c.Key]
|
||||
if !inBank {
|
||||
return bankBasisVerdict{Reason: basisAbsent}
|
||||
}
|
||||
if row.Status != "draft" || strings.TrimSpace(row.Dst) == "" {
|
||||
return bankBasisVerdict{Reason: basisUnsettled}
|
||||
}
|
||||
// ⛔ THE MEMORY AND THE BANK MUST STILL AGREE, and this is a CONDITION rather than a mechanism — two
|
||||
// lines, on the money path, making an existing claim true.
|
||||
//
|
||||
// The predicate's sentence is «this book decided this and the bank still holds that decision». Nothing
|
||||
// above checks the second half: the answer is served from the record, and it then travels straight into
|
||||
// the delta and overwrites whatever the bank held. On the run that wrote them the two agree by
|
||||
// construction — the auto-bank is emitted from the same consolidation the record was built from — so
|
||||
// this cannot fire on ordinary material, and it is NOT measured on the bought runs.
|
||||
//
|
||||
// ⚠ THE POPULATION IS NAMED RATHER THAN CLAIMED: the auto-bank is a FILE beside the database
|
||||
// (18-bank-ontology.md lists it among the SOURCES), so a hand edit of it is the reachable way for the
|
||||
// two to diverge. Out of contract, yes — and precisely the case where serving the stored answer would
|
||||
// silently undo somebody's edit while the sheet reported the row as settled earlier. The cheap
|
||||
// direction of a doubt on this path is always to re-ask.
|
||||
if text.NormalizeTargetForm(row.Dst) != text.NormalizeTargetForm(rec.Answer.Dst) {
|
||||
return bankBasisVerdict{Reason: basisDiverged}
|
||||
}
|
||||
return bankBasisVerdict{Answer: rec.Answer, Reason: basisServed}
|
||||
}
|
||||
|
||||
// bankBasisRows indexes a book's bank by the key a candidate is matched on — the same normalized key the
|
||||
// role is asked about and consolidatedRows gives a proposal, so the predicate and the emission cannot come
|
||||
// to disagree about which row belongs to which candidate. It returns the surfaces the bank answers
|
||||
// UNAMBIGUOUSLY, and separately the ones it does not.
|
||||
//
|
||||
// ⛔ A SURFACE WITH MORE THAN ONE ROW IS NOT INDEXED, IT IS REFUSED. The bank's uniqueness key is the
|
||||
// FOUR-tuple (src, sense, since_ch, until_ch): two senses of one surface are two laws by design, and a term
|
||||
// whose rendering changes across a spoiler boundary is two rows with different windows. A candidate carries
|
||||
// none of those — it has a Key and nothing else — so a map written as `out[key] = row` keeps whichever row
|
||||
// the store happened to return last (it orders by src, sense, since_ch, until_ch, so the LATEST window
|
||||
// wins) and silently answers for the other.
|
||||
//
|
||||
// ⚠ THE CONSEQUENCE WAS NOT THEORETICAL, WHICH IS WHY IT IS A REFUSAL AND NOT A WARNING: a surface the
|
||||
// owner had SIGNED on one window, holding an engine draft on another, indexed to the draft — so the
|
||||
// predicate read «status draft, rendering present», served the engine's word, and the sheet announced it as
|
||||
// the book's own earlier decision. The owner's signature was invisible to the mechanism that was supposed
|
||||
// to defer to it. Found by adversarial review of this pack, on the tree.
|
||||
//
|
||||
// Aliases are NOT indexed here, unlike bankSettledSurfaces. That filter asks «does the bank already RENDER
|
||||
// this surface», which an alias answers; this one asks «is THIS row the engine's decision», which an alias
|
||||
// of another row is not.
|
||||
func bankBasisRows(bank []store.GlossaryEntry, norm func(string) string) (rows map[string]store.GlossaryEntry, ambiguous map[string]bool) {
|
||||
seen := make(map[string][]store.GlossaryEntry, len(bank))
|
||||
for _, e := range bank {
|
||||
k := norm(e.Src)
|
||||
seen[k] = append(seen[k], e)
|
||||
}
|
||||
rows, ambiguous = make(map[string]store.GlossaryEntry, len(seen)), map[string]bool{}
|
||||
for k, es := range seen {
|
||||
if len(es) != 1 {
|
||||
ambiguous[k] = true
|
||||
continue
|
||||
}
|
||||
rows[k] = es[0]
|
||||
}
|
||||
return rows, ambiguous
|
||||
}
|
||||
|
||||
// basisSeriesParams is the pair's series morphology as the unit pass needs it — resolved from the SOURCE
|
||||
// language's declared data, exactly as the batcher resolves it, so the two passes cannot disagree about
|
||||
// what a series is.
|
||||
func basisSeriesParams(sourceLang string) terminology.SeriesParams {
|
||||
enabled, headFinal := lang.SeriesMorphology(sourceLang)
|
||||
return terminology.SeriesParams{Enabled: enabled, HeadFinal: headFinal}
|
||||
}
|
||||
|
||||
// basisReasonNames is the fixed print order of the re-ask classes, so a report of the same run always
|
||||
// lists them the same way and two reports can be diffed.
|
||||
var basisReasonNames = []basisReason{basisNew, basisKin, basisMoved, basisUnsettled, basisAbsent, basisUnit, basisSigned, basisOwnerRow, basisDiverged, basisAmbiguous,
|
||||
basisProducerMoved, basisWidthMoved, basisShapeMoved}
|
||||
|
||||
// countBasisReasons tallies a verdict map by class, in the fixed order above.
|
||||
func countBasisReasons(v map[string]bankBasisVerdict) (served int, byReason []struct {
|
||||
Reason basisReason
|
||||
N int
|
||||
}) {
|
||||
n := map[basisReason]int{}
|
||||
for _, verdict := range v {
|
||||
n[verdict.Reason]++
|
||||
}
|
||||
served = n[basisServed]
|
||||
for _, r := range basisReasonNames {
|
||||
byReason = append(byReason, struct {
|
||||
Reason basisReason
|
||||
N int
|
||||
}{r, n[r]})
|
||||
}
|
||||
return served, byReason
|
||||
}
|
||||
|
||||
// bankBasisPass is the money gate: it runs BEFORE the first paid call, decides which candidates this book
|
||||
// has already answered, and returns both the fingerprints (for the boundary writer) and the answers (for
|
||||
// every reader of a consolidation).
|
||||
//
|
||||
// ⛔ THE ANSWERS ARE RETURNED, NOT MERELY A FILTER, and that is the whole shape of this mechanism. A pass
|
||||
// that only removed settled candidates from the paid set would leave them with NO rendering downstream —
|
||||
// attachConsolidatedDst stamps nothing for a term the role did not answer — and the emission would fall to
|
||||
// the raw first-chunk banknote guess at status `auto`, or to nothing. The saving would be visible and the
|
||||
// loss would be silent, which is the exact trade this codebase refuses.
|
||||
//
|
||||
// ⚠ THE SERVED ANSWERS ARE JUDGED BY THE DISPUTE CHECKS like any other consolidation, and that is decided
|
||||
// rather than inherited. ConsolidationKeyConflicts skips the engine's own unsigned row on an unchanged
|
||||
// uniqueness key, so a basis answer cannot contradict itself by construction; CanonConflicts fires only on
|
||||
// strict containment of a SIGNED source, and every signed row related to a candidate is inside its
|
||||
// fingerprint — so a served row's canon has not moved and its verdict is the same one the purchase that
|
||||
// bought it produced. Exempting them instead would silence the marks for precisely the population the
|
||||
// two-condition filter is two-condition for.
|
||||
//
|
||||
// A read failure degrades to «nothing is settled»: the run then pays for the pass exactly as it did before
|
||||
// this mechanism existed. Loud, because that is money — and the cheap direction of any doubt here is to
|
||||
// re-buy, never to serve an answer whose evidence nobody could check.
|
||||
func (r *Runner) bankBasisPass(ctx context.Context, cands []terminology.Candidate, consider map[int]bool,
|
||||
bank []store.GlossaryEntry, res *terminologyResult) (fps map[string]bankBasisFP, answers map[string]bankBasisAnswer,
|
||||
rows map[string]store.GlossaryEntry, ambiguous map[string]bool) {
|
||||
|
||||
width := r.basisWidth()
|
||||
producer := r.bankRoleProducer()
|
||||
canon := approvedNeighbours(bank)
|
||||
nsource := ""
|
||||
if r.bankSrc != nil {
|
||||
nsource = r.bankSrc.book
|
||||
}
|
||||
fps = make(map[string]bankBasisFP, len(cands))
|
||||
for _, c := range cands {
|
||||
fps[c.Key] = candidateBasisFingerprint(c, nsource, canon, width)
|
||||
}
|
||||
rows, ambiguous = bankBasisRows(bank, text.NormalizeSourceKey)
|
||||
if nsource == "" {
|
||||
// Without the source every fingerprint is taken over an empty text, so surfaces that differ in the
|
||||
// book collide here. Said out loud and treated as «nothing is settled»: a collision in this map
|
||||
// serves one term's rendering for another's, which is worse than any amount of re-buying.
|
||||
r.Log.WarnContext(ctx, "the settled basis is INERT this run: the book's normalized source was not built, so the evidence behind a decision cannot be fingerprinted and every candidate goes to the paid role",
|
||||
"book", r.Book.BookID, "candidates", len(cands))
|
||||
return fps, nil, rows, ambiguous
|
||||
}
|
||||
stored, err := r.Store.BankBasisForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
r.Log.WarnContext(ctx, "could not read the settled basis, so every already-decided term is being bought AGAIN out of this book's LIFETIME bank-role budget; the pass still runs and the basis is rewritten at this run's boundary",
|
||||
"book", r.Book.BookID, "err", err)
|
||||
return fps, nil, rows, ambiguous
|
||||
}
|
||||
shape := basisShape(width, producer)
|
||||
basis, shapeChanged, oldShapes := bankBasisRecordsFrom(stored, shape)
|
||||
if shapeChanged > 0 {
|
||||
// BOTH shapes are printed, because that is what turns «your bank is being re-bought» into a sentence
|
||||
// the operator can act on: the shape carries the evidence WIDTH and the PRODUCER, so the line names
|
||||
// what moved rather than leaving them to infer it from a count.
|
||||
was := make([]string, 0, len(oldShapes))
|
||||
for s, n := range oldShapes {
|
||||
was = append(was, fmt.Sprintf("%s×%d", s, n))
|
||||
}
|
||||
sort.Strings(was)
|
||||
r.Log.WarnContext(ctx, "the settled basis was written under a DIFFERENT fingerprint shape and cannot be compared against this one: those terms are bought again, ONCE, and stored under the new shape — this is the announced price of changing what counts as the same evidence, and it is NOT «the source moved»",
|
||||
"book", r.Book.BookID, "dropped", shapeChanged, "of_stored", len(stored),
|
||||
"shape_now", shape, "shape_stored", strings.Join(was, " "))
|
||||
}
|
||||
units := bankBasisUnits(cands, basisSeriesParams(r.Book.SourceLang), r.familyParams)
|
||||
verdicts := bankBasisSettle(cands, fps, basis, rows, ambiguous, units, shape)
|
||||
|
||||
answers = map[string]bankBasisAnswer{}
|
||||
res.BasisReAsked = map[basisReason]int{}
|
||||
for i, c := range cands {
|
||||
// ⛔ ONLY THE CANDIDATES THE BANK FILTER KEPT. dropBankSettled has already taken the surfaces the
|
||||
// SEED renders and every draft agreed with, and those are a different fact with a different
|
||||
// sentence on the sheet («the bank already renders this surface» versus «this book decided it
|
||||
// earlier»). Asked about both, one candidate ends up carrying BOTH marks: the sheet prints two
|
||||
// mutually exclusive sentences about it, the read-out counts it twice, and — worst of the three —
|
||||
// SettledByBank lifts it out of the contested queue, which is precisely the exemption the comment
|
||||
// on that queue refuses for a basis row. Found by adversarial review, reproduced end to end.
|
||||
if !consider[i] {
|
||||
continue
|
||||
}
|
||||
v := verdicts[c.Key]
|
||||
if v.Reason == basisServed {
|
||||
answers[c.Key] = v.Answer
|
||||
res.BasisServedKeys = append(res.BasisServedKeys, c.Key)
|
||||
continue
|
||||
}
|
||||
res.BasisReAsked[v.Reason]++
|
||||
}
|
||||
res.BasisServed = len(answers)
|
||||
if len(answers) == 0 {
|
||||
return fps, nil, rows, ambiguous
|
||||
}
|
||||
// The saving with its denominator, and the re-ask classes beside it: «57 skipped» is unreadable without
|
||||
// «of 69», and a purchase that costs money owes the operator the reason it did.
|
||||
r.Log.InfoContext(ctx, "the settled basis answered terms this book had already decided, so they were NOT sent to the paid roles; their stored rendering, type and gender are carried into the delta exactly as the purchase that bought them left them",
|
||||
"book", r.Book.BookID, "served", len(answers), "of_candidates", len(cands),
|
||||
"re_asked_new", res.BasisReAsked[basisNew], "re_asked_kin_of_signed", res.BasisReAsked[basisKin],
|
||||
"re_asked_evidence_moved", res.BasisReAsked[basisMoved], "re_asked_unsettled", res.BasisReAsked[basisUnsettled],
|
||||
"re_asked_no_bank_row", res.BasisReAsked[basisAbsent], "re_asked_unit_incomplete", res.BasisReAsked[basisUnit],
|
||||
"signed_by_owner", res.BasisReAsked[basisSigned], "owners_unsigned_row", res.BasisReAsked[basisOwnerRow],
|
||||
"bank_disagrees", res.BasisReAsked[basisDiverged],
|
||||
"producer_changed", res.BasisReAsked[basisProducerMoved], "width_changed", res.BasisReAsked[basisWidthMoved], "ambiguous_surface", res.BasisReAsked[basisAmbiguous],
|
||||
"fingerprint_shape", shape)
|
||||
return fps, answers, rows, ambiguous
|
||||
}
|
||||
|
||||
// basisRowsToStore is what the book has decided as of the end of this pass — fresh answers and served ones
|
||||
// together, each with the fingerprint of the evidence it stands on.
|
||||
//
|
||||
// Only a CONSOLIDATED rendering is recorded. A term the role declined, or never answered, has not been
|
||||
// decided: storing it would let a later purchase serve an empty rendering back, which is the regression
|
||||
// this whole mechanism exists to prevent, arriving through its own memory.
|
||||
//
|
||||
// ⛔ AND A SURFACE THE OWNER HAS SIGNED IS NOT RECORDED EITHER, which is a second refusal and a harder one.
|
||||
// A signed surface still goes to the paid role on purpose (the two-condition filter keeps the dispute marks
|
||||
// on the sheet, backlog row 447), so its consolidation DOES arrive here — and recording it would put the
|
||||
// engine's word into the memory of decisions under a surface whose law is the owner's. The day that
|
||||
// signature is lifted, or its window changes, the predicate would find a record, see an unsigned draft row,
|
||||
// and serve the ENGINE's rendering as «this book decided it earlier» — over the owner's own. That is the
|
||||
// exact merging of the two words the bank ontology keeps apart, arriving through the door this pack built.
|
||||
// Found by adversarial review, on the tree, with the un-signing reproduced.
|
||||
//
|
||||
// ⛔ AN AMBIGUOUS SURFACE IS NOT RECORDED FOR THE MIRROR REASON: the carrier's key is (book, surface), so
|
||||
// two senses or two windows of one surface cannot be remembered apart, and whichever was written last would
|
||||
// answer for both. Refused at the write as well as at the read, because a guarantee kept on one side only
|
||||
// is one the next caller can walk around.
|
||||
func basisRowsToStore(cands []terminology.Candidate, consolidated, classified, gendered map[string]string,
|
||||
fps map[string]bankBasisFP, rows map[string]store.GlossaryEntry, ambiguous map[string]bool, shape string) []store.BankBasisRow {
|
||||
|
||||
out := make([]store.BankBasisRow, 0, len(consolidated))
|
||||
for _, c := range cands {
|
||||
dst := consolidated[c.Key]
|
||||
if strings.TrimSpace(dst) == "" {
|
||||
continue
|
||||
}
|
||||
if ambiguous[c.Key] {
|
||||
continue // the bank answers this surface with several rows: nothing here can say which
|
||||
}
|
||||
if row, inBank := rows[c.Key]; inBank && !membank.IsEngineUnsigned(row) {
|
||||
continue // the owner's word — signed or hand-written and unsigned — is not the engine's memory to keep
|
||||
}
|
||||
fp, ok := fps[c.Key]
|
||||
if !ok || fp.Whole == "" {
|
||||
continue // no fingerprint, no comparable memory: better re-bought than served blind
|
||||
}
|
||||
out = append(out, store.BankBasisRow{
|
||||
SrcKey: c.Key, FPVersion: shape, FPWhole: fp.Whole, FPCanon: fp.Canon,
|
||||
Dst: dst, Type: classified[c.Key], Gender: gendered[c.Key],
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// writeBankBasis publishes the run's decisions at an output boundary. `at` names the boundary and rides the
|
||||
// log line, exactly as the bank read-out's does.
|
||||
//
|
||||
// ⛔ AT THE OUTPUT BOUNDARIES, AND AT NO OTHER MOMENT. Written straight after the paid pass — before the
|
||||
// run reaches a boundary — a basis would be present on a CRASH RESUME: the resume's candidate set would
|
||||
// then be narrower than the one that paid, the batch composition would differ, the checkpoints addressed by
|
||||
// request hash would not be found, and the resume would buy again what was already bought. The two
|
||||
// boundaries are `run-finished` and the signature stop, and there are two because one is unreachable from
|
||||
// the other: the signature stop returns as an ERROR VALUE, so a run that stops for signature never reaches
|
||||
// `run-finished` at all, and a resume after the owner signs would re-buy the entire pass.
|
||||
//
|
||||
// ⚠ FAILURE IS LOUD, AND LOUDER THAN ITS NEIGHBOUR'S. The stop's flag memory may fail quietly: losing it
|
||||
// costs one benign extra stop. Losing this costs a re-purchase of both bank roles out of a LIFETIME
|
||||
// per-book ceiling whose exhaustion is irreversible — so it is an ERROR line naming the money, never a
|
||||
// quiet note. The run itself is NOT failed: the work is done and paid for, and killing a finished run over
|
||||
// a bookkeeping row would trade a real book for a row that the next boundary rewrites anyway.
|
||||
func (r *Runner) writeBankBasis(ctx context.Context, at string) {
|
||||
t := r.lastTerminology
|
||||
if t == nil || len(t.BasisRows) == 0 {
|
||||
return
|
||||
}
|
||||
if err := r.Store.PutBankBasis(r.Book.BookID, t.BasisRows); err != nil {
|
||||
r.Log.ErrorContext(ctx, "could not record the settled basis: the terms this run decided are NOT remembered, so the NEXT purchase will pay both bank roles for them again out of this book's LIFETIME budget (exhausting it leaves the book without a consolidated bank permanently). The run itself is unaffected and its bank is on disk",
|
||||
"book", r.Book.BookID, "at", at, "rows", len(t.BasisRows), "err", err)
|
||||
return
|
||||
}
|
||||
r.Log.InfoContext(ctx, "settled basis recorded", "book", r.Book.BookID, "at", at,
|
||||
"rows", len(t.BasisRows), "served_this_run", t.BasisServed, "shape", basisFingerprintVersion)
|
||||
}
|
||||
|
||||
// stampBasisTypes writes the basis's banked type onto the candidates it answered, and returns how many it
|
||||
// changed. It is deliberately NOT applyTypes: that one's return value is published as `reclassified`, which
|
||||
// means «how many types the CLASSIFIER changed», and a term the classifier never saw must not appear in it.
|
||||
//
|
||||
// ⚠ IT RUNS AFTER THE FINGERPRINTS ARE TAKEN, and the order is load-bearing rather than incidental: the
|
||||
// fingerprint hashes the MINER'S heuristic type on purpose, so that learning whether the paid roles may be
|
||||
// skipped never requires buying the classifier first. Stamp before the hash and that property is gone —
|
||||
// silently, because everything would still work and every book would simply pay the classifier forever.
|
||||
func stampBasisTypes(cands []terminology.Candidate, answers map[string]bankBasisAnswer) int {
|
||||
n := 0
|
||||
for i := range cands {
|
||||
a, ok := answers[cands[i].Key]
|
||||
if !ok || a.Type == "" || a.Type == cands[i].Type {
|
||||
continue
|
||||
}
|
||||
cands[i].Type = a.Type
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
326
backend/internal/pipeline/bankbasis_answers_test.go
Normal file
326
backend/internal/pipeline/bankbasis_answers_test.go
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/seed"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// bankbasis_answers_test.go: the pin of the settled basis's ONE silent regression — the one §4.2 of the
|
||||
// pack calls the quietest thing that can go wrong here.
|
||||
//
|
||||
// ⛔ WHAT IT GUARDS. A basis that merely REMOVED settled candidates from the paid set would look like a
|
||||
// saving and be a loss. attachConsolidatedDst stamps nothing for a candidate the role did not answer, so
|
||||
// the emission falls back to the raw first-chunk banknote guess and the row drops from `draft` to `auto`
|
||||
// (miner.DeltaFile: `st.Dst = props[0].Dst` under `props[0].Via == ""`, and `st.Status` is only raised to
|
||||
// «draft» by a CONSOLIDATED dst). Where the drafts proposed nothing on the surface itself, the rendering
|
||||
// disappears entirely.
|
||||
//
|
||||
// ⚠ AND THE PAIR IS COMPARED, NOT THE RENDERING ALONE. A zeroed dst is visible — an empty cell on the
|
||||
// sheet. A row that quietly swapped its consolidated rendering for the first chunk's guess and its status
|
||||
// for `auto` looks NORMAL and travels on as an ordinary undecided row. Comparing `dst` alone would go
|
||||
// green on exactly the worse half, so every assertion below is over (dst, status) together.
|
||||
func TestTheBasisCarriesTheBoughtAnswerRatherThanDroppingIt(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isClassifierBody(body) {
|
||||
// ⚠ THE CLASSIFIER IS ON, AND THE FIXTURE ANSWERS IT, because the answer has TWO halves bought
|
||||
// on the same money and reaching the row through different doors. With the classifier off,
|
||||
// `type` and `gender` are empty on every row, so a basis that dropped the classifier's half
|
||||
// entirely would leave this pin green — it would be comparing "" with "". Measured: with the
|
||||
// phase off, a planting that discards the type and the gender survived the pin unchanged.
|
||||
return "方源\tname\tmale\n青茅山\tterm\tnone", "stop"
|
||||
}
|
||||
if isTerminologyBody(body) {
|
||||
return "方源\tФан Юань\n青茅山\tгора Цинмао", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл к горе Цинмао.\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, classify: true})
|
||||
|
||||
// Run 1: the roles are paid, and what they decided is what the book now holds.
|
||||
r1 := newRunner(t, bookPath)
|
||||
if _, err := r1.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r1.lastTerminology == nil || r1.lastTerminology.Consolidated == 0 {
|
||||
t.Fatalf("premise broken: run 1 must consolidate something, or there is no bought answer to lose: %+v", r1.lastTerminology)
|
||||
}
|
||||
bought := autoBankPairs(t, r1.autoBankPath())
|
||||
decided := 0
|
||||
for _, p := range bought {
|
||||
if p.status == "draft" && p.dst != "" {
|
||||
decided++
|
||||
}
|
||||
}
|
||||
if decided == 0 {
|
||||
t.Fatalf("premise broken: run 1 emitted no `draft` row with a rendering, so the loss this test is about cannot happen in this fixture: %+v", bought)
|
||||
}
|
||||
// The classifier's half must actually be PRESENT, or the type/gender comparisons below are "" against
|
||||
// "" — the exact way this pin was measured to pass over a planting that threw both away.
|
||||
withType, withGender := 0, 0
|
||||
for _, p := range bought {
|
||||
if p.typ != "" {
|
||||
withType++
|
||||
}
|
||||
if p.gender != "" {
|
||||
withGender++
|
||||
}
|
||||
}
|
||||
if withType == 0 || withGender == 0 {
|
||||
t.Fatalf("premise broken: the classifier's half of the answer is absent from run 1 (rows with a type: %d, with a gender: %d), so this pin cannot see it being lost: %+v", withType, withGender, bought)
|
||||
}
|
||||
r1.Close()
|
||||
|
||||
// Run 2 meets the basis run 1 wrote and pays its one-time re-pack; run 3 is the steady state where the
|
||||
// basis actually answers. Both are walked, because the property must hold in the steady state and not
|
||||
// merely survive one hop.
|
||||
var last map[string]bankPair
|
||||
for i := 2; i <= 3; i++ {
|
||||
r := newRunner(t, bookPath)
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatalf("run %d: %v", i, err)
|
||||
}
|
||||
if r.lastTerminology == nil || r.lastTerminology.BasisServed == 0 {
|
||||
t.Fatalf("premise broken: run %d must be served by the basis, or this test never exercises it: %+v", i, r.lastTerminology)
|
||||
}
|
||||
last = autoBankPairs(t, r.autoBankPath())
|
||||
r.Close()
|
||||
}
|
||||
|
||||
// THE ASSERTION: every row the book had bought still says exactly what it said.
|
||||
for src, was := range bought {
|
||||
now, present := last[src]
|
||||
if !present {
|
||||
t.Errorf("%q vanished from the auto-bank once the basis answered it: the book lost a rendering it had PAID for", src)
|
||||
continue
|
||||
}
|
||||
if now != was {
|
||||
t.Errorf("%q changed under the basis: was dst=%q status=%q type=%q gender=%q, now dst=%q status=%q type=%q gender=%q.\n"+
|
||||
" A `draft` row falling to `auto` is the silent half of this regression: the row still LOOKS ordinary, "+
|
||||
"it simply carries the first chunk's guess instead of the rendering the roles were paid for.",
|
||||
src, was.dst, was.status, was.typ, was.gender, now.dst, now.status, now.typ, now.gender)
|
||||
}
|
||||
}
|
||||
// The control that keeps those comparisons from being vacuous: the second reading is not empty, and it
|
||||
// is not a copy of a file nobody rewrote.
|
||||
if len(last) == 0 {
|
||||
t.Fatal("control: the steady-state auto-bank holds NO rows, so «everything matches» is a statement about an empty map")
|
||||
}
|
||||
}
|
||||
|
||||
// bankPair is one auto-bank row reduced to the two fields that decide whether an answer survived.
|
||||
// ⚠ TYPE AND GENDER RIDE HERE TOO. They are the CLASSIFIER's half of the answer, bought on the same
|
||||
// money, and they reach the row by a different door (attachClassifiedType / attachClassifiedGender). A
|
||||
// pin over the rendering alone would go green on a basis that carried the terminologist's word and
|
||||
// dropped the classifier's — and a row that loses its gender loses a wire directive the engine already
|
||||
// promised.
|
||||
type bankPair struct{ dst, status, typ, gender string }
|
||||
|
||||
// autoBankPairs reads the engine's own auto-bank document — the artifact the emission actually produces —
|
||||
// rather than a projection of it, so the pin cannot pass on a view that agrees with itself while the
|
||||
// document beneath disagrees.
|
||||
func autoBankPairs(t *testing.T, path string) map[string]bankPair {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("the run must leave an auto-bank behind, or the emission never happened: %v", err)
|
||||
}
|
||||
var f seed.File
|
||||
if err := yaml.Unmarshal(raw, &f); err != nil {
|
||||
t.Fatalf("parse the auto-bank %s: %v", path, err)
|
||||
}
|
||||
out := map[string]bankPair{}
|
||||
for _, term := range f.Terms {
|
||||
out[strings.TrimSpace(term.Src)] = bankPair{dst: term.Dst, status: term.Status, typ: term.Type, gender: term.Gender}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
t.Fatalf("the auto-bank at %s holds no terms (%d bytes): every comparison against it would be vacuous", path, len(raw))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestTheBasisIsNotWrittenBeforeTheRunReachesABoundary is the pin §4.3 of the pack asks for: a planting
|
||||
// that records the basis straight after the paid pass — before the run forks into «stop» or «continue» —
|
||||
// has to turn a FREE resume into a paid one.
|
||||
//
|
||||
// ⛔ THE MECHANISM, because the failure is not obvious. A run that dies after the bank roles have been paid
|
||||
// leaves their checkpoints behind, and a resume rebuilds the SAME batch composition and replays them for
|
||||
// $0. Write the basis before the fork and the crashed run has already recorded its decisions: the resume
|
||||
// then serves them, builds a NARROWER composition, finds no checkpoint addressed by that request hash, and
|
||||
// buys the whole pass again — money out of a lifetime ceiling, for work already done, with nothing red.
|
||||
//
|
||||
// The crash is produced by the BOOK CEILING rather than by a killed process: the draft wave and the bank
|
||||
// roles fit under it, the edit wave's reservation does not, so TranslateBook returns a refusal at exactly
|
||||
// the position this pin needs — after the paid pass, before either output boundary.
|
||||
func TestTheBasisIsNotWrittenBeforeTheRunReachesABoundary(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
return "方源\tФан Юань\n青茅山\tгора Цинмао", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл к горе Цинмао.\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
|
||||
tightenBookCeiling(t, bookPath, 0.004)
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
_, err := r1.TranslateBook(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("premise broken: the tightened ceiling must STOP this run before it reaches an output boundary, or nothing here is about a crash")
|
||||
}
|
||||
if r1.lastTerminology == nil || r1.lastTerminology.CostUSD == 0 {
|
||||
t.Fatalf("premise broken: the bank roles must have been PAID before the run died, or there are no checkpoints for the resume to replay: %+v", r1.lastTerminology)
|
||||
}
|
||||
paidCalls := countTerminologyCalls(rec)
|
||||
// THE STATE THIS PIN IS ABOUT: the run died, so no boundary ran, so nothing was recorded.
|
||||
stored, serr := r1.Store.BankBasisForBook(r1.Book.BookID)
|
||||
if serr != nil {
|
||||
t.Fatal(serr)
|
||||
}
|
||||
if len(stored) != 0 {
|
||||
t.Fatalf("a run that never reached an output boundary recorded %d basis row(s): the next resume will serve them, narrow the batch composition, miss every checkpoint and re-buy the pass it already paid for", len(stored))
|
||||
}
|
||||
r1.Close()
|
||||
|
||||
// The consequence, asserted rather than inferred: the resume replays for $0.
|
||||
r2 := newRunner(t, bookPath)
|
||||
defer r2.Close()
|
||||
if _, err := r2.TranslateBook(context.Background()); err == nil {
|
||||
t.Fatal("premise broken: the resume must meet the same ceiling, or it is not the run this pin is about")
|
||||
}
|
||||
if r2.lastTerminology == nil {
|
||||
t.Fatal("premise broken: the resumed run must reach the bank pass again")
|
||||
}
|
||||
if r2.lastTerminology.CostUSD != 0 {
|
||||
t.Errorf("the resume re-bought the bank pass ($%f) although the crashed run had paid for it: the basis is being recorded before the run reaches a boundary", r2.lastTerminology.CostUSD)
|
||||
}
|
||||
if n := countTerminologyCalls(rec); n != paidCalls {
|
||||
t.Errorf("the resume reached the provider for the bank role: calls went %d → %d", paidCalls, n)
|
||||
}
|
||||
}
|
||||
|
||||
// tightenBookCeiling rewrites the fixture's book ceiling in place. It edits the generated book.yaml rather
|
||||
// than adding an option to the fixture builder, because the ceiling is the only thing this one pin needs
|
||||
// and a knob nothing else sets is a knob that goes stale unread.
|
||||
func tightenBookCeiling(t *testing.T, bookPath string, usd float64) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(bookPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, replaced := []string{}, false
|
||||
for _, ln := range strings.Split(string(raw), "\n") {
|
||||
if strings.Contains(ln, "book_usd") {
|
||||
out = append(out, fmt.Sprintf("ceilings: { book_usd: %g, day_usd: 10.0 }", usd))
|
||||
replaced = true
|
||||
continue
|
||||
}
|
||||
out = append(out, ln)
|
||||
}
|
||||
if !replaced {
|
||||
t.Fatalf("the fixture's book.yaml declares no book_usd, so this helper silently changed nothing:\n%s", raw)
|
||||
}
|
||||
if err := os.WriteFile(bookPath, []byte(strings.Join(out, "\n")), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestABookWithTheGateOffTouchesNoBasisAtAll pins the promise every new mechanism owes the books already in
|
||||
// the field: a project that does not buy bank roles must behave exactly as it did before this one existed.
|
||||
//
|
||||
// ⛔ WHY IT IS A PIN AND NOT AN OBSERVATION. The basis lives on the paid path, and the paid path is gated
|
||||
// (`gates.terminology.enabled`). A gate whose OFF branch grew a side effect is the quiet way a mechanism
|
||||
// reaches books nobody enabled it for — here it would mean rows in a table for a contour that never ran,
|
||||
// and a later purchase serving answers nobody bought. The engine's own refusals are written against that
|
||||
// shape everywhere else; this is the same refusal, asserted.
|
||||
func TestABookWithTheGateOffTouchesNoBasisAtAll(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
return "Фан Юань пришёл к горе Цинмао.\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
// terminology OFF: the mining stop still runs, the delta is still emitted, nothing is bought.
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{})
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The premise: the run really did reach the bank-mining stop and emit a delta. Without it the zeros
|
||||
// below would be a statement about a run that never got near the contour.
|
||||
if r.lastMinedCount == 0 {
|
||||
t.Fatalf("premise broken: the run mined nothing, so «the gate-off path touched no basis» is about a path that was never walked")
|
||||
}
|
||||
if r.lastTerminology != nil {
|
||||
t.Fatalf("premise broken: the terminology pass ran with the gate OFF: %+v", r.lastTerminology)
|
||||
}
|
||||
stored, err := r.Store.BankBasisForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
// The TABLE must exist — the migration is additive and runs for every project — so a read error
|
||||
// here is a real failure, not an absent feature.
|
||||
t.Fatalf("the basis table must exist on every project, gate or no gate: %v", err)
|
||||
}
|
||||
if len(stored) != 0 {
|
||||
t.Fatalf("a run with the bank gate OFF recorded %d basis row(s): the mechanism reached a book nobody enabled it for, and a later purchase would serve answers nobody bought", len(stored))
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheSignatureStopWritesTheBasisToo pins the boundary nothing was guarding, and acceptance proved the
|
||||
// hole with a control rather than by reading: removing the write at `run-finished` turned the package RED,
|
||||
// removing it at the SIGNATURE STOP left it entirely `ok`.
|
||||
//
|
||||
// ⛔ WHY THE UNGUARDED ONE IS THE EXPENSIVE ONE. The two boundaries are not interchangeable and the stop is
|
||||
// the one that cannot be reached from the other: the signature stop returns as an ERROR VALUE
|
||||
// (`WaveSignatureStop`), so a run that stops for the owner's signature never arrives at `run-finished` at
|
||||
// all. Lose the write there and the resume after the owner signs re-buys the ENTIRE bank pass — out of a
|
||||
// lifetime per-book ceiling — while every fixture that only ever auto-continues stays green.
|
||||
func TestTheSignatureStopWritesTheBasisToo(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, miningStopProvider(t, rec, ""))
|
||||
defer srv.Close()
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
|
||||
|
||||
r := newVerifyRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
stop := runToSignatureStop(t, r)
|
||||
if stop.Terms == 0 {
|
||||
t.Fatal("premise broken: the stop fired with zero terms")
|
||||
}
|
||||
// The premise that makes the assertion about the STOP and not about a finished run: this run ended as
|
||||
// an error value, so `run-finished` never ran.
|
||||
if r.lastTerminology == nil || len(r.lastTerminology.BasisRows) == 0 {
|
||||
t.Fatalf("premise broken: the stopping run must have decided something to record: %+v", r.lastTerminology)
|
||||
}
|
||||
stored, err := r.Store.BankBasisForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(stored) == 0 {
|
||||
t.Fatalf("the signature stop recorded NOTHING although it decided %d row(s): the resume after the owner signs would re-buy the whole bank pass out of a lifetime ceiling, and no fixture that auto-continues would ever notice",
|
||||
len(r.lastTerminology.BasisRows))
|
||||
}
|
||||
if len(stored) != len(r.lastTerminology.BasisRows) {
|
||||
t.Errorf("the stop recorded %d row(s) of %d decided — a partial write is the same loss, only harder to see",
|
||||
len(stored), len(r.lastTerminology.BasisRows))
|
||||
}
|
||||
// AND THE ORDER, which is the other half of this boundary's contract: the basis goes in BEFORE the flag
|
||||
// memory, because a crash between them must cost the cheaper loss. Asserted on the durable state: the
|
||||
// memory is present too, so «basis written» is not true merely because the stop died early.
|
||||
presented, err := r.Store.StopPresentedSurfaces(r.Book.BookID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(presented) == 0 {
|
||||
t.Fatal("control: the flag memory is empty, so this stop did not complete its boundary and the assertion above is about a half-written state")
|
||||
}
|
||||
}
|
||||
881
backend/internal/pipeline/bankbasis_m1_probe_test.go
Normal file
881
backend/internal/pipeline/bankbasis_m1_probe_test.go
Normal file
|
|
@ -0,0 +1,881 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/chunk"
|
||||
"textmachine/backend/internal/miner"
|
||||
"textmachine/backend/internal/obs"
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
"textmachine/backend/internal/text"
|
||||
)
|
||||
|
||||
// bankbasis_m1_probe_test.go: measurement M1 — an opt-in INSTRUMENT over BOUGHT material, not a unit test.
|
||||
// It answers the question the settled basis has to be designed against before a carrier is written: on the
|
||||
// runs this project actually paid for, how many candidates would the predicate serve from memory, and for
|
||||
// which reasons would the rest be bought again.
|
||||
//
|
||||
// ⛔ $0. Nothing here reaches a provider. The runner is opened on the READ path (openRunner forWrite=false),
|
||||
// which skips models.CheckKeys entirely, and the only engine calls made are the $0 assembly ones — ingest,
|
||||
// split, mine, merge, KWIC, score. runTerminologist, runClassifier and TranslateBook are never called.
|
||||
//
|
||||
// HOW TO PREPARE THE MATERIAL (the instrument refuses to guess, and every path is named):
|
||||
//
|
||||
// cp -a /home/ubuntu/tm-coldrun-a/stand/books/bk_ROEHZBD46ALFI43E /tmp/m1/A
|
||||
// cp -a /home/ubuntu/tm-coldrun-b/stand/books/bk_NH6275ZIRFFTLZHG /tmp/m1/B
|
||||
// rm -f /tmp/m1/*/project.db.lock
|
||||
// sed -i 's#/home/ubuntu-26/#/home/ubuntu/#g' /tmp/m1/*/book.yaml
|
||||
//
|
||||
// # ⚠ RUN A NEEDS ONE MORE STEP, and without it the instrument FAILS on A while passing on B — which
|
||||
// # reads as a defect of the pack and is a gap in the ARCHIVE. A's pipeline resolves its role prompts
|
||||
// # relative to its own directory (`../prompts/zh-ru/…`) and that directory was not archived beside the
|
||||
// # config mirror; B's was. The prompts themselves are in the freeze and are byte-identical to B's
|
||||
// # archived copy (`diff -rq` empty, 11 files against 11), and they touch no number here — the
|
||||
// # instrument makes no calls — so assembling them is restoring the archive, not choosing an input:
|
||||
// mkdir -p /tmp/m1/A/cfg && cp -a /home/ubuntu/tm-coldrun-a/mirror/cfg/. /tmp/m1/A/cfg/
|
||||
// cp -a /home/ubuntu/tm-coldrun-a/freeze/backend/prompts /tmp/m1/A/prompts
|
||||
// sed -i 's#^pipeline: .*#pipeline: /tmp/m1/A/cfg/pipeline-c1.yaml#' /tmp/m1/A/book.yaml
|
||||
//
|
||||
// TM_BASIS_M1=/tmp/m1/A,/tmp/m1/B go test ./internal/pipeline/ -run TestProbeBankBasisM1 -v
|
||||
//
|
||||
// ⚠ COPIES, ALWAYS. The material is bought and unrecoverable, and run A carries store schema 16 against a
|
||||
// head of 18: OpenReadOnly REFUSES it, so the instrument migrates the copy with store.Open first (v17/v18
|
||||
// are additive — CREATE TABLE IF NOT EXISTS and ADD COLUMN, no DROP/RENAME/DELETE). Run this against an
|
||||
// original and the original is migrated in place.
|
||||
//
|
||||
// ⚠ THE book.yaml OF EVERY BOUGHT RUN POINTS AT /home/ubuntu-26, WHICH DOES NOT EXIST HERE. Re-point it on
|
||||
// the copy, as the recipe does. Leaving it is not a loud failure in the interesting direction: the config
|
||||
// loader would take the HEAD configs and the instrument would silently measure a different pipeline than
|
||||
// the one the money was spent under, which is why the run below PRINTS the knobs it resolved.
|
||||
func TestProbeBankBasisM1(t *testing.T) {
|
||||
roots := os.Getenv("TM_BASIS_M1")
|
||||
if roots == "" {
|
||||
t.Skip("instrument: set TM_BASIS_M1=<copy of a bought run dir>[,<another>] to measure the settled basis (see the file docblock)")
|
||||
}
|
||||
for _, dir := range strings.Split(roots, ",") {
|
||||
dir = strings.TrimSpace(dir)
|
||||
if dir == "" {
|
||||
continue
|
||||
}
|
||||
t.Run(filepath.Base(dir), func(t *testing.T) { probeBasisOneRun(t, dir) })
|
||||
}
|
||||
}
|
||||
|
||||
func probeBasisOneRun(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
m := openBasisMaterial(t, dir)
|
||||
defer m.r.Close()
|
||||
|
||||
// ── THE KNOBS THIS MEASUREMENT WAS TAKEN UNDER ──────────────────────────────────────────────────
|
||||
// Printed, not assumed: «the config of the bought run» and «the config of HEAD, silently substituted»
|
||||
// produce the same numbers-shaped output, and only these lines tell them apart.
|
||||
batchRunes, kwicPer, kwicWidth := m.r.terminologyOpts()
|
||||
t.Logf("KNOBS book=%s pair=%s contrast=%s batch_runes=%d kwic_per_term=%d kwic_width=%d basis_width=%d model=%s",
|
||||
m.r.Book.BookID, m.r.Book.LangPair(), m.r.Pipeline.Mining.ContrastPath,
|
||||
batchRunes, kwicPer, kwicWidth, m.r.basisWidth(), m.r.Pipeline.Gates.Terminology.Model)
|
||||
|
||||
// ── FIDELITY OF THE RECONSTRUCTION ─────────────────────────────────────────────────────────────
|
||||
// The instrument rebuilds the candidate list from the source; the run that was paid for left its own
|
||||
// list behind in the stop table sidecar. If the two disagree, every number below is about a different
|
||||
// book than the one the money was spent on — so this is asserted, not hoped for.
|
||||
m.assertMatchesBoughtTable(t)
|
||||
|
||||
// ── THE BASELINE: the basis written by this very purchase, read by the next one on unchanged source ─
|
||||
// ⚠ NOT A CONTROL. Two identical runs agreeing measures the determinism of the harness, not the force
|
||||
// of the predicate; it is reported as the SAVING, with its denominator, and the discriminating
|
||||
// controls are below.
|
||||
base := m.verdicts(m.basisFromRun(), m.nsource)
|
||||
m.report(t, "BASELINE unchanged-source", base)
|
||||
m.controlUnitShape(t, base)
|
||||
m.measureAutoTail(t, base)
|
||||
m.measureRechunkResidue(t)
|
||||
m.measureUnitRuleVariants(t)
|
||||
|
||||
// ── DISCRIMINATING CONTROL 1: the source moves around ONE occurrence of ONE candidate ──────────────
|
||||
// Exactly that candidate and its unit must be bought again, and nothing else. A predicate that ignored
|
||||
// the source would show 0 here; one that hashed too much would show the whole book.
|
||||
m.controlOneOccurrenceMoved(t, base)
|
||||
|
||||
// ── DISCRIMINATING CONTROL 2: two passes under ONE basis produce byte-identical batches ────────────
|
||||
m.controlBatchesStable(t, base)
|
||||
|
||||
// ── KNOB INVARIANCE: the window SELECTION must not move the settled set ────────────────────────────
|
||||
// This is the property the whole fingerprint shape exists for: the day a stratified selection lands,
|
||||
// or anyone tunes how much context the model is shown, no book may lose its basis.
|
||||
m.controlKnobsDoNotMatter(t, base)
|
||||
|
||||
// ── THE GROWING BOOK, AS A NUMBER RATHER THAN AS A GUARD ───────────────────────────────────────────
|
||||
// The premise «the source arrives whole» is an OPEN question to the owner, so the engine must not
|
||||
// answer it with a watchdog. What it can do is say what growth costs.
|
||||
m.measureGrowth(t)
|
||||
}
|
||||
|
||||
// ── material ────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// basisMaterial is one bought run, reconstructed far enough to ask the predicate a question.
|
||||
type basisMaterial struct {
|
||||
dir string
|
||||
r *Runner
|
||||
doc *chunk.Document
|
||||
chunks []chunk.Chunk
|
||||
nsource string // the book's normalized source — the one definition, r.bankSrc's own
|
||||
cands []terminology.Candidate // as the stop would have built them, before any paid call
|
||||
bank []store.GlossaryEntry // the bank as the bought run left it
|
||||
rows map[string]store.GlossaryEntry
|
||||
ambiguous map[string]bool
|
||||
// consider is the candidates the BANK filter kept — the only population production offers the basis.
|
||||
consider map[string]bool
|
||||
bankSettled int
|
||||
unitID map[string]int
|
||||
width int
|
||||
}
|
||||
|
||||
func openBasisMaterial(t *testing.T, dir string) *basisMaterial {
|
||||
t.Helper()
|
||||
bookPath := filepath.Join(dir, "book.yaml")
|
||||
if _, err := os.Stat(bookPath); err != nil {
|
||||
t.Fatalf("material: %s has no book.yaml — point TM_BASIS_M1 at a COPY of a bought run dir: %v", dir, err)
|
||||
}
|
||||
// Migrate the copy first: run A is schema 16 and the read door refuses it outright (backlog row 475).
|
||||
// Done explicitly rather than by opening for write, so nothing here takes the run lock or a key check.
|
||||
if st, err := store.Open(filepath.Join(dir, "project.db")); err != nil {
|
||||
t.Fatalf("material: could not open/migrate the COPY at %s: %v", dir, err)
|
||||
} else {
|
||||
st.Close()
|
||||
}
|
||||
r, err := openRunner(bookPath, obs.NewLogger(), false)
|
||||
if err != nil {
|
||||
t.Fatalf("material: %s does not load — did you re-point its /home/ubuntu-26 paths? %v", dir, err)
|
||||
}
|
||||
if r.pack == nil {
|
||||
t.Fatalf("material: %s loaded no langpack for pair %q, so the miner cannot run and every count below would be a zero about the instrument", dir, r.Book.LangPair())
|
||||
}
|
||||
m := &basisMaterial{dir: dir, r: r, width: r.basisWidth()}
|
||||
m.build(t)
|
||||
return m
|
||||
}
|
||||
|
||||
// build runs the $0 half of the bank-mining stop — the same sequence runBankMiningStop takes before its
|
||||
// first paid call, in the same order and through the same functions.
|
||||
func (m *basisMaterial) build(t *testing.T) {
|
||||
t.Helper()
|
||||
doc, err := m.r.ingestSource()
|
||||
if err != nil {
|
||||
t.Fatalf("ingest %s: %v", m.dir, err)
|
||||
}
|
||||
m.doc = doc
|
||||
m.chunks, _, _ = chunk.SplitChunksWithChapters(doc.Chapters, m.r.segBudget(), m.r.chapterRule(), m.r.sentenceAbbrevs())
|
||||
if len(m.chunks) == 0 {
|
||||
t.Fatalf("premise broken: %s produced no chunks", m.dir)
|
||||
}
|
||||
m.r.bankSrc = newBankSourceIndex(m.chunks)
|
||||
m.nsource = m.r.bankSrc.book
|
||||
|
||||
m.bank, err = m.r.Store.GlossaryForBook(m.r.Book.BookID)
|
||||
if err != nil {
|
||||
t.Fatalf("read the bank of %s: %v", m.dir, err)
|
||||
}
|
||||
if len(m.bank) == 0 {
|
||||
t.Fatalf("premise broken: %s holds an EMPTY bank, so every «settled» count below would be zero for a reason that has nothing to do with the predicate", m.dir)
|
||||
}
|
||||
m.rows, m.ambiguous = bankBasisRows(m.bank, text.NormalizeSourceKey)
|
||||
m.cands = m.candidatesOver(t, m.chunks)
|
||||
m.unitID = bankBasisUnits(m.cands, basisSeriesParams(m.r.Book.SourceLang), m.r.familyParams)
|
||||
// ⛔ THE INSTRUMENT ASKS THE PREDICATE EXACTLY WHAT THE ENGINE ASKS IT, AND NOT ONE CANDIDATE MORE.
|
||||
// Production offers the basis only the candidates the BANK filter kept: a surface the seed renders and
|
||||
// every draft agreed with is a different saving with a different sentence on the sheet, and offering it
|
||||
// to both is the double-marking this pack had to fix. An instrument that measured over ALL candidates
|
||||
// would report a saving the engine never claims — a number about a question nobody asks.
|
||||
kept, dropped := m.r.dropBankSettled(context.Background(), m.cands)
|
||||
m.consider = map[string]bool{}
|
||||
for _, i := range kept {
|
||||
m.consider[m.cands[i].Key] = true
|
||||
}
|
||||
m.bankSettled = dropped
|
||||
}
|
||||
|
||||
// candidatesOver rebuilds the merged candidate list over a given chunk set — parameterised so the growth
|
||||
// and re-chunk measurements can ask the same question of a different source.
|
||||
func (m *basisMaterial) candidatesOver(t *testing.T, chunks []chunk.Chunk) []terminology.Candidate {
|
||||
t.Helper()
|
||||
f, err := os.Open(m.r.Pipeline.Mining.ContrastPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open the mining contrast %s: %v", m.r.Pipeline.Mining.ContrastPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
contrast, err := miner.LoadContrast(f)
|
||||
if err != nil {
|
||||
t.Fatalf("load the mining contrast: %v", err)
|
||||
}
|
||||
minerChunks := make([]miner.Chunk, len(chunks))
|
||||
tchunks := make([]terminology.Chunk, len(chunks))
|
||||
for i, ch := range chunks {
|
||||
n := text.NormalizeSourceKey(ch.Text)
|
||||
minerChunks[i] = miner.Chunk{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, NSource: n}
|
||||
tchunks[i] = terminology.Chunk{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, NSource: n}
|
||||
}
|
||||
rejects, err := m.r.loadMinedRejects()
|
||||
if err != nil {
|
||||
t.Fatalf("load the mined rejects: %v", err)
|
||||
}
|
||||
mined, _ := miner.MineBankStats(minerChunks, contrast, unsignedEngineSurfaces(m.bank), rejects, miner.FrozenConfig(), m.r.pack)
|
||||
observed, _, err := m.r.bankObservedForBook()
|
||||
if err != nil {
|
||||
t.Fatalf("read the banknote proposals: %v", err)
|
||||
}
|
||||
return m.r.buildBankCandidates(mined, observed, tchunks)
|
||||
}
|
||||
|
||||
// ── fidelity ────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// assertMatchesBoughtTable compares the reconstructed candidate list against the one the PAID run wrote to
|
||||
// its stop-table sidecar.
|
||||
//
|
||||
// ⛔ WHY THIS IS AN ASSERT AND NOT A LOG LINE. Every number this file prints is a statement about the runs
|
||||
// the project paid for. Reconstructed from a source with the wrong chunker, the wrong contrast or the HEAD
|
||||
// config, the instrument still prints a full, plausible, well-formed table — about a book nobody bought.
|
||||
// The sidecar is the bought run's own account of which candidates existed, so agreeing with it is the one
|
||||
// cheap proof that the subject here is the subject there.
|
||||
func (m *basisMaterial) assertMatchesBoughtTable(t *testing.T) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(m.r.bankStopTablePath())
|
||||
if err != nil {
|
||||
t.Fatalf("fidelity: %s has no bought stop table beside its DB, so the reconstruction cannot be checked against anything: %v", m.dir, err)
|
||||
}
|
||||
bought := map[string]bool{}
|
||||
for _, ln := range strings.Split(string(raw), "\n") {
|
||||
// One term opens a block with «src<TAB>dst»; every other line of the block is indented.
|
||||
if ln == "" || strings.HasPrefix(ln, " ") || !strings.Contains(ln, "\t") {
|
||||
continue
|
||||
}
|
||||
bought[text.NormalizeSourceKey(strings.SplitN(ln, "\t", 2)[0])] = true
|
||||
}
|
||||
if len(bought) == 0 {
|
||||
t.Fatalf("fidelity: parsed ZERO terms out of the bought stop table %s (%d bytes) — the parser, not the book", m.r.bankStopTablePath(), len(raw))
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, c := range m.cands {
|
||||
got[c.Key] = true
|
||||
}
|
||||
missing, extra := diffKeys(bought, got), diffKeys(got, bought)
|
||||
t.Logf("FIDELITY bought_table_terms=%d reconstructed_candidates=%d missing=%d extra=%d",
|
||||
len(bought), len(got), len(missing), len(extra))
|
||||
if len(missing) > 0 || len(extra) > 0 {
|
||||
t.Fatalf("fidelity: the reconstruction is NOT the bought candidate set — every number below would be about another book.\n in the bought table, not reconstructed (%d): %s\n reconstructed, not in the bought table (%d): %s",
|
||||
len(missing), strings.Join(capList(missing, 12), ", "), len(extra), strings.Join(capList(extra, 12), ", "))
|
||||
}
|
||||
}
|
||||
|
||||
// ── the predicate ───────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// basisFromRun is the memory the bought purchase WOULD have written at its boundary: for every candidate
|
||||
// the bank holds as a decided engine row, the answer it holds and the fingerprint of the evidence as it
|
||||
// stands now.
|
||||
//
|
||||
// ⚠ THE FINGERPRINT IS TAKEN ON THE PRE-ROLE CANDIDATES on purpose, and the production writer must do the
|
||||
// same: the classifier pass MUTATES Candidate.Type in place (applyTypes), so a fingerprint computed after
|
||||
// the roles have run would hash the CLASSIFIER's type — and then learning whether the terminologist may be
|
||||
// skipped would require buying the classifier first, every purchase, which is the cost the basis exists
|
||||
// to remove.
|
||||
func (m *basisMaterial) basisFromRun() map[string]bankBasisRecord {
|
||||
canon := approvedNeighbours(m.bank)
|
||||
out := make(map[string]bankBasisRecord, len(m.cands))
|
||||
for _, c := range m.cands {
|
||||
row, ok := m.rows[c.Key]
|
||||
if !ok || row.Status != "draft" || strings.TrimSpace(row.Dst) == "" {
|
||||
continue // the purchase settled nothing about this surface, so it writes no record
|
||||
}
|
||||
out[c.Key] = bankBasisRecord{
|
||||
Key: c.Key,
|
||||
FP: candidateBasisFingerprint(c, m.nsource, canon, m.width),
|
||||
Answer: bankBasisAnswer{Dst: row.Dst, Type: row.Type, Gender: row.Gender},
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// verdicts runs the predicate over the current candidates against a given basis and source.
|
||||
func (m *basisMaterial) verdicts(basis map[string]bankBasisRecord, nsource string) map[string]bankBasisVerdict {
|
||||
canon := approvedNeighbours(m.bank)
|
||||
fps := make(map[string]bankBasisFP, len(m.cands))
|
||||
for _, c := range m.cands {
|
||||
fps[c.Key] = candidateBasisFingerprint(c, nsource, canon, m.width)
|
||||
}
|
||||
return m.scoped(bankBasisSettle(m.cands, fps, basis, m.rows, m.ambiguous, m.unitID, "s"))
|
||||
}
|
||||
|
||||
// scoped drops the candidates the bank filter already took, so every count below is about the population
|
||||
// the engine actually offers the basis.
|
||||
func (m *basisMaterial) scoped(v map[string]bankBasisVerdict) map[string]bankBasisVerdict {
|
||||
out := make(map[string]bankBasisVerdict, len(v))
|
||||
for k, verdict := range v {
|
||||
if !m.consider[k] {
|
||||
continue
|
||||
}
|
||||
out[k] = verdict
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// report prints the counts with every denominator beside them.
|
||||
func (m *basisMaterial) report(t *testing.T, label string, v map[string]bankBasisVerdict) {
|
||||
t.Helper()
|
||||
served, byReason := countBasisReasons(v)
|
||||
parts := make([]string, 0, len(byReason))
|
||||
for _, r := range byReason {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", r.Reason, r.N))
|
||||
}
|
||||
signed := 0
|
||||
for _, e := range m.bank {
|
||||
if e.Status == "approved" {
|
||||
signed++
|
||||
}
|
||||
}
|
||||
t.Logf("%s candidates=%d offered_to_the_basis=%d (bank filter took %d) bank_rows=%d signed_rows=%d | SERVED=%d re-asked=%d | %s",
|
||||
label, len(m.cands), len(m.consider), m.bankSettled, len(m.bank), signed, served, len(m.consider)-served, strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
// ── controls ────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// controlOneOccurrenceMoved is the first discriminating control: the source moves around ONE occurrence of
|
||||
// ONE settled candidate, and the set bought again afterwards must be EXACTLY the set the edit can reach.
|
||||
//
|
||||
// ⛔ WHAT «EXACTLY» MEANS HERE, and why the obvious formulation is wrong. The first version of this control
|
||||
// asserted «the victim and its unit, and nobody else», and it failed on real material — correctly. An edit
|
||||
// is a point in the text, and it lands inside the evidence window of every surface whose occurrence is
|
||||
// near it, including surfaces the victim CONTAINS (三转蛊师 contains 蛊师) and neighbours a few runes away.
|
||||
// Re-asking those is not coupling, it is the mechanism working: their evidence really did change. Asserting
|
||||
// otherwise would have forced the fingerprint to read LESS of the source than it must.
|
||||
//
|
||||
// So the assertion is a CROSS-CHECK between two independent routes to the same set: the predicate's verdict
|
||||
// (a hash over evidence + canon + identity) against a direct positional walk of the windows alone. They
|
||||
// agree only if nothing except the windows contributed to the change — which is the property that matters,
|
||||
// and which the naive formulation could not see at all.
|
||||
func (m *basisMaterial) controlOneOccurrenceMoved(t *testing.T, base map[string]bankBasisVerdict) {
|
||||
t.Helper()
|
||||
var victim terminology.Candidate
|
||||
for _, c := range m.cands {
|
||||
if base[c.Key].Reason == basisServed && strings.Contains(m.nsource, c.Key) {
|
||||
victim = c
|
||||
break
|
||||
}
|
||||
}
|
||||
if victim.Key == "" {
|
||||
t.Fatalf("control 1: no served candidate occurs in the source, so this control would pass without measuring anything")
|
||||
}
|
||||
at := strings.Index(m.nsource, victim.Key)
|
||||
moved := m.nsource[:at] + "ЗЗЗ" + m.nsource[at:]
|
||||
|
||||
// ROUTE A — the direct positional truth: whose windows actually moved. Uses no hash, no basis, no unit.
|
||||
touched := map[string]bool{}
|
||||
for _, c := range m.cands {
|
||||
if !equalStrings(terminology.EvidenceWindows(m.nsource, c.Key, m.width),
|
||||
terminology.EvidenceWindows(moved, c.Key, m.width)) {
|
||||
touched[c.Key] = true
|
||||
}
|
||||
}
|
||||
if !touched[victim.Key] {
|
||||
t.Fatalf("control 1: the edit did not even move the windows of its own victim %q — the fixture, not the predicate", victim.Src)
|
||||
}
|
||||
if len(touched) == len(m.cands) {
|
||||
t.Fatalf("control 1: a single edit moved the windows of EVERY candidate (%d) — the control cannot separate the classes on this material", len(m.cands))
|
||||
}
|
||||
// …expanded by the unit rule, which is the predicate's own promise: a family is decided whole.
|
||||
hotUnits := map[int]bool{}
|
||||
for k := range touched {
|
||||
hotUnits[m.unitID[k]] = true
|
||||
}
|
||||
expected := map[string]bool{}
|
||||
for _, c := range m.cands {
|
||||
if hotUnits[m.unitID[c.Key]] && base[c.Key].Reason == basisServed {
|
||||
expected[c.Key] = true
|
||||
}
|
||||
}
|
||||
|
||||
// ROUTE B — the predicate.
|
||||
after := m.verdicts(m.basisFromRun(), moved)
|
||||
observed := map[string]bool{}
|
||||
for _, c := range m.cands {
|
||||
if base[c.Key].Reason == basisServed && after[c.Key].Reason != basisServed {
|
||||
observed[c.Key] = true
|
||||
}
|
||||
}
|
||||
missing, extra := diffKeys(expected, observed), diffKeys(observed, expected)
|
||||
t.Logf("CONTROL-1 one-occurrence-moved victim=%q windows_moved=%d expected_re-asked=%d observed_re-asked=%d missing=%d extra=%d | %s",
|
||||
victim.Src, len(touched), len(expected), len(observed), len(missing), len(extra),
|
||||
strings.Join(capList(sortedSet(observed), 8), ", "))
|
||||
if len(observed) == 0 {
|
||||
t.Fatalf("control 1: moving the source around %q re-asked NOTHING — the fingerprint does not read the source, and the whole predicate is blind", victim.Src)
|
||||
}
|
||||
if len(missing) > 0 || len(extra) > 0 {
|
||||
t.Fatalf("control 1: the predicate's re-ask set is NOT the set the edit can reach.\n windows moved, predicate kept serving (%d): %s\n predicate re-asked, windows did not move (%d): %s",
|
||||
len(missing), strings.Join(capList(missing, 10), ", "), len(extra), strings.Join(capList(extra, 10), ", "))
|
||||
}
|
||||
}
|
||||
|
||||
// controlUnitShape is not a pass/fail control but the measurement the unit rule has to be judged by: how
|
||||
// much of the saving the «a family is decided whole» rule costs, and whether the cost comes from the rule
|
||||
// or from the unit DETECTION being run over a different population than the batcher's.
|
||||
//
|
||||
// ⚠ IT PRINTS BOTH POPULATIONS ON PURPOSE. The predicate detects units over ALL candidates (bankBasisUnits)
|
||||
// while the batcher detects them over the PAID subset; that difference is a decision of this pack, and a
|
||||
// decision whose price is not printed is a decision nobody can review.
|
||||
func (m *basisMaterial) controlUnitShape(t *testing.T, base map[string]bankBasisVerdict) {
|
||||
t.Helper()
|
||||
sizes := map[int]int{}
|
||||
for _, c := range m.cands {
|
||||
sizes[m.unitID[c.Key]]++
|
||||
}
|
||||
biggest, singles := 0, 0
|
||||
for _, n := range sizes {
|
||||
if n > biggest {
|
||||
biggest = n
|
||||
}
|
||||
if n == 1 {
|
||||
singles++
|
||||
}
|
||||
}
|
||||
// What the answer would be with no unit rule at all — the ceiling the rule is spending against.
|
||||
own := 0
|
||||
for _, c := range m.cands {
|
||||
if base[c.Key].Reason == basisServed || base[c.Key].Reason == basisUnit {
|
||||
own++
|
||||
}
|
||||
}
|
||||
// Which candidates do the blocking, and why: the operator's question is «who is holding my family open».
|
||||
blockers := map[basisReason][]string{}
|
||||
blocked := map[int]bool{}
|
||||
for _, c := range m.cands {
|
||||
if r := base[c.Key].Reason; r != basisServed && r != basisUnit {
|
||||
blocked[m.unitID[c.Key]] = true
|
||||
blockers[r] = append(blockers[r], c.Src)
|
||||
}
|
||||
}
|
||||
t.Logf("UNIT-SHAPE units=%d singletons=%d largest_unit=%d | settled_on_their_own=%d served_after_unit_rule=%d cost_of_the_rule=%d | units_blocked=%d",
|
||||
len(sizes), singles, biggest, own, countServed(base), own-countServed(base), len(blocked))
|
||||
for _, r := range basisReasonNames {
|
||||
if names := blockers[r]; len(names) > 0 {
|
||||
sort.Strings(names)
|
||||
t.Logf("UNIT-SHAPE blocked by %-16s n=%d | %s", r, len(names), strings.Join(capList(names, 10), ", "))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// controlBatchesStable is the second discriminating control: with one basis, two passes build byte-identical
|
||||
// batches. This is what makes a resume replay for $0 — a bank batch is addressed by the hash of its request,
|
||||
// so a composition that differs between passes finds no checkpoint and buys the pass again.
|
||||
func (m *basisMaterial) controlBatchesStable(t *testing.T, base map[string]bankBasisVerdict) {
|
||||
t.Helper()
|
||||
batchRunes, _, _ := m.r.terminologyOpts()
|
||||
render := func() string {
|
||||
var paid []terminology.Candidate
|
||||
for _, c := range m.cands {
|
||||
if base[c.Key].Reason != basisServed {
|
||||
paid = append(paid, c)
|
||||
}
|
||||
}
|
||||
// The batcher exactly as production runs it: units detected over the PAID subset, not over all
|
||||
// candidates. The predicate uses its own unit pass (bankBasisUnits) and must not have moved this.
|
||||
seriesID := terminology.DetectSeries(paid, basisSeriesParams(m.r.Book.SourceLang))
|
||||
fams := terminology.DetectFamilies(paid, m.r.familyParams)
|
||||
unitID, _ := terminology.MergeUnits(seriesID, fams, m.r.familyParams)
|
||||
var b strings.Builder
|
||||
for i, batch := range terminology.Batch(paid, batchRunes, unitID) {
|
||||
fmt.Fprintf(&b, "#%d\n%s\n", i, terminology.RenderBatch(batch))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
first, second := render(), render()
|
||||
t.Logf("CONTROL-2 batches-under-one-basis paid_candidates=%d rendered_bytes=%d identical=%v",
|
||||
len(m.cands)-countServed(base), len(first), first == second)
|
||||
if first != second {
|
||||
t.Fatalf("control 2: two passes under ONE basis rendered DIFFERENT batches — every resume would re-buy the pass")
|
||||
}
|
||||
if len(first) == 0 {
|
||||
t.Fatalf("control 2: the batcher rendered ZERO bytes, so identical-to-itself proves nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// controlKnobsDoNotMatter is the property the fingerprint shape exists for: turning the knobs that decide
|
||||
// WHICH contexts the model is shown must not move the settled set.
|
||||
//
|
||||
// ⛔ THE ONE KNOB THAT IS ALLOWED TO MOVE IT is basis_width, and that is stated here rather than discovered:
|
||||
// it is the pair's statement of what counts as the same evidence, so changing it IS a change of the
|
||||
// question. Every other window knob — how many contexts, how wide the model's own view — is prompt shape.
|
||||
func (m *basisMaterial) controlKnobsDoNotMatter(t *testing.T, base map[string]bankBasisVerdict) {
|
||||
t.Helper()
|
||||
basis := m.basisFromRun()
|
||||
for _, knob := range []struct {
|
||||
name string
|
||||
kwicPer int
|
||||
kwicWidth int
|
||||
basisWidth int
|
||||
mustNotChange bool
|
||||
}{
|
||||
{"kwic_per_term 3→1", 1, 0, 0, true},
|
||||
{"kwic_per_term 3→40", 40, 0, 0, true},
|
||||
{"kwic_width 40→120", 0, 120, 0, true},
|
||||
{"basis_width 40→80 (ALLOWED to move it)", 0, 0, 80, false},
|
||||
} {
|
||||
_, per, wid := m.r.terminologyOpts()
|
||||
if knob.kwicPer > 0 {
|
||||
per = knob.kwicPer
|
||||
}
|
||||
if knob.kwicWidth > 0 {
|
||||
wid = knob.kwicWidth
|
||||
}
|
||||
// Rebuild the candidates with the knob turned: KWIC is attached during the merge, so a knob that
|
||||
// leaks into the fingerprint can only be caught by rebuilding, never by re-hashing the same slice.
|
||||
turned := m.rebuiltWithKWIC(t, per, wid)
|
||||
width := m.width
|
||||
if knob.basisWidth > 0 {
|
||||
width = knob.basisWidth
|
||||
}
|
||||
canon := approvedNeighbours(m.bank)
|
||||
fps := make(map[string]bankBasisFP, len(turned))
|
||||
for _, c := range turned {
|
||||
fps[c.Key] = candidateBasisFingerprint(c, m.nsource, canon, width)
|
||||
}
|
||||
after := m.scoped(bankBasisSettle(turned, fps, basis, m.rows, m.ambiguous, m.unitID, "s"))
|
||||
lost := 0
|
||||
for _, c := range m.cands {
|
||||
if base[c.Key].Reason == basisServed && after[c.Key].Reason != basisServed {
|
||||
lost++
|
||||
}
|
||||
}
|
||||
t.Logf("CONTROL-3 knob=%-40s served_before=%d lost_from_settled=%d", knob.name, countServed(base), lost)
|
||||
if knob.mustNotChange && lost > 0 {
|
||||
t.Fatalf("control 3: turning %s lost %d settled candidate(s) — a prompt-shape knob is re-buying banks", knob.name, lost)
|
||||
}
|
||||
if !knob.mustNotChange && lost == 0 {
|
||||
t.Fatalf("control 3: turning %s lost NOTHING, so this row proves the instrument cannot see the class at all", knob.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rebuiltWithKWIC re-attaches KWIC at other settings over the same merged candidates.
|
||||
func (m *basisMaterial) rebuiltWithKWIC(t *testing.T, per, width int) []terminology.Candidate {
|
||||
t.Helper()
|
||||
tchunks := make([]terminology.Chunk, len(m.chunks))
|
||||
for i, ch := range m.chunks {
|
||||
tchunks[i] = terminology.Chunk{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, NSource: text.NormalizeSourceKey(ch.Text)}
|
||||
}
|
||||
out := make([]terminology.Candidate, len(m.cands))
|
||||
copy(out, m.cands)
|
||||
for i := range out {
|
||||
out[i].KWIC = nil
|
||||
}
|
||||
return terminology.AttachKWIC(out, tchunks, per, width)
|
||||
}
|
||||
|
||||
// measureGrowth answers the growing-book question with a number instead of a watchdog: decide the bank on a
|
||||
// PREFIX of the book, then let the book grow, and count how many of the decided rows the predicate buys
|
||||
// again.
|
||||
//
|
||||
// ⚠ THIS IS THE COST OF THE DESIGN, NOT A DEFECT OF IT. A fingerprint over every occurrence necessarily
|
||||
// moves when new chapters add occurrences of an old term — and that is the honest reading: a term the new
|
||||
// chapters talk about HAS new evidence. The number below is what the owner needs in order to answer whether
|
||||
// a growing serial wants a different rule, and it is the one thing the engine must not decide for them.
|
||||
func (m *basisMaterial) measureGrowth(t *testing.T) {
|
||||
t.Helper()
|
||||
chapters := map[int]bool{}
|
||||
for _, ch := range m.chunks {
|
||||
chapters[ch.Chapter] = true
|
||||
}
|
||||
if len(chapters) < 2 {
|
||||
t.Logf("GROWTH skipped: the book has %d chapter(s), so a prefix cannot be formed — the class is unmeasured here, not absent", len(chapters))
|
||||
return
|
||||
}
|
||||
last := 0
|
||||
for c := range chapters {
|
||||
if c > last {
|
||||
last = c
|
||||
}
|
||||
}
|
||||
var prefix []chunk.Chunk
|
||||
for _, ch := range m.chunks {
|
||||
if ch.Chapter < last {
|
||||
prefix = append(prefix, ch)
|
||||
}
|
||||
}
|
||||
prefixSrc := newBankSourceIndex(prefix).book
|
||||
// ⛔ THE PREFIX HAS TO BE SMALLER, AND IT IS ASSERTED RATHER THAN ASSUMED. A chapter rule that did not
|
||||
// fire leaves every chunk in «chapter 0», the prefix then equals the whole book, and the measurement
|
||||
// below prints a beautifully plausible «growth costs nothing» — a right-looking number from a method
|
||||
// that never grew anything.
|
||||
if len(prefix) == 0 || len(prefix) == len(m.chunks) {
|
||||
t.Fatalf("growth: the prefix is %d chunk(s) of %d — the chapter split did not separate a prefix, so nothing below would be about growth", len(prefix), len(m.chunks))
|
||||
}
|
||||
prefixBytes, wholeBytes := len(prefixSrc), len(m.nsource)
|
||||
prefixCands := m.candidatesOver(t, prefix)
|
||||
canon := approvedNeighbours(m.bank)
|
||||
|
||||
// The basis as the PREFIX purchase would have written it.
|
||||
basis := map[string]bankBasisRecord{}
|
||||
for _, c := range prefixCands {
|
||||
row, ok := m.rows[c.Key]
|
||||
if !ok || row.Status != "draft" || strings.TrimSpace(row.Dst) == "" {
|
||||
continue
|
||||
}
|
||||
basis[c.Key] = bankBasisRecord{Key: c.Key,
|
||||
FP: candidateBasisFingerprint(c, prefixSrc, canon, m.width),
|
||||
Answer: bankBasisAnswer{Dst: row.Dst, Type: row.Type, Gender: row.Gender}}
|
||||
}
|
||||
// The next purchase: the whole book, the prefix's basis.
|
||||
after := m.verdicts(basis, m.nsource)
|
||||
kept, reasked := 0, 0
|
||||
for key := range basis {
|
||||
if after[key].Reason == basisServed {
|
||||
kept++
|
||||
} else {
|
||||
reasked++
|
||||
}
|
||||
}
|
||||
// The breakdown matters more than the total: a decision re-asked because ITS OWN evidence grew is the
|
||||
// mechanism working, while one re-asked because a 44-member unit holds a single grown member is the
|
||||
// UNIT RULE's bill, and the two have entirely different cures.
|
||||
own, byUnit := 0, 0
|
||||
for key := range basis {
|
||||
switch after[key].Reason {
|
||||
case basisServed:
|
||||
case basisUnit:
|
||||
byUnit++
|
||||
default:
|
||||
own++
|
||||
}
|
||||
}
|
||||
t.Logf("GROWTH chapters=%d prefix_chapters=%d prefix_chunks=%d/%d prefix_bytes=%d/%d prefix_candidates=%d decided_in_prefix=%d | still_served=%d re-asked=%d (own_evidence_grew=%d held_by_unit=%d)",
|
||||
len(chapters), len(chapters)-1, len(prefix), len(m.chunks), prefixBytes, wholeBytes,
|
||||
len(prefixCands), len(basis), kept, reasked, own, byUnit)
|
||||
if len(basis) == 0 {
|
||||
t.Fatalf("growth: the prefix purchase decided NOTHING, so both numbers above are zeros about the fixture rather than about growth")
|
||||
}
|
||||
}
|
||||
|
||||
// ── small helpers ───────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
func countServed(v map[string]bankBasisVerdict) int {
|
||||
n := 0
|
||||
for _, x := range v {
|
||||
if x.Reason == basisServed {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func diffKeys(a, b map[string]bool) []string {
|
||||
var out []string
|
||||
for k := range a {
|
||||
if !b[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func capList(xs []string, n int) []string {
|
||||
if len(xs) <= n {
|
||||
return xs
|
||||
}
|
||||
return append(append([]string{}, xs[:n]...), fmt.Sprintf("…(+%d more)", len(xs)-n))
|
||||
}
|
||||
|
||||
func equalStrings(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func sortedSet(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// measureUnitRuleVariants prices the «a family is decided whole» rule against the alternatives, so the rule
|
||||
// is CHOSEN with its cost printed rather than inherited from a sentence.
|
||||
//
|
||||
// ⛔ ONE OF THESE ROWS WAS A LIE IN ITS FIRST VERSION, and the correction is the reason the row survives.
|
||||
// It was labelled «signed rows count as settled» and implemented by handing the predicate a basis RECORD
|
||||
// for each signed row — which measures nothing, because the predicate then rejects the row on its bank
|
||||
// STATUS (`approved`, not `draft`) and the unit stays blocked exactly as before. The row printed the same
|
||||
// number as the row above it and read like a finding. A variant that does not implement its own label is
|
||||
// not a weak measurement, it is a false one.
|
||||
//
|
||||
// What the row asks now is the question that is actually open: a signed row IS still bought today, on
|
||||
// purpose (the two-condition filter pays for banked surfaces so the sheet keeps its dispute marks, backlog
|
||||
// row 447) — so should it also HOLD ITS FAMILY OPEN? Being bought and blocking a family are two different
|
||||
// consequences, and only the second is free to change without touching product behaviour.
|
||||
func (m *basisMaterial) measureUnitRuleVariants(t *testing.T) {
|
||||
t.Helper()
|
||||
basis := m.basisFromRun()
|
||||
canon := approvedNeighbours(m.bank)
|
||||
fps := make(map[string]bankBasisFP, len(m.cands))
|
||||
for _, c := range m.cands {
|
||||
fps[c.Key] = candidateBasisFingerprint(c, m.nsource, canon, m.width)
|
||||
}
|
||||
familyOnly := func() map[string]int {
|
||||
fams := terminology.DetectFamilies(m.cands, m.r.familyParams)
|
||||
id := map[string]int{}
|
||||
for i, f := range fams {
|
||||
for _, k := range f.Keys {
|
||||
if _, taken := id[k]; !taken {
|
||||
id[k] = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return totalUnits(m.cands, id)
|
||||
}()
|
||||
signed := map[string]bool{}
|
||||
for _, c := range m.cands {
|
||||
if row, ok := m.rows[c.Key]; ok && row.Status == "approved" && strings.TrimSpace(row.Dst) != "" {
|
||||
signed[c.Key] = true
|
||||
}
|
||||
}
|
||||
// own-verdicts: the per-candidate half, computed once — every variant differs only in how units block.
|
||||
own := make(map[string]bankBasisVerdict, len(m.cands))
|
||||
for _, c := range m.cands {
|
||||
own[c.Key] = basisVerdictFor(c, fps[c.Key], basis, m.rows, m.ambiguous, "s")
|
||||
}
|
||||
// ⛔ basisSigned IS EXEMPT IN EVERY ROW, because production exempts it (bankBasisSettle) and a row
|
||||
// labelled «the design» that does not implement the design is not a weak measurement, it is a false
|
||||
// one — the very thing this function's own docblock refuses. Measured before the fix: the row printed
|
||||
// 62 of 66 where the engine gives 64, so a reader concluded the signature exemption was an unbuilt
|
||||
// alternative worth +2, when it was already built. Found by adversarial review.
|
||||
count := func(units map[string]int, exempt map[string]bool) int {
|
||||
blocked := map[int]bool{}
|
||||
for _, c := range m.cands {
|
||||
if r := own[c.Key].Reason; r != basisServed && r != basisSigned && !exempt[c.Key] {
|
||||
blocked[units[c.Key]] = true
|
||||
}
|
||||
}
|
||||
n := 0
|
||||
for _, c := range m.cands {
|
||||
if own[c.Key].Reason == basisServed && !blocked[units[c.Key]] {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
none := map[string]int{}
|
||||
for i, c := range m.cands {
|
||||
none[c.Key] = i + 1
|
||||
}
|
||||
for _, v := range []struct {
|
||||
name string
|
||||
units map[string]int
|
||||
exempt map[string]bool
|
||||
}{
|
||||
{"none — no unit rule at all (the ceiling)", none, nil},
|
||||
{"family-only", familyOnly, nil},
|
||||
{"merged series+family (THE DESIGN, signature exempt)", m.unitID, nil},
|
||||
{"merged, a signed row ALSO counted as a hole (pre-17.09 shape)", m.unitID, nil},
|
||||
} {
|
||||
n := count(v.units, v.exempt)
|
||||
t.Logf("UNIT-RULE %-48s SERVED=%2d of %d (%.0f%%)", v.name, n, len(m.cands),
|
||||
100*float64(n)/float64(len(m.cands)))
|
||||
}
|
||||
t.Logf("UNIT-RULE signed rows on this book: %d (%s) — they are BOUGHT either way (backlog row 447); the rows above only ask whether they hold a family open",
|
||||
len(signed), strings.Join(capList(sortedSet(signed), 6), ", "))
|
||||
}
|
||||
|
||||
// totalUnits turns a PARTIAL unit map into a total one, giving every unlisted candidate a unit of its own.
|
||||
// Same trap as bankBasisUnits guards, same cure: a missing key reads as unit 0, and unit 0 is a unit.
|
||||
func totalUnits(cands []terminology.Candidate, partial map[string]int) map[string]int {
|
||||
out := make(map[string]int, len(cands))
|
||||
next := 1 << 30
|
||||
for _, c := range cands {
|
||||
if id, ok := partial[c.Key]; ok && id != 0 {
|
||||
out[c.Key] = id
|
||||
continue
|
||||
}
|
||||
next++
|
||||
out[c.Key] = next
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// measureAutoTail prints the price the consilium named and nobody had put a number on: a surface the model
|
||||
// steadily declines to render is never SETTLED, so it goes to the paid role on EVERY purchase, on input
|
||||
// that has not changed by a byte. The basis cannot help it — «no rendering» is not a decision, and storing
|
||||
// it would let a later purchase serve emptiness back — so this is a standing cost of the mechanism rather
|
||||
// than a defect of it, and a cost with no number attached is one nobody can decide about.
|
||||
func (m *basisMaterial) measureAutoTail(t *testing.T, base map[string]bankBasisVerdict) {
|
||||
t.Helper()
|
||||
tail, noRow, byStatus := 0, 0, map[string]int{}
|
||||
for _, c := range m.cands {
|
||||
row, ok := m.rows[c.Key]
|
||||
if !ok {
|
||||
noRow++
|
||||
continue
|
||||
}
|
||||
byStatus[row.Status]++
|
||||
if row.Status != "approved" && (row.Status != "draft" || strings.TrimSpace(row.Dst) == "") {
|
||||
tail++
|
||||
}
|
||||
}
|
||||
// ⚠ THE DENOMINATOR AND THE STATUS HISTOGRAM RIDE WITH IT, because a zero here has two readings that
|
||||
// look identical: «this book has no undecided tail» and «this book's rows are not where the instrument
|
||||
// looked». The histogram is what separates them.
|
||||
t.Logf("AUTO-TAIL candidates=%d bank_rows=%d with_no_bank_row=%d | re-asked EVERY purchase on unchanged input=%d | statuses=%v served_now=%d",
|
||||
len(m.cands), len(m.bank), noRow, tail, byStatus, countServed(base))
|
||||
}
|
||||
|
||||
// measureRechunkResidue puts a NUMBER on the one residue of the fingerprint's shape its author can name but
|
||||
// could not, at first, measure.
|
||||
//
|
||||
// ⚠ THE RESIDUE. The windows are taken from the book's normalized source as the engine holds it —
|
||||
// r.bankSrc.book, the chunks' normalized texts joined by a newline. That is ONE definition of «the book's
|
||||
// source», shared with the banknote attestation check, and keeping one definition is worth more than the
|
||||
// precision it costs. What it costs is this: an occurrence sitting within basis_width runes of a CHUNK JOIN
|
||||
// carries that join's newline inside its window, so re-chunking the book at another segmentation budget
|
||||
// moves that window and re-asks the term. A re-cut of CHAPTERS is a named, priced event; a change of the
|
||||
// chunk budget is not, and «I named the residue but did not measure it» is a blind spot rather than a
|
||||
// property. This is the measurement.
|
||||
func (m *basisMaterial) measureRechunkResidue(t *testing.T) {
|
||||
t.Helper()
|
||||
base := m.r.segBudget()
|
||||
canon := approvedNeighbours(m.bank)
|
||||
for _, factor := range []float64{0.5, 2.0} {
|
||||
b := base
|
||||
b.DraftBudgetOut = base.DraftBudgetOut * factor
|
||||
other, _, _ := chunk.SplitChunksWithChapters(m.doc.Chapters, b, m.r.chapterRule(), m.r.sentenceAbbrevs())
|
||||
if len(other) == len(m.chunks) {
|
||||
t.Logf("RECHUNK budget×%.1f produced the SAME %d chunk(s): this factor measures nothing on this book", factor, len(other))
|
||||
continue
|
||||
}
|
||||
src := newBankSourceIndex(other).book
|
||||
moved, attested := 0, 0
|
||||
for _, c := range m.cands {
|
||||
if len(terminology.EvidenceWindows(m.nsource, c.Key, m.width)) > 0 {
|
||||
attested++
|
||||
}
|
||||
if candidateBasisFingerprint(c, m.nsource, canon, m.width).Whole !=
|
||||
candidateBasisFingerprint(c, src, canon, m.width).Whole {
|
||||
moved++
|
||||
}
|
||||
}
|
||||
// The denominator and the attested count ride along: a zero over candidates the source does not
|
||||
// contain would be a zero about the fixture, not about the residue.
|
||||
t.Logf("RECHUNK budget×%.1f chunks %d→%d bytes %d→%d | fingerprints moved=%d of %d candidates (attested in the source: %d)",
|
||||
factor, len(m.chunks), len(other), len(m.nsource), len(src), moved, len(m.cands), attested)
|
||||
}
|
||||
}
|
||||
648
backend/internal/pipeline/bankbasis_review_test.go
Normal file
648
backend/internal/pipeline/bankbasis_review_test.go
Normal file
|
|
@ -0,0 +1,648 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/config"
|
||||
"textmachine/backend/internal/lang"
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
)
|
||||
|
||||
// bankbasis_review_test.go: the pins an ADVERSARIAL pass over this pack's own finished work asked for.
|
||||
// Every one of them guards a decision the pack had DECLARED and nothing had been asserting — which is the
|
||||
// state where a declaration is prose, not a guarantee.
|
||||
|
||||
// TestTheBankedTypeReachesTheSheetWhenTheBasisAnswers pins the reader the pack's first reader-table missed.
|
||||
//
|
||||
// ⛔ THE SHAPE OF THE MISS. The sheet, the stop table and the sidecar's `kind` all read Candidate.Type, and
|
||||
// applyTypes — the only thing that writes it — runs BEFORE the basis answers reach the maps it reads. So the
|
||||
// banked type travelled to the auto-bank and not to the sheet: the row the owner signs by said `place` while
|
||||
// the bank row for the same term said `term`, in the steady state, for as long as the basis kept answering.
|
||||
func TestTheBankedTypeReachesTheSheetWhenTheBasisAnswers(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isClassifierBody(body) {
|
||||
// The heuristic drafted 青茅山 as a place; the classifier says otherwise. THAT is the byte that
|
||||
// has to survive being answered from memory.
|
||||
return "青茅山\tterm\tnone", "stop"
|
||||
}
|
||||
if isTerminologyBody(body) {
|
||||
return "青茅山\tгора Цинмао", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, classify: true})
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
if _, err := r1.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r1.lastTerminology.Reclassified != 1 {
|
||||
t.Fatalf("premise broken: the classifier must CHANGE a type on run 1, or there is nothing for the basis to carry: %+v", r1.lastTerminology)
|
||||
}
|
||||
want := sheetTypeOf(t, sheetRowsOf(t, r1), "青茅山")
|
||||
if want != "term" {
|
||||
t.Fatalf("premise broken: run 1's sheet must already carry the classifier's type, got %q", want)
|
||||
}
|
||||
reclassifiedOnRun1 := r1.lastTerminology.Reclassified
|
||||
r1.Close()
|
||||
|
||||
// Run 2 pays the one-time re-pack; run 3 is the steady state where the basis answers.
|
||||
for i := 2; i <= 3; i++ {
|
||||
r := newRunner(t, bookPath)
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatalf("run %d: %v", i, err)
|
||||
}
|
||||
if r.lastTerminology.BasisServed == 0 {
|
||||
t.Fatalf("premise broken: run %d must be served by the basis: %+v", i, r.lastTerminology)
|
||||
}
|
||||
if got := sheetTypeOf(t, sheetRowsOf(t, r), "青茅山"); got != want {
|
||||
t.Errorf("run %d: the SHEET says type=%q while the answer the basis carries says %q — the owner signs by a row that disagrees with the bank row for the same term",
|
||||
i, got, want)
|
||||
}
|
||||
// The other half of the same decision: the counter must NOT have absorbed the stamp. `reclassified`
|
||||
// means «how many types the CLASSIFIER changed», and the classifier saw nothing this run.
|
||||
if r.lastTerminology.Reclassified != 0 {
|
||||
t.Errorf("run %d: reclassified=%d, want 0 — the basis's stamp inflated a counter that answers a different question (run 1 legitimately had %d)",
|
||||
i, r.lastTerminology.Reclassified, reclassifiedOnRun1)
|
||||
}
|
||||
r.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// sheetRowsOf reads the sheet the owner actually gets — the stop-table SIDECAR beside the database, which
|
||||
// every run writes — rather than r.lastBankStopRows, which is set only on the stopping branch and is empty
|
||||
// on an auto-continuing run. Reading the shipped artifact also puts its parser under the same assertion.
|
||||
func sheetRowsOf(t *testing.T, r *Runner) []BankStopRow {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(r.bankStopTablePath())
|
||||
if err != nil {
|
||||
t.Fatalf("every run writes the stop table sidecar; this one did not: %v", err)
|
||||
}
|
||||
rows, err := ParseBankStopTable(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse the stop table the run just wrote: %v", err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
t.Fatalf("the stop table holds no rows (%d bytes): every assertion against it would be vacuous", len(raw))
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func sheetTypeOf(t *testing.T, rows []BankStopRow, src string) string {
|
||||
t.Helper()
|
||||
for _, r := range rows {
|
||||
if r.Src == src {
|
||||
return r.Type
|
||||
}
|
||||
}
|
||||
t.Fatalf("%q is not on the sheet at all (%d row(s)) — the fixture, not the subject", src, len(rows))
|
||||
return ""
|
||||
}
|
||||
|
||||
// TestTheOwnersSignedRowIsNeverRecordedInTheBasis pins the refusal at the WRITE. A signed surface still goes
|
||||
// to the paid role on purpose (backlog row 447), so its consolidation does reach the recorder — and recording
|
||||
// it puts the ENGINE's word into the memory of decisions under a surface whose law is the OWNER's. The day
|
||||
// that signature is lifted, the predicate finds a record, sees an unsigned draft row, and serves the
|
||||
// engine's rendering as «this book decided it earlier», over the owner's own.
|
||||
func TestTheOwnersSignedRowIsNeverRecordedInTheBasis(t *testing.T) {
|
||||
cands := []terminology.Candidate{{Key: "方源"}, {Key: "青茅山"}, {Key: "元石"}}
|
||||
out := map[string]string{"方源": "Фан Юань", "青茅山": "гора Цинмао", "元石": "юаньши"}
|
||||
fps := map[string]bankBasisFP{"方源": {Whole: "a"}, "青茅山": {Whole: "b"}, "元石": {Whole: "c"}}
|
||||
rows := map[string]store.GlossaryEntry{
|
||||
"方源": {Src: "方源", Dst: "Фан Юань", Status: "approved"}, // the owner's word
|
||||
"青茅山": {Src: "青茅山", Dst: "гора Цинмао", Status: "draft", Source: "mined"},
|
||||
}
|
||||
ambiguous := map[string]bool{"元石": true} // two senses or two windows: nothing can say which is the answer
|
||||
|
||||
got := basisRowsToStore(cands, out, nil, nil, fps, rows, ambiguous, basisShape(40, "p"))
|
||||
keys := map[string]bool{}
|
||||
for _, r := range got {
|
||||
keys[r.SrcKey] = true
|
||||
}
|
||||
if keys["方源"] {
|
||||
t.Error("a surface the OWNER signed was recorded in the basis: lift the signature and the engine's word is served over his, which merges the two words the bank ontology keeps apart")
|
||||
}
|
||||
if keys["元石"] {
|
||||
t.Error("an AMBIGUOUS surface was recorded: the carrier's key is (book, surface), so two senses of one surface cannot be remembered apart and whichever was written last answers for both")
|
||||
}
|
||||
if !keys["青茅山"] {
|
||||
t.Fatalf("control: the ordinary engine-decided row was NOT recorded either (%d row(s)) — this test would then pass on a recorder that stores nothing", len(got))
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("recorded %d row(s), want exactly the one: %+v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnAmbiguousSurfaceIsReAskedRatherThanGuessed pins the read side of the same class. The bank's
|
||||
// uniqueness key is the FOUR-tuple (src, sense, since_ch, until_ch); a candidate carries none of sense or
|
||||
// window. A map written as `out[key] = row` keeps whichever row the store returned last — it orders by that
|
||||
// tuple, so the LATEST window wins — and answers for the other.
|
||||
func TestAnAmbiguousSurfaceIsReAskedRatherThanGuessed(t *testing.T) {
|
||||
bank := []store.GlossaryEntry{
|
||||
// One surface, two LEGITIMATE rows: the owner signed chapters 1–10, the engine drafted 11 onward.
|
||||
{Src: "古月", Dst: "Гуюэ", Status: "approved", SinceCh: 1, UntilCh: 10},
|
||||
{Src: "古月", Dst: "клан Гу Юэ", Status: "draft", SinceCh: 11, Source: "mined"},
|
||||
{Src: "元石", Dst: "юаньши", Status: "draft", Source: "mined"},
|
||||
}
|
||||
rows, ambiguous := bankBasisRows(bank, func(s string) string { return s })
|
||||
if !ambiguous["古月"] {
|
||||
t.Fatal("a surface the bank answers with TWO rows was indexed as if it had one: the predicate would serve whichever row sorted last, and a signature on the other window would be invisible to it")
|
||||
}
|
||||
if _, indexed := rows["古月"]; indexed {
|
||||
t.Fatal("the ambiguous surface is ALSO in the unambiguous index: a caller reading that map gets the collapsed answer anyway")
|
||||
}
|
||||
if _, ok := rows["元石"]; !ok || len(rows) != 1 {
|
||||
t.Fatalf("control: the unambiguous surface must still be indexed, got %d row(s) %+v — otherwise the refusal above is «the index is empty», not «this one is ambiguous»", len(rows), rows)
|
||||
}
|
||||
// And the predicate says WHY, in its own class: «ambiguous» and «the source moved» call for different
|
||||
// reactions, and a re-purchase that cannot name its cause is one nobody can act on.
|
||||
fp := bankBasisFP{Whole: "x"}
|
||||
basis := map[string]bankBasisRecord{"古月": {Key: "古月", FP: fp, Answer: bankBasisAnswer{Dst: "клан Гу Юэ"}}}
|
||||
v := basisVerdictFor(terminology.Candidate{Key: "古月"}, fp, basis, rows, ambiguous, "s")
|
||||
if v.Reason != basisAmbiguous {
|
||||
t.Fatalf("verdict = %q, want %q", v.Reason, basisAmbiguous)
|
||||
}
|
||||
if v.Answer.Dst != "" {
|
||||
t.Fatalf("an ambiguous surface was handed an answer (%q) — the guess this class exists to refuse", v.Answer.Dst)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBasisWidthIsReadFromThePairWhenThePairStatesIt pins the branch the canon's standing review question
|
||||
// turns on — «would a pair that is not in the repository work without a Go edit» — and it exists because the
|
||||
// pack's first pin for it exercised only the DEFAULT: it built a Runner with no pack at all, so a planting
|
||||
// that read kwic_width instead of basis_width survived every test in the tree.
|
||||
func TestBasisWidthIsReadFromThePairWhenThePairStatesIt(t *testing.T) {
|
||||
r := &Runner{pack: &lang.Pack{Terminology: &lang.TerminologySizing{KWICWidth: 11, BasisWidth: 7}}}
|
||||
if got := r.basisWidth(); got != 7 {
|
||||
t.Fatalf("basisWidth() = %d, want the pair's basis_width (7). Reading %d would mean the KWIC width — a prompt-shape knob — decides which stored decisions survive, so tuning the prompt re-buys every book's bank",
|
||||
got, r.pack.Terminology.KWICWidth)
|
||||
}
|
||||
// The other half: a pair that states nothing falls through to the engine default, so a pair with no
|
||||
// terminology.txt keeps working.
|
||||
bare := &Runner{pack: &lang.Pack{Terminology: &lang.TerminologySizing{KWICWidth: 11}}}
|
||||
if got := bare.basisWidth(); got != terminologyDefaultBasisWidth {
|
||||
t.Fatalf("a pair stating no basis_width must fall through to the engine default, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestASettledRowIsNotAlsoMarkedNeverAsked pins the exclusivity of the two savings. They are different facts
|
||||
// with different sentences on the sheet — «the bank already renders this surface and every draft agreed»
|
||||
// versus «this book decided it earlier» — and one candidate carrying both prints two mutually exclusive
|
||||
// sentences, is counted twice in the read-out, and, worst of the three, is lifted out of the contested queue
|
||||
// by SettledByBank, which is exactly the exemption the comment on that queue refuses for a basis row.
|
||||
func TestASettledRowIsNotAlsoMarkedNeverAsked(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
// BOTH surfaces are answered, and that is what puts an ENGINE row — and therefore a basis
|
||||
// record — on 方源, while 青茅山 becomes the SEED surface the bank filter takes. One run, two
|
||||
// savings, which is the state this pin needs and the only state in which «no row carries
|
||||
// both marks» is a measurement rather than a tautology.
|
||||
return "方源\tФан Юань\n青茅山\tгора Цинмао", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл." + "\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
// ⚠ THE FIXTURE HAS TO BUILD A STATE NEITHER SAVING REACHES ALONE, and it does it the way a real book
|
||||
// does. Run 1: the seed DISAGREES with the drafts (Цинмаошань against гора Цинмао), so the bank filter
|
||||
// leaves 青茅山 to the paid role — which is what puts a BASIS record on a surface the seed also holds.
|
||||
// Then the owner edits the seed to the rendering the drafts actually use, and from run 2 the bank
|
||||
// filter takes the same surface as well. One candidate, both savings, which is the overlap this pin is
|
||||
// about.
|
||||
seed := "terms:\n - { src: 青茅山, dst: Цинмаошань, status: draft }\n"
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, glossarySeed: seed})
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
if _, err := r1.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r1.lastTerminology == nil || r1.lastTerminology.Consolidated == 0 {
|
||||
t.Fatalf("premise broken: run 1 must BUY 青茅山, or no basis record exists for it: %+v", r1.lastTerminology)
|
||||
}
|
||||
seedPath := r1.Book.GlossarySeed
|
||||
r1.Close()
|
||||
writeFile(t, seedPath, "terms:\n - { src: 青茅山, dst: гора Цинмао, status: draft }\n")
|
||||
|
||||
var last *Runner
|
||||
for i := 2; i <= 3; i++ {
|
||||
r := newRunner(t, bookPath)
|
||||
// Editing the seed legitimately moves the edit-wave snapshot, and the engine refuses to re-pay for
|
||||
// it without consent — correct, loud behaviour that has nothing to do with this pin. Consent, as
|
||||
// the owner would.
|
||||
r.Resnapshot = true
|
||||
r.AcceptRebill = RebillConsent{Given: true}
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatalf("run %d: %v", i, err)
|
||||
}
|
||||
if i == 3 {
|
||||
last = r
|
||||
break
|
||||
}
|
||||
r.Close()
|
||||
}
|
||||
defer last.Close()
|
||||
|
||||
rows := sheetRowsOf(t, last)
|
||||
both := 0
|
||||
for _, row := range rows {
|
||||
if row.SettledByBank && row.SettledByBasis {
|
||||
both++
|
||||
t.Errorf("%q carries BOTH marks: the sheet prints «the bank already renders this surface and every draft agreed» AND «this book decided it earlier» about one row, the read-out counts it twice, and SettledByBank lifts it out of the contested queue — the exemption stopcontest.go explicitly refuses for a basis row",
|
||||
row.Src)
|
||||
}
|
||||
}
|
||||
tres := last.lastTerminology
|
||||
if tres.BankSettled+tres.BasisServed > len(rows) {
|
||||
t.Errorf("the two savings sum to %d over %d candidates: they overlap, so the read-out's `never_asked` and `settled_earlier` double-count",
|
||||
tres.BankSettled+tres.BasisServed, len(rows))
|
||||
}
|
||||
_ = both
|
||||
|
||||
// ⛔ THE PREMISE IS THAT BOTH SAVINGS ARE LIVE IN THIS RUN — not that both claimed one surface, which
|
||||
// is now impossible for a reason deeper than this test, and the reason is worth stating because an
|
||||
// earlier revision of this pin DID assert it and had to be rewritten.
|
||||
//
|
||||
// The bank filter takes only SEED surfaces (dropBankSettled → unsignedEngineSurfaces), and the basis
|
||||
// records only ENGINE rows (basisRowsToStore → membank.IsEngineUnsigned). Those two populations are
|
||||
// DISJOINT BY CONSTRUCTION, so no surface can carry a record and be taken by the filter. Before the
|
||||
// «whose word is this» fix they were not disjoint: the recorder asked `Status == "approved"` instead,
|
||||
// so an owner's hand-written unsigned seed row read as the engine's, got a basis record, and became
|
||||
// exactly the contended surface this test used to build. ⇒ the fixture that built it is now
|
||||
// unbuildable, and asserting an unbuildable state would be asserting nothing.
|
||||
if tres.BankSettled == 0 || tres.BasisServed == 0 {
|
||||
t.Fatalf("premise broken: this run must produce BOTH savings for the exclusivity to be about anything (bank_settled=%d basis_served=%d)", tres.BankSettled, tres.BasisServed)
|
||||
}
|
||||
// And the disjointness itself, asserted rather than trusted: nothing the bank filter took this run has
|
||||
// a row in the memory of decisions.
|
||||
stored, err := last.Store.BankBasisForBook(last.Book.BookID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(stored) == 0 {
|
||||
t.Fatalf("premise broken: the basis holds nothing at all, so «no overlap» is a statement about an empty memory")
|
||||
}
|
||||
// ⛔ AND THE CLASSES MUST ACCOUNT FOR THE OFFERED POPULATION AND NOTHING MORE. This is what the scoping
|
||||
// still guards now that «whose row is this» is asked properly: the two savings can no longer collide on
|
||||
// one surface — the bank filter takes SEED surfaces, the recorder keeps ENGINE rows, and those are
|
||||
// disjoint — so the visible damage of asking the predicate about candidates nobody offered it is no
|
||||
// longer a double MARK but a double COUNT: every bank-taken candidate lands in a re-ask class, and the
|
||||
// run reports terms as «bought again» that were never a question for this mechanism.
|
||||
offered := len(rows) - tres.BankSettled
|
||||
counted := tres.BasisServed
|
||||
for _, n := range tres.BasisReAsked {
|
||||
counted += n
|
||||
}
|
||||
if counted != offered {
|
||||
t.Errorf("the basis reported on %d candidate(s) while only %d were offered to it (%d of %d taken by the bank filter): its classes are counting a population it was never asked about, so every re-ask number the run prints is inflated by the OTHER saving",
|
||||
counted, offered, tres.BankSettled, len(rows))
|
||||
}
|
||||
for _, k := range tres.BankSettledKeys {
|
||||
if _, held := stored[k]; held {
|
||||
t.Errorf("%q was taken by the BANK filter and also holds a basis record (%d stored: %v): the two populations are meant to be disjoint by construction — the filter takes seed surfaces, the recorder keeps engine rows — and this surface is in both",
|
||||
k, len(stored), basisKeysOf(stored))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func basisKeysOf(m map[string]store.BankBasisRow) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestTheThreeDeclaredDecisionsAboutABasisRowAreGuarded pins three choices this pack DECLARED in comments
|
||||
// and nothing was asserting — the state in which a declaration is prose rather than a guarantee. All three
|
||||
// were found by planting: each mutation survived the whole of ./internal/pipeline/ and ./cmd/tmctl/.
|
||||
func TestTheThreeDeclaredDecisionsAboutABasisRowAreGuarded(t *testing.T) {
|
||||
basisRow := BankStopRow{
|
||||
Src: "元石", Dst: "юаньши", Type: "name", Origin: "mined", Freq: 9, Conf: -1,
|
||||
SettledByBasis: true,
|
||||
// The drafts of THIS run proposed something else entirely — which is the whole point: the rendering
|
||||
// came from an earlier purchase, so «no draft proposed it» is true and meaningless.
|
||||
Variants: []BankStopVariant{{Dst: "камень юань", Chunks: 3}},
|
||||
}
|
||||
|
||||
// (a) INVENTED must not fire. It means «the engine produced a rendering no draft proposed» and is a
|
||||
// finding about THIS run's drafts; on a row answered from memory it is a false accusation printed
|
||||
// against the book's own established canon, on the sheet the owner signs by.
|
||||
rows := bankStopRows([]terminology.Candidate{{
|
||||
Key: "元石", Src: "元石", Type: "name", Origin: terminology.OriginMined, Freq: 9,
|
||||
Variants: []terminology.Variant{{Dst: "камень юань", Chunks: 3}},
|
||||
}}, map[string]string{"元石": "юаньши"}, &terminologyResult{BasisServedKeys: []string{"元石"}})
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("premise broken: projected %d row(s), want 1", len(rows))
|
||||
}
|
||||
if !rows[0].SettledByBasis {
|
||||
t.Fatal("premise broken: the row must be marked as answered by the basis, or (a) tests nothing")
|
||||
}
|
||||
if rows[0].Invented {
|
||||
t.Error("(a) a row answered from the basis was flagged INVENTED: the drafts of THIS run were never offered the term, so the flag accuses the book's own canon of being made up")
|
||||
}
|
||||
// The control: the flag is NOT simply dead — an ordinary row with an unproposed rendering still raises it.
|
||||
ctl := bankStopRows([]terminology.Candidate{{
|
||||
Key: "元石", Src: "元石", Origin: terminology.OriginMined,
|
||||
Variants: []terminology.Variant{{Dst: "камень юань", Chunks: 3}},
|
||||
}}, map[string]string{"元石": "юаньши"}, &terminologyResult{})
|
||||
if !ctl[0].Invented {
|
||||
t.Fatal("control: INVENTED did not fire on an ordinary row with an unproposed rendering, so the assertion above holds for the wrong reason")
|
||||
}
|
||||
|
||||
// (b) The contested queue must NOT exempt it. SettledByBank means «the bank renders the surface and
|
||||
// every draft agreed» — genuinely nothing to doubt. A basis row carries a real rendering chosen by a
|
||||
// paid role, and neither half of that sentence is true of it, so it is judged on the same signals as
|
||||
// any other answered row.
|
||||
opts := StopContestOpts{CountInvented: true, CountUnresolved: true, UseConf: true, MaxConf: 50, CountConflicts: true}
|
||||
doubted := basisRow
|
||||
doubted.Contradicts = []string{`this run also renders 元 as "юань"`}
|
||||
if got := doubted.Contest(opts); !got.Contested {
|
||||
t.Error("(b) a basis row with a live contradiction was NOT contested: the basis was exempted by its resemblance to SettledByBank, and the queue goes silent over most of a settled book")
|
||||
}
|
||||
// The control: a BANK-settled row still IS exempt, so (b) is about the basis and not about the queue.
|
||||
bankSettled := basisRow
|
||||
bankSettled.SettledByBasis, bankSettled.SettledByBank = false, true
|
||||
bankSettled.Contradicts = []string{`this run also renders 元 as "юань"`}
|
||||
if got := bankSettled.Contest(opts); got.Contested {
|
||||
t.Fatal("control: a BANK-settled row became contested, so the exemption this pack kept is gone and (b) proves nothing about the basis")
|
||||
}
|
||||
|
||||
// (c) The stop table must round-trip BOTH marks. They are trailing clauses of one line, so a reader that
|
||||
// cuts at the first label it finds swallows the other — and the renderer prints NOT ASKED first, so the
|
||||
// order is not theoretical.
|
||||
both := BankStopRow{Src: "青茅山", Dst: "гора Цинмао", Type: "place", Origin: "mined", Conf: -1,
|
||||
SettledByBank: true, SettledByBasis: true}
|
||||
parsed, err := ParseBankStopTable([]byte(renderBankStopTable([]BankStopRow{both})))
|
||||
if err != nil {
|
||||
t.Fatalf("(c) the table this engine just wrote did not parse: %v", err)
|
||||
}
|
||||
if len(parsed) != 1 {
|
||||
t.Fatalf("(c) parsed %d row(s), want 1", len(parsed))
|
||||
}
|
||||
if !parsed[0].SettledByBank || !parsed[0].SettledByBasis {
|
||||
t.Errorf("(c) a row carrying BOTH labels round-tripped as bank=%v basis=%v: one label was swallowed by the other's cut, and a reader of the shipped table loses a state the sheet shows",
|
||||
parsed[0].SettledByBank, parsed[0].SettledByBasis)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAProducerChangeReportsItsOwnCause pins BOTH halves of the producer rule, and the second half is the
|
||||
// one the pack's first attempt missed.
|
||||
//
|
||||
// ⛔ HALF ONE — it must move something. Before the settled basis, improving the terminologist's prompt or
|
||||
// moving it to another model missed every checkpoint (RequestHash folds model, effort and snapshot) and the
|
||||
// book re-bought its bank from the new producer. Leave the producer out entirely and a settled row is never
|
||||
// asked again no matter who is asking now: a better prompt reaches the unsettled TAIL of every book and
|
||||
// nothing else, permanently. Improving a bank role's prompt is a direct lever on translationese.
|
||||
//
|
||||
// ⛔ HALF TWO — it must move the RIGHT thing, and the pack's first attempt hashed the producer in with the
|
||||
// EVIDENCE. The re-purchase then arrived as `evidence-moved`, «the source around this surface changed» —
|
||||
// told to the operator on every row, about a book nobody had touched. A producer is part of the RULE that
|
||||
// decided a row, exactly as the evidence width is, and the rule is compared before the evidence so each
|
||||
// part can name itself. The first version of this pin asserted only that the hash MOVED and was green and
|
||||
// empty on precisely this.
|
||||
func TestAProducerChangeReportsItsOwnCause(t *testing.T) {
|
||||
c := terminology.Candidate{Key: "元石"}
|
||||
fp := candidateBasisFingerprint(c, basisSource, nil, 40)
|
||||
rows := map[string]store.GlossaryEntry{"元石": {Src: "元石", Dst: "юаньши", Status: "draft", Source: "mined"}}
|
||||
rec := func(shape string) map[string]bankBasisRecord {
|
||||
return map[string]bankBasisRecord{"元石": {Key: "元石", Shape: shape, FP: fp, Answer: bankBasisAnswer{Dst: "юаньши"}}}
|
||||
}
|
||||
was := basisShape(40, "producer-A")
|
||||
|
||||
// The premise: under the SAME rule the row is served, or every assertion below holds trivially.
|
||||
if v := basisVerdictFor(c, fp, rec(was), rows, nil, was); v.Reason != basisServed {
|
||||
t.Fatalf("premise broken: an unchanged rule must serve, got %q", v.Reason)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
now string
|
||||
want basisReason
|
||||
}{
|
||||
{"the role's prompt or model changed", basisShape(40, "producer-B"), basisProducerMoved},
|
||||
{"the pair's evidence width changed", basisShape(80, "producer-A"), basisWidthMoved},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
v := basisVerdictFor(c, fp, rec(was), rows, nil, tc.now)
|
||||
if v.Reason != tc.want {
|
||||
t.Fatalf("verdict = %q, want %q. The evidence did not move — the RULE did, and reporting it as «the source around this term changed» sends the owner looking at the book for a change somebody made to a prompt or to a knob",
|
||||
v.Reason, tc.want)
|
||||
}
|
||||
if v.Answer.Dst != "" {
|
||||
t.Fatalf("a row decided under another rule was served anyway (%q)", v.Answer.Dst)
|
||||
}
|
||||
})
|
||||
}
|
||||
// AND the cause must not be «never decided»: a record whose rule moved is kept, not dropped, because a
|
||||
// dropped record reads as `new` — false of every one of these rows, and the wrong cause printed per row
|
||||
// while the summary line states the right one.
|
||||
if v := basisVerdictFor(c, fp, rec(basisShape(40, "producer-B")), rows, nil, was); v.Reason == basisNew {
|
||||
t.Fatal("a row whose producer changed was reported as never decided")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheProducerIdentityIsResolvedInOnePlace pins that the identity is one function, and that it is built
|
||||
// from what the role's system turn ACTUALLY says on this book rather than from the template's own hash.
|
||||
//
|
||||
// ⛔ THE RENDERED TEXT IS THE POINT, not tidiness. The pair's terminologist prompt reads the BRIEF —
|
||||
// `{{title}}`, `{{audience}}`, `{{honorifics}}`, `{{transcription}}`, `{{venuti}}`. Change `honorifics: keep`
|
||||
// to `adapt`, or the transcription convention, and both waves of the book are re-bought through BriefHash,
|
||||
// while an identity keyed on the template's SHA would go on serving renderings decided under the OLD law
|
||||
// into waves paid for under the new one. Neither the evidence nor the model would have moved; the law the
|
||||
// role was told to apply would have. ⚠ And it is pair-agnostic BY SHAPE rather than by a list of brief
|
||||
// fields in Go: whatever a pair's own prompt chooses to read enters because it enters the rendered text.
|
||||
func TestTheProducerIdentityIsResolvedInOnePlace(t *testing.T) {
|
||||
newRun := func(book *config.Book, model string) *Runner {
|
||||
r := &Runner{Pipeline: &config.Pipeline{}, Book: book}
|
||||
r.Pipeline.Gates.Terminology.Model = model
|
||||
r.Pipeline.Gates.Terminology.Reasoning = "low"
|
||||
r.terminologyTemplate = &PromptTemplate{
|
||||
System: "Книга: «{{title}}». Хонорифики: {{honorifics}}.", SHA256: "fixed-template-sha"}
|
||||
return r
|
||||
}
|
||||
keep := &config.Book{BookID: "b", Title: "蛊真人", Honorifics: "keep"}
|
||||
adapt := &config.Book{BookID: "b", Title: "蛊真人", Honorifics: "adapt"}
|
||||
|
||||
base := newRun(keep, "deepseek-v4-flash").bankRoleProducer()
|
||||
if base == "" {
|
||||
t.Fatal("a configured role must produce a comparable identity")
|
||||
}
|
||||
if same := newRun(keep, "deepseek-v4-flash").bankRoleProducer(); same != base {
|
||||
t.Fatal("the same configuration produced two identities: every purchase would re-buy everything")
|
||||
}
|
||||
// THE BRIEF: the template's own SHA is identical in both runs, so a producer keyed on it could not tell
|
||||
// these apart — and the law the role applies is not the same.
|
||||
if brief := newRun(adapt, "deepseek-v4-flash").bankRoleProducer(); brief == base {
|
||||
t.Fatal("changing the book's `honorifics` left the producer identity unmoved although the role's system turn renders it: decisions taken under the old law would keep being served into waves bought under the new one")
|
||||
}
|
||||
// THE MODEL.
|
||||
if other := newRun(keep, "deepseek-v4-pro").bankRoleProducer(); other == base {
|
||||
t.Fatal("changing the bank role's model left the producer identity unmoved")
|
||||
}
|
||||
// THE CLASSIFIER, which authors the type and gender halves of every stored answer.
|
||||
withClassifier := newRun(keep, "deepseek-v4-flash")
|
||||
withClassifier.classifierTemplate = &PromptTemplate{System: "classify", SHA256: "c"}
|
||||
if withClassifier.bankRoleProducer() == base {
|
||||
t.Fatal("turning the CLASSIFIER on left the producer identity unmoved, although it authors the type and gender halves of every stored answer")
|
||||
}
|
||||
// A TEMPLATE THAT CANNOT RENDER must not hash like an absent role: «could not be described» and «off»
|
||||
// are different states, and the first must re-buy rather than serve.
|
||||
broken := newRun(keep, "deepseek-v4-flash")
|
||||
broken.terminologyTemplate = &PromptTemplate{System: "{{no_such_field}}", SHA256: "fixed-template-sha"}
|
||||
off := newRun(keep, "deepseek-v4-flash")
|
||||
off.terminologyTemplate = nil
|
||||
if broken.bankRoleProducer() == off.bankRoleProducer() {
|
||||
t.Fatal("an unrenderable prompt hashed the same as an absent one")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnOwnersUNSIGNEDRowIsNotTheEnginesMemoryEither pins the other half of «whose word is this», and it is
|
||||
// the half the pack's first fix missed: it asked `Status == "approved"` — a SECOND spelling of a question
|
||||
// membank.IsEngineUnsigned already answers — so a term the owner wrote BY HAND and left unsigned
|
||||
// (`source: seed`, `status: draft`, a shape the loader admits and even requires a rendering for) read as
|
||||
// «not the owner's», and the engine recorded its own consolidation under it.
|
||||
func TestAnOwnersUNSIGNEDRowIsNotTheEnginesMemoryEither(t *testing.T) {
|
||||
cands := []terminology.Candidate{{Key: "青茅山"}, {Key: "元石"}}
|
||||
out := map[string]string{"青茅山": "гора Цинмао", "元石": "юаньши"}
|
||||
fps := map[string]bankBasisFP{"青茅山": {Whole: "a"}, "元石": {Whole: "b"}}
|
||||
rows := map[string]store.GlossaryEntry{
|
||||
// The owner's own hand, unsigned: Source "seed" is what makes it his, NOT the status.
|
||||
"青茅山": {Src: "青茅山", Dst: "Цинмаошань", Status: "draft", Source: "seed"},
|
||||
// The engine's own unsigned row: same STATUS, different author.
|
||||
"元石": {Src: "元石", Dst: "юаньши", Status: "draft", Source: "mined"},
|
||||
}
|
||||
// THE WRITE side: the owner's row must not enter the memory of decisions.
|
||||
got := basisRowsToStore(cands, out, nil, nil, fps, rows, nil, basisShape(40, "p"))
|
||||
if len(got) != 1 || got[0].SrcKey != "元石" {
|
||||
t.Fatalf("recorded %+v, want only the ENGINE's row: a seed term the owner left unsigned is his word, and storing the engine's answer under it means serving that answer back as «this book decided it earlier»", got)
|
||||
}
|
||||
// THE READ side: and it is not served, with a class of its own — «you signed it» and «you wrote it and
|
||||
// left it unsigned» are different sentences to the owner.
|
||||
basis := map[string]bankBasisRecord{"青茅山": {Key: "青茅山", FP: fps["青茅山"], Answer: bankBasisAnswer{Dst: "гора Цинмао"}}}
|
||||
v := basisVerdictFor(cands[0], fps["青茅山"], basis, rows, nil, "s")
|
||||
if v.Reason != basisOwnerRow {
|
||||
t.Fatalf("verdict = %q, want %q", v.Reason, basisOwnerRow)
|
||||
}
|
||||
if v.Answer.Dst != "" {
|
||||
t.Fatalf("the engine's rendering was handed back for the owner's own row (%q)", v.Answer.Dst)
|
||||
}
|
||||
// The control: the ENGINE's row on the same status IS served, so the refusal above is about authorship
|
||||
// and not about `draft`.
|
||||
basis["元石"] = bankBasisRecord{Key: "元石", FP: fps["元石"], Answer: bankBasisAnswer{Dst: "юаньши"}}
|
||||
if e := basisVerdictFor(cands[1], fps["元石"], basis, rows, nil, "s"); e.Reason != basisServed {
|
||||
t.Fatalf("control: the engine's own draft row was refused too (%q) — the predicate is reading the STATUS, not the author", e.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAWidthChangeReportsItsOwnCauseAndNotTheSources pins D1: the evidence width lives in the stored SHAPE,
|
||||
// so turning it is reported as «the rule that decided this is no longer the rule» — with both widths — and
|
||||
// never as «the source around this term changed».
|
||||
//
|
||||
// ⛔ WHY THE CAUSE MATTERS MORE THAN THE COUNT. Both readings re-buy the same rows; they ask the owner for
|
||||
// opposite reactions. «The source moved» is nothing to do — the book changed, the engine noticed. «The width
|
||||
// changed» is a knob somebody turned, and the owner is the only one who can say whether that was meant. A
|
||||
// re-purchase reported under the wrong cause spends money and teaches the wrong lesson.
|
||||
func TestAWidthChangeReportsItsOwnCauseAndNotTheSources(t *testing.T) {
|
||||
if basisShape(40, "p") == basisShape(80, "p") {
|
||||
t.Fatal("the shape does not carry the width: a change of basis_width would be indistinguishable from a change of the source, and every re-purchase it causes would be reported under the wrong cause")
|
||||
}
|
||||
if !strings.Contains(basisShape(40, "p"), basisFingerprintVersion) {
|
||||
t.Fatalf("the shape must still carry the version it extends, got %q", basisShape(40, "p"))
|
||||
}
|
||||
// And the two halves stay independent: the same width under a different version, and the same version
|
||||
// under a different width, must both be distinguishable — otherwise one of them silently masks the other.
|
||||
if basisShape(40, "p") == basisShape(41, "p") {
|
||||
t.Fatal("two adjacent widths collapse to one shape")
|
||||
}
|
||||
// The row is STORED under the shape, not under the bare version — checked on the value the store gets,
|
||||
// because that is the string a later purchase compares against.
|
||||
rows := basisRowsToStore(
|
||||
[]terminology.Candidate{{Key: "元石"}},
|
||||
map[string]string{"元石": "юаньши"},
|
||||
nil, nil,
|
||||
map[string]bankBasisFP{"元石": {Whole: "h"}},
|
||||
map[string]store.GlossaryEntry{"元石": {Src: "元石", Dst: "юаньши", Status: "draft", Source: "mined"}},
|
||||
nil, basisShape(40, "p"))
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("premise broken: recorded %d row(s), want 1", len(rows))
|
||||
}
|
||||
if rows[0].FPVersion != basisShape(40, "p") {
|
||||
t.Fatalf("stored shape = %q, want %q — a row written under one width and read under another must report a SHAPE change, and it can only do that if the width is in the string it stores",
|
||||
rows[0].FPVersion, basisShape(40, "p"))
|
||||
}
|
||||
if rows[0].FPVersion == basisFingerprintVersion {
|
||||
t.Fatal("the row stored the bare version: the width is gone from the comparison, and a width change would be reported as «the source moved»")
|
||||
}
|
||||
}
|
||||
|
||||
// TestABankRowThatDisagreesWithTheMemoryIsReAsked pins D3: the predicate's sentence is «this book decided
|
||||
// this AND the bank still holds that decision», and until this condition existed only the first half was
|
||||
// checked — the stored answer was served and then travelled into the delta, overwriting whatever the bank
|
||||
// held, with the sheet reporting the row as settled earlier.
|
||||
//
|
||||
// ⚠ ITS POPULATION IS NAMED, NOT CLAIMED: on the run that writes them the two agree by construction (the
|
||||
// auto-bank is emitted from the same consolidation the record was built from), so this cannot fire on
|
||||
// ordinary material and is NOT measured on the bought runs. The reachable way to diverge is a hand edit of
|
||||
// `<project_db>.auto-bank.yaml`, which the bank ontology lists among the SOURCES.
|
||||
func TestABankRowThatDisagreesWithTheMemoryIsReAsked(t *testing.T) {
|
||||
c := terminology.Candidate{Key: "元石"}
|
||||
fp := bankBasisFP{Whole: "h"}
|
||||
basis := map[string]bankBasisRecord{"元石": {Key: "元石", FP: fp, Answer: bankBasisAnswer{Dst: "юаньши"}}}
|
||||
|
||||
agreeing := map[string]store.GlossaryEntry{"元石": {Src: "元石", Dst: "юаньши", Status: "draft", Source: "mined"}}
|
||||
if v := basisVerdictFor(c, fp, basis, agreeing, nil, "s"); v.Reason != basisServed {
|
||||
t.Fatalf("premise broken: an AGREEING row must still be served, got %q — otherwise the assertion below holds for the wrong reason", v.Reason)
|
||||
}
|
||||
diverged := map[string]store.GlossaryEntry{"元石": {Src: "元石", Dst: "камень юань", Status: "draft", Source: "mined"}}
|
||||
v := basisVerdictFor(c, fp, basis, diverged, nil, "s")
|
||||
if v.Reason != basisDiverged {
|
||||
t.Fatalf("verdict = %q, want %q: the memory would have been served over a bank row that says something else, and the sheet would have called it «decided earlier»", v.Reason, basisDiverged)
|
||||
}
|
||||
if v.Answer.Dst != "" {
|
||||
t.Fatalf("a diverged surface was handed an answer (%q) — the overwrite this condition exists to refuse", v.Answer.Dst)
|
||||
}
|
||||
// ⚠ AND THE COMPARISON IS UNDER THE TARGET-FORM FOLD, the same one the vote is counted with: a case or
|
||||
// ё difference is ONE decision, and re-asking over it would spend money on a spelling.
|
||||
folded := map[string]store.GlossaryEntry{"元石": {Src: "元石", Dst: "Юаньши", Status: "draft", Source: "mined"}}
|
||||
if f := basisVerdictFor(c, fp, basis, folded, nil, "s"); f.Reason != basisServed {
|
||||
t.Fatalf("a row differing only by case was treated as a disagreement (%q): the engine would buy a term again over a spelling", f.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// TestARecordWhoseRuleMovedIsKeptAndNamed pins the loading rule directly, because inside bankBasisPass it
|
||||
// needs a store, a book and a runner — and a money rule that can only be exercised end to end is one nobody
|
||||
// plants a mutation into.
|
||||
func TestARecordWhoseRuleMovedIsKeptAndNamed(t *testing.T) {
|
||||
now := basisShape(40, "producer-A")
|
||||
stored := map[string]store.BankBasisRow{
|
||||
"元石": {SrcKey: "元石", FPVersion: now, FPWhole: "a", Dst: "юаньши"},
|
||||
"青茅山": {SrcKey: "青茅山", FPVersion: basisShape(40, "producer-B"), FPWhole: "b", Dst: "гора Цинмао"},
|
||||
"古月": {SrcKey: "古月", FPVersion: basisShape(80, "producer-A"), FPWhole: "c", Dst: "Гуюэ"},
|
||||
}
|
||||
basis, changed, old := bankBasisRecordsFrom(stored, now)
|
||||
if len(basis) != 3 {
|
||||
t.Fatalf("kept %d record(s) of 3: a record whose rule moved was DROPPED, and a dropped record reads as «never decided» — the wrong cause on every one of those rows", len(basis))
|
||||
}
|
||||
if changed != 2 {
|
||||
t.Fatalf("counted %d moved shape(s), want 2", changed)
|
||||
}
|
||||
if len(old) != 2 {
|
||||
t.Fatalf("the operator line needs the shapes it is comparing against, got %v", old)
|
||||
}
|
||||
// Each kept record carries ITS OWN shape, which is what lets the verdict name the part that moved.
|
||||
if basis["青茅山"].Shape == now || basis["古月"].Shape == now {
|
||||
t.Fatal("a moved record was stamped with the CURRENT shape: the verdict can no longer tell which part of the rule changed, and every such row would report «the source moved»")
|
||||
}
|
||||
if basis["元石"].Shape != now {
|
||||
t.Fatalf("the unmoved record lost its shape (%q): it would be re-asked for a rule change that did not happen", basis["元石"].Shape)
|
||||
}
|
||||
// The control that keeps the counts readable: with nothing moved, nothing is reported.
|
||||
if _, c, o := bankBasisRecordsFrom(map[string]store.BankBasisRow{"元石": stored["元石"]}, now); c != 0 || len(o) != 0 {
|
||||
t.Fatalf("control: an unchanged shape reported %d change(s) %v — the counter fires on its own", c, o)
|
||||
}
|
||||
}
|
||||
343
backend/internal/pipeline/bankbasis_test.go
Normal file
343
backend/internal/pipeline/bankbasis_test.go
Normal file
|
|
@ -0,0 +1,343 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
)
|
||||
|
||||
// bankbasis_test.go: the pins of the settled basis's two pure halves. Every one of them guards a MONEY
|
||||
// property — a fingerprint that moves too easily re-buys a bank out of a lifetime ceiling, one that moves
|
||||
// too rarely serves an answer decided on evidence the book no longer holds.
|
||||
|
||||
// basisCand is the fixture candidate: enough fields that a pin can move ONE of them and see what happens.
|
||||
func basisCand() terminology.Candidate {
|
||||
return terminology.Candidate{
|
||||
Key: "元石", Src: "元石", Type: "name", Freq: 4, SinceCh: 2,
|
||||
Aliases: []string{"元石头"}, Related: []string{"元海"}, Evidence: []string{"surname:元"},
|
||||
Origin: terminology.OriginMined,
|
||||
Variants: []terminology.Variant{
|
||||
{Dst: "юаньши", Chunks: 2}, {Dst: "камень юань", Chunks: 1},
|
||||
},
|
||||
KWIC: []string{"…元石…"},
|
||||
}
|
||||
}
|
||||
|
||||
const basisSource = "他拿出元石,看了看。元石在手中发光。这是元海的东西。"
|
||||
|
||||
// TestBasisFingerprintIgnoresWhatTheRoleMerelyGUESSED is the pin of what must NOT be in the hash. Draft
|
||||
// variants, frequency and the first chapter of appearance all reach the role's prompt, and all three move
|
||||
// between purchases for reasons that are not new knowledge about the term — model noise, a cap, a re-cut.
|
||||
// Hashing any of them turns ordinary variation into a re-purchase of the whole bank.
|
||||
func TestBasisFingerprintIgnoresWhatTheRoleMerelyGUESSED(t *testing.T) {
|
||||
base := candidateBasisFingerprint(basisCand(), basisSource, nil, 8)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
mut func(c *terminology.Candidate)
|
||||
}{
|
||||
{"the drafts proposed something else", func(c *terminology.Candidate) {
|
||||
c.Variants = []terminology.Variant{{Dst: "энергетический камень", Chunks: 9}}
|
||||
}},
|
||||
{"the drafts proposed nothing at all", func(c *terminology.Candidate) { c.Variants = nil }},
|
||||
{"the detector counted more occurrences", func(c *terminology.Candidate) { c.Freq = 400 }},
|
||||
{"the term now first appears in another chapter", func(c *terminology.Candidate) { c.SinceCh = 77 }},
|
||||
{"the KWIC subset shown to the model changed", func(c *terminology.Candidate) {
|
||||
c.KWIC = []string{"совсем", "другие", "окна"}
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := basisCand()
|
||||
tc.mut(&c)
|
||||
got := candidateBasisFingerprint(c, basisSource, nil, 8)
|
||||
if got.Whole != base.Whole {
|
||||
t.Fatalf("the fingerprint MOVED on %q, so this book and every other one would re-buy its bank out of a lifetime ceiling for a change that is not new knowledge about the term\n before %s\n after %s",
|
||||
tc.name, base.Whole[:16], got.Whole[:16])
|
||||
}
|
||||
})
|
||||
}
|
||||
// The control that makes those five zeros readable: the fingerprint is NOT constant. Without this the
|
||||
// whole test passes on a function that returns the same string for everything.
|
||||
moved := candidateBasisFingerprint(basisCand(), strings.Replace(basisSource, "他拿出元石", "他慢慢拿出元石", 1), nil, 8)
|
||||
if moved.Whole == base.Whole {
|
||||
t.Fatal("control: the fingerprint did not move when the SOURCE around the key changed — it is constant, and the five assertions above proved nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBasisFingerprintReadsEveryOccurrenceNotTheFirstFew is the pin the whole shape exists for: the
|
||||
// fingerprint must not depend on how many contexts the model is shown, or on which of them are chosen.
|
||||
// Otherwise the stratified selection that is already on the backlog, or any tuning of kwic_per_term, zeroes
|
||||
// the settled set of every book on earth at once.
|
||||
func TestBasisFingerprintReadsEveryOccurrenceNotTheFirstFew(t *testing.T) {
|
||||
src := strings.Repeat("前面的话。元石在这里。", 6)
|
||||
full := terminology.EvidenceWindows(src, "元石", 4)
|
||||
if len(full) != 6 {
|
||||
t.Fatalf("premise broken: the fixture must hold 6 occurrences for «uncapped» to mean anything, got %d", len(full))
|
||||
}
|
||||
// KWIC over the same text, capped — the subset the role actually sees.
|
||||
capped := terminology.AttachKWIC([]terminology.Candidate{{Key: "元石"}},
|
||||
[]terminology.Chunk{{NSource: src}}, 3, 4)
|
||||
if len(capped[0].KWIC) != 3 {
|
||||
t.Fatalf("premise broken: KWIC must be CAPPED here, or the two views are the same view; got %d", len(capped[0].KWIC))
|
||||
}
|
||||
// The last occurrence is beyond the cap: an edit there is invisible to KWIC and must be visible here.
|
||||
last := strings.LastIndex(src, "元石")
|
||||
edited := src[:last] + "新" + src[last:]
|
||||
if equalStrings(full, terminology.EvidenceWindows(edited, "元石", 4)) {
|
||||
t.Fatal("an edit at the LAST occurrence — past the KWIC cap — left the evidence windows unchanged; the fingerprint would keep serving an answer whose evidence moved")
|
||||
}
|
||||
after := terminology.AttachKWIC([]terminology.Candidate{{Key: "元石"}},
|
||||
[]terminology.Chunk{{NSource: edited}}, 3, 4)
|
||||
if !equalStrings(capped[0].KWIC, after[0].KWIC) {
|
||||
t.Fatal("control: the KWIC subset ALSO changed, so this fixture does not separate «what the model is shown» from «what the book says» and the assertion above measures nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBasisFingerprintNamesTheSignedRowsItDependsOn pins both halves of the canon component: a signature
|
||||
// that relates to the candidate MOVES the fingerprint (so the term is re-asked, paid for, and judged
|
||||
// against the new law), and the Canon hash moves with it — which is what lets the re-purchase state its own
|
||||
// cause instead of being an unexplained charge.
|
||||
func TestBasisFingerprintNamesTheSignedRowsItDependsOn(t *testing.T) {
|
||||
bare := candidateBasisFingerprint(basisCand(), basisSource, nil, 8)
|
||||
related := []terminology.Neighbour{{Src: "元", Dst: "юань"}}
|
||||
withKin := candidateBasisFingerprint(basisCand(), basisSource, related, 8)
|
||||
if withKin.Whole == bare.Whole {
|
||||
t.Fatal("signing a RELATED row left the fingerprint unmoved: the book's own law could change under a term and the engine would keep serving the rendering decided before it")
|
||||
}
|
||||
if withKin.Canon == bare.Canon {
|
||||
t.Fatal("the Canon half did not move, so a re-purchase caused by a signature would be reported as «the source moved» — a message lying about its own cause")
|
||||
}
|
||||
// The other half: a signature with NOTHING to do with this candidate must not move it. Otherwise every
|
||||
// act of signing re-buys the whole book, which is the opposite of what a signature is for.
|
||||
unrelated := []terminology.Neighbour{{Src: "春天", Dst: "весна"}}
|
||||
if got := candidateBasisFingerprint(basisCand(), basisSource, unrelated, 8); got.Whole != bare.Whole {
|
||||
t.Fatal("signing an UNRELATED row moved the fingerprint: every signature would re-buy every term of the book")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBasisUnitsGiveEveryCandidateAUnitOfItsOwn pins the zero-value trap that produced a false measurement
|
||||
// before it was found: MergeUnits returns a PARTIAL map and the engine's convention reads a missing key as
|
||||
// «singleton» (terminology.Batch checks `id != 0`). Indexed raw, every unit-less candidate shares unit 0 —
|
||||
// one unsettled row among them re-buys them all.
|
||||
func TestBasisUnitsGiveEveryCandidateAUnitOfItsOwn(t *testing.T) {
|
||||
// Three candidates with no series and no family between them: nothing should join them.
|
||||
cands := []terminology.Candidate{{Key: "春天"}, {Key: "石头"}, {Key: "海"}}
|
||||
units := bankBasisUnits(cands, terminology.SeriesParams{}, terminology.FamilyParams{})
|
||||
seen := map[int]string{}
|
||||
for _, c := range cands {
|
||||
id, ok := units[c.Key]
|
||||
if !ok {
|
||||
t.Fatalf("%q got no unit id at all, so a caller indexing the map reads 0 for it", c.Key)
|
||||
}
|
||||
if id == 0 {
|
||||
t.Fatalf("%q got unit id 0, which the engine's own convention means «no unit» — every such candidate would share one unit and block each other", c.Key)
|
||||
}
|
||||
if other, clash := seen[id]; clash {
|
||||
t.Fatalf("%q and %q share unit id %d although neither is in a series or a family: one being re-asked would re-buy the other", other, c.Key, id)
|
||||
}
|
||||
seen[id] = c.Key
|
||||
}
|
||||
}
|
||||
|
||||
// TestBasisServesOnlyADecidedRow pins the predicate's conditions one at a time, and each row of the table
|
||||
// is a way the engine could pay for, or wrongly skip, a term.
|
||||
//
|
||||
// ⛔ THE `auto`-WITH-A-RENDERING ROW IS THE ONE TO READ. The miner really does produce it — the raw
|
||||
// first-chunk banknote guess lands as a dst while the status stays `auto` — a pair four engine comments
|
||||
// declare impossible and backlog row 487 has open. This pack does NOT fix that; it pins the premise it
|
||||
// depends on, so the day 487 is fixed the settled set does not move by a step nobody priced.
|
||||
func TestBasisServesOnlyADecidedRow(t *testing.T) {
|
||||
c := basisCand()
|
||||
fp := candidateBasisFingerprint(c, basisSource, nil, 8)
|
||||
rec := bankBasisRecord{Key: c.Key, FP: fp, Answer: bankBasisAnswer{Dst: "юаньши", Type: "name"}}
|
||||
draft := store.GlossaryEntry{Src: c.Key, Dst: "юаньши", Status: "draft", Source: "mined"}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
rec *bankBasisRecord
|
||||
row *store.GlossaryEntry
|
||||
fp bankBasisFP
|
||||
want basisReason
|
||||
}{
|
||||
{"decided, evidence unchanged", &rec, &draft, fp, basisServed},
|
||||
{"never decided", nil, &draft, fp, basisNew},
|
||||
{"the source around it moved", &rec, &draft, bankBasisFP{Whole: "other", Canon: fp.Canon}, basisMoved},
|
||||
{"a relative was signed", &rec, &draft, bankBasisFP{Whole: "other", Canon: "other"}, basisKin},
|
||||
{"the bank no longer holds the row", &rec, nil, fp, basisAbsent},
|
||||
{"status auto WITH a rendering (backlog row 487)", &rec,
|
||||
&store.GlossaryEntry{Src: c.Key, Dst: "юаньши", Status: "auto", Source: "mined"}, fp, basisUnsettled},
|
||||
{"status draft with NO rendering", &rec,
|
||||
&store.GlossaryEntry{Src: c.Key, Dst: " ", Status: "draft", Source: "mined"}, fp, basisUnsettled},
|
||||
{"signed by the owner — its own class", &rec,
|
||||
&store.GlossaryEntry{Src: c.Key, Dst: "Юаньши", Status: "approved"}, fp, basisSigned},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
basis := map[string]bankBasisRecord{}
|
||||
if tc.rec != nil {
|
||||
basis[c.Key] = *tc.rec
|
||||
}
|
||||
rows := map[string]store.GlossaryEntry{}
|
||||
if tc.row != nil {
|
||||
rows[c.Key] = *tc.row
|
||||
}
|
||||
got := basisVerdictFor(c, tc.fp, basis, rows, nil, "s")
|
||||
if got.Reason != tc.want {
|
||||
t.Fatalf("verdict = %q, want %q", got.Reason, tc.want)
|
||||
}
|
||||
if tc.want == basisServed && got.Answer.Dst != "юаньши" {
|
||||
t.Fatalf("a served candidate must carry the stored answer, got %+v — a basis that serves an EMPTY rendering is the silent regression this mechanism exists to avoid", got.Answer)
|
||||
}
|
||||
if tc.want != basisServed && got.Answer.Dst != "" {
|
||||
t.Fatalf("a re-asked candidate must carry NO answer, got %q: it would be stamped as consolidated without anyone consolidating it", got.Answer.Dst)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBasisServesAUnitOnlyWhole pins the family rule in both directions: one unsettled member holds its
|
||||
// unit, and a member of ANOTHER unit does not.
|
||||
func TestBasisServesAUnitOnlyWhole(t *testing.T) {
|
||||
kin := []terminology.Candidate{{Key: "一转"}, {Key: "三转"}, {Key: "春天"}}
|
||||
fps := map[string]bankBasisFP{}
|
||||
basis := map[string]bankBasisRecord{}
|
||||
rows := map[string]store.GlossaryEntry{}
|
||||
for _, c := range kin {
|
||||
fps[c.Key] = bankBasisFP{Whole: "fp-" + c.Key}
|
||||
rows[c.Key] = store.GlossaryEntry{Src: c.Key, Dst: "передача", Status: "draft", Source: "mined"}
|
||||
basis[c.Key] = bankBasisRecord{Key: c.Key, FP: fps[c.Key], Answer: bankBasisAnswer{Dst: "передача"}}
|
||||
}
|
||||
units := map[string]int{"一转": 7, "三转": 7, "春天": 9}
|
||||
if v := bankBasisSettle(kin, fps, basis, rows, nil, units, "s"); countServed(v) != 3 {
|
||||
t.Fatalf("premise broken: with everything settled all three must be served, got %d — the rows below would then pass for the wrong reason", countServed(v))
|
||||
}
|
||||
// Now one member of unit 7 loses its record.
|
||||
delete(basis, "一转")
|
||||
v := bankBasisSettle(kin, fps, basis, rows, nil, units, "s")
|
||||
if v["三转"].Reason != basisUnit {
|
||||
t.Fatalf("its unit-mate was NOT held: verdict %q — a family would be decided in halves, which is the chimera co-batching exists to prevent", v["三转"].Reason)
|
||||
}
|
||||
if v["一转"].Reason != basisNew {
|
||||
t.Fatalf("the member that actually lost its record must report its OWN reason, got %q", v["一转"].Reason)
|
||||
}
|
||||
if v["春天"].Reason != basisServed {
|
||||
t.Fatalf("a candidate in ANOTHER unit was held too (%q): the unit rule is not keyed on the unit at all", v["春天"].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBasisWidthIsPairDataWithAnEngineDefault answers the canon's standing review question — «would a pair
|
||||
// that is not in the repository work without a Go edit» — by execution rather than by assertion in a
|
||||
// report. It also pins that the width REACHES the hash: a pair datum nothing reads is a key that looks
|
||||
// like a decision and is none.
|
||||
func TestBasisWidthIsPairDataWithAnEngineDefault(t *testing.T) {
|
||||
r := &Runner{}
|
||||
if got := r.basisWidth(); got != terminologyDefaultBasisWidth {
|
||||
t.Fatalf("a book with no pack must fall through to the engine default, got %d", got)
|
||||
}
|
||||
narrow := candidateBasisFingerprint(basisCand(), basisSource, nil, 2)
|
||||
wide := candidateBasisFingerprint(basisCand(), basisSource, nil, 30)
|
||||
if narrow.Whole == wide.Whole {
|
||||
t.Fatal("the evidence width does not reach the fingerprint: the pair datum would be a key that reads as a decision and decides nothing")
|
||||
}
|
||||
// ⛔ AND THE WIDTH IS NAMED IN THE HASH ITSELF, not merely reflected through the windows it produced.
|
||||
// A planted mutation that dropped the width from the hash survived the two lines above, because for an
|
||||
// ATTESTED surface the windows differ by width anyway. They do not for a surface the source does not
|
||||
// contain: its window list is empty at every width, so the two fingerprints collide and the book keeps
|
||||
// serving a decision taken under a rule that is no longer the rule. The population is real — a
|
||||
// draft-invented term is exactly a candidate with no occurrence — and the cure costs one hashed field.
|
||||
absent := basisCand()
|
||||
absent.Key, absent.Src = "这个词不在原文里", "这个词不在原文里"
|
||||
if got := terminology.EvidenceWindows(basisSource, absent.Key, 30); len(got) != 0 {
|
||||
t.Fatalf("premise broken: the fixture key must NOT occur in the source, got %d window(s)", len(got))
|
||||
}
|
||||
if candidateBasisFingerprint(absent, basisSource, nil, 2).Whole ==
|
||||
candidateBasisFingerprint(absent, basisSource, nil, 30).Whole {
|
||||
t.Fatal("two different evidence widths produced ONE fingerprint for an unattested surface: the rule that decided it is not part of what is stored, so changing the rule leaves those rows silently settled under the old one")
|
||||
}
|
||||
}
|
||||
|
||||
// TestASignedRowDoesNotHoldItsFamilyOpen pins the owner-signature rule in both of its halves, and it exists
|
||||
// because the two halves are easy to confuse into one.
|
||||
//
|
||||
// ⚠ ITS PREMISE, NAMED IN THE TEST RATHER THAN ASSUMED BY IT: backlog row 447 is OPEN, so a signed surface
|
||||
// IS still sent to the paid role — the two-condition filter pays for banked surfaces on purpose, to keep
|
||||
// the dispute marks on the sheet the owner signs by. This pin does not assert that it is bought (that is
|
||||
// the filter's subject, not the basis's); it asserts that being bought and holding a family open have been
|
||||
// separated. If row 447 closes, the first half stops being true and the second half must still hold —
|
||||
// which is why they are two assertions and not one.
|
||||
func TestASignedRowDoesNotHoldItsFamilyOpen(t *testing.T) {
|
||||
fam := []terminology.Candidate{{Key: "古月山寨"}, {Key: "古月"}, {Key: "古月族长"}}
|
||||
units := map[string]int{"古月山寨": 3, "古月": 3, "古月族长": 3}
|
||||
fps := map[string]bankBasisFP{}
|
||||
basis := map[string]bankBasisRecord{}
|
||||
rows := map[string]store.GlossaryEntry{}
|
||||
for _, c := range fam {
|
||||
fps[c.Key] = bankBasisFP{Whole: "fp-" + c.Key}
|
||||
rows[c.Key] = store.GlossaryEntry{Src: c.Key, Dst: "решение движка", Status: "draft", Source: "mined"}
|
||||
basis[c.Key] = bankBasisRecord{Key: c.Key, FP: fps[c.Key],
|
||||
Answer: bankBasisAnswer{Dst: "решение движка"}}
|
||||
}
|
||||
// The owner signs ONE member, and the engine's memory of it is dropped: signing is not a decision the
|
||||
// basis recorded, so this is the state a real book reaches — exactly what both bought runs held.
|
||||
rows["古月山寨"] = store.GlossaryEntry{Src: "古月山寨", Dst: "горная крепость Гуюэ", Status: "approved"}
|
||||
delete(basis, "古月山寨")
|
||||
|
||||
v := bankBasisSettle(fam, fps, basis, rows, nil, units, "s")
|
||||
if v["古月山寨"].Reason != basisSigned {
|
||||
t.Fatalf("the signed row is in class %q, want %q — the owner's signature must be its own class, not «unsettled»", v["古月山寨"].Reason, basisSigned)
|
||||
}
|
||||
// HALF ONE: the owner's word is never handed back as if the engine had decided it.
|
||||
if got := v["古月山寨"].Answer; got.Dst != "" {
|
||||
t.Fatalf("the basis handed back %q for a SIGNED row: the owner's canon would be republished as an engine decision, and the two words the bank ontology keeps apart would have merged", got.Dst)
|
||||
}
|
||||
// HALF TWO: the rest of the family is served WHOLE, which is the whole point of the exemption.
|
||||
for _, k := range []string{"古月", "古月族长"} {
|
||||
if v[k].Reason != basisServed {
|
||||
t.Fatalf("%q was re-asked (%q) although the only member missing from the basis is one the OWNER SIGNED — his signature would re-buy the family it belongs to", k, v[k].Reason)
|
||||
}
|
||||
if v[k].Answer.Dst != "решение движка" {
|
||||
t.Fatalf("%q was served with %q instead of its stored answer", k, v[k].Answer.Dst)
|
||||
}
|
||||
}
|
||||
// The control that keeps the exemption from being a hole: an UNSIGNED member missing from the basis
|
||||
// still holds the family. Without this row the test passes on a predicate with no unit rule at all.
|
||||
delete(basis, "古月")
|
||||
rows["古月"] = store.GlossaryEntry{Src: "古月", Dst: "решение движка", Status: "draft", Source: "mined"}
|
||||
v2 := bankBasisSettle(fam, fps, basis, rows, nil, units, "s")
|
||||
if v2["古月族长"].Reason != basisUnit {
|
||||
t.Fatalf("control: an UNSIGNED member missing from the basis left its family served (%q) — the exemption has swallowed the unit rule", v2["古月族长"].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBasisRowsToStoreRecordsOnlyDecidedRows pins the pipeline half of the same guarantee the store keeps
|
||||
// at its write: a term the role declined, never answered, or has no fingerprint for is NOT recorded.
|
||||
//
|
||||
// ⚠ TWO GUARDS FOR ONE PROPERTY IS DELIBERATE, and each is pinned separately rather than one standing for
|
||||
// both. The store's refusal is the one that cannot be forgotten by a future caller; this one keeps the
|
||||
// decision where the reason lives — «undecided» and «decided» are a pipeline distinction, and a row
|
||||
// dropped here never has to be explained to a layer that does not know what a role is.
|
||||
func TestBasisRowsToStoreRecordsOnlyDecidedRows(t *testing.T) {
|
||||
cands := []terminology.Candidate{{Key: "元石"}, {Key: "青茅山"}, {Key: "方源"}, {Key: "无指纹"}}
|
||||
consolidated := map[string]string{
|
||||
"元石": "юаньши",
|
||||
"青茅山": "", // the role declined it
|
||||
"无指纹": "передача", // answered, but no fingerprint was computed for it
|
||||
// 方源 is absent entirely: no reply line covered it
|
||||
}
|
||||
fps := map[string]bankBasisFP{
|
||||
"元石": {Whole: "fp1", Canon: "c1"}, "青茅山": {Whole: "fp2"}, "方源": {Whole: "fp3"},
|
||||
}
|
||||
rows := basisRowsToStore(cands, consolidated, map[string]string{"元石": "name"}, map[string]string{"元石": "male"}, fps, nil, nil, basisShape(40, "p"))
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("recorded %d row(s), want exactly the one that was DECIDED: %+v", len(rows), rows)
|
||||
}
|
||||
got := rows[0]
|
||||
if got.SrcKey != "元石" || got.Dst != "юаньши" || got.Type != "name" || got.Gender != "male" {
|
||||
t.Fatalf("the decided row lost part of its answer: %+v", got)
|
||||
}
|
||||
// ⚠ The SHAPE, not the bare version: since D1 the stored string carries the effective evidence width,
|
||||
// so that a width change reports its own cause instead of «the source moved».
|
||||
if got.FPVersion != basisShape(40, "p") || got.FPWhole != "fp1" || got.FPCanon != "c1" {
|
||||
t.Fatalf("the row carries no comparable fingerprint, so a later purchase could not tell whether the evidence moved: %+v", got)
|
||||
}
|
||||
}
|
||||
|
|
@ -184,9 +184,10 @@ func TestAClassifierCutIsNotAnIncompleteBank(t *testing.T) {
|
|||
func TestEveryConsolidationFieldIsCarried(t *testing.T) {
|
||||
cut := projectBankConsolidation(&terminologyResult{
|
||||
BatchesDropped: 1, ClassifyBatchesDropped: 2, Consolidated: 3, Declined: 4, Unanswered: 5, BankSettled: 6,
|
||||
BasisServed: 7,
|
||||
})
|
||||
whole := projectBankConsolidation(&terminologyResult{
|
||||
ClassifyBatchesDropped: 2, Consolidated: 3, Declined: 4, Unanswered: 5, BankSettled: 6,
|
||||
ClassifyBatchesDropped: 2, Consolidated: 3, Declined: 4, Unanswered: 5, BankSettled: 6, BasisServed: 7,
|
||||
})
|
||||
a, b := reflect.ValueOf(*cut), reflect.ValueOf(*whole)
|
||||
for i := range a.NumField() {
|
||||
|
|
|
|||
|
|
@ -103,6 +103,16 @@ type BankConsolidation struct {
|
|||
// NeverAsked is the opposite of a gap and must not be read as one: these candidates never reached the
|
||||
// paid role because the bank already renders the surface and every draft agreed with it.
|
||||
NeverAsked int `json:"never_asked"`
|
||||
// SettledEarlier is the OTHER way a candidate skips the paid role: this book decided its rendering in
|
||||
// an earlier purchase and the evidence has not moved since, so the settled basis answered it.
|
||||
//
|
||||
// ⛔ AN ADDITIVE FIELD RATHER THAN A WIDER `never_asked`, and the addition is the whole point. That
|
||||
// field's published meaning is «the seed holds the surface and every draft agreed» — a sentence a
|
||||
// consumer is entitled to act on — and both of its halves are FALSE for a row the basis answered. Adding
|
||||
// these rows to it would change what a shipped number means without changing its name, which is the
|
||||
// shape this engine refuses everywhere else. Consumers that do not know the new field keep reading the
|
||||
// old one correctly; that is what makes the change a minor one.
|
||||
SettledEarlier int `json:"settled_earlier"`
|
||||
}
|
||||
|
||||
// projectBankConsolidation turns the pass's counters into the section. NIL IN, NIL OUT, and that is the
|
||||
|
|
@ -115,7 +125,7 @@ func projectBankConsolidation(t *terminologyResult) *BankConsolidation {
|
|||
Complete: t.BatchesDropped == 0,
|
||||
RenderBatchesDropped: t.BatchesDropped, ClassifyBatchesDropped: t.ClassifyBatchesDropped,
|
||||
Consolidated: t.Consolidated, Declined: t.Declined, Unanswered: t.Unanswered,
|
||||
NeverAsked: t.BankSettled,
|
||||
NeverAsked: t.BankSettled, SettledEarlier: t.BasisServed,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -147,7 +157,15 @@ type BankExportProposal struct {
|
|||
// and «the role said 0 %» are different rows on a signing sheet.
|
||||
Conf int `json:"conf"`
|
||||
// Invented says the rendering is not one the drafts proposed — legitimate, and the class to read first.
|
||||
//
|
||||
// ⚠ It is NOT set on a row the basis settled, and the absence there is a decision: the flag is a finding
|
||||
// about THIS run's drafts, and a rendering carried over from an earlier purchase was never offered to
|
||||
// them. Read `settled_earlier` beside it or the missing flag reads as «the drafts proposed this».
|
||||
Invented bool `json:"invented,omitempty"`
|
||||
// SettledEarlier says this book decided the rendering in an earlier purchase and the roles were not paid
|
||||
// to decide it again. It is NOT the same as the summary's `never_asked`, whose sentence — «the bank
|
||||
// renders the surface and every draft agreed» — is false of this row in both halves.
|
||||
SettledEarlier bool `json:"settled_earlier,omitempty"`
|
||||
// Contradicts names this run's OTHER consolidations the rendering breaks; BankHolds names rows the bank
|
||||
// already carries for the same firing surface with a different rendering. Two fields because "the run
|
||||
// disagreed with itself" and "the book already calls it something else" are different decisions.
|
||||
|
|
@ -248,7 +266,7 @@ func projectBankProposals(rows []BankStopRow) []BankExportProposal {
|
|||
p := BankExportProposal{
|
||||
Src: row.Src, Dst: row.Dst, Kind: row.Type, Channel: row.Origin,
|
||||
Freq: row.Freq, Spread: row.Spread, Conventions: row.Conventions, Conf: row.Conf, Invented: row.Invented,
|
||||
Contradicts: row.Contradicts, BankHolds: row.BankHolds,
|
||||
Contradicts: row.Contradicts, BankHolds: row.BankHolds, SettledEarlier: row.SettledByBasis,
|
||||
}
|
||||
for _, v := range row.Variants {
|
||||
p.Variants = append(p.Variants, v.Label())
|
||||
|
|
|
|||
|
|
@ -70,7 +70,17 @@ func TestEveryDecidingFieldSurvivesTheProjection(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
got := projectBankProposals([]BankStopRow{decided, undecided})
|
||||
// The row the SETTLED BASIS answered: a real rendering, carried over from an earlier purchase. It is a
|
||||
// third state rather than a flag on one of the two above, because the combination is production's own:
|
||||
// `Invented` is deliberately NOT computed for such a row (it is a finding about THIS run's drafts, which
|
||||
// were never offered the term), so a fixture that set both would encode a state the engine cannot reach.
|
||||
settledEarlier := BankStopRow{
|
||||
Src: "元石", Dst: "юаньши", Type: "name", Origin: "mined",
|
||||
Freq: 9, Spread: 1, Conf: -1, SettledByBasis: true,
|
||||
Variants: []BankStopVariant{{Dst: "юаньши", Chunks: 2}},
|
||||
}
|
||||
|
||||
got := projectBankProposals([]BankStopRow{decided, undecided, settledEarlier})
|
||||
want := []BankExportProposal{
|
||||
{
|
||||
Src: "方源", Dst: "Фан-Юань", Kind: "name", Channel: "banknote",
|
||||
|
|
@ -87,6 +97,11 @@ func TestEveryDecidingFieldSurvivesTheProjection(t *testing.T) {
|
|||
Freq: 5, Spread: 2, Conf: -1,
|
||||
Variants: []string{"гора Цинмао ×3"},
|
||||
},
|
||||
{
|
||||
Src: "元石", Dst: "юаньши", Kind: "name", Channel: "mined",
|
||||
Freq: 9, Spread: 1, Conf: -1, SettledEarlier: true,
|
||||
Variants: []string{"юаньши ×2"},
|
||||
},
|
||||
}
|
||||
// Compared as a WHOLE SLICE, not row by row found by Src: the ranking is the information, so the
|
||||
// position each row holds is part of what is being asserted.
|
||||
|
|
@ -104,11 +119,27 @@ func TestEveryDecidingFieldSurvivesTheProjection(t *testing.T) {
|
|||
// ⚠ Its limit, named: this catches a field the PROPOSAL grew and the fold ignored. A field BankStopRow
|
||||
// grows that ought to be projected and is not cannot be caught mechanically — whether a new fact is
|
||||
// one an owner signs on is a judgement, and it belongs to whoever adds the field.
|
||||
v := reflect.ValueOf(got[0])
|
||||
for i := range v.NumField() {
|
||||
if v.Field(i).IsZero() {
|
||||
t.Errorf("field %s is zero in the projection of a row where every projected field was set — the fold does not carry it",
|
||||
v.Type().Field(i).Name)
|
||||
//
|
||||
// ⚠ ACROSS THE ROWS AND NOT WITHIN ONE, for the same reason its sibling over the consolidation section
|
||||
// gives: some pairs of fields cannot both be set on one row. `Invented` and `SettledEarlier` are such a
|
||||
// pair — the engine refuses to compute the first for a row the basis answered — so a single all-fields
|
||||
// fixture would have to encode an unreachable state or carry an exclusion list, and an exclusion list is
|
||||
// the thing that goes stale in silence. Requiring each field in AT LEAST ONE row needs no list.
|
||||
rows := make([]reflect.Value, 0, len(got))
|
||||
for _, p := range got {
|
||||
rows = append(rows, reflect.ValueOf(p))
|
||||
}
|
||||
for i := range rows[0].NumField() {
|
||||
carried := false
|
||||
for _, v := range rows {
|
||||
if !v.Field(i).IsZero() {
|
||||
carried = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !carried {
|
||||
t.Errorf("field %s is zero in EVERY projected row, including the ones built to set it — the fold does not carry it",
|
||||
rows[0].Type().Field(i).Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -84,31 +84,99 @@ func TestTheRunTotalIsThisRunsSpendIncludingTheClassifier(t *testing.T) {
|
|||
}
|
||||
_ = r1.Close()
|
||||
|
||||
// --- leg 2: the resume pays nothing, and the total has to say nothing ---
|
||||
r2 := newRunner(t, bookPath)
|
||||
defer func() { _ = r2.Close() }()
|
||||
res2, err := r2.TranslateBook(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}))
|
||||
}
|
||||
|
||||
// TestAReplayedBankContourIsFreeWhileItsCumulativeFigureRemembers is the second half of the run total's
|
||||
// contract — «the total reports what THIS run spent» — and it lives in its own fixture rather than as a
|
||||
// second leg of the test above.
|
||||
//
|
||||
// ⛔ WHY IT WAS SPLIT OUT, because a split is the kind of change that hides a weakening. The test above
|
||||
// needs a render budget that DROPS a batch: that asymmetry against the classifier's own budget is the only
|
||||
// lever its fixture has, since the fake model bills a fixed price per call and two passes making the same
|
||||
// number of calls cost the same to the cent. This test needs the opposite — a book with enough budget left
|
||||
// that a re-asked batch can be bought and then REPLAYED. With the settled basis in the tree the two
|
||||
// requirements became arithmetically incompatible on one fixture: admitting two of three batches costs the
|
||||
// whole ceiling, so the run that meets the basis's one-time re-pack can buy nothing, and a contour that
|
||||
// buys nothing has nothing to replay either. Both assertions survive; each now stands on a fixture that can
|
||||
// actually produce the state it is about, and each states its own premise.
|
||||
func TestAReplayedBankContourIsFreeWhileItsCumulativeFigureRemembers(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isTerminologyBody(body) {
|
||||
// ONE of the three batches' terms is answered; the rest stay unanswered on every run, so there
|
||||
// is always a batch the basis cannot serve and the contour keeps something to replay.
|
||||
return "方源\tФан Юань\n青茅山\tгора Цинмао", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл к горе Цинмао.\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{
|
||||
terminology: true, classify: true, batchRunes: 1, budgetUSD: 0.008,
|
||||
})
|
||||
ctx := func() context.Context {
|
||||
return obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||||
}
|
||||
|
||||
// Run 1 buys the pass. Run 2 meets the basis run 1 wrote, re-packs what is left and buys that ONCE.
|
||||
// Run 3 is the subject: the composition is stable, so the contour replays for $0 while its cumulative
|
||||
// figure still remembers what run 2 paid.
|
||||
r1 := newRunner(t, bookPath)
|
||||
if _, err := r1.TranslateBook(ctx()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r1.lastTerminology == nil || r1.lastTerminology.CostUSD <= 0 {
|
||||
t.Fatalf("premise broken: run 1 must BUY the pass, or a free replay later means nothing: %+v", r1.lastTerminology)
|
||||
}
|
||||
committedAfter1, _, err := r1.Store.SpentUSD("test-book")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r2.lastTerminology == nil {
|
||||
t.Fatal("premise broken: the resumed run must still run the pass (replaying it), or leg 2 is vacuous")
|
||||
_ = r1.Close()
|
||||
|
||||
r2 := newRunner(t, bookPath)
|
||||
if _, err := r2.TranslateBook(ctx()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The premise that makes the leg non-degenerate: this run's spend and the contour's cumulative spend
|
||||
// must now be DIFFERENT numbers. On leg 1 they are equal and the distinction cannot be tested.
|
||||
fresh, cumulative := r2.lastTerminology.CostUSD, r2.lastTerminology.CumUSD
|
||||
if r2.lastTerminology == nil || r2.lastTerminology.BasisServed == 0 {
|
||||
t.Fatalf("premise broken: run 2 must be served by the basis, or the re-pack has another cause: %+v", r2.lastTerminology)
|
||||
}
|
||||
if r2.lastTerminology.CostUSD <= 0 {
|
||||
t.Fatalf("premise broken: run 2 must pay the one-time re-pack, or run 3 has nothing to replay: fresh=$%.9f", r2.lastTerminology.CostUSD)
|
||||
}
|
||||
committedAfter2, _, err := r2.Store.SpentUSD("test-book")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = r2.Close()
|
||||
|
||||
r3 := newRunner(t, bookPath)
|
||||
defer func() { _ = r3.Close() }()
|
||||
res3, err := r3.TranslateBook(ctx())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r3.lastTerminology == nil {
|
||||
t.Fatal("premise broken: the resumed run must still run the pass (replaying it), or this test is vacuous")
|
||||
}
|
||||
fresh, cumulative := r3.lastTerminology.CostUSD, r3.lastTerminology.CumUSD
|
||||
if fresh != 0 || cumulative <= 0 {
|
||||
t.Fatalf("premise broken: a resume must replay the contour for $0 while the cumulative figure keeps "+
|
||||
"what it cost then: fresh=$%.9f cumulative=$%.9f", fresh, cumulative)
|
||||
}
|
||||
committed2, _, err := r2.Store.SpentUSD("test-book")
|
||||
committedAfter3, _, err := r3.Store.SpentUSD("test-book")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d := res2.TotalUSD - (committed2 - committed1); d > 1e-9 || d < -1e-9 {
|
||||
if d := res3.TotalUSD - (committedAfter3 - committedAfter2); d > 1e-9 || d < -1e-9 {
|
||||
t.Fatalf("the total reports what THIS run spent, not what the book has cost: TOTAL=$%.9f, this run "+
|
||||
"added $%.9f to the ledger (%.9f → %.9f); the contour's cumulative figure is $%.9f and is NOT it",
|
||||
res2.TotalUSD, committed2-committed1, committed1, committed2, cumulative)
|
||||
res3.TotalUSD, committedAfter3-committedAfter2, committedAfter2, committedAfter3, cumulative)
|
||||
}
|
||||
// The denominator that keeps the zero above readable: the book DID cost something over the three runs,
|
||||
// so «this run added nothing» is a statement about the run and not about an idle fixture.
|
||||
if committedAfter3 <= committedAfter1 {
|
||||
t.Fatalf("the ledger never grew past run 1 ($%.9f → $%.9f): nothing was ever re-packed, and the free replay is free for the wrong reason",
|
||||
committedAfter1, committedAfter3)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -234,24 +234,58 @@ func TestAResumeFindsItsCheckpointsAfterTheFilter(t *testing.T) {
|
|||
if _, err := r2.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatalf("the resumed run: %v", err)
|
||||
}
|
||||
// ⚠ THE ONE-TIME-COST WARNING MUST NOT FIRE HERE, and this assertion exists because the first version
|
||||
// of it did. It was emitted before the pass, on "this book has paid before" — which is true of every
|
||||
// ordinary resume of a filtered book — so on a live run it announced a re-purchase while the pass made
|
||||
// zero calls. An operator message that cries money on a $0 resume is the same defect this pack closes.
|
||||
if strings.Contains(log.String(), logKeyReconsolidated+"=true") {
|
||||
t.Errorf("the migration warning fired on a resume that re-bought nothing:\n%s", log.String())
|
||||
// ⛔ RUN 2 IS THE ONE-TIME RE-PACK, AND IT IS THE PACK'S ORDERED PRICE — NOT A REGRESSION.
|
||||
//
|
||||
// Until the settled basis existed, run 2 was $0: the filter is unconditional, so the composition was a
|
||||
// pure function of the candidates and the bank, identical on every run, and every checkpoint replayed.
|
||||
// The basis adds a THIRD input, and that input goes from empty to full at run 1's output boundary. So
|
||||
// run 2 sees a composition run 1 never had, finds no checkpoint for it, and buys the pass ONCE. From
|
||||
// run 3 on the basis is stable and every run replays for $0 — which is the half this test now pins, and
|
||||
// the half the old two-run shape could not express at all.
|
||||
//
|
||||
// ⚠ THE CURE THAT LOOKS OBVIOUS IS THE BUG THIS FILE ALREADY DOCUMENTS: «stand down for a book that has
|
||||
// already paid» makes the rule decide differently on the second run, and a rule that decides differently
|
||||
// on the second run is not a resume rule (see dropBankSettled). The one-time cost is bounded by
|
||||
// gates.terminology.budget_usd and ANNOUNCED — the reconsolidation warning below is that announcement,
|
||||
// and run 2 is exactly the case its sentence describes.
|
||||
if !strings.Contains(log.String(), logKeyReconsolidated+"=true") {
|
||||
t.Errorf("run 2 re-bought the pass and said nothing: the one-time cost must announce itself, or an operator meets it as an unexplained charge\n%s", log.String())
|
||||
}
|
||||
if r2.lastTerminology.BankSettled != 1 {
|
||||
t.Errorf("the filter must decide the same way on every run, or the composition moves: skipped=%d", r2.lastTerminology.BankSettled)
|
||||
}
|
||||
if r2.lastTerminology.CostUSD != 0 {
|
||||
t.Errorf("the resumed pass must replay from its checkpoints for $0, got $%f", r2.lastTerminology.CostUSD)
|
||||
if r2.lastTerminology.BasisServed == 0 {
|
||||
t.Errorf("premise broken: run 2 must be served by the basis run 1 wrote, or the re-pack below has another cause entirely")
|
||||
}
|
||||
// Counted on the BANK ROLE's own calls: the second run still runs the edit wave the stop held back, so
|
||||
// a count of all provider calls would move for a reason that has nothing to do with this filter.
|
||||
if n := countTerminologyCalls(rec); n != terminologyCallsAfterFirst {
|
||||
t.Errorf("the resume must reach the provider zero times for the bank role: role calls went %d → %d",
|
||||
terminologyCallsAfterFirst, n)
|
||||
callsAfterSecond := countTerminologyCalls(rec)
|
||||
// The re-pack is a PURCHASE, and it is asserted rather than left to the cost field: «run 2 paid» and
|
||||
// «run 2 called the provider» are different facts, and the second is what makes run 3's zero below mean
|
||||
// «replayed» instead of «never ran».
|
||||
if callsAfterSecond <= terminologyCallsAfterFirst {
|
||||
t.Errorf("run 2 announced a re-purchase but reached the provider %d → %d times: either the announcement is false or the pass never ran",
|
||||
terminologyCallsAfterFirst, callsAfterSecond)
|
||||
}
|
||||
r2.Close()
|
||||
|
||||
// --- run 3: the basis is stable now, so the composition is, so every checkpoint replays ---
|
||||
r3 := newRunner(t, bookPath)
|
||||
defer r3.Close()
|
||||
var log3 bytes.Buffer
|
||||
r3.Log = slog.New(slog.NewTextHandler(&log3, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
if _, err := r3.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatalf("the third run: %v", err)
|
||||
}
|
||||
if r3.lastTerminology.CostUSD != 0 {
|
||||
t.Errorf("run 3 must replay from its checkpoints for $0, got $%f — the basis is not stable between runs and EVERY purchase would re-buy the pass", r3.lastTerminology.CostUSD)
|
||||
}
|
||||
// Counted on the BANK ROLE's own calls: the run still walks the edit wave, so a count of all provider
|
||||
// calls would move for a reason that has nothing to do with this contour.
|
||||
if n := countTerminologyCalls(rec); n != callsAfterSecond {
|
||||
t.Errorf("run 3 must reach the provider zero times for the bank role: role calls went %d → %d",
|
||||
callsAfterSecond, n)
|
||||
}
|
||||
if strings.Contains(log3.String(), logKeyReconsolidated+"=true") {
|
||||
t.Errorf("the one-time-cost warning fired on a run that re-bought nothing — it would then cry money on every ordinary resume, which is the defect this message was rewritten to remove:\n%s", log3.String())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -159,6 +159,13 @@ func parseStopFields(r *BankStopRow, body string) error {
|
|||
r.Invented = true
|
||||
rest = strings.Replace(rest, " INVENTED(no draft proposed it)", "", 1)
|
||||
}
|
||||
// ⚠ SETTLED EARLIER is read BEFORE NOT ASKED, and the order is load-bearing: both labels are trailing
|
||||
// clauses, so cutting at the first one found would swallow the other and the round trip would lose a
|
||||
// state the sheet shows. They are different facts and a reader must be able to tell them apart.
|
||||
if i := strings.Index(rest, " SETTLED EARLIER("); i >= 0 {
|
||||
r.SettledByBasis = true
|
||||
rest = rest[:i]
|
||||
}
|
||||
if i := strings.Index(rest, " NOT ASKED("); i >= 0 {
|
||||
r.SettledByBank = true
|
||||
rest = rest[:i]
|
||||
|
|
|
|||
|
|
@ -256,6 +256,10 @@ func (r *Runner) translateBook(ctx context.Context) (*BookResult, error) {
|
|||
// they do not cover: a run that finished, whose last bank change was the auto-continue re-seed or a
|
||||
// signed mined-delta loaded at start.
|
||||
r.exportBank(ctx, "run-finished")
|
||||
// The settled basis at the run's OTHER output boundary. Two are needed and neither is redundant:
|
||||
// this one is under `err == nil`, and the signature stop returns as an error VALUE, so a run that
|
||||
// stops for the owner's signature never arrives here — its basis is written at the stop itself.
|
||||
r.writeBankBasis(ctx, "run-finished")
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -268,6 +268,13 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
|
|||
// The signature stop is the boundary the signing screen reads at (row 125): refresh the bank read-out
|
||||
// so the state behind the decisions is the state as of this stop, not as of the last run.
|
||||
r.exportBank(ctx, "bank-mining/signature-stop")
|
||||
// THE SETTLED BASIS, and it goes BEFORE the flag memory rather than last, which is a departure from the
|
||||
// consilium's letter taken on the authority of the rule the next comment states. The order between
|
||||
// carriers is decided by WHAT A CRASH BETWEEN THEM COSTS, and the two costs are not comparable: losing
|
||||
// the flag memory costs one benign extra stop, losing the basis costs a re-purchase of both bank roles
|
||||
// out of a LIFETIME per-book ceiling whose exhaustion is permanent (backlog row 356). The cheaper loss
|
||||
// goes last.
|
||||
r.writeBankBasis(ctx, "bank-mining/signature-stop")
|
||||
// THE MEMORY, and strictly after the map: a crash between the two costs one benign re-stop, while
|
||||
// the reverse order can mark surfaces presented whose map never reached disk — a silent loss with no
|
||||
// retry. The same asymmetry decides the failure branch: the stop STANDS when the memory write fails
|
||||
|
|
@ -413,6 +420,16 @@ type BankStopRow struct {
|
|||
// «undecided». Rendered by the text table only: the sidecar's proposal section is contract-pinned
|
||||
// (backlog row 353) and is deliberately not touched.
|
||||
SettledByBank bool
|
||||
// SettledByBasis marks a row whose rendering came from the SETTLED BASIS — this book decided it in an
|
||||
// earlier purchase and the evidence has not moved, so the roles were not paid to decide it again.
|
||||
//
|
||||
// ⛔ IT IS NOT SettledByBank AND THE TWO MUST NOT BE MERGED, however alike the names read. That one
|
||||
// means «the bank already renders this surface and every draft agreed» — a statement about the SEED and
|
||||
// about what the drafts proposed THIS run. For a basis row both halves are FALSE: the bank holds no
|
||||
// seed surface for it, and the drafts were never asked to agree with anything. A sheet that printed one
|
||||
// sentence for both would tell the owner the bank renders a surface it does not, which is exactly the
|
||||
// class the engine refuses elsewhere — a message lying about its own cause.
|
||||
SettledByBasis bool
|
||||
}
|
||||
|
||||
// BankStopVariant is ONE rendering the drafts produced, kept in its PARTS rather than as the sentence a
|
||||
|
|
@ -492,14 +509,20 @@ func bankStopRows(cands []terminology.Candidate, consolidated map[string]string,
|
|||
Freq: c.Freq, Spread: c.Spread(), Conventions: c.Conventions(),
|
||||
Contexts: c.KWIC, Evidence: c.Evidence,
|
||||
}
|
||||
row.Conf, row.Contradicts, row.BankHolds, row.SettledByBank = findings.findingsFor(c)
|
||||
row.Conf, row.Contradicts, row.BankHolds, row.SettledByBank, row.SettledByBasis = findings.findingsFor(c)
|
||||
for i, v := range c.Variants {
|
||||
row.Variants = append(row.Variants, BankStopVariant{Dst: v.Dst, Chunks: v.Chunks, Via: v.Via})
|
||||
if i == 0 {
|
||||
row.Signals = v.Signals
|
||||
}
|
||||
}
|
||||
row.Invented = dst != "" && !proposedByDrafts(dst, c.Variants)
|
||||
// ⛔ NOT COMPUTED FOR A BASIS ROW, because the predicate behind it is false there by construction.
|
||||
// `Invented` means «the engine produced a rendering no draft proposed», and it is a finding about THIS
|
||||
// run's drafts. A rendering carried over from an earlier purchase was of course not proposed by this
|
||||
// run's drafts — it was never offered to them — so the flag would fire on every settled row and print
|
||||
// «INVENTED (no draft proposed it)» against the book's own established canon, on the sheet the owner
|
||||
// signs by. A false accusation is worse than a missing flag.
|
||||
row.Invented = dst != "" && !row.SettledByBasis && !proposedByDrafts(dst, c.Variants)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out
|
||||
|
|
@ -520,7 +543,7 @@ func bankStopRows(cands []terminology.Candidate, consolidated map[string]string,
|
|||
// surface the role was asked about (candKeys), and the surface ConsolidationConflicts compares on. A
|
||||
// banknote candidate's raw Src is a different string and its firing key may be an alias, so either of
|
||||
// those would lose the whole banknote population silently.
|
||||
func (t terminologyResult) findingsFor(c terminology.Candidate) (conf int, contradicts, bankHolds []string, settled bool) {
|
||||
func (t terminologyResult) findingsFor(c terminology.Candidate) (conf int, contradicts, bankHolds []string, settled, fromBasis bool) {
|
||||
// The role's stated confidence, or -1 when the reply carried none. A plain zero would merge the two,
|
||||
// and they are opposites: one is the first row to review, the other is silence.
|
||||
conf = -1
|
||||
|
|
@ -542,7 +565,12 @@ func (t terminologyResult) findingsFor(c terminology.Candidate) (conf int, contr
|
|||
settled = true
|
||||
}
|
||||
}
|
||||
return conf, contradicts, bankHolds, settled
|
||||
for _, k := range t.BasisServedKeys {
|
||||
if k == c.Key {
|
||||
fromBasis = true
|
||||
}
|
||||
}
|
||||
return conf, contradicts, bankHolds, settled, fromBasis
|
||||
}
|
||||
|
||||
// proposedByDrafts reports whether the consolidated rendering is one the drafts actually produced, compared
|
||||
|
|
@ -578,6 +606,9 @@ func renderBankStopTable(rows []BankStopRow) string {
|
|||
if r.SettledByBank {
|
||||
b.WriteString(" NOT ASKED(the bank already renders this surface and every draft agreed — nothing to decide)")
|
||||
}
|
||||
if r.SettledByBasis {
|
||||
b.WriteString(" SETTLED EARLIER(this book decided this rendering in an earlier purchase and the source around the term has not moved since, so the roles were not paid to decide it again — it is still unsigned and you may replace it)")
|
||||
}
|
||||
b.WriteString("\n")
|
||||
if len(r.Signals) > 0 {
|
||||
fmt.Fprintf(&b, " why: %s\n", strings.Join(r.Signals, ", "))
|
||||
|
|
|
|||
|
|
@ -1411,9 +1411,31 @@ func TestAutoWireIsDeterministicAndDoesNotMoveTheDraftWave(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls1 := rec.count()
|
||||
callsAfterFirst := rec.count()
|
||||
r1.Close()
|
||||
|
||||
// ⛔ THE SETTLED BASIS COSTS ONE RE-PACK, ONCE, AND THE $0 CLAIM BELONGS TO THE RUN AFTER IT. Run 1
|
||||
// writes the basis at its output boundary, so run 2 meets a composition run 1 never had and buys the
|
||||
// bank pass once (bounded by gates.terminology.budget_usd and announced by the reconsolidation
|
||||
// warning). From run 3 on the third input is stable and the whole run replays for $0 — which is what
|
||||
// «a resumed auto-mode run must be $0» was always about, and what this test now asserts of a run that
|
||||
// can actually deliver it.
|
||||
rMid := newRunner(t, bookPath)
|
||||
if _, err := rMid.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatalf("the basis re-pack run must complete cleanly, got %v", err)
|
||||
}
|
||||
if rMid.lastTerminology == nil || rMid.lastTerminology.BasisServed == 0 {
|
||||
t.Fatalf("premise broken: the re-pack run must be served by the basis run 1 wrote, or the $0 claim below is about something else: %+v", rMid.lastTerminology)
|
||||
}
|
||||
// The premise, asserted rather than assumed: the re-pack run really did buy something. Without this the
|
||||
// «$0 from here on» below could be true because the contour never ran at all, and the two are the same
|
||||
// zero on the page.
|
||||
calls1 := rec.count()
|
||||
if calls1 <= callsAfterFirst {
|
||||
t.Fatalf("premise broken: the basis re-pack bought nothing (provider calls %d → %d), so the $0 claim below is about a contour that never ran", callsAfterFirst, calls1)
|
||||
}
|
||||
rMid.Close()
|
||||
|
||||
r2 := newRunner(t, bookPath)
|
||||
defer r2.Close()
|
||||
res2, err := r2.TranslateBook(context.Background())
|
||||
|
|
|
|||
|
|
@ -60,6 +60,15 @@ func (r *Runner) seedGlossary(ctx context.Context) error {
|
|||
r.Log.WarnContext(ctx, "voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)",
|
||||
"book", r.Book.BookID, "gaps", strings.Join(gaps, "; "))
|
||||
}
|
||||
// The same lint on the TERM side, and it had no guard at all until now (backlog row 490). The loader
|
||||
// refuses an OVERLAP of windows; a HOLE between them — «until chapter 3» followed by «since chapter 5» —
|
||||
// is not a schema violation and passes everything, leaving chapter 4 with no law for that term inside a
|
||||
// book whose whole point is one word per term. Logged and never fatal, exactly like its voice sibling:
|
||||
// a deliberate gap is a real editorial shape, and refusing the load would turn one into a broken book.
|
||||
if gaps := membank.TermWindowGaps(rows); len(gaps) > 0 {
|
||||
r.Log.WarnContext(ctx, "bank term windows leave chapters uncovered — those chapters have NO law for the term and their drafts will render it however they like (deliberate is fine; a typo is not)",
|
||||
"book", r.Book.BookID, "gaps", strings.Join(gaps, "; "))
|
||||
}
|
||||
r.materializeBanks(rows, storedVoices, storedPairs)
|
||||
if cols := membank.InjectivityCollisions(rows); len(cols) > 0 {
|
||||
r.Log.WarnContext(ctx, "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)",
|
||||
|
|
|
|||
|
|
@ -70,6 +70,14 @@ type StopContest struct {
|
|||
// ⚠ A row the bank SETTLED (never asked) is never contested: there is no answer to doubt. Reading it as
|
||||
// contested would put the cheapest rows — the ones a seed already decided — at the head of the queue for a
|
||||
// second paid opinion.
|
||||
//
|
||||
// ⛔ A ROW THE BASIS SETTLED IS A DIFFERENT CASE AND IS NOT EXEMPT, and that is DECIDED here rather than
|
||||
// inherited from the name next to it. SettledByBank means the bank renders the surface and every draft
|
||||
// agreed — there is genuinely nothing to doubt. A basis row carries a real rendering, chosen by a paid
|
||||
// role in an earlier purchase, and neither half of the other sentence is true of it: the bank holds no
|
||||
// seed surface for it and the drafts were never asked. So it is judged on the same signals as any other
|
||||
// answered row — the doubt it deserves is the doubt its own rendering deserves. Exempting it by the
|
||||
// resemblance of the two field names would silence the queue for most of a settled book.
|
||||
func (r BankStopRow) Contest(o StopContestOpts) StopContest {
|
||||
var out StopContest
|
||||
if r.SettledByBank {
|
||||
|
|
|
|||
|
|
@ -140,6 +140,23 @@ type terminologyResult struct {
|
|||
// asked about instead of printing an empty rendering — which reads as «undecided» beside three other
|
||||
// facts that share the glyph.
|
||||
BankSettledKeys []string
|
||||
// BasisServed is how many candidates the SETTLED BASIS answered — surfaces this book had already
|
||||
// decided, whose answer is handed back instead of bought again. It is a DIFFERENT fact from
|
||||
// BankSettled beside it: that one means «the bank renders this surface and every draft agreed», this
|
||||
// one means «the engine decided this and the evidence has not moved since». Both skip the paid role
|
||||
// and the two must never be summed into one number, because the sheet says opposite things about them.
|
||||
BasisServed int
|
||||
BasisServedKeys []string
|
||||
// BasisReAsked counts, by class, the candidates the basis did NOT answer — new surfaces, a signed
|
||||
// relative, moved evidence, an unsettled row, an incomplete unit, the owner's own signature. Carried
|
||||
// rather than only logged so a purchase can state WHY it cost money instead of only that it did.
|
||||
BasisReAsked map[basisReason]int
|
||||
// BasisRows is what this run has decided, ready for the boundary writer: every candidate that now
|
||||
// carries a CONSOLIDATED rendering, with the fingerprint of the evidence it was decided on. Built here
|
||||
// because this is the only place both halves exist at once, and carried rather than written here
|
||||
// because writing it before the run reaches an output boundary is the one thing that must not happen —
|
||||
// a basis present on a crash-resume shrinks the batch composition and the resume re-buys what was paid.
|
||||
BasisRows []store.BankBasisRow
|
||||
// Conf is the role's own stated confidence per key. It sorts the review list «least sure first» and does
|
||||
// nothing else — never a weight, a threshold or a cross-model comparison (D39.102).
|
||||
Conf map[string]int
|
||||
|
|
@ -364,6 +381,30 @@ func approvedNeighbours(rows []store.GlossaryEntry) []terminology.Neighbour {
|
|||
return out
|
||||
}
|
||||
|
||||
// rescoreCandidates re-ranks EVERY candidate's renderings under the current scoring options, for the caller
|
||||
// that stamps a BANKED type onto a candidate the settled basis answered. A type has just changed and the
|
||||
// §C2-3 conformance factor routes on type, so a stale ranking would put the wrong rendering first on the
|
||||
// sheet the owner reads.
|
||||
//
|
||||
// ⛔ OVER `cands` AND NEVER OVER A FILTERED COPY. ScoreVariants REPLACES a candidate's Variants slice, so
|
||||
// re-scoring a copy leaves the caller's own slice — the one the signing sheet is rendered from — holding
|
||||
// the ranking computed before the type was known. The operator then reads a ranking factor the
|
||||
// classification has retired, and nothing says so.
|
||||
//
|
||||
// ⚠ IT DELIBERATELY DID NOT ABSORB THE TWO OLDER RE-SCORING SITES, and the restraint is the point rather
|
||||
// than an oversight. Folding them in looked like the obvious tidy-up and was measured wrong: the money pin
|
||||
// that guards exactly this defect plants into the older site's LITERAL THREE LINES, so absorbing them left
|
||||
// its anchor unresolvable — and re-aiming the mutation at this function made it attack «no re-scoring at
|
||||
// all», which is a different defect that nothing pins. A refactor that moves a pin's subject out from under
|
||||
// its planting silently retires the pin. So the older sites keep their shape, and this one does not
|
||||
// pretend to be them.
|
||||
func (r *Runner) rescoreCandidates(cands []terminology.Candidate) {
|
||||
opts := r.scoreOpts()
|
||||
for i := range cands {
|
||||
terminology.ScoreVariants(&cands[i], opts)
|
||||
}
|
||||
}
|
||||
|
||||
// runTerminologist calls the role over the candidate list and returns key → consolidated rendering. With
|
||||
// the gate off it is a no-op returning an empty map and a zero result, so every existing book takes a
|
||||
// byte-identical path and pays nothing.
|
||||
|
|
@ -411,9 +452,56 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
}
|
||||
}
|
||||
}
|
||||
if len(paidIdx) == 0 {
|
||||
// The bank is read once HERE and shared by everything below it — never per batch, since it is the same
|
||||
// law for every batch. The signed rows are the anchor the model is shown; the whole set is what the
|
||||
// conflict checks read, and, since the settled basis, what the predicate reads to learn which rows the
|
||||
// book has decided. Moved above the basis pass so those three cannot come to disagree about what the
|
||||
// bank held this run.
|
||||
//
|
||||
// ⚠ NOT the only read of the run: dropBankSettled above takes its own (glossaryRows again), as it did
|
||||
// before this pack. That is pre-existing and harmless — the glossary's only writer is seedGlossary, and
|
||||
// it does not run between the two — but the claim "read ONCE for the whole role" would be false, and a
|
||||
// comment that overstates its own guarantee is one a later reader relies on.
|
||||
bank := r.glossaryRows()
|
||||
canon := approvedNeighbours(bank)
|
||||
|
||||
// THE SETTLED BASIS, and its position is load-bearing twice over. It stands BEFORE the first paid call,
|
||||
// so a decision already taken never reaches a provider; and it stands BEFORE the classifier pass, which
|
||||
// MUTATES Candidate.Type in place (applyTypes) — a fingerprint taken after it would hash the
|
||||
// classifier's type, and then learning whether the terminologist may be skipped would require buying
|
||||
// the classifier first, on every purchase, which is the cost this mechanism exists to remove.
|
||||
consider := make(map[int]bool, len(paidIdx))
|
||||
for _, i := range paidIdx {
|
||||
consider[i] = true
|
||||
}
|
||||
basisFPs, basisAnswers, basisRows, basisAmbiguous := r.bankBasisPass(ctx, cands, consider, bank, res)
|
||||
if len(basisAnswers) > 0 {
|
||||
kept := paidIdx[:0]
|
||||
for _, i := range paidIdx {
|
||||
if _, served := basisAnswers[cands[i].Key]; !served {
|
||||
kept = append(kept, i)
|
||||
}
|
||||
}
|
||||
paidIdx = kept
|
||||
}
|
||||
if len(paidIdx) == 0 && len(basisAnswers) == 0 {
|
||||
return nil, nil, nil, res, nil
|
||||
}
|
||||
// ⛔ THE BASIS'S TYPE IS STAMPED ONTO THE CANDIDATES HERE, and NOT through the classifier's own merge
|
||||
// further down. The sheet, the stop table and the sidecar's `kind` all read Candidate.Type, and
|
||||
// applyTypes — the only thing that writes it — has already run by the time the basis answers reach the
|
||||
// maps it reads. Merged there, the banked type reaches the auto-bank and NOT the sheet: the row the
|
||||
// owner signs by would keep saying `place` while the bank row for the same term says `term`, for as
|
||||
// long as the basis keeps answering it. Measured on the tree by adversarial review, in the steady state.
|
||||
//
|
||||
// ⚠ It is a STAMP and not applyTypes, because applyTypes returns the number the run publishes as
|
||||
// `reclassified` — «how many types the CLASSIFIER changed». A term the classifier never saw must not
|
||||
// inflate it; a counter that counts two different events is one nobody can act on.
|
||||
if n := stampBasisTypes(cands, basisAnswers); n > 0 {
|
||||
r.rescoreCandidates(cands)
|
||||
r.Log.InfoContext(ctx, "the settled basis restored the BANKED type of terms it answered, so the signing sheet and the bank row agree about what they are; the run's `reclassified` counter is untouched because the classifier did not see them",
|
||||
"book", r.Book.BookID, "terms", n)
|
||||
}
|
||||
paid := pickCandidates(cands, paidIdx)
|
||||
// ⛔ ASKED HERE, BEFORE THE FIRST PAID CALL, and the position is the whole point. The question is
|
||||
// whether this book had bank-role checkpoints BEFORE this run — and after the passes below it is
|
||||
|
|
@ -505,10 +593,6 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
}
|
||||
}
|
||||
|
||||
// The bank is read ONCE for the whole role, not per batch: it is the same law for every batch. The
|
||||
// signed rows are the anchor the model is shown; the whole set is what the conflict checks below read.
|
||||
bank := r.glossaryRows()
|
||||
canon := approvedNeighbours(bank)
|
||||
plan := bankRolePlan{
|
||||
role: roleTerminologist, budgetUSD: r.Pipeline.Gates.Terminology.BudgetUSD,
|
||||
messages: func(b []terminology.Candidate) ([]llm.Message, error) { return r.terminologyMessages(b, canon) },
|
||||
|
|
@ -538,7 +622,7 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
// gets the same warning with a reason that is simply false for it, and a message lying about its
|
||||
// own cause is what D39.93 п.2 forbids. Both branches leave the operator with the same action, so
|
||||
// naming both costs nothing and claiming one costs the truth.
|
||||
r.Log.WarnContext(ctx, "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — the earlier calls' checkpoints could not be reused, either because the batch composition changed (the already-banked filter does not reproduce it) or because they recorded money with no result and cannot be replayed. This is a ONE-TIME cost; every later run replays for $0",
|
||||
r.Log.WarnContext(ctx, "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — the earlier calls' checkpoints could not be reused, either because the batch composition changed (what the bank and the settled basis answer is not what they answered last time) or because they recorded money with no result and cannot be replayed. It is bought ONCE per change, not once per book: re-running THIS purchase replays for $0 from here on, while the next purchase that settles a new row re-packs the remaining batches and pays for them again",
|
||||
logKeyReconsolidated, true,
|
||||
"book", r.Book.BookID, "skipped", res.BankSettled, "of_candidates", res.Candidates, "cost_usd", fmt.Sprintf("%.6f", run.costUSD))
|
||||
}
|
||||
|
|
@ -605,6 +689,23 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
res.Conf[k] = v
|
||||
}
|
||||
}
|
||||
// ⛔ THE BASIS ANSWERS JOIN `out` HERE — after the batches, before every reader below — and the
|
||||
// position is chosen against two opposite failures.
|
||||
//
|
||||
// Joining them LATER (or never) is the silent regression this mechanism is built against: `out` is what
|
||||
// attachConsolidatedDst stamps, and a candidate absent from it keeps no rendering at all, so the
|
||||
// emission falls back to the raw first-chunk banknote guess at status `auto`. The run would look
|
||||
// cheaper and the bank would be poorer, with nothing red anywhere.
|
||||
//
|
||||
// Joining them EARLIER — before the count loop below — would change what a published number means:
|
||||
// `consolidated` is the RENDER PASS's account of itself, and folding rows nobody asked about into it
|
||||
// would report the role as having answered terms it never saw. The loop runs over `paid`, which the
|
||||
// basis pass has already narrowed, so the counters stay a fact about the money spent this run.
|
||||
//
|
||||
// Judged by the dispute checks, deliberately: see bankBasisPass.
|
||||
for k, a := range basisAnswers {
|
||||
out[k] = a.Dst
|
||||
}
|
||||
// Over `paid`, not over every candidate: a candidate nobody was asked about has not gone UNANSWERED —
|
||||
// that word means the role was given it and said nothing, which is a fact about the model.
|
||||
for _, c := range paid {
|
||||
|
|
@ -706,6 +807,29 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
"families", res.Families, "families_refused", res.FamiliesRefused, "families_held", res.FamiliesHeld,
|
||||
"cost_usd", fmt.Sprintf("%.6f", res.CostUSD),
|
||||
"classify_cost_usd", fmt.Sprintf("%.6f", res.ClassifyCostUSD))
|
||||
|
||||
// The basis's TYPE and GENDER reach their stampers the same way the classifier's do — and they are
|
||||
// merged HERE, after `res.Reclassified` has been counted, on purpose: that counter answers «how many
|
||||
// types did the classifier change», and a row the classifier never saw must not inflate it. A basis row
|
||||
// whose purchase produced no type or no gender carries none now either, exactly as it did then.
|
||||
for k, a := range basisAnswers {
|
||||
if a.Type != "" {
|
||||
if classified == nil {
|
||||
classified = map[string]string{}
|
||||
}
|
||||
classified[k] = a.Type
|
||||
}
|
||||
if a.Gender != "" {
|
||||
if gendered == nil {
|
||||
gendered = map[string]string{}
|
||||
}
|
||||
gendered[k] = a.Gender
|
||||
}
|
||||
}
|
||||
// WHAT THIS BOOK HAS DECIDED AS OF NOW — this run's fresh answers and the ones it was handed, together,
|
||||
// each with the fingerprint of the evidence it stands on. Built here because this is the only moment
|
||||
// both halves are in hand; WRITTEN at the run's output boundaries and nowhere else (see writeBankBasis).
|
||||
res.BasisRows = basisRowsToStore(cands, out, classified, gendered, basisFPs, basisRows, basisAmbiguous, basisShape(r.basisWidth(), r.bankRoleProducer()))
|
||||
return out, classified, gendered, res, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ attemptladder.go admitLadderStep "the next attempt was NOT bought: the caller's
|
|||
attemptladder.go walkAttemptLadder "echo flagged, regenerating before escalation (echo is stochastic per call, D39.61)"
|
||||
attemptladder.go walkAttemptLadder "the call returned nothing at the full budget, regenerating with less thinking at the SAME budget"
|
||||
attemptladder.go walkAttemptLadder "the call was flagged, regenerating with a larger budget"
|
||||
bankbasis.go bankBasisPass "could not read the settled basis, so every already-decided term is being bought AGAIN out of this book's LIFETIME bank-role budget; the pass still runs and the basis is rewritten at this run's boundary"
|
||||
bankbasis.go bankBasisPass "the settled basis is INERT this run: the book's normalized source was not built, so the evidence behind a decision cannot be fingerprinted and every candidate goes to the paid role"
|
||||
bankbasis.go bankBasisPass "the settled basis was written under a DIFFERENT fingerprint shape and cannot be compared against this one: those terms are bought again, ONCE, and stored under the new shape — this is the announced price of changing what counts as the same evidence, and it is NOT «the source moved»"
|
||||
bankbasis.go writeBankBasis "could not record the settled basis: the terms this run decided are NOT remembered, so the NEXT purchase will pay both bank roles for them again out of this book's LIFETIME budget (exhausting it leaves the book without a consolidated bank permanently). The run itself is unaffected and its bank is on disk"
|
||||
bankexport.go exportBank "bank export: could not marshal the bank; the export artifact was NOT refreshed and may be stale"
|
||||
bankexport.go exportBank "bank export: could not read the bank; the export artifact was NOT refreshed and may be stale"
|
||||
bankexport.go exportBank "bank export: could not write the export artifact; it was NOT refreshed and may be stale"
|
||||
|
|
@ -88,6 +92,7 @@ runner.go loadLangPack "langpack declares no source-morphology channel — the n
|
|||
runner.go openRunner "a stage sends the SOURCE text to a model whose wire SUPPRESSES thinking — the ratified echo zone (D19.1 point 2: reasoning-off + dense CJK is a property of the model class, not a vendor quirk). Not a stop: the downstream echo gate (cjk_artifact) still catches an echo on every configuration (D19.2), but it costs a paid call per hit"
|
||||
runner.go openRunner "content routing is not runnable for this book; read-only projections continue (translate/redrive would refuse)"
|
||||
runner.go openRunner "the banknote channel is ON but this book cannot reach the bank-mining stop, which is the only reader of what it collects: the pair's prompt asks the translator for a term table on every draft call, those output tokens are billed, and nothing on this configuration will ever read them (add the mining inputs — book.yaml `langpack_root` for this pair and pipeline.yaml `mining.contrast_path` — or drop `gates.banknote` together with the banknote prompt override)"
|
||||
seeding.go seedGlossary "bank term windows leave chapters uncovered — those chapters have NO law for the term and their drafts will render it however they like (deliberate is fine; a typo is not)"
|
||||
seeding.go seedGlossary "bank: two rows render ONE firing surface differently and at least one is unsigned (the deterministic matcher cannot pick between them; reconcile the seed, the delta or the proposal)"
|
||||
seeding.go seedGlossary "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)"
|
||||
seeding.go seedGlossary "voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)"
|
||||
|
|
@ -133,7 +138,7 @@ terminologist.go runTerminologist "terminology: the TYPE classifier ended short
|
|||
terminologist.go runTerminologist "terminology: the model answered in another script; those lines are REFUSED (the terms stay unconsolidated) — a book with few signed rows gives the model no target-language anchor"
|
||||
terminologist.go runTerminologist "terminology: the role declined term(s) in WORDS rather than with the engine sentinel; they are treated as declined (NOT banked), and the pair's prompt is what needs the look"
|
||||
terminologist.go runTerminologist "terminology: this bank is PARTIALLY consolidated — a budget cut the RENDER pass, so some terms were never offered to the role at all; `unanswered` below counts them together with terms the role saw and did not answer"
|
||||
terminologist.go runTerminologist "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — the earlier calls' checkpoints could not be reused, either because the batch composition changed (the already-banked filter does not reproduce it) or because they recorded money with no result and cannot be replayed. This is a ONE-TIME cost; every later run replays for $0"
|
||||
terminologist.go runTerminologist "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — the earlier calls' checkpoints could not be reused, either because the batch composition changed (what the bank and the settled basis answer is not what they answered last time) or because they recorded money with no result and cannot be replayed. It is bought ONCE per change, not once per book: re-running THIS purchase replays for $0 from here on, while the next purchase that settles a new row re-packs the remaining batches and pays for them again"
|
||||
volume.go deliveredUnits <dynamic>
|
||||
volume.go planVolume "a VOLUME ceiling on a book that MINES its bank: each purchase drafts more, so the bank-mining stop writes a larger auto-bank, and the next purchase moves the edit-wave snapshot the previous purchase's edit jobs are pinned to. Expect that run to need --resnapshot and to re-pay the units the new terms actually touch (the re-payment consent gate still bounds it)"
|
||||
volume.go rescopeEditWave "the bank moved between planning and the edit wave, so units judged FREE are no longer free: they have been re-judged against the snapshot the edit wave actually uses"
|
||||
|
|
|
|||
86
backend/internal/store/bankbasis.go
Normal file
86
backend/internal/store/bankbasis.go
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
package store
|
||||
|
||||
// bankbasis.go: the settled basis (v19) — what this book has already decided, and the fingerprint of the
|
||||
// evidence each decision was taken on. Dumb storage like stopmemory.go and ruby.go: the fingerprint, the
|
||||
// predicate and every rule about WHEN a row may be served live in the pipeline; this layer writes rows and
|
||||
// reads them back.
|
||||
//
|
||||
// ⚠ IT REPLACES, WHERE ITS NEIGHBOUR ONLY ACCUMULATES. bank_stop_presented has no delete on purpose —
|
||||
// trimming it re-arms a stop the owner has already been shown. This table is the opposite: a term decided
|
||||
// again must overwrite its row, or the predicate keeps comparing against evidence the book has moved past
|
||||
// and serves an answer that was right two purchases ago. Same shape, deliberately different guarantee.
|
||||
|
||||
// BankBasisRow is one decided surface as the store holds it.
|
||||
type BankBasisRow struct {
|
||||
SrcKey string
|
||||
FPVersion string
|
||||
FPWhole string
|
||||
FPCanon string
|
||||
Dst string
|
||||
Type string
|
||||
Gender string
|
||||
}
|
||||
|
||||
// BankBasisForBook returns the book's settled basis keyed by the normalized source key. An empty table is
|
||||
// an empty map: nothing has been decided yet, which is the state of every book before its first purchase.
|
||||
func (s *Store) BankBasisForBook(bookID string) (map[string]BankBasisRow, error) {
|
||||
ctx, cancel := opContext()
|
||||
defer cancel()
|
||||
rows, err := s.r.QueryContext(ctx, `
|
||||
SELECT src_key, fp_version, fp_whole, fp_canon, dst, type, gender
|
||||
FROM bank_basis WHERE book_id = ?`, bookID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]BankBasisRow{}
|
||||
for rows.Next() {
|
||||
var r BankBasisRow
|
||||
if err := rows.Scan(&r.SrcKey, &r.FPVersion, &r.FPWhole, &r.FPCanon, &r.Dst, &r.Type, &r.Gender); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[r.SrcKey] = r
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// PutBankBasis writes the run's decisions, in ONE transaction, replacing whatever the book held for the
|
||||
// same surfaces.
|
||||
//
|
||||
// ⛔ A ROW WITH NO RENDERING IS REFUSED RATHER THAN STORED. The basis exists to hand an answer back, and an
|
||||
// empty one handed back is the silent regression the whole mechanism is built against: the emission would
|
||||
// fall to the raw first-chunk guess at status `auto` and the run would look cheaper while the bank got
|
||||
// poorer. Refusing here means the predicate cannot serve emptiness even if a caller upstream stops
|
||||
// checking — the guarantee lives at the write, where a caller cannot forget it.
|
||||
func (s *Store) PutBankBasis(bookID string, rows []BankBasisRow) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := opContext()
|
||||
defer cancel()
|
||||
tx, err := s.w.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, r := range rows {
|
||||
if r.SrcKey == "" || r.Dst == "" || r.FPWhole == "" || r.FPVersion == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO bank_basis (book_id, src_key, fp_version, fp_whole, fp_canon, dst, type, gender, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
|
||||
ON CONFLICT(book_id, src_key) DO UPDATE SET
|
||||
fp_version = excluded.fp_version,
|
||||
fp_whole = excluded.fp_whole,
|
||||
fp_canon = excluded.fp_canon,
|
||||
dst = excluded.dst,
|
||||
type = excluded.type,
|
||||
gender = excluded.gender,
|
||||
updated_at = excluded.updated_at`,
|
||||
bookID, r.SrcKey, r.FPVersion, r.FPWhole, r.FPCanon, r.Dst, r.Type, r.Gender); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
76
backend/internal/store/bankbasis_test.go
Normal file
76
backend/internal/store/bankbasis_test.go
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestBasisStoreRefusesAnEmptyRendering pins the guarantee at the WRITE, which is where it has to live.
|
||||
//
|
||||
// ⛔ WHY AT THE WRITE AND NOT ONLY AT THE CALLER. The basis exists to hand an answer back. An empty
|
||||
// rendering handed back is the silent regression the whole mechanism is built against: the emission drops
|
||||
// to the raw first-chunk guess at status `auto`, the run looks cheaper and the bank gets poorer with
|
||||
// nothing red. The pipeline also skips such rows before it calls this — belt AND braces — but a guarantee
|
||||
// kept only by a caller is one the next caller can forget, and this is the money path.
|
||||
func TestBasisStoreRefusesAnEmptyRendering(t *testing.T) {
|
||||
s, _ := openTemp(t)
|
||||
good := BankBasisRow{SrcKey: "元石", FPVersion: "v1", FPWhole: "abc", FPCanon: "def", Dst: "юаньши", Type: "name"}
|
||||
bad := []BankBasisRow{
|
||||
{SrcKey: "空", FPVersion: "v1", FPWhole: "abc", Dst: ""}, // no rendering: nothing to serve
|
||||
{SrcKey: "", FPVersion: "v1", FPWhole: "abc", Dst: "что-то"}, // no key: it would answer for nobody
|
||||
{SrcKey: "无", FPVersion: "v1", FPWhole: "", Dst: "что-то"}, // no fingerprint: nothing to compare
|
||||
{SrcKey: "版", FPVersion: "", FPWhole: "abc", Dst: "что-то"}, // no shape: cannot say which rule decided it
|
||||
}
|
||||
if err := s.PutBankBasis("b", append([]BankBasisRow{good}, bad...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := s.BankBasisForBook("b")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The control first: the GOOD row landed, so «nothing was written» cannot masquerade as «the bad rows
|
||||
// were refused».
|
||||
if _, ok := got["元石"]; !ok {
|
||||
t.Fatalf("the well-formed row did not land at all (%d row(s) stored): this test would then pass on a store that writes nothing", len(got))
|
||||
}
|
||||
for _, r := range bad {
|
||||
if _, landed := got[r.SrcKey]; landed && r.SrcKey != "" {
|
||||
t.Errorf("a row with src=%q dst=%q fp=%q shape=%q was STORED: a later purchase would serve it back, and an empty or unattributable answer served back is the loss this mechanism exists to prevent",
|
||||
r.SrcKey, r.Dst, r.FPWhole, r.FPVersion)
|
||||
}
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Errorf("stored %d row(s), want exactly the one well-formed row: %+v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBasisStoreReplacesRatherThanAccumulates pins the guarantee this table does NOT inherit from its
|
||||
// neighbour. bank_stop_presented only grows and has no delete on purpose; a decision re-taken must
|
||||
// OVERWRITE, or the predicate keeps comparing against evidence the book has moved past and serves an answer
|
||||
// that was right two purchases ago.
|
||||
func TestBasisStoreReplacesRatherThanAccumulates(t *testing.T) {
|
||||
s, _ := openTemp(t)
|
||||
first := BankBasisRow{SrcKey: "元石", FPVersion: "v1", FPWhole: "old", FPCanon: "oldc", Dst: "юаньши", Type: "name", Gender: "none"}
|
||||
if err := s.PutBankBasis("b", []BankBasisRow{first}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := BankBasisRow{SrcKey: "元石", FPVersion: "v2", FPWhole: "new", FPCanon: "newc", Dst: "камень юань", Type: "term", Gender: "male"}
|
||||
if err := s.PutBankBasis("b", []BankBasisRow{second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := s.BankBasisForBook("b")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("the same surface produced %d rows: the table accumulates where it must replace, and the predicate would meet two answers for one term", len(got))
|
||||
}
|
||||
if got["元石"] != second {
|
||||
t.Fatalf("the row was not replaced: got %+v, want %+v", got["元石"], second)
|
||||
}
|
||||
// A different BOOK must not be touched by either write: the key is (book, surface), and a basis that
|
||||
// leaked across books would answer one book's terms with another's canon.
|
||||
if err := s.PutBankBasis("other", []BankBasisRow{first}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := s.BankBasisForBook("b"); again["元石"] != second {
|
||||
t.Fatalf("writing another book's basis changed this one: %+v", again["元石"])
|
||||
}
|
||||
}
|
||||
|
|
@ -522,6 +522,47 @@ var migrations = []string{
|
|||
`
|
||||
ALTER TABLE request_log ADD COLUMN reasoning_in_completion INTEGER;
|
||||
`,
|
||||
// v19 (row 130): WHAT THE BOOK HAS ALREADY DECIDED — the settled basis the bank roles are not paid to
|
||||
// decide twice.
|
||||
//
|
||||
// THE CLASS IS NEW, and it is new on purpose. The bank ontology names four roles; the errata of
|
||||
// 17.09-д adds a fifth — «MEMORY OF DECISIONS»: written by the RUN at its output boundaries, read by a
|
||||
// money predicate BEFORE the first paid call. It is deliberately not filed under the existing
|
||||
// «flag memory» (bank_stop_presented), whose signature is narrower — «what the stop has already
|
||||
// shown», written by the stop — and widening that signature to fit would have made one row of the
|
||||
// ontology mean two things.
|
||||
//
|
||||
// ⛔ WHY A TABLE AND NOT A FILE BESIDE THE DB, which is what the consilium's letter said. A file beside
|
||||
// the DB belongs to the ontology's SOURCES role — the documents the bank view is rebuilt from
|
||||
// (auto-bank, the mined delta, the reject list). The basis is not a source: not one row of the bank is
|
||||
// assembled from it, and the view would be byte-identical if it vanished. Filing a memory of decisions
|
||||
// among the sources would create a source that produces nothing, which the same page judges as
|
||||
// strictly as it judges a projection nobody reads.
|
||||
//
|
||||
// ⚠ WHAT IT DOES NOT INHERIT FROM ITS NEIGHBOUR, said here because copying a mechanism's shape without
|
||||
// its guarantees is how this codebase has been bitten before. bank_stop_presented only GROWS and has
|
||||
// no delete on purpose; this table must REPLACE a row when a term is decided again, or the predicate
|
||||
// would compare against a fingerprint the book has moved past. Hence the upsert in PutBankBasis — a
|
||||
// new property, not an inherited one.
|
||||
//
|
||||
// fp_version is stored PER ROW rather than assumed global: changing the fingerprint's shape invalidates
|
||||
// every stored row, and a row that fails on its version must be able to SAY «the rule that decided this
|
||||
// is no longer the rule» instead of reporting itself, indistinguishably, as «the source moved». A
|
||||
// re-purchase whose message names the wrong cause is worse than one that says nothing.
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS bank_basis (
|
||||
book_id TEXT NOT NULL,
|
||||
src_key TEXT NOT NULL, -- text.NormalizeSourceKey form: the key a candidate is matched on
|
||||
fp_version TEXT NOT NULL, -- the fingerprint SHAPE that produced the hashes below
|
||||
fp_whole TEXT NOT NULL, -- what the predicate compares
|
||||
fp_canon TEXT NOT NULL, -- the signed-rows half alone, so a re-ask can name its cause
|
||||
dst TEXT NOT NULL, -- the CONSOLIDATED rendering, never a raw per-chunk guess
|
||||
type TEXT NOT NULL DEFAULT '', -- the classifier's corrected type, '' when it did not answer
|
||||
gender TEXT NOT NULL DEFAULT '', -- the classifier's gender datum, '' when none
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
PRIMARY KEY (book_id, src_key)
|
||||
);
|
||||
`,
|
||||
}
|
||||
|
||||
// DESIGN NOTE (D21.10 → BUILT by pack-19 / D39.55; kept as the record of what each type is FOR).
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
@ -71,8 +72,25 @@ func TestTheThinkingColumnKeepsSilenceApartFromZero(t *testing.T) {
|
|||
func TestRowsWrittenBeforeTheThinkingColumnReadAsUnanswered(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.db")
|
||||
|
||||
// One vintage back: every migration except the head, recorded exactly the way applyStep records
|
||||
// them, so the real migrator sees a project that is genuinely behind rather than one pretending.
|
||||
// The vintage is the one JUST BEFORE THIS COLUMN, located by looking for the migration that adds it —
|
||||
// not `len(migrations)-1`.
|
||||
//
|
||||
// ⛔ IT USED TO BE «one back from the head», and that was correct only while this column happened to BE
|
||||
// the head. The first migration to land after it broke the fixture: head-1 already carried the column,
|
||||
// so the premise guard below fired and the test stopped being about anything. Worse is the version of
|
||||
// that failure where the guard is absent — the fixture then silently measures «a column that already
|
||||
// exists survives an ALTER», passes, and the class it was written for goes unwatched. A fixture must
|
||||
// name its own subject, not borrow the tree's current shape to express it.
|
||||
before := -1
|
||||
for i, m := range migrations {
|
||||
if strings.Contains(m, "reasoning_in_completion") {
|
||||
before = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if before < 0 {
|
||||
t.Fatal("no migration adds reasoning_in_completion — this test names a column the schema no longer grows, and its subject is gone rather than green")
|
||||
}
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -80,7 +98,7 @@ func TestRowsWrittenBeforeTheThinkingColumnReadAsUnanswered(t *testing.T) {
|
|||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for v := 0; v < len(migrations)-1; v++ {
|
||||
for v := 0; v < before; v++ {
|
||||
if _, err := db.Exec(migrations[v]); err != nil {
|
||||
t.Fatalf("build vintage %d: %v", v+1, err)
|
||||
}
|
||||
|
|
@ -88,10 +106,10 @@ func TestRowsWrittenBeforeTheThinkingColumnReadAsUnanswered(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The column must NOT exist yet, or the fixture is not one vintage back and the test is about
|
||||
// The column must NOT exist yet, or the fixture is not standing before it and the test is about
|
||||
// nothing.
|
||||
if columnExists(t, db, "request_log", "reasoning_in_completion") {
|
||||
t.Fatal("the fixture is meant to be ONE VINTAGE BEHIND: request_log already carries the column the head migration adds")
|
||||
t.Fatalf("the fixture is meant to stand at the vintage BEFORE this column (migration %d of %d), and request_log already carries it", before+1, len(migrations))
|
||||
}
|
||||
// A row written by that older binary: the INSERT it had, without the column.
|
||||
if _, err := db.Exec(`INSERT INTO request_log (book_id, chapter, chunk_idx, stage, role, completion_tokens, ok)
|
||||
|
|
|
|||
79
backend/internal/terminology/evidence.go
Normal file
79
backend/internal/terminology/evidence.go
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
package terminology
|
||||
|
||||
import "strings"
|
||||
|
||||
// evidence.go: the EVIDENCE view of a candidate — every context the source holds for a surface, as opposed
|
||||
// to the subset AttachKWIC hands the model.
|
||||
//
|
||||
// ⛔ WHY THIS IS NOT KWIC, AND WHY THE TWO MUST NOT SHARE A KNOB. KWIC answers "how much context does the
|
||||
// model need to translate this well", and it is a PROMPT-SHAPE question: it is capped (maxPer), it takes
|
||||
// the FIRST occurrences in (chapter, chunk, offset) order, and the day a stratified selection lands it
|
||||
// will take a different subset of the same text. EvidenceWindows answers a different question — "what does
|
||||
// the book say about this surface" — so it is uncapped and selection-free by construction.
|
||||
//
|
||||
// The distinction is money. A fingerprint built over the KWIC subset would move for every book on earth
|
||||
// the moment anyone turned kwic_per_term, changed the selection rule, or widened the model's context, and
|
||||
// each of those moves would re-buy a whole bank out of a LIFETIME role budget. Built over every
|
||||
// occurrence, it moves only when the SOURCE around the surface moves — which is the event that genuinely
|
||||
// invalidates a decision.
|
||||
|
||||
// EvidenceWindows returns the ±width-rune context around EVERY occurrence of key in nsource, in document
|
||||
// order, non-overlapping — the same window shape KWIC shows, without the cap and without the selection.
|
||||
//
|
||||
// nsource is the book's NORMALIZED source (the same space keys live in), so a caller must not pass raw
|
||||
// text: a surface would then be looked up in one space and matched in another.
|
||||
//
|
||||
// An empty result means the surface does not occur in the source at all, which is honest rather than
|
||||
// empty-by-accident: it is what a draft-invented term looks like, and a caller that hashes it gets a
|
||||
// value distinct from every attested surface.
|
||||
//
|
||||
// ⛔ IT DOES NOT REUSE window(), AND THAT IS ABOUT COST RATHER THAN TASTE. window() re-derives the rune
|
||||
// view of its input on every call — `[]rune(s)` plus two prefix counts — which is free at the size its own
|
||||
// comment declares ("the strings are chunk-sized") and quadratic here, where the input is the WHOLE book
|
||||
// and the call happens once per occurrence. Measured on a synthetic source at the occurrence density of the
|
||||
// bought cold run: a source four times longer took FIFTEEN times as long, and a million-rune book would
|
||||
// have spent minutes — plus tens of gigabytes of garbage — inside a $0 pass that runs before every
|
||||
// purchase, whether or not it ends up saving anything. So the source is converted ONCE per call and the
|
||||
// windows are cut from that view.
|
||||
//
|
||||
// ⚠ The window CONTENT is identical to window()'s by construction: same ±width in runes, same clamp at the
|
||||
// ends, same TrimSpace. That equality is pinned rather than asserted here, because a fingerprint built on
|
||||
// a subtly different window would be a silent, book-wide re-purchase the first time anything compared them.
|
||||
func EvidenceWindows(nsource, key string, width int) []string {
|
||||
if key == "" || width <= 0 {
|
||||
return nil
|
||||
}
|
||||
runes := []rune(nsource)
|
||||
// Byte offset → rune index, built by ONE forward pass. `range` over a string visits rune STARTS, which
|
||||
// is exactly the set of offsets this function indexes: strings.Index of a valid UTF-8 key inside valid
|
||||
// UTF-8 text returns a rune-aligned offset, and so does that offset plus the key's byte length.
|
||||
// Continuation bytes are left at zero on purpose — indexing one would be a bug in the caller, not a
|
||||
// case to handle here, and giving them a plausible value would hide it.
|
||||
runeAt := make([]int, len(nsource)+1)
|
||||
n := 0
|
||||
for bi := range nsource {
|
||||
runeAt[bi] = n
|
||||
n++
|
||||
}
|
||||
runeAt[len(nsource)] = n
|
||||
|
||||
var out []string
|
||||
for from := 0; from < len(nsource); {
|
||||
i := strings.Index(nsource[from:], key)
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
at := from + i
|
||||
lo := runeAt[at] - width
|
||||
if lo < 0 {
|
||||
lo = 0
|
||||
}
|
||||
hi := runeAt[at+len(key)] + width
|
||||
if hi > len(runes) {
|
||||
hi = len(runes)
|
||||
}
|
||||
out = append(out, strings.TrimSpace(string(runes[lo:hi])))
|
||||
from = at + len(key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
82
backend/internal/terminology/evidence_test.go
Normal file
82
backend/internal/terminology/evidence_test.go
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
package terminology
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestEvidenceWindowsCutsTheSameWindowAsKWIC pins the equality the fingerprint's meaning rests on.
|
||||
//
|
||||
// ⛔ WHY IT IS PINNED RATHER THAN TRUSTED. EvidenceWindows stopped calling window() for cost — window()
|
||||
// re-derives the whole rune view per call, which is quadratic over a book — so the two now compute the same
|
||||
// shape by two routes. Let them drift by one rune and every stored fingerprint on earth stops matching the
|
||||
// one computed next purchase: a silent, book-wide re-purchase out of a LIFETIME budget, with nothing red.
|
||||
func TestEvidenceWindowsCutsTheSameWindowAsKWIC(t *testing.T) {
|
||||
for _, src := range []string{
|
||||
"他拿出元石,看了看。元石在手中发光。这是元海的东西。",
|
||||
"元石", // the whole string IS the key: both ends clamp
|
||||
" 元石 ", // surrounding space: both must TrimSpace
|
||||
"aa元石bb元石cc", // adjacent occurrences
|
||||
strings.Repeat("前面。元石。", 5), // many
|
||||
"ǎǎǎ元石ǎǎǎ", // multi-byte NON-CJK neighbours: byte≠rune arithmetic
|
||||
} {
|
||||
for _, width := range []int{1, 3, 40} {
|
||||
got := EvidenceWindows(src, "元石", width)
|
||||
// The reference: window() over EVERY occurrence, the loop EvidenceWindows replaced.
|
||||
var want []string
|
||||
for from := 0; from < len(src); {
|
||||
i := strings.Index(src[from:], "元石")
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
at := from + i
|
||||
want = append(want, window(src, at, at+len("元石"), width))
|
||||
from = at + len("元石")
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("src=%q width=%d: %d window(s), want %d\n got=%q\nwant=%q", src, width, len(got), len(want), got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("src=%q width=%d window %d: got %q, want %q", src, width, i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
// The control that keeps the equality from being vacuous: there IS a window to compare.
|
||||
if len(want) == 0 {
|
||||
t.Fatalf("premise broken: the reference produced no windows for src=%q", src)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvidenceWindowsDoesNotGrowQUADRATICALLYWithTheBook pins the cost, because the cost is what forced the
|
||||
// rewrite and a later "simplification" back to window() would restore it invisibly: the result would stay
|
||||
// correct and the $0 pass before every purchase would start taking minutes on a real book.
|
||||
//
|
||||
// ⚠ IT ASSERTS A RATIO, NOT A DURATION. A wall-clock threshold on a shared machine is a flake; the shape of
|
||||
// the curve is the property. Quadratic means ×4 input → ≈×16 time; linear means ≈×4. The gate sits far above
|
||||
// linear and far below quadratic, so it survives a noisy host and still fails the regression it is for.
|
||||
func TestEvidenceWindowsDoesNotGrowQUADRATICALLYWithTheBook(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("cost pin: -short")
|
||||
}
|
||||
unit := strings.Repeat("前面的话,还有更多的话。", 9) + "元石。"
|
||||
measure := func(reps int) (time.Duration, int) {
|
||||
src := strings.Repeat(unit, reps)
|
||||
start := time.Now()
|
||||
n := len(EvidenceWindows(src, "元石", 40))
|
||||
return time.Since(start), n
|
||||
}
|
||||
small, nSmall := measure(2000)
|
||||
large, nLarge := measure(8000) // ×4 the source and ×4 the occurrences
|
||||
if nSmall == 0 || nLarge != 4*nSmall {
|
||||
t.Fatalf("premise broken: occurrences %d → %d, want a clean ×4 so the ratio below is about the source size", nSmall, nLarge)
|
||||
}
|
||||
ratio := float64(large) / float64(small)
|
||||
t.Logf("source ×4 (occurrences %d → %d): %s → %s, ratio ×%.1f (linear ≈4, quadratic ≈16)", nSmall, nLarge, small, large, ratio)
|
||||
if ratio > 8 {
|
||||
t.Fatalf("time grew ×%.1f for a ×4 source (%s → %s): the scan is super-linear again, and a $0 pass that runs before EVERY purchase would spend minutes on a real book. The usual cause: cutting each window with a helper that re-derives []rune over the WHOLE source per occurrence",
|
||||
ratio, small, large)
|
||||
}
|
||||
}
|
||||
|
|
@ -4,11 +4,17 @@
|
|||
> при конфликте побеждает журнал решений. Собрано оркестратором №19 по консилиуму со сторонним
|
||||
> архитектором при приёмке четырёх паков входной двери шва.
|
||||
>
|
||||
> **Зачем страница.** Вокруг одного понятия — «состояние банка этой книги» — живут десять носителей.
|
||||
> **Зачем страница.** Вокруг одного понятия — «состояние банка этой книги» — живут одиннадцать носителей.
|
||||
> Форма СТРОГАЯ: поток однонаправленный, у каждого источника единственный писатель, вид пересобирается
|
||||
> полностью на каждой границе прогона (D39.158 п.8).
|
||||
|
||||
## Четыре роли, и они не смешиваются
|
||||
## Пять ролей, и они не смешиваются
|
||||
|
||||
⚠ **Было ЧЕТЫРЕ до 17.09.** Пятую — «ПАМЯТЬ РЕШЕНИЙ» — завела эррата 17.09-д и построил пак базиса
|
||||
решённого. Заводить новый класс пришлось именно потому, что подпись существующего у́же: «ПАМЯТЬ ФЛАЖКА»
|
||||
определена как «что стоп уже предъявлял», писатель — стоп, а базис пишет ПРОГОН на своих границах и читает
|
||||
его денежный предикат ДО первой траты. Расширить чужую подпись значило бы сделать одну строку таблицы
|
||||
двузначной, чего эта страница не допускает нигде.
|
||||
|
||||
| Роль | Носители | Писатель | Момент фиксации |
|
||||
|---|---|---|---|
|
||||
|
|
@ -16,6 +22,7 @@
|
|||
| **ВИД** — банк, каким его видят роли | строки глоссария в хранилище | только `seedGlossary`, и только полным пересбором | каждая граница прогона; один загрузчик, все гарды |
|
||||
| **ПРОЕКЦИИ** — то, что банк рассказывает наружу | `<project_db>.bank.json` (единственный канал банка наружу) · карта подписи `<project_db>.mined-signature.yaml` · человеческая таблица стопа `.bank-stop.txt` | прогон, на границе | ⚠ не одинаков: `bank.json` — ПОСЛЕ смены вида; карта подписи и `.bank-stop.txt` — при вычислении стопа, ДО пере-сида, а на стоп-границе вид вообще не меняется (остановившийся прогон авто-банк не пишет) |
|
||||
| **ПАМЯТЬ ФЛАЖКА** — что стоп уже предъявлял | таблица `bank_stop_presented` | стоп | ПОСЛЕ карты, никогда до |
|
||||
| **ПАМЯТЬ РЕШЕНИЙ** — что книга уже решила и на каких уликах | таблица `bank_basis` (отпечаток улик + консолидированный ответ) | **прогон**, и границ у него ДВЕ: `run-finished` и ветка стопа подписи | на выходной границе прогона, ПЕРЕД памятью флажка |
|
||||
|
||||
**Поток однонаправленный: источники → вид → проекции.** Отсюда главное свойство, из-за которого
|
||||
рефактор не нужен: **расхождение источников с видом структурно невозможно дольше одной границы
|
||||
|
|
@ -28,6 +35,21 @@
|
|||
слово владельца. Однонаправленность формулируется точно так: **петля есть, но она не может изменить
|
||||
подписанное.**
|
||||
|
||||
⚠ **ВТОРАЯ ПЕТЛЯ (пак базиса решённого, 17.09): базис → предикат → авто-банк.** Прежняя редакция называла
|
||||
авто-провод ЕДИНСТВЕННОЙ петлёй, и с этого дня это неверно: прогон пишет память решений на своей границе,
|
||||
а СЛЕДУЮЩАЯ покупка читает её предикатом до первого платного вызова и подставляет оттуда ответ, который
|
||||
уезжает в авто-банк. Защищена она тем же самым — слово движка не может присвоить себе слово владельца:
|
||||
подписанная строка записи в базис не получает (предикат отвергает её по статусу) и её передача базисом НЕ
|
||||
подменяется никогда. Формулировка однонаправленности не меняется, меняется число её членов: **петель две,
|
||||
и ни одна не может изменить подписанное.**
|
||||
|
||||
⚠ **Порядок на границе стопа — карта → таблица стопа → проекция банка → БАЗИС → память флажка**, и базис
|
||||
стоит ПЕРЕД памятью флажка по тому же правилу, которым правило 3 ниже объясняет саму асимметрию: последней
|
||||
пишется запись, чья потеря ДЕШЕВЛЕ. Потеря памяти флажка стоит одного доброкачественного лишнего стопа;
|
||||
потеря базиса стоит пере-покупки обеих банк-ролей из ПОЖИЗНЕННОГО потолка книги, исчерпание которого
|
||||
необратимо (ряд 356). ⛔ И писать базис РАНЬШЕ развилки нельзя: базис, оказавшийся на краш-резюме, сжимает
|
||||
состав батчей, чекпойнты по хешу запроса не находятся, и резюм перекупает уже оплаченное.
|
||||
|
||||
## Почему слово владельца и слово движка не сливаются
|
||||
|
||||
Соблазн понятный: одобрения владельца и авто-строки движка — одинаковые по форме строки банка. Но
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue