package pipeline import ( "context" "crypto/sha256" "encoding/hex" "fmt" "io" "sort" "strings" "textmachine/backend/internal/lang" "textmachine/backend/internal/membank" "textmachine/backend/internal/store" "textmachine/backend/internal/terminology" "textmachine/backend/internal/text" ) // bankbasis.go: the SETTLED BASIS — the memory that a row the engine already decided is not bought again. // // WHAT PROBLEM IT IS FOR. Every purchase of chapters re-ran both bank roles over surfaces whose rendering // the book had already settled, and paid for them out of a LIFETIME per-book ceiling // (gates.terminology.budget_usd / classify_budget_usd). Exhausting that ceiling is irreversible: the book // then keeps whatever bank it has, forever. So this is not an optimisation — it is the difference between // a long book that can afford a consolidated bank and one that cannot. // // WHAT IS IN HERE. The two PURE halves: the fingerprint of what the role reads about one candidate, and // the predicate that reads a fingerprint back and says whether the stored answer still holds. Storage, // boundaries and wiring live elsewhere; nothing in this file touches a store, a clock or a provider. // // ⛔ THE BASIS IS A SOURCE OF ANSWERS, NOT A FILTER, and the distinction is the whole money risk of the // mechanism. Dropping a settled candidate from the paid set and stopping there does NOT leave the book // where it was: attachConsolidatedDst stamps nothing for a candidate the role did not answer, and the // emission then falls back to the raw first-chunk banknote guess at status `auto` // (miner_emit.go, `st.Dst = props[0].Dst` under `props[0].Via == ""`), or to no rendering at all. The book // would look CHEAPER and be POORER, and nothing in the run would say so. So a settled candidate must be // handed its stored answer, and the answer handed over is the CONSOLIDATED one — never a raw guess, for // exactly the reason the emission gives for its own two-mode rule. // basisFingerprintVersion versions the SHAPE of the fingerprint, and it is load-bearing rather than // hygienic: every stored fingerprint was produced by one shape, so changing what goes into the hash // invalidates every book's basis at once. Bumping this is therefore a DECISION with a price — one // re-consolidation per book, out of the lifetime ceiling — and the version makes that decision explicit // and one-time instead of silent and repeated. const basisFingerprintVersion = "bank-basis-fp-v1" // basisShape is the fingerprint's SHAPE as it is stored beside every row: the version above plus the // EFFECTIVE evidence width the row was decided under. // // ⛔ THE WIDTH BELONGS IN THE SHAPE, NOT ONLY INSIDE THE HASH, and the difference is which sentence the // operator reads when money moves. Hashed only inside the evidence, a change of `basis_width` moves every // fingerprint of every book — and the predicate reports that as `evidence-moved`, i.e. «the source around // this term changed». That is the WRONG CAUSE for every one of those rows: the source did not move, the // rule for reading it did, and the owner's reaction to the two is opposite (one is nothing to do, the other // is a knob somebody turned). Carried in the shape, the same change is reported by the version check as // «the rule that decided this is no longer the rule», with the old and the new width printed. // // ⚠ It is the EFFECTIVE width — what terminologyOpts actually resolved — and not the pair's declared value: // a pair that states nothing runs on the engine default, and the day that default moves those rows must // report a shape change too, not silence. func basisShape(width int, producer string) string { return fmt.Sprintf("%s|w%d|p%s", basisFingerprintVersion, width, producer) } // basisShapeParts splits a stored shape back into its components, so a mismatch can say WHICH of them moved // instead of reporting «the shape changed» and leaving the operator to guess between a knob they turned and // a prompt somebody edited. func basisShapeParts(shape string) (version, width, producer string) { p := strings.SplitN(shape, "|", 3) for len(p) < 3 { p = append(p, "") } return p[0], p[1], p[2] } // terminologyDefaultBasisWidth is the engine's fallback for the evidence width when the pair states none. // It equals today's KWIC default by coincidence of judgement, not by construction: the two are separate // knobs (lang.TerminologySizing.BasisWidth) precisely so that tuning the prompt cannot re-buy a bank. const terminologyDefaultBasisWidth = 40 // bankBasisFP is one candidate's evidence reduced to hashes. // // ⚠ TWO VALUES AND NOT ONE, and the second is not decoration. `Whole` is what the predicate compares; // `Canon` is the part contributed by the book's SIGNED rows alone. Without it the engine can say "this // term is being bought again" and not say WHY, and the two reasons call for opposite reactions from the // owner: «the source around this term changed» is nothing to do, «you signed a relative of this term» is // the consequence of an act they just performed. A message that cannot name its own cause is the class // this codebase already refuses elsewhere. type bankBasisFP struct { Whole string Canon string } // bankBasisAnswer is what the paid roles decided about one surface — the thing the basis exists to hand // back. Gender rides with it because the classifier is the only automatic producer of that field, and a // settled row that lost its gender would silently drop a directive the wire already promised. type bankBasisAnswer struct { Dst string Type string Gender string } // bankBasisRecord is one row of the memory of decisions: what was decided, and the fingerprint of the // evidence it was decided on. type bankBasisRecord struct { Key string FP bankBasisFP Answer bankBasisAnswer Shape string // the fingerprint SHAPE this row was decided under: version | evidence width | producer UnitMembers []string // the unit this key was decided as part of, so a later split is visible } // basisReason is why a candidate was NOT served from the basis — the class the report counts by, and the // line the operator reads when a purchase costs money. type basisReason string const ( basisServed basisReason = "" // served from the basis: not bought basisNew basisReason = "new" // no record: this surface has never been decided basisKin basisReason = "kin-of-signed" // the canon anchors of THIS candidate moved (a relative was signed) basisMoved basisReason = "evidence-moved" // the source around this surface changed basisUnsettled basisReason = "unsettled" // the bank row is not a decided one (status != draft, or no rendering) basisAbsent basisReason = "no-bank-row" // decided once, but the bank no longer holds the row at all basisUnit basisReason = "unit-incomplete" // this member is settled, another member of its unit is not // basisSigned: the OWNER signed this row. Not served (his word is not the engine's memory to hand // back) and not a hole either (it is settled harder than any draft row) — see basisVerdictFor. basisSigned basisReason = "signed-by-owner" // basisOwnerRow: the bank row for this surface is not the ENGINE's at all — it came from the owner's // hand (a seed term) and he has not signed it. Not served and not a hole, exactly like a signed row: // see basisVerdictFor. basisOwnerRow basisReason = "owners-unsigned-row" // basisProducerMoved / basisWidthMoved / basisShapeMoved: the RULE that decided this row is not the rule // being applied now — a different prompt or model, a different evidence width, or a different // fingerprint shape. Three classes and not one, because the owner's reaction differs: somebody edited a // prompt, somebody turned a pair datum, or the engine's own shape was bumped. basisProducerMoved basisReason = "producer-changed" basisWidthMoved basisReason = "evidence-width-changed" basisShapeMoved basisReason = "fingerprint-shape-changed" // basisDiverged: the memory of decisions and the bank disagree about this surface's rendering. Serving // the stored answer would overwrite the bank's row and report the result as «decided earlier». basisDiverged basisReason = "bank-disagrees-with-the-memory" // basisAmbiguous: the bank holds MORE THAN ONE row for this surface — two senses, or two chapter // windows — and a candidate carries no sense and no window, so nothing here can say WHICH row is the // answer. Re-asked rather than guessed: the cheap direction of a doubt on this path is always to buy. basisAmbiguous basisReason = "ambiguous-surface" ) // basisWidth resolves the evidence width in ONE place: the pair's own datum, else the engine default. // // ⚠ TWO STEPS, WHERE terminologyOpts HAS THREE (explicit config → pair → engine default), and the missing // one is deliberate: there is no `gates.terminology.basis_width`. A per-book config override of what counts // as the same evidence would let one book's pipeline.yaml silently decide which of its stored decisions // survive, and that is a decision about the SOURCE LANGUAGE, not about a book. Said here because an earlier // revision of this comment claimed it "mirrors terminologyOpts", which the tree does not support — and a // comment describing a resolution order the code does not have is how the next edit adds the key without // anyone deciding to. // // ⛔ NOT A Go CONSTANT, and the reason is ratified rather than stylistic: D39.50 п.4 settled that a width // counted in RUNES is a fact about the source language, so freezing one in Go makes a PAIR-BLIND decision // — and here it would make it on the money path, where being pair-blind means buying the wrong bank for // every pair that is not the one the constant was chosen for. The review question this pack answers to: // a pair that is not in the repository states `basis_width` in its own terminology.txt and needs no Go edit. func (r *Runner) basisWidth() int { if pair := r.packTerminology(); pair != nil && pair.BasisWidth > 0 { return pair.BasisWidth } return terminologyDefaultBasisWidth } // bankRoleProducer is WHO would answer a bank-role question on this configuration, as one comparable // string: each role's model, the effort they share, and the canonical SHA256 of the pair's prompt for it. // // ⚠ IT IS RESOLVED IN ONE PLACE AND HASHED WHOLE, rather than assembled field by field at the call site, // because a producer identity spelled twice is two identities the day the two spellings disagree — the same // rule membank.IsEngineUnsigned keeps for «whose row is this». The classifier rides here too: it produces // the TYPE and the GENDER halves of a stored answer, so a book whose classifier changed has decisions the // new one would take differently. // // A role that is not configured contributes an empty field rather than being skipped: «off» and «absent» // must not hash the same, or turning a phase on would leave every decision it now co-authors untouched. func (r *Runner) bankRoleProducer() string { g := r.Pipeline.Gates.Terminology return hashOf(func(h io.Writer) { fmt.Fprintf(h, "tm-bank-producer-v2\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1e", g.Model, r.roleSystemPrompt(r.terminologyTemplate), g.ClassifierModel(), r.roleSystemPrompt(r.classifierTemplate), g.Reasoning) }) } // roleSystemPrompt is what a bank role's system turn ACTUALLY says on this book — the template with the // book's brief rendered into it — reduced to a comparable string. // // ⛔ THE RENDERED TEXT, NOT THE TEMPLATE'S SHA, and the difference is a whole class of law. The pair's // terminologist prompt reads the BRIEF: `{{title}}`, `{{audience}}`, `{{honorifics}}`, `{{transcription}}`, // `{{venuti}}`. Change `honorifics: keep` to `adapt`, or the transcription convention, and both waves of the // book are re-bought through BriefHash — while a basis keyed on the template's SHA would go on serving // renderings decided under the OLD law into waves paid for under the new one. Neither the evidence nor the // model would have moved; the law the role was told to apply would have. // // ⚠ AND IT IS PAIR-AGNOSTIC BY CONSTRUCTION, which is why it is one line rather than a list of brief // fields in Go: whatever a pair's own prompt chooses to read enters the identity because it enters the // rendered text. A pair that is not in this repository needs no edit here — the canon's standing review // question, answered by shape instead of by enumeration. // // A template that fails to render contributes the error text rather than falling back to silence: «this // producer could not be described» must not hash the same as «this role is off», and a run in that state // re-buys rather than serves. func (r *Runner) roleSystemPrompt(tpl *PromptTemplate) string { if tpl == nil { return "" // the role is not configured; an empty FIELD, never an absent one (see bankRoleProducer) } out, err := Render(tpl.System, RenderVars{Book: r.Book}) if err != nil { return "unrenderable:" + err.Error() } return out } // candidateBasisFingerprint hashes what the role reads ABOUT ONE CANDIDATE, in a form invariant to how the // contexts shown to the role are selected. // // WHAT GOES IN, and why each: // // - the candidate's key, its MINER-heuristic type, aliases, related surfaces and evidence tags — the // identity and the detector's account of it. The type is the heuristic one on purpose: hashing the // CLASSIFIER's type would mean buying the classifier on every purchase merely to learn whether the // terminologist may be skipped, which is the cost this whole mechanism exists to avoid; // - the context around EVERY occurrence of the key in the source, at the pair's evidence width; // - the SIGNED bank rows related to this candidate, uncapped. // // ⚠ WHO WOULD ANSWER is NOT here — it lives in the stored SHAPE (basisShape), for the same reason the // evidence width does: a producer is part of the RULE that decided a row, not part of the evidence, and // hashing it in with the evidence would report every prompt edit as «the source around this term changed». // // ⛔ THE PRODUCER IS IN, AND LEAVING IT OUT WAS A SILENT QUALITY REGRESSION. The fingerprint summarises the // evidence a decision stands on, and a decision stands on WHO READ that evidence as much as on the evidence // itself. Before this mechanism, improving the terminologist's prompt or moving it to another model missed // every checkpoint (RequestHash folds model, effort and snapshot) and the book simply re-bought its bank // from the new producer. With the producer out of the fingerprint, a settled row would never be asked again // no matter who is asking now — so a better prompt would reach the unsettled TAIL of every book and nothing // else, permanently, and the only exit would be bumping basisFingerprintVersion, which zeroes every book at // once. Improving a bank role's prompt is a direct lever on translationese, which is goal №1 of this // project; a money mechanism must not take that lever away to save a purchase it was never asked to save. // // ⚠ THE PRODUCER IS NOT THE SNAPSHOT, and the narrowing is deliberate. SnapshotID folds the BANK among many // other things, so hashing it would move this fingerprint every time any row of the bank changed — the // mechanism would save nothing on exactly the books it exists for. What identifies a producer is: the model // each bank role will be called on, the effort they share, and the canonical SHA256 of the pair's prompt for // each of them. Change any of those and the answer can differ; change the bank's contents and it cannot. // // WHAT STAYS OUT, and why each: // // - the draft variants. They leave the TRIGGER and stay EVIDENCE: a rendering the drafts happened to // produce this time is not new information about the term, and hashing it would re-buy a bank on // ordinary model noise. They are still printed on the sheet and in the log; // - freq and since_ch — measured constant between purchases, and since_ch is a chapter number, which a // re-cut moves without changing a word of the book; // - the KWIC subset, its cap and its selection rule — see terminology.EvidenceWindows; // - the batch this candidate ends up in. The fingerprint is per CANDIDATE, so it cannot inherit a // dependency on who its batch-mates were — which is the dependency the pack exists to remove. // // ⚠ AND TWO THINGS NEITHER THE EVIDENCE NOR THE RULE CAN SEE, named here because a blind spot that is not // written down is one the next reader assumes away: // // - A MODEL SLUG IS NOT A MODEL. The producer identity carries the name the call will be addressed to, // and a vendor that moves the weights behind that name changes who answers without changing anything // this engine can hash. «Слаг живой» is not «модель та же» (docs/experiments/00-provider-quirks.md), // and there is no mechanism here that would notice. // - AN EMPTY `reasoning` MEANS «the vendor's default», which is a value this project has already watched // move. The shipping pipelines pin the effort explicitly, so the identity is stable for them; a book // that leaves it empty inherits a producer whose behaviour can change under a constant string. // // ⚠ THE NARROWING THIS ACCEPTS, said out loud because it is invisible and deliberate. The role is shown // the anchor of its whole BATCH, so a signed row that relates to a NEIGHBOUR changes what the role reads // about this candidate and does NOT change this fingerprint. That is the measured case — a batch where // the anchor related to none of the six terms whose rendering moved and was there because of three other // members — and re-asking a term because somebody else's relative was signed is exactly the re-purchase // this mechanism refuses. func candidateBasisFingerprint(c terminology.Candidate, nsource string, canon []terminology.Neighbour, width int) bankBasisFP { canonPart := hashOf(func(h io.Writer) { // CanonFor with no cap: terminologyCanonCap bounds what one PROMPT may carry, which is a wire // concern. Applied here it could drop the very signed row CanonConflicts reads, and the fingerprint // would then call a term settled whose canon had moved. for _, pair := range terminology.CanonFor([]terminology.Candidate{c}, canon, len(canon)) { fmt.Fprintf(h, "canon\x00%s\x00%s\x1e", pair[0], pair[1]) } }) whole := hashOf(func(h io.Writer) { fmt.Fprintf(h, "v\x00%s\x1e", basisFingerprintVersion) fmt.Fprintf(h, "key\x00%s\x1e", c.Key) fmt.Fprintf(h, "type\x00%s\x1e", c.Type) fmt.Fprintf(h, "width\x00%d\x1e", width) for _, a := range c.Aliases { fmt.Fprintf(h, "alias\x00%s\x1e", a) } for _, rel := range c.Related { fmt.Fprintf(h, "related\x00%s\x1e", rel) } for _, e := range c.Evidence { fmt.Fprintf(h, "evidence\x00%s\x1e", e) } for _, w := range terminology.EvidenceWindows(nsource, c.Key, width) { fmt.Fprintf(h, "ctx\x00%s\x1e", w) } fmt.Fprintf(h, "canon\x00%s\x1e", canonPart) }) return bankBasisFP{Whole: whole, Canon: canonPart} } // hashOf runs write against a hash and returns it hex-encoded. Full-width sha256 rather than a truncation: // a collision here does not read as a bug, it reads as «already settled» — the engine would hand back an // answer decided on OTHER evidence and never buy the term again. The cheap direction of a wrong guess is // to re-buy, never to keep. func hashOf(write func(io.Writer)) string { h := sha256.New() write(h) return hex.EncodeToString(h.Sum(nil)) } // bankBasisRecordsFrom turns the stored rows into records, KEEPING the ones whose shape has moved. // // ⛔ KEPT, NOT DROPPED, and this is the whole reason the loop is a function. A dropped record leaves the // predicate with nothing, and nothing reads as `new` — «this surface has never been decided» — which is // false of every one of those rows. The summary line would then state the right cause ONCE while every row // carried the wrong one, and the two would disagree inside one run. The record carries its own shape so the // verdict can name which part of the rule moved: the producer, the evidence width, or the engine's shape. // // It is also the only way this rule can be asked a question: inside bankBasisPass it needs a store, a book // and a runner, and a money rule that can only be exercised end to end is one nobody plants a mutation into. func bankBasisRecordsFrom(stored map[string]store.BankBasisRow, shape string) (map[string]bankBasisRecord, int, map[string]int) { basis := make(map[string]bankBasisRecord, len(stored)) changed, old := 0, map[string]int{} for k, row := range stored { if row.FPVersion != shape { old[row.FPVersion]++ changed++ } basis[k] = bankBasisRecord{Key: k, Shape: row.FPVersion, FP: bankBasisFP{Whole: row.FPWhole, Canon: row.FPCanon}, Answer: bankBasisAnswer{Dst: row.Dst, Type: row.Type, Gender: row.Gender}} } return basis, changed, old } // bankBasisUnits assigns every candidate a unit id over the WHOLE candidate list — the series and families // a term must be decided together with. // // ⛔ OVER ALL CANDIDATES, NOT OVER THE PAID SUBSET, and this is a deliberate departure from the batcher's // own rule rather than an oversight. The batcher detects units over `paid` because co-batching has to // agree with the batches built from it — a statement about BATCHES. The predicate asks a different // question: «is this term's whole family settled». Answer it over `paid` and the unit is defined by the // very filter it feeds, so a family would count as whole precisely because its other members had been // filtered out — the circularity would make the guarantee vacuous exactly when it matters. // // The batcher is left untouched: it still detects over `paid` and its batches stay byte-identical. // // ⛔ THE RESULT IS TOTAL: EVERY candidate gets a unit of its own when it belongs to none, and that is a // correction of a real defect rather than tidiness. MergeUnits returns a PARTIAL map — a candidate in no // series and no family is simply absent from it — and the engine's own convention reads a missing key as // «a singleton» (terminology.Batch: `if id := unitID[...]; id != 0`). A predicate that indexed the raw map // instead would give every unit-less candidate the id 0, i.e. ONE shared unit, so a single unsettled // candidate anywhere in that crowd would re-buy all of them. // // ⚠ IT IS NOT A THEORETICAL HAZARD: the first run of this measurement did exactly that and reported 22 of // 69 candidates served where the truth is 65. The number looked entirely plausible — a strict-sounding // rule with a visible cost — and it was found only because a second view of the same rule, written with // the ids assigned explicitly, disagreed with it. A zero value that means «no unit» and a zero value that // means «unit number zero» are the same integer, and only one of them is a decision. func bankBasisUnits(cands []terminology.Candidate, sp terminology.SeriesParams, fp terminology.FamilyParams) map[string]int { seriesID := terminology.DetectSeries(cands, sp) fams := terminology.DetectFamilies(cands, fp) merged, _ := terminology.MergeUnits(seriesID, fams, fp) out := make(map[string]int, len(cands)) // Singleton ids are handed out from a region the detectors never reach, so a real unit id and a // synthesised one can never collide however many units a long book grows. next := 1 << 30 for _, c := range cands { if id, ok := merged[c.Key]; ok && id != 0 { out[c.Key] = id continue } next++ out[c.Key] = next } return out } // bankBasisVerdict is one candidate's outcome of the predicate. type bankBasisVerdict struct { Answer bankBasisAnswer Reason basisReason // basisServed when the answer is served from memory } // bankBasisSettle is THE PREDICATE: for each candidate, does the basis already answer it? // // A candidate is served only when ALL of these hold, and each condition is one that has already cost // somebody money to learn: // // - the basis holds a record for its key, and the record's fingerprint equals the one computed now; // - the bank holds an ENGINE row for it with status `draft` and a non-empty rendering. Status is asked // EXPLICITLY and «non-empty dst» is not enough: the miner produces `status:auto` WITH a rendering // (the raw first-chunk guess), a pair four engine comments declare impossible and no pin guards, so a // predicate reading the rendering alone would serve an answer nobody consolidated; // - every other member of its unit satisfies the same. A family decided in halves is the chimera the // co-batching exists to prevent, and half a family served from memory while the other half is re-asked // produces exactly it. // // Pure: no store, no clock, no map-order iteration in the output. func bankBasisSettle( cands []terminology.Candidate, fps map[string]bankBasisFP, basis map[string]bankBasisRecord, rows map[string]store.GlossaryEntry, ambiguous map[string]bool, unitID map[string]int, shape string, ) map[string]bankBasisVerdict { // Pass one: the per-candidate verdict, unit ignored. own := make(map[string]bankBasisVerdict, len(cands)) for _, c := range cands { own[c.Key] = basisVerdictFor(c, fps[c.Key], basis, rows, ambiguous, shape) } // Pass two: a unit is served only whole. Computed over the units' members rather than by re-walking // candidates per member, so the cost is linear and the answer cannot depend on candidate order. blocked := map[int]bool{} for _, c := range cands { // A signed row is neither served nor a hole: it does not hold its family open. Written as an // explicit second case rather than folded into the test above, because «not served» and «blocks» // stopped being the same statement the moment the owner's signature acquired a class of its own. if r := own[c.Key].Reason; r != basisServed && r != basisSigned && r != basisOwnerRow { blocked[unitID[c.Key]] = true } } out := make(map[string]bankBasisVerdict, len(cands)) for _, c := range cands { v := own[c.Key] if v.Reason == basisServed && blocked[unitID[c.Key]] { // The member itself is settled; its unit is not. Named as its own class rather than folded // into «evidence moved», because the two are different facts and only this one is a // consequence of a NEIGHBOUR — the operator who sees it should look at the family, not at // the source. v = bankBasisVerdict{Reason: basisUnit} } out[c.Key] = v } return out } // basisVerdictFor is the per-candidate half of the predicate, split out so the unit rule above reads as // the one thing it is and so the class of a re-ask is decided in exactly one place. func basisVerdictFor(c terminology.Candidate, fp bankBasisFP, basis map[string]bankBasisRecord, rows map[string]store.GlossaryEntry, ambiguous map[string]bool, shape string) bankBasisVerdict { // A surface the bank answers with SEVERAL rows is refused before anything else is asked: a candidate // carries no sense and no window, so there is no honest way to choose one of them — see bankBasisRows. if ambiguous[c.Key] { return bankBasisVerdict{Reason: basisAmbiguous} } // THE OWNER'S SIGNATURE IS ANSWERED FIRST, before the memory of engine decisions is consulted at all, // and it is answered with a class of its own. Both halves of what follows are deliberate: // // - the basis NEVER hands back a rendering for a signed row. That rendering is the owner's canon, not // a decision this mechanism took, and a memory of engine decisions that starts serving the owner's // words has merged the two things the bank ontology keeps apart on purpose; // - the row is still sent to the paid role, because the two-condition filter pays for banked surfaces // ON PURPOSE — that is what keeps the dispute marks on the sheet the owner signs by. ⚠ PREMISE: // backlog row 447 is OPEN, and it is the row that would change this. Should it close, a signed // surface stops being bought — and that must be a decision somebody takes, not a behaviour that // shifts because a predicate here was written to assume one answer. // // What it does NOT do is hold its family open. «Bought» and «blocks its unit» are two consequences of // one cause, and only the second is free to change without touching product behaviour: a row the owner // has signed is settled harder than any draft one, so counting it as a hole in its own family made the // owner's signature re-buy the family it belongs to. Measured on both bought runs: the two signed rows // were the ONLY blockers, and exempting them serves 67 of 69 and 64 of 66 instead of 65 and 62. if row, inBank := rows[c.Key]; inBank && !membank.IsEngineUnsigned(row) && strings.TrimSpace(row.Dst) != "" { // ⛔ ASKED THROUGH membank.IsEngineUnsigned AND NOT BY A SECOND SPELLING. The engine has ONE // definition of «whose row is this» — `Source == "mined" && Status != "approved"` — and it has // other callers; a private `Status == "approved"` here is a SECOND spelling of the same question, // and two spellings are two answers the day they disagree. They already did: a term the OWNER // wrote by hand and left unsigned (`source: seed`, `status: draft` — a shape the loader admits, // and for which it even REQUIRES a rendering) answered «not approved», so the engine recorded its // own consolidation under that surface and later served it back as «this book decided it earlier». // Found by acceptance, on a population this pack's own fixture builds. if row.Status == "approved" { return bankBasisVerdict{Reason: basisSigned} } return bankBasisVerdict{Reason: basisOwnerRow} } rec, held := basis[c.Key] if !held { return bankBasisVerdict{Reason: basisNew} } // THE RULE IS COMPARED BEFORE THE EVIDENCE, and each part of it names itself. A row decided under // another width, or by another producer, has evidence that never moved — reporting it as «the source // around this term changed» would send the owner looking at the book for a change somebody made to a // knob or to a prompt, and those three call for three different reactions. if rec.Shape != "" && rec.Shape != shape { wantV, wantW, wantP := basisShapeParts(shape) gotV, gotW, gotP := basisShapeParts(rec.Shape) switch { case gotP != wantP: return bankBasisVerdict{Reason: basisProducerMoved} case gotW != wantW: return bankBasisVerdict{Reason: basisWidthMoved} case gotV != wantV: return bankBasisVerdict{Reason: basisShapeMoved} } } if rec.FP.Whole != fp.Whole { // WHICH PART moved, so the re-purchase can state its own cause. Canon first: a signed relative is // an act the owner performed and can recognise, while «the source moved» is not. if rec.FP.Canon != fp.Canon { return bankBasisVerdict{Reason: basisKin} } return bankBasisVerdict{Reason: basisMoved} } row, inBank := rows[c.Key] if !inBank { return bankBasisVerdict{Reason: basisAbsent} } if row.Status != "draft" || strings.TrimSpace(row.Dst) == "" { return bankBasisVerdict{Reason: basisUnsettled} } // ⛔ THE MEMORY AND THE BANK MUST STILL AGREE, and this is a CONDITION rather than a mechanism — two // lines, on the money path, making an existing claim true. // // The predicate's sentence is «this book decided this and the bank still holds that decision». Nothing // above checks the second half: the answer is served from the record, and it then travels straight into // the delta and overwrites whatever the bank held. On the run that wrote them the two agree by // construction — the auto-bank is emitted from the same consolidation the record was built from — so // this cannot fire on ordinary material, and it is NOT measured on the bought runs. // // ⚠ THE POPULATION IS NAMED RATHER THAN CLAIMED: the auto-bank is a FILE beside the database // (18-bank-ontology.md lists it among the SOURCES), so a hand edit of it is the reachable way for the // two to diverge. Out of contract, yes — and precisely the case where serving the stored answer would // silently undo somebody's edit while the sheet reported the row as settled earlier. The cheap // direction of a doubt on this path is always to re-ask. if text.NormalizeTargetForm(row.Dst) != text.NormalizeTargetForm(rec.Answer.Dst) { return bankBasisVerdict{Reason: basisDiverged} } return bankBasisVerdict{Answer: rec.Answer, Reason: basisServed} } // bankBasisRows indexes a book's bank by the key a candidate is matched on — the same normalized key the // role is asked about and consolidatedRows gives a proposal, so the predicate and the emission cannot come // to disagree about which row belongs to which candidate. It returns the surfaces the bank answers // UNAMBIGUOUSLY, and separately the ones it does not. // // ⛔ A SURFACE WITH MORE THAN ONE ROW IS NOT INDEXED, IT IS REFUSED. The bank's uniqueness key is the // FOUR-tuple (src, sense, since_ch, until_ch): two senses of one surface are two laws by design, and a term // whose rendering changes across a spoiler boundary is two rows with different windows. A candidate carries // none of those — it has a Key and nothing else — so a map written as `out[key] = row` keeps whichever row // the store happened to return last (it orders by src, sense, since_ch, until_ch, so the LATEST window // wins) and silently answers for the other. // // ⚠ THE CONSEQUENCE WAS NOT THEORETICAL, WHICH IS WHY IT IS A REFUSAL AND NOT A WARNING: a surface the // owner had SIGNED on one window, holding an engine draft on another, indexed to the draft — so the // predicate read «status draft, rendering present», served the engine's word, and the sheet announced it as // the book's own earlier decision. The owner's signature was invisible to the mechanism that was supposed // to defer to it. Found by adversarial review of this pack, on the tree. // // Aliases are NOT indexed here, unlike bankSettledSurfaces. That filter asks «does the bank already RENDER // this surface», which an alias answers; this one asks «is THIS row the engine's decision», which an alias // of another row is not. func bankBasisRows(bank []store.GlossaryEntry, norm func(string) string) (rows map[string]store.GlossaryEntry, ambiguous map[string]bool) { seen := make(map[string][]store.GlossaryEntry, len(bank)) for _, e := range bank { k := norm(e.Src) seen[k] = append(seen[k], e) } rows, ambiguous = make(map[string]store.GlossaryEntry, len(seen)), map[string]bool{} for k, es := range seen { if len(es) != 1 { ambiguous[k] = true continue } rows[k] = es[0] } return rows, ambiguous } // basisSeriesParams is the pair's series morphology as the unit pass needs it — resolved from the SOURCE // language's declared data, exactly as the batcher resolves it, so the two passes cannot disagree about // what a series is. func basisSeriesParams(sourceLang string) terminology.SeriesParams { enabled, headFinal := lang.SeriesMorphology(sourceLang) return terminology.SeriesParams{Enabled: enabled, HeadFinal: headFinal} } // basisReasonNames is the fixed print order of the re-ask classes, so a report of the same run always // lists them the same way and two reports can be diffed. var basisReasonNames = []basisReason{basisNew, basisKin, basisMoved, basisUnsettled, basisAbsent, basisUnit, basisSigned, basisOwnerRow, basisDiverged, basisAmbiguous, basisProducerMoved, basisWidthMoved, basisShapeMoved} // countBasisReasons tallies a verdict map by class, in the fixed order above. func countBasisReasons(v map[string]bankBasisVerdict) (served int, byReason []struct { Reason basisReason N int }) { n := map[basisReason]int{} for _, verdict := range v { n[verdict.Reason]++ } served = n[basisServed] for _, r := range basisReasonNames { byReason = append(byReason, struct { Reason basisReason N int }{r, n[r]}) } return served, byReason } // bankBasisPass is the money gate: it runs BEFORE the first paid call, decides which candidates this book // has already answered, and returns both the fingerprints (for the boundary writer) and the answers (for // every reader of a consolidation). // // ⛔ THE ANSWERS ARE RETURNED, NOT MERELY A FILTER, and that is the whole shape of this mechanism. A pass // that only removed settled candidates from the paid set would leave them with NO rendering downstream — // attachConsolidatedDst stamps nothing for a term the role did not answer — and the emission would fall to // the raw first-chunk banknote guess at status `auto`, or to nothing. The saving would be visible and the // loss would be silent, which is the exact trade this codebase refuses. // // ⚠ THE SERVED ANSWERS ARE JUDGED BY THE DISPUTE CHECKS like any other consolidation, and that is decided // rather than inherited. ConsolidationKeyConflicts skips the engine's own unsigned row on an unchanged // uniqueness key, so a basis answer cannot contradict itself by construction; CanonConflicts fires only on // strict containment of a SIGNED source, and every signed row related to a candidate is inside its // fingerprint — so a served row's canon has not moved and its verdict is the same one the purchase that // bought it produced. Exempting them instead would silence the marks for precisely the population the // two-condition filter is two-condition for. // // A read failure degrades to «nothing is settled»: the run then pays for the pass exactly as it did before // this mechanism existed. Loud, because that is money — and the cheap direction of any doubt here is to // re-buy, never to serve an answer whose evidence nobody could check. func (r *Runner) bankBasisPass(ctx context.Context, cands []terminology.Candidate, consider map[int]bool, bank []store.GlossaryEntry, res *terminologyResult) (fps map[string]bankBasisFP, answers map[string]bankBasisAnswer, rows map[string]store.GlossaryEntry, ambiguous map[string]bool) { width := r.basisWidth() producer := r.bankRoleProducer() canon := approvedNeighbours(bank) nsource := "" if r.bankSrc != nil { nsource = r.bankSrc.book } fps = make(map[string]bankBasisFP, len(cands)) for _, c := range cands { fps[c.Key] = candidateBasisFingerprint(c, nsource, canon, width) } rows, ambiguous = bankBasisRows(bank, text.NormalizeSourceKey) if nsource == "" { // Without the source every fingerprint is taken over an empty text, so surfaces that differ in the // book collide here. Said out loud and treated as «nothing is settled»: a collision in this map // serves one term's rendering for another's, which is worse than any amount of re-buying. r.Log.WarnContext(ctx, "the settled basis is INERT this run: the book's normalized source was not built, so the evidence behind a decision cannot be fingerprinted and every candidate goes to the paid role", "book", r.Book.BookID, "candidates", len(cands)) return fps, nil, rows, ambiguous } stored, err := r.Store.BankBasisForBook(r.Book.BookID) if err != nil { r.Log.WarnContext(ctx, "could not read the settled basis, so every already-decided term is being bought AGAIN out of this book's LIFETIME bank-role budget; the pass still runs and the basis is rewritten at this run's boundary", "book", r.Book.BookID, "err", err) return fps, nil, rows, ambiguous } shape := basisShape(width, producer) basis, shapeChanged, oldShapes := bankBasisRecordsFrom(stored, shape) if shapeChanged > 0 { // BOTH shapes are printed, because that is what turns «your bank is being re-bought» into a sentence // the operator can act on: the shape carries the evidence WIDTH and the PRODUCER, so the line names // what moved rather than leaving them to infer it from a count. was := make([]string, 0, len(oldShapes)) for s, n := range oldShapes { was = append(was, fmt.Sprintf("%s×%d", s, n)) } sort.Strings(was) r.Log.WarnContext(ctx, "the settled basis was written under a DIFFERENT fingerprint shape and cannot be compared against this one: those terms are bought again, ONCE, and stored under the new shape — this is the announced price of changing what counts as the same evidence, and it is NOT «the source moved»", "book", r.Book.BookID, "dropped", shapeChanged, "of_stored", len(stored), "shape_now", shape, "shape_stored", strings.Join(was, " ")) } units := bankBasisUnits(cands, basisSeriesParams(r.Book.SourceLang), r.familyParams) verdicts := bankBasisSettle(cands, fps, basis, rows, ambiguous, units, shape) answers = map[string]bankBasisAnswer{} res.BasisReAsked = map[basisReason]int{} for i, c := range cands { // ⛔ ONLY THE CANDIDATES THE BANK FILTER KEPT. dropBankSettled has already taken the surfaces the // SEED renders and every draft agreed with, and those are a different fact with a different // sentence on the sheet («the bank already renders this surface» versus «this book decided it // earlier»). Asked about both, one candidate ends up carrying BOTH marks: the sheet prints two // mutually exclusive sentences about it, the read-out counts it twice, and — worst of the three — // SettledByBank lifts it out of the contested queue, which is precisely the exemption the comment // on that queue refuses for a basis row. Found by adversarial review, reproduced end to end. if !consider[i] { continue } v := verdicts[c.Key] if v.Reason == basisServed { answers[c.Key] = v.Answer res.BasisServedKeys = append(res.BasisServedKeys, c.Key) continue } res.BasisReAsked[v.Reason]++ } res.BasisServed = len(answers) if len(answers) == 0 { return fps, nil, rows, ambiguous } // The saving with its denominator, and the re-ask classes beside it: «57 skipped» is unreadable without // «of 69», and a purchase that costs money owes the operator the reason it did. r.Log.InfoContext(ctx, "the settled basis answered terms this book had already decided, so they were NOT sent to the paid roles; their stored rendering, type and gender are carried into the delta exactly as the purchase that bought them left them", "book", r.Book.BookID, "served", len(answers), "of_candidates", len(cands), "re_asked_new", res.BasisReAsked[basisNew], "re_asked_kin_of_signed", res.BasisReAsked[basisKin], "re_asked_evidence_moved", res.BasisReAsked[basisMoved], "re_asked_unsettled", res.BasisReAsked[basisUnsettled], "re_asked_no_bank_row", res.BasisReAsked[basisAbsent], "re_asked_unit_incomplete", res.BasisReAsked[basisUnit], "signed_by_owner", res.BasisReAsked[basisSigned], "owners_unsigned_row", res.BasisReAsked[basisOwnerRow], "bank_disagrees", res.BasisReAsked[basisDiverged], "producer_changed", res.BasisReAsked[basisProducerMoved], "width_changed", res.BasisReAsked[basisWidthMoved], "ambiguous_surface", res.BasisReAsked[basisAmbiguous], "fingerprint_shape", shape) return fps, answers, rows, ambiguous } // basisRowsToStore is what the book has decided as of the end of this pass — fresh answers and served ones // together, each with the fingerprint of the evidence it stands on. // // Only a CONSOLIDATED rendering is recorded. A term the role declined, or never answered, has not been // decided: storing it would let a later purchase serve an empty rendering back, which is the regression // this whole mechanism exists to prevent, arriving through its own memory. // // ⛔ AND A SURFACE THE OWNER HAS SIGNED IS NOT RECORDED EITHER, which is a second refusal and a harder one. // A signed surface still goes to the paid role on purpose (the two-condition filter keeps the dispute marks // on the sheet, backlog row 447), so its consolidation DOES arrive here — and recording it would put the // engine's word into the memory of decisions under a surface whose law is the owner's. The day that // signature is lifted, or its window changes, the predicate would find a record, see an unsigned draft row, // and serve the ENGINE's rendering as «this book decided it earlier» — over the owner's own. That is the // exact merging of the two words the bank ontology keeps apart, arriving through the door this pack built. // Found by adversarial review, on the tree, with the un-signing reproduced. // // ⛔ AN AMBIGUOUS SURFACE IS NOT RECORDED FOR THE MIRROR REASON: the carrier's key is (book, surface), so // two senses or two windows of one surface cannot be remembered apart, and whichever was written last would // answer for both. Refused at the write as well as at the read, because a guarantee kept on one side only // is one the next caller can walk around. func basisRowsToStore(cands []terminology.Candidate, consolidated, classified, gendered map[string]string, fps map[string]bankBasisFP, rows map[string]store.GlossaryEntry, ambiguous map[string]bool, shape string) []store.BankBasisRow { out := make([]store.BankBasisRow, 0, len(consolidated)) for _, c := range cands { dst := consolidated[c.Key] if strings.TrimSpace(dst) == "" { continue } if ambiguous[c.Key] { continue // the bank answers this surface with several rows: nothing here can say which } if row, inBank := rows[c.Key]; inBank && !membank.IsEngineUnsigned(row) { continue // the owner's word — signed or hand-written and unsigned — is not the engine's memory to keep } fp, ok := fps[c.Key] if !ok || fp.Whole == "" { continue // no fingerprint, no comparable memory: better re-bought than served blind } out = append(out, store.BankBasisRow{ SrcKey: c.Key, FPVersion: shape, FPWhole: fp.Whole, FPCanon: fp.Canon, Dst: dst, Type: classified[c.Key], Gender: gendered[c.Key], }) } return out } // writeBankBasis publishes the run's decisions at an output boundary. `at` names the boundary and rides the // log line, exactly as the bank read-out's does. // // ⛔ AT THE OUTPUT BOUNDARIES, AND AT NO OTHER MOMENT. Written straight after the paid pass — before the // run reaches a boundary — a basis would be present on a CRASH RESUME: the resume's candidate set would // then be narrower than the one that paid, the batch composition would differ, the checkpoints addressed by // request hash would not be found, and the resume would buy again what was already bought. The two // boundaries are `run-finished` and the signature stop, and there are two because one is unreachable from // the other: the signature stop returns as an ERROR VALUE, so a run that stops for signature never reaches // `run-finished` at all, and a resume after the owner signs would re-buy the entire pass. // // ⚠ FAILURE IS LOUD, AND LOUDER THAN ITS NEIGHBOUR'S. The stop's flag memory may fail quietly: losing it // costs one benign extra stop. Losing this costs a re-purchase of both bank roles out of a LIFETIME // per-book ceiling whose exhaustion is irreversible — so it is an ERROR line naming the money, never a // quiet note. The run itself is NOT failed: the work is done and paid for, and killing a finished run over // a bookkeeping row would trade a real book for a row that the next boundary rewrites anyway. func (r *Runner) writeBankBasis(ctx context.Context, at string) { t := r.lastTerminology if t == nil || len(t.BasisRows) == 0 { return } if err := r.Store.PutBankBasis(r.Book.BookID, t.BasisRows); err != nil { r.Log.ErrorContext(ctx, "could not record the settled basis: the terms this run decided are NOT remembered, so the NEXT purchase will pay both bank roles for them again out of this book's LIFETIME budget (exhausting it leaves the book without a consolidated bank permanently). The run itself is unaffected and its bank is on disk", "book", r.Book.BookID, "at", at, "rows", len(t.BasisRows), "err", err) return } r.Log.InfoContext(ctx, "settled basis recorded", "book", r.Book.BookID, "at", at, "rows", len(t.BasisRows), "served_this_run", t.BasisServed, "shape", basisFingerprintVersion) } // stampBasisTypes writes the basis's banked type onto the candidates it answered, and returns how many it // changed. It is deliberately NOT applyTypes: that one's return value is published as `reclassified`, which // means «how many types the CLASSIFIER changed», and a term the classifier never saw must not appear in it. // // ⚠ IT RUNS AFTER THE FINGERPRINTS ARE TAKEN, and the order is load-bearing rather than incidental: the // fingerprint hashes the MINER'S heuristic type on purpose, so that learning whether the paid roles may be // skipped never requires buying the classifier first. Stamp before the hash and that property is gone — // silently, because everything would still work and every book would simply pay the classifier forever. func stampBasisTypes(cands []terminology.Candidate, answers map[string]bankBasisAnswer) int { n := 0 for i := range cands { a, ok := answers[cands[i].Key] if !ok || a.Type == "" || a.Type == cands[i].Type { continue } cands[i].Type = a.Type n++ } return n }