package pipeline import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "os" "path/filepath" "sort" "strings" "unicode" "unicode/utf8" "textmachine/backend/internal/checks" "textmachine/backend/internal/chunk" "textmachine/backend/internal/config" "textmachine/backend/internal/llm" "textmachine/backend/internal/membank" "textmachine/backend/internal/store" ) // repair.go: the addressable-defect repair sub-step (pack-16, ratified D39.24) — the loop that closes // «checker → typed error → targeted repair of the minimal span → deterministic re-gate → otherwise the // previous behaviour». It is OPT-IN and OFF by default; with the gate off every function here returns // immediately and the runner's bytes, snapshot and golden capture are unchanged. // // WHERE IT SITS. Inside runStage, after the attempt loop and the escalation resolve, BEFORE chunk_status is // written — a sub-step of the FINAL stage rather than a stage of its own. A new stage would join the edit // wave, steal isFinal from the editor, move the edit-wave snapshot (re-billing every existing book), add a // chunk_status row per unit and wedge status's `expected == ok` arithmetic forever. The sub-step disturbs // none of that. // // WHAT IT MAY DO. Replace a BOUNDED span of a text the stage already resolved OK. It never changes a // disposition — not to flagged, not from it. A failure is not a new failure mode: the text stays exactly as // it was and the defect stays exactly as visible as it is today (observability). That is what keeps D2 // intact: the loop cannot make garbage ship, because it only ever runs on output the gates already passed. // // WHAT PROTECTS IT. Six guards on the reply (see repairGuardsVersion) and a re-gate over the ASSEMBLED text. // The re-gate is deliberately POSITIVE where it can be and conservative everywhere else: the shipped // checkers all go silent when the offending text is simply DELETED, so "the class stopped firing" is not on // its own evidence of repair — deletion must be refused by a content guard, not rewarded by a green re-gate. // roleRepair is the synthetic stage role every repair call is addressed under. It keeps the repair call on // its own request-hash axis (Role is a hash field), gives its checkpoints and telemetry rows their own // queryable class — which is the cost marker, so no migration is needed to answer "what did repair spend" — // and keeps it out of every role-gated path (banknote and the coverage gate are translator-only). const roleRepair = "repair" // repairVersion versions the repair ALGORITHM (candidate selection → span expansion → call → apply). It is // folded into the snapshot only when the gate is enabled, so a change re-pins repaired books loudly instead // of silently re-resolving them, and costs a disabled book nothing. const repairVersion = "repair-v1-span-replace" // repairGuardsVersion versions the ACCEPTANCE rules (the reply guards + the re-gate). It is separate from // repairVersion because a guard can be tightened without touching the selection algorithm, and either // change alters which repairs survive — i.e. the resolved bytes. const repairGuardsVersion = "repair-guards-v1-nochange+corridor+nodel+regate" // repairNoChange is the ENGINE sentinel a repair prompt instructs the model to return when the flagged text // is in fact correct. It is deliberately a bracketed engine token (the ⟦TM-BANK-v1⟧ idiom) rather than a // natural-language phrase: a phrase would be pair data leaking into Go, and any wording would collide with // prose. A first-class "no edits" answer is load-bearing, not a nicety — our own research records that on a // strong draft no APE system beat doing nothing, and that LLM editors edit correct input when not offered // the option. A decline is therefore a SUCCESS of the protocol and is counted separately. const repairNoChange = "⟦TM-NOCHANGE⟧" // Reply guards (versioned by repairGuardsVersion). const ( // repairMinSpanRatio / repairMaxSpanRatio bound the replacement against the original span in RUNES. The // lower bound is an anti-deletion floor: every shipped checker goes quiet when the offending phrase is // deleted, so without it the cheapest possible "repair" — dropping the sentence's content — would pass // the re-gate. It does NOT catch a deletion small relative to the span, which is why the token and digit // guards below exist as well. repairMinSpanRatio = 0.6 repairMaxSpanRatio = 2.0 // repairMinWordKeepRatio requires the replacement to keep nearly all of the span's word tokens. The span // judged here is the SENTENCE-expanded one, so a permissive ratio would license deleting a whole clause // while "repairing" a two-word unit expression: at 0.6 a 21-token sentence could come back as 13 tokens. // The design's rule is «tokens(reply) >= tokens(original) - 1» — one token of slack for the conversion // itself (e.g. «полчаса» → «час» merges nothing, «три часа» → «шесть часов» keeps the count). repairMaxTokenLoss = 1 // repairMaxSpanBytes bounds a single repair span absolutely: sentence expansion degenerates to the whole // text on a target whose terminators are not in the data, and a whole-unit rewrite is not a repair. repairMaxSpanBytes = 1200 ) // repairResult is one unit's repair outcome, derived entirely from what actually happened (never stored as // its own row): counters for the report and the run's spend on this class. type repairResult struct { Calls int Applied int Declined int // the model answered ⟦TM-NOCHANGE⟧ — our flag was a false positive Rejected int // a guard or the re-gate refused the reply // CostUSD is what THIS run paid; CumUSD is what the calls cost in total (a replayed checkpoint costs $0 // this run but itsoriginal cost still belongs to the unit's honest total, exactly as retries do). CostUSD float64 CumUSD float64 // Fresh marks that at least one repair call actually reached the provider this run (a replayed // checkpoint is $0 and must not make a resumed stage look like a fresh one). Fresh bool } func (rr repairResult) touched() bool { return rr.Calls > 0 } // defaultRepairClasses is the engine's ratified default ACTUATOR set — deliberately the smallest one. // // ABSENT, and why (design §16.1/§16.2, ratified): DC2 (千万/数十万) and 成-percent fire on whole-unit substring // probes with no positional relation to the source match, so a span derived from them can point at innocent // prose. BOTH DC1 classes are absent for a different reason: their positive post-invariant cannot be // expressed on the pair data as it stands (the hours table carries 9 forms, so a CORRECT repair rendered // «шести часов» does not parse), and the fractional probe's target side is an unanchored substring scan — // so a unit whose 半个时辰 was translated CORRECTLY, but which mentions «полчаса» elsewhere, would have that // unrelated sentence rewritten. They remain DETECTORS (the residual scan counts them) and become actuators // only when an operator names them explicitly, having accepted that trade. var defaultRepairClasses = []checks.RepairClass{ checks.RepairLatinResidue, checks.RepairBrokenWord, } // knownRepairClasses is the validation surface for gates.repair.classes. var knownRepairClasses = map[string]checks.RepairClass{ string(checks.RepairDC1TimeUnits): checks.RepairDC1TimeUnits, string(checks.RepairDC1Fractional): checks.RepairDC1Fractional, string(checks.RepairLatinResidue): checks.RepairLatinResidue, string(checks.RepairBrokenWord): checks.RepairBrokenWord, } // repairClasses resolves the enabled class set: the configured subset, else the ratified default. Sorted, so // the snapshot fold is deterministic. func (r *Runner) repairClasses() []string { var out []string if len(r.Pipeline.Gates.Repair.Classes) > 0 { out = append(out, r.Pipeline.Gates.Repair.Classes...) } else { for _, c := range defaultRepairClasses { out = append(out, string(c)) } } sort.Strings(out) return out } // loadRepairTemplates resolves one prompt per enabled class by CONVENTION — `//repair/ // .md` — and fails LOUD naming the path when it is missing, exactly as the stage convention does. // The prompt carries the typed error: which file is chosen IS the error type, so the class-specific wording // («1 时辰 = 2 часа») lives in the pair's own language as data, and Go passes only spans. An unknown class // name is a config error here rather than in config/, which cannot import the class vocabulary without an // import cycle (internal/checks imports internal/config). func (r *Runner) loadRepairTemplates() error { if !r.Pipeline.Gates.Repair.Enabled { return nil } dir := r.Pipeline.Gates.Repair.PromptsDir r.repairTemplates = map[checks.RepairClass]*PromptTemplate{} for _, name := range r.repairClasses() { cls, ok := knownRepairClasses[name] if !ok { return fmt.Errorf("pipeline: gates.repair.classes lists unknown class %q — known: %s", name, strings.Join(sortedClassNames(), ", ")) } // A class whose POSITIVE post-condition cannot be asserted on this pair's data must not be enabled: // otherwise every repair of that class is paid for and then rejected, silently burning money. The // fractional-unit class needs the pair's bare hour word to verify that the replacement still states a // duration (rather than deleting it), so its absence is a loud load-time stop naming the missing key. if cls == checks.RepairDC1Fractional && !checks.HasHourWordProbe(r.checkers) { return fmt.Errorf("pipeline: gates.repair enables class %q, but the pair pack ships no `hour_word_re` — without it the repair's positive post-condition (the replacement must still state a duration) cannot be asserted and every repair of this class would be paid for and rejected; add the key to the pair's dc-checkers data or drop the class", name) } path := filepath.Join(dir, name+".md") if _, err := os.Stat(path); err != nil { return fmt.Errorf("pipeline: gates.repair is enabled for class %q but its prompt is missing — expected %s (a repair prompt is authored in the pair's own language, like the role prompts); add that file or drop the class from gates.repair.classes", name, path) } tpl, err := LoadPromptTemplate(path) if err != nil { return err } r.repairTemplates[cls] = tpl } return nil } func sortedClassNames() []string { out := make([]string, 0, len(knownRepairClasses)) for k := range knownRepairClasses { out = append(out, k) } sort.Strings(out) return out } // maybeRepair is the sub-step. It returns the (possibly repaired) final text plus the outcome counters; the // original text is returned unchanged on every path that is not a fully-verified repair. It returns an error // ONLY on an infra failure that is not this optional step's business to swallow — a USD ceiling denial is // explicitly NOT one of those (see below). func (r *Runner) maybeRepair(ctx context.Context, st config.Stage, snapID string, ch chunk.Chunk, job *store.Job, draft, finalText string, injected []membank.PickedEntry) (string, repairResult, error) { var res repairResult gate := r.Pipeline.Gates.Repair if !gate.Enabled || strings.TrimSpace(finalText) == "" { return finalText, res, nil } cfg := r.cheapGateConfig() enabled := map[checks.RepairClass]bool{} // The Latin-residue class is pure Go (no pair data can make it inert), so on a LATIN-script target every // ordinary word reads as a leak. It gates on the target's DECLARED word script (data): meaningful only for // a NON-Latin target, inert by absence for a Latin-written or dataless target — no language predicate. latinOK := r.checkers.TargetScriptNonLatin() for _, n := range r.repairClasses() { enabled[knownRepairClasses[n]] = true } var cands []checks.RepairCandidate for _, c := range checks.RepairCandidates(ch.Text, finalText, cfg) { if !enabled[c.Class] || (c.Class == checks.RepairLatinResidue && !latinOK) { continue } cands = append(cands, c) } // Expansion + disjointness happen BEFORE any call: an overlapping candidate can never be applied (two // splices into one range corrupt the text), so paying for its reply would be paying for nothing. cands = checks.DisjointCandidates(finalText, cands) // ABSOLUTE SPAN CAP: sentence expansion falls back to the WHOLE text when the target's terminators are // absent from the data, or when the unit is one long unpunctuated line — and a span of several thousand // bytes is a whole-unit rewrite wearing a repair's clothes. The bound is absolute rather than relative to // the unit: a legitimately SHORT unit may well be a single sentence, and rewriting that one sentence is // exactly the intended behaviour. Dropped BEFORE the call, so an unusable candidate is never paid for. kept := cands[:0] for _, c := range cands { if c.DstSpan[1]-c.DstSpan[0] > repairMaxSpanBytes { r.Log.InfoContext(ctx, "repair candidate dropped: span too large to be a targeted repair", "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "class", string(c.Class), "span_bytes", c.DstSpan[1]-c.DstSpan[0]) continue } kept = append(kept, c) } cands = kept if len(cands) > gate.MaxCallsPerUnit { cands = cands[:gate.MaxCallsPerUnit] } if len(cands) == 0 { return finalText, res, nil } // PRE-CALL soft budget cap, read WITHOUT serialisation: parallel wave workers may overshoot it by up to // (workers-1) calls. That is a deliberate deviation from the escalation cap's mutex — escalation is rare // (its own doc says so) while repair is common-path, and a lock spanning the provider call (which is what // makes the escalation bound exact) would put every worker behind one slow transport retry. // // The cap is applied PER CANDIDATE and only to a FRESH call (see the loop below): an ALREADY-PAID repair // must replay from its checkpoint regardless of the remaining budget, or a crash between the paid call // and the chunk_status write would make the resumed run ship DIFFERENT bytes than the run that paid — // exactly the idempotency escalation.go:99-114 exists to protect. budgetExhausted := false if spent, err := r.Store.RepairSpentUSD(r.Book.BookID); err != nil { return finalText, res, err } else if spent >= gate.BudgetUSD { budgetExhausted = true r.Log.InfoContext(ctx, "repair budget exhausted; only already-paid repairs will replay", "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "spent_usd", fmt.Sprintf("%.6f", spent), "budget_usd", gate.BudgetUSD) } type accepted struct { span [2]int replacement string } var applied []accepted for i, c := range cands { tpl := r.repairTemplates[c.Class] if tpl == nil { continue // not enabled / no prompt — cannot happen after loadRepairTemplates, defensive } srcSpan := "" if c.SrcSpan[1] > c.SrcSpan[0] && c.SrcSpan[1] <= len(ch.Text) { s := checks.ExpandToSentence(ch.Text, c.SrcSpan) srcSpan = ch.Text[s[0]:s[1]] } dstSpan := finalText[c.DstSpan[0]:c.DstSpan[1]] msgs, err := Messages(tpl, RenderVars{Book: r.Book, Text: srcSpan, Draft: dstSpan}) if err != nil { return finalText, res, err } // Checkpoint FIRST, budget second (escalation.go's idempotency ordering): a paid reply replays free. paid, perr := r.repairCheckpointExists(st, snapID, ch, i, msgs) if perr != nil { return finalText, res, perr } if budgetExhausted && !paid { continue } att, err := r.runRepairAttempt(ctx, st, snapID, ch, job, i, msgs) if err != nil { // A ceiling denial must NOT abort the book: this step is optional and sits BEFORE the // chunk_status write, so propagating would discard the row of an already-paid, successful // editor stage — and would do it again on every resume. Mirrors escalation.go's degrade. if errors.Is(err, errReserveCeiling) { r.Log.WarnContext(ctx, "repair call denied by a USD ceiling; leaving the defect flagged", "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "class", string(c.Class)) break } return finalText, res, err } res.Calls++ res.CostUSD += att.runCost res.CumUSD += att.cumCost res.Fresh = res.Fresh || att.freshCall reply, verdict := repairReplyVerdict(att, dstSpan, c.Class, cfg.Checkers) switch verdict { case repairDeclined: res.Declined++ r.Log.InfoContext(ctx, "repair declined by the model (the flagged text is correct)", "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "class", string(c.Class)) case repairAccepted: applied = append(applied, accepted{span: c.DstSpan, replacement: reply}) default: res.Rejected++ r.Log.WarnContext(ctx, "repair reply rejected by a guard", "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "class", string(c.Class), "reason", string(verdict)) } } if len(applied) == 0 { return finalText, res, nil } // Apply in DESCENDING offset order so earlier spans keep their offsets. The spans are provably disjoint // (DisjointCandidates), so this is a pure splice — asserted rather than assumed. sort.Slice(applied, func(i, j int) bool { return applied[i].span[0] > applied[j].span[0] }) out := finalText for _, a := range applied { if a.span[0] < 0 || a.span[1] > len(out) || a.span[0] >= a.span[1] { res.Rejected += len(applied) return finalText, res, nil // never splice a range the text cannot hold } out = out[:a.span[0]] + a.replacement + out[a.span[1]:] } if !utf8.ValidString(out) || !r.repairReGate(ch.Text, draft, finalText, out, injected) { res.Rejected += len(applied) r.Log.WarnContext(ctx, "repair discarded: the re-gate refused the assembled text (original kept)", "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "repairs", len(applied)) return finalText, res, nil } res.Applied = len(applied) r.Log.InfoContext(ctx, "repair applied", "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "repairs", res.Applied, "declined", res.Declined, "rejected", res.Rejected, "cost_usd", fmt.Sprintf("%.6f", res.CostUSD)) return out, res, nil } // repairCheckpointExists reports whether THIS repair request was already paid for in an earlier run. It // mirrors runRepairAttempt's request identity BY CONSTRUCTION — same derived stage, same helper — because // the two must address the same checkpoint; the doccomment used to promise that mirroring while the two // field lists were maintained apart, which held only while the dropped fields were zero. func (r *Runner) repairCheckpointExists(st config.Stage, snapID string, ch chunk.Chunk, ordinal int, msgs []llm.Message) (bool, error) { model, maxTokens := r.repairCallBudget(msgs) rst := r.repairStage(st, model) cp, err := r.Store.GetCheckpoint(RequestHash(r.attemptRequest(rst, model, snapID, ch, ordinal, maxTokens, msgs))) return cp != nil, err } // repairStage derives the stage of a repair call from the stage whose output is being repaired. It takes // the NAME from that stage (the accounting axis) and everything else from the repair gate. // // The effort is the GATE's, NOT inherited from the parent stage — see RepairGate.Reasoning for why: the // value's meaning is per-control, the model is a different one, and inheriting would have moved the // request hash of every already-paid repair call. func (r *Runner) repairStage(st config.Stage, model string) config.Stage { return config.InternalCall{ Name: st.Name, Role: roleRepair, Model: model, Reasoning: r.Pipeline.Gates.Repair.Reasoning, }.Stage() } // repairCallBudget resolves the model and the max_tokens of a repair call — ONE definition, so the // checkpoint probe and the call itself can never address different request hashes. func (r *Runner) repairCallBudget(msgs []llm.Message) (string, int) { model := r.Pipeline.Gates.Repair.Model est := 0 for _, m := range msgs { est += EstimateTokens(m.Content) } maxTokens := int(float64(est) * r.Pipeline.Defaults.MaxOutputRatio) if maxTokens < r.Pipeline.Defaults.MinMaxTokens { maxTokens = r.Pipeline.Defaults.MinMaxTokens } return model, r.applyModelFloor(maxTokens, model) } // runRepairAttempt performs ONE repair call on the shared money path: reserve → call → settle+checkpoint, // with a checkpoint hit replayed for free. It reuses runAttempt rather than re-implementing the money // sequence — a second copy of that sequence is exactly the drift the snapshot fold helper was extracted to // prevent. The derived stage carries the repair ROLE and MODEL (both request-hash fields, so a repair call // can never collide with the stage's own attempts) and INHERITS the repaired stage's effort. // // It used to carry no reasoning setting at all, on a doccomment that read "the provider default holds — on // DeepSeek that means thinking stays ON and the echo mine is not armed". The premise was falsified on the // wire (D39.86): riding the vendor default is not a safe resting place, because the vendor MOVED it — to // effort `high`, which walls a dense-Han call at max_tokens with an empty body. Thinking staying on was // never the whole question; how much of the budget it eats is the other half. func (r *Runner) runRepairAttempt(ctx context.Context, st config.Stage, snapID string, ch chunk.Chunk, job *store.Job, ordinal int, msgs []llm.Message) (stageAttempt, error) { model, maxTokens := r.repairCallBudget(msgs) rst := r.repairStage(st, model) // isFinal=false: the reply is a SPAN, not a shipping text, so the output sanitizer (which reasons about // whole chunks — preambles, trailing note blocks) must not judge it. The intrinsic classifier still runs // inside runAttempt and is a real guard: a model answering «Не могу помочь…» would otherwise pass every // content guard below and be spliced into the prose. return r.runAttempt(ctx, rst, model, snapID, ch, job, ordinal, maxTokens, msgs, false, false) } // repairVerdict is the reply-guard outcome. type repairVerdict string const ( repairAccepted repairVerdict = "accepted" repairDeclined repairVerdict = "declined" repairBadVerdict repairVerdict = "classifier_rejected" repairBadShape repairVerdict = "shape_rejected" repairBadContent repairVerdict = "content_rejected" ) // repairReplyVerdict applies the reply guards to one repair answer against the span it would replace. The // guards exist because the re-gate CANNOT see the damage a bad replacement does: every deterministic checker // goes silent when the defective text is deleted, so acceptance has to be earned on the reply's own shape. func repairReplyVerdict(att stageAttempt, original string, cls checks.RepairClass, c *checks.Checkers) (string, repairVerdict) { if !att.cls.ok() { return "", repairBadVerdict // empty, truncated, refusal or echo — the intrinsic classifier caught it } reply := strings.TrimSpace(checks.StripThink(att.text)) if reply == "" { return "", repairBadShape } if reply == repairNoChange { return "", repairDeclined } if strings.Contains(reply, repairNoChange) { return "", repairBadShape // the sentinel mixed with prose is an unparseable answer, not a repair } origRunes, replyRunes := []rune(original), []rune(reply) if len(replyRunes) < int(float64(len(origRunes))*repairMinSpanRatio) || len(replyRunes) > int(float64(len(origRunes))*repairMaxSpanRatio) { return "", repairBadShape // a rewrite far outside the span's size is not a targeted repair } if !strings.Contains(original, "\n") && strings.Contains(reply, "\n") { return "", repairBadShape // a repair must not re-paragraph the unit } // Script guard: the reply may not introduce SOURCE-script runes the span did not have. A few source // glyphs inside one sentence sit far below the intrinsic echo threshold, so nothing downstream would catch // them. The script set is the run's DECLARED source (c.SourceScripts()) — the same notion the echo // detector uses, so ko/en are covered, not only Han+kana. Empty (unknown source) → the guard is inert. srcScripts := c.SourceScripts() had := map[rune]bool{} for _, ru := range original { had[ru] = true } for _, ru := range reply { if len(srcScripts) > 0 && unicode.In(ru, srcScripts...) && !had[ru] { return "", repairBadContent } } // Anti-deletion: the replacement must keep the span's word tokens, minus at most one for the conversion // itself. Deleting content satisfies every class re-check, so preservation is asserted here rather than // inferred from a checker that has gone quiet. if ow, rw := countWordTokens(original), countWordTokens(reply); ow > 0 && rw < ow-repairMaxTokenLoss { return "", repairBadContent } // PER-CLASS POSITIVE POST-CONDITION: the class invariant must be RESTORED, not merely silenced. Every // shipped checker goes quiet when the offending text is deleted or mangled, so "the class stopped firing" // is not evidence of a repair — the replacement has to positively satisfy what the class is about. if !classInvariantRestored(cls, c, original, reply) { return "", repairBadContent } // The span's FIGURES must line up one-for-one, with at most ONE value changed — the substitution a // unit/scale repair legitimately makes. Anything else is the addition/omission class the loop exists to // reduce rather than to create: a dropped figure is silent data loss, and an INVENTED one is fabrication // that no downstream check would catch (the reflow number-drift guard is opt-in and enabled in no shipping // config, so it is structurally zero here). if !digitsPreserved(original, reply) { return "", repairBadContent } return reply, repairAccepted } // classInvariantRestored is the POSITIVE half of the acceptance rule (design §15.2 A): for the class that // fired, the replacement must positively satisfy the class's own invariant, not merely stop matching its // detector. Deletion, mangling and "changed to a different wrong value" all silence a detector; only these // assertions distinguish a repair from a silencing. // // The assertions are expressed through the SAME pair/target data the detector uses, so a new pair inherits // them for free. Where an assertion is not expressible on today's data (the counted-hours class — see the // class's exclusion from defaultRepairClasses), the conservative answer is to REJECT rather than to accept: // a rejected correct repair costs one wasted call, an accepted wrong one costs the reader. func classInvariantRestored(cls checks.RepairClass, c *checks.Checkers, original, reply string) bool { switch cls { case checks.RepairLatinResidue: // The leaked Latin token must be gone, and no NEW Latin token may take its place. before, _ := lintLatinResidueCount(original) after, _ := lintLatinResidueCount(reply) return after < before case checks.RepairBrokenWord: // The malformed form must be gone and none introduced. return checks.BrokenWordCount(c, reply) < checks.BrokenWordCount(c, original) case checks.RepairDC1Fractional: // The halved duration must be gone AND the replacement must still state a duration: a reply that // simply drops the time expression is a deletion, not a conversion. return checks.FractionalUnitPresent(c, reply) == false && checks.MentionsHourWord(c, reply) case checks.RepairDC1TimeUnits: // The counted-hours class: assert the hours count POSITIVELY equals twice the source count. On the // pair data as it stands this also rejects a correct repair phrased in a case form the hours table // does not carry — accepted, because the alternative is accepting «три часа» → «пять часов». return checks.HoursCountDoubled(c, original, reply) } return true } // lintLatinResidueCount is the package-local view of the Latin-residue count (the lint lives in checks and // takes an allowlist; the guard compares before/after on the same allowlist-free basis). func lintLatinResidueCount(s string) (int, []string) { return checks.LatinResidueCount(s), nil } // countWordTokens counts maximal letter runs (any script) — the coarse content measure the anti-deletion // guard uses. Language-general: it makes no assumption about which alphabet the target uses. func countWordTokens(s string) int { n, in := 0, false for _, r := range s { if unicode.IsLetter(r) { if !in { n++ in = true } continue } in = false } return n } // digitRuns returns the maximal ASCII digit runs of s. func digitRuns(s string) []string { var out []string cur := strings.Builder{} for _, r := range s { if r >= '0' && r <= '9' { cur.WriteRune(r) continue } if cur.Len() > 0 { out = append(out, cur.String()) cur.Reset() } } if cur.Len() > 0 { out = append(out, cur.String()) } return out } // digitsPreserved reports whether the reply keeps the span's figures one-for-one, changing at most one // VALUE. Counting positionally (rather than asking "did each original figure survive somewhere") is what // catches an INVENTED figure: a reply may not carry more numbers than the text it replaces. func digitsPreserved(original, reply string) bool { o, r := digitRuns(original), digitRuns(reply) if len(o) != len(r) { return false // a figure appeared or disappeared — never a targeted repair } diff := 0 for i := range o { if o[i] != r[i] { diff++ } } return diff <= 1 } // repairReGate re-runs the deterministic verdicts over the ASSEMBLED text and reports whether the repair may // stand. It is conservative by construction: every check must be no worse than before, and the repaired // classes must actually be gone. It deliberately does NOT treat "the class stopped firing" as sufficient — // that condition is satisfiable by deletion, which the reply guards refuse first. func (r *Runner) repairReGate(source, draft, before, after string, injected []membank.PickedEntry) bool { cfg := r.cheapGateConfig() // 1. The addressable defects must not have grown, and the classes we attacked must be gone. beforeCands := len(checks.RepairCandidates(source, before, cfg)) afterCands := len(checks.RepairCandidates(source, after, cfg)) if afterCands >= beforeCands { return false } // 2. The cheap observability suite must not have gained anything. if checks.RunCheapGates(source, draft, after, cfg).Total() > checks.RunCheapGates(source, draft, before, cfg).Total() { return false } // 3. The intrinsic classifier must still accept the whole text. if !classify(classifyInput{Source: source, Output: after, Finish: llm.FinishStop, TargetLang: r.Book.TargetLang, SourceScripts: r.checkers.SourceScripts()}).ok() { return false } // 4. The output sanitizer must not have been made dirty (only when the gate is on — it is the gate that // decides those classes are verdict-bearing for this book). if r.Pipeline.Gates.Sanitizer.Enabled && r.checkers.TargetActive() { if r.checkers.SanitizeOutput(after).Total() > r.checkers.SanitizeOutput(before).Total() { return false } } // 5. The glossary post-check must not have gained a CONFIRMED miss: a repair that rewrites a canonical // term would otherwise be invisible here and, with the post-check gate on, would flip the unit to // flagged and ship an EMPTY export — the repair destroying the chapter it was meant to improve. if r.memory != nil && len(injected) > 0 { if r.memory.Postcheck(injected, after).ConfirmedCount() > r.memory.Postcheck(injected, before).ConfirmedCount() { return false } } return true } // repairDerivedNS is the derived-checkpoint id namespace for a repaired final text, mirroring the sanitized // and banknote exports: content-addressed, prefixed so it can never collide with a real hex request_hash, // and re-derived identically on resume for free. const repairDerivedNS = "tm-repair-v1" // repairDerivedHash is the EXACT id formula: sha256(namespace \0 terminal-attempt hash \0 repaired text). func repairDerivedHash(reqHash, repaired string) string { sum := sha256.Sum256([]byte(repairDerivedNS + "\x00" + reqHash + "\x00" + repaired)) return repairDerivedNS + ":" + hex.EncodeToString(sum[:]) } // commitRepairExport persists the repaired final text as a $0 derived checkpoint and returns its id, so // chunk_status.final_hash points at the bytes that actually ship and a resume re-serves them without // re-running (or re-paying for) the repair. Written BEFORE chunk_status references it, like its siblings. func (r *Runner) commitRepairExport(st config.Stage, ch chunk.Chunk, job *store.Job, att stageAttempt, repaired string) (string, error) { derived := repairDerivedHash(att.reqHash, repaired) if err := r.Store.PutDerivedCheckpoint(store.Checkpoint{ RequestHash: derived, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: att.attempt, Stage: st.Name, Role: st.Role, ModelRequested: att.modelActual, ModelActual: att.modelActual, ResponseText: repaired, UsageJSON: "{}", FinishReason: "repair_export", }); err != nil { return "", fmt.Errorf("pipeline: commit repair export ch%d/chunk%d/%s: %w", ch.Chapter, ch.ChunkIdx, st.Name, err) } return derived, nil }