package pipeline import ( "crypto/sha256" "encoding/hex" "encoding/json" "textmachine/backend/internal/config" ) // snapshot.go: материализация snapshotID (§3.2/D5.2/D8) — контент-хеш всего, что // влияет на wire-байты ИЛИ на резолв вердикта чекпоинта. Правка любого входа — // громкий --resnapshot (= переоплата книги, D15), никогда тихий false-hit. // contextSnap freezes the context-assembly knobs (§3.8) inside the snapshot. // Fixed field order — it is part of a content hash. WS2 dropped STMDepth/OverlapTokens // (dead carryover knobs — §2а), a structural change folded loudly (§8 manifest). type contextSnap struct { GlossaryInjection string `json:"glossary_injection"` GlossaryTokenBudget int `json:"glossary_token_budget"` CacheTTL string `json:"cache_ttl"` } // segmentationSnap freezes the WS2 output-token chunking budget (draft/edit ceilings + per-pair // fertility) inside the snapshot, mirroring coverageSnap's discipline: a budget or fertility edit // re-chunks the book (changes chunk boundaries → the wire), so it is a loud --resnapshot (D30.9), // never a silent divergence. Unlike coverageSnap it is ALWAYS folded — segmentation always runs and // its values are wire-determining. Fixed field order — part of a content hash. type segmentationSnap struct { DraftBudgetOut int `json:"draft_budget_out"` EditCeilingOut int `json:"edit_ceiling_out"` FertilityCJK float64 `json:"fertility_cjk"` FertilityOther float64 `json:"fertility_other"` } func (r *Runner) segmentationSnapshot() segmentationSnap { seg := r.Pipeline.Segmentation return segmentationSnap{ DraftBudgetOut: seg.DraftBudgetOut, EditCeilingOut: seg.EditCeilingOut, FertilityCJK: seg.Fertility.CJK, FertilityOther: seg.Fertility.Other, } } // coverageSnap freezes the excision coverage-gate config inside the snapshot, so // enabling the gate OR tuning its thresholds is a loud --resnapshot, never a silent // mismatch between an old checkpoint's stored disposition and a changed gate. HONEST // COST (D15): the snapshotID is folded into every call's RequestHash (render.go), so a // --resnapshot changes EVERY request_hash → every checkpoint misses → the whole book is // RE-BILLED, even for byte-identical wire requests whose verdict merely needs // re-classifying. There is no free "re-classify from the checkpoint" here: the // content-addressed reuse that COULD make an unchanged wire request free (msgsContentHash, // render.go) is gated on an UNCHANGED snapshot (stagerun.go), so it never fires across a // resnapshot. This all-or-nothing re-pay is ACCEPTED for the static acceptance book (the // gate is loud, divergences do not occur) but is exactly why content-addressed checkpoint // reuse must be designed before the ongoing/append-chapters mode (D15.2). Only folded when // ENABLED: tweaking a disabled gate's bounds must not force a re-pin (the gate produces no // verdicts while off). json.Marshal sorts the LenRatio map keys → deterministic render. // Mirrors the estimator/max_tokens-policy discipline (render.go) for a NON-wire input that // still determines a checkpoint's resolved verdict. type coverageSnap struct { Enabled bool `json:"enabled"` Version string `json:"version,omitempty"` LenRatio map[string][]float64 `json:"len_ratio,omitempty"` SentCovMin float64 `json:"sent_cov_min,omitempty"` MinChunkChars int `json:"min_chunk_chars,omitempty"` } // coverageSnapshot renders the gate's snapshot component: {enabled:false} when off // (so toggling on is a visible change), the full config + algorithm version when on. func (r *Runner) coverageSnapshot() coverageSnap { cov := coverageSnap{Enabled: r.Pipeline.Gates.Coverage.Enabled} if cov.Enabled { cov.Version = coverageGateVersion cov.LenRatio = r.Pipeline.Gates.Coverage.LenRatio cov.SentCovMin = r.Pipeline.Gates.Coverage.SentCovMin cov.MinChunkChars = r.Pipeline.Gates.Coverage.MinChunkChars } return cov } // sanitizerSnap freezes the output-sanitizer gate (D30.3) inside the snapshot, mirroring // coverageSnap: {enabled:false} when off (so toggling on is a visible --resnapshot), the // algorithm version when on. It changes a checkpoint's RESOLVED disposition (a defect flips // the chunk to flagged), never the wire, so a rule edit is a loud re-pin, not a silent // re-verdict on resume. Folded ONLY when enabled — tweaking a disabled gate must not force a // re-pin (it produces no verdicts while off). Moves to verdictSnapshotID with D15.2. type sanitizerSnap struct { Enabled bool `json:"enabled"` Version string `json:"version,omitempty"` } func (r *Runner) sanitizerSnapshot() sanitizerSnap { s := sanitizerSnap{Enabled: r.Pipeline.Gates.Sanitizer.Enabled} if s.Enabled { s.Version = sanitizerVersion } return s } // banknoteSnap freezes the banknote channel (WS4 §4в, F9) inside the snapshot — the VERDICT-AXIS that // governs the resolved draft (the slice + the derived-checkpoint), mirroring sanitizerSnap. parser_version // is the split/parse algorithm version: bumping it re-resolves the stripped draft, so it must be a loud // --resnapshot even without a prompt edit (§4а point 6). DEVIATION-with-rationale from "mirror // sanitizerSnap exactly": it is folded via a POINTER with omitempty (nil when the channel is off) rather // than an unconditional {enabled:false} — so ADDING this optional feature does NOT invalidate every // banknote-OFF book (incl. the golden fixture), while ENABLING the channel or bumping the parser is still // a loud snapshot move (the substantive loudness point 6 requires). A snapshot re-pin is never billed for // a feature a book does not use. type banknoteSnap struct { Enabled bool `json:"enabled"` ParserVersion string `json:"parser_version,omitempty"` // TokenBudget folds bankTokenBudget (FL-1): the extra max_tokens the enabled channel adds to the // translator draft (stagerun.go, integration point 5) enters request_hash, but bankParserVersion // versions only the split/parse ALGORITHM — so editing bankMaxLines (or the per-line multiplier) // would change the wire without bumping the parser version = a silent re-bill on resume. Folding // the resolved budget makes any such edit a loud --resnapshot by mechanism, not discipline. TokenBudget int `json:"token_budget,omitempty"` } // banknoteSnapshot returns the fold ONLY when the channel is enabled (nil otherwise → omitempty drops it). func (r *Runner) banknoteSnapshot() *banknoteSnap { if !r.Pipeline.Gates.Banknote.Enabled { return nil } return &banknoteSnap{Enabled: true, ParserVersion: bankParserVersion, TokenBudget: bankTokenBudget} } // memoryVersion is the content-hash of the deterministically materialized injected // memory (the frozen APPROVED glossary rows + the normalization/matcher algorithm // versions), the memory component of the snapshot (D5.2/D8, F1 CLOSED). It is the // Version() of the bank materialized once before the loop, so a change to the approved // glossary — or to the deterministic machinery (trad→simp table, matcher) — fires the // resnapshot gate LOUDLY instead of a stale checkpoint re-paying a diverged translation. // nil bank (report path / a book with no glossary) → the empty-materialization hash, a // stable constant. STM is excluded (rebuilt from checkpoints, §3.2). Auto/draft rows are // excluded per D8 (their injected-content changes are caught at the per-chunk // content_hash level; a future autopopulation milestone revisits this). func (r *Runner) memoryVersion() string { if r.memory != nil { return r.memory.Version() } return computeMemoryVersion(nil, r.Pipeline.Gates.Glossary.PostcheckGate) } // wave selects which pipeline wave a per-wave snapshot folds (WS1 §1в). type wave int const ( waveDraft wave = iota // DRAFT wave: translator-role stages + the BASE bank version (excl Source:mined) waveEdit // EDIT wave: non-translator stages + the ENRICHED bank version (all approved incl mined) ) // snapshotID materializes the BOOK-GLOBAL job-context snapshot (§3.2): brief_hash, chunker version, // context-assembly knobs, the (enriched) memory version and the FULL stage plan. It is the snapshot the // SEQUENTIAL driver pins every job to; the byte output is unchanged (buildSnapshotID over all stages + // the enriched memory version reproduces it exactly). Payload is a fixed-field-order content hash. func (r *Runner) snapshotID() (id, payload string, err error) { return r.buildSnapshotID(r.Pipeline.Stages, r.memoryVersion()) } // snapshotIDForWave computes the PER-WAVE snapshot (WS1 §1в, ADDITIVE — the wave executor residual pins // each wave's jobs to it; the sequential driver still uses snapshotID). the draft wave folds the DRAFT stages // (translator role) + the BASE bank version (Source∈{seed,ruby,auto}, EXCL mined); the edit wave folds the EDIT // stages (non-translator) + the ENRICHED bank version (all approved incl Source:mined). A the bank-mining stop bank // enrichment (adding mined rows) moves ONLY the enriched version → ONLY edit-wave snapshot, keeping the draft wave // checkpoints valid («переоплата ОДНА»). Byte-consistent with snapshotID's building — the SAME payload // struct, only the stage subset and the memory version differ. func (r *Runner) snapshotIDForWave(w wave) (id, payload string, err error) { memVer := r.memoryVersion() // enriched (the edit wave) if w == waveDraft { memVer = r.baseMemoryVersion() } return r.buildSnapshotID(waveStages(r.Pipeline.Stages, w), memVer) } // finalStageWave is the wave that owns the SHIPPING (last) stage: the edit wave when the last stage is an editor/other // role (the normal 2-stage pipeline — the edit unit ships), the draft wave when the pipeline is draft-only (the draft // itself ships). The wave-aware read-models (export/status) use it to compare a stored final-stage row // against the correct per-wave snapshot and to know whether the shipping unit is an edit unit or a chunk. func (r *Runner) finalStageWave() wave { n := len(r.Pipeline.Stages) if n > 0 && r.Pipeline.Stages[n-1].Role != roleTranslator { return waveEdit } return waveDraft } // waveStages partitions the pipeline stages by role for a wave: the draft wave = the translator-role (draft) stages, // the edit wave = every non-translator (editor/other) stage. Order preserved. The prod C1 core is one draft + one // edit, so this yields [draft] and [edit] respectively. func waveStages(stages []config.Stage, w wave) []config.Stage { var out []config.Stage for _, st := range stages { isDraft := st.Role == roleTranslator if (w == waveDraft) == isDraft { out = append(out, st) } } return out } // baseMemoryVersion is the DRAFT-wave (the draft wave) memory component: the BASE bank version (excl Source:mined). // nil bank → the empty-materialization base hash (a stable constant), mirroring memoryVersion(). func (r *Runner) baseMemoryVersion() string { if r.memory != nil { return r.memory.BaseVersion() } return computeMemoryVersionScoped(nil, r.Pipeline.Gates.Glossary.PostcheckGate, true) } // buildSnapshotID materializes the snapshot payload for a given stage subset + memory version (§3.2). // Extracted from snapshotID so the book-global snapshot (all stages + enriched version) AND the two // per-wave snapshots (WS1 §1в) share one byte-exact builder — a stage subset / memory version is the // ONLY axis that differs. Fixed field order (a content hash). func (r *Runner) buildSnapshotID(stages []config.Stage, memVersion string) (id, payload string, err error) { type stageSnap struct { Name string `json:"name"` Role string `json:"role"` Model string `json:"model"` PromptVersion string `json:"prompt_version"` PromptSHA256 string `json:"prompt_sha256"` Temperature float64 `json:"temperature"` Reasoning string `json:"reasoning"` // FewShot folds the stage's few_shot on/off state (D38.4) — but ONLY for a prompt // that HAS a ---FEWSHOT--- block (nil/omitted otherwise, so stages without examples // keep a byte-identical snapshot). The block IS inside the raw file already folded via // PromptSHA256, yet dropping it at render (few_shot:false) leaves PromptSHA256 unchanged // while the wire changes, so the resolved flag must be folded to keep a flip a loud // --resnapshot rather than a silent false-hit — same discipline as Temperature/Reasoning. FewShot *bool `json:"few_shot,omitempty"` // ExtraBody модели меняет ТЕЛО запроса (GLM thinking-off и т.п.) — // без него правка ручки в models.yaml молча не инвалидировала бы // чекпоинты (находка ревью). json.Marshal карты сортирует ключи → // детерминированный рендер. ModelExtra json.RawMessage `json:"model_extra,omitempty"` // Провайдер-уровневые оверрайды (local kind переопределяет wire // temperature/max_tokens ПОСЛЕ вычисления request-hash) — тоже влияют // на фактический запрос; без них правка providers.local.max_tokens // давала бы тот же snapshot и ложный checkpoint-hit на стендовом // local-пути (находка внешнего ревью F2). ProviderTemp float64 `json:"provider_temp,omitempty"` ProviderMaxTok int `json:"provider_max_tok,omitempty"` // ProviderModel — the local-kind backend tag the provider swaps onto the wire // AFTER the request-hash (the actual model that answers). The MOST impactful // local override, yet it was missing here while its weaker temp/max_tok siblings // were folded: a local swap 8b→14b mid-book keeps the same snapID and resume // serves the old model (external-review). Folded so it is a loud --resnapshot. ProviderModel string `json:"provider_model,omitempty"` // Capability — резолвнутая wire-форма модели (D3.1): budget-ключ, // temperature-режим, reasoning-контроль. Меняет ТЕЛО запроса (max_tokens // vs max_completion_tokens, отправлять ли temperature, thinking-выключа- // тель), поэтому обязана входить в снапшот — иначе правка каппы молча // false-хитит чекпоинты (тот же класс D5.2, что и payload ниже). Capability json.RawMessage `json:"capability,omitempty"` // EscalateTo + EscalateCapability — the single-hop fallback model and its // resolved wire shape (Веха 2.5). The fallback's request_hash uses ITS model, // and its wire body uses ITS capability — both must be in the snapshot, or // changing the escalation model / its caps would silently false-hit an // escalated chunk's checkpoint (closes the D5.2 escalation-capability techdebt: // stages were folded, escalation models were not, until the cycle landed). EscalateTo string `json:"escalate_to,omitempty"` EscalateCapability json.RawMessage `json:"escalate_capability,omitempty"` // The fallback model's OTHER wire-affecting inputs, mirroring the primary // fold above: its top-level extra_body (merged into the escalation wire body) // and its provider-level temperature/max_tokens overrides (local kind). Without // these, editing the fallback's extra_body / provider knobs would change the // escalation wire but leave the snapshot identical → a silent false-hit on a // resumed escalated chunk (self-review: the primary path guards this, escalation // did not). EscalateExtra json.RawMessage `json:"escalate_extra,omitempty"` EscalateProviderTemp float64 `json:"escalate_provider_temp,omitempty"` EscalateProviderMaxTok int `json:"escalate_provider_max_tok,omitempty"` EscalateProviderModel string `json:"escalate_provider_model,omitempty"` } snap := struct { BriefHash string `json:"brief_hash"` ChunkerVersion string `json:"chunker_version"` EstimatorVersion string `json:"estimator_version"` // MaxTokensPolicy versions the attempt→max_tokens scaling (Веха 2): a // change to the retry doubling shifts attempt≥1 request_hashes, so it // belongs in the snapshot as a loud invalidation (same class as // estimator_version, applied to the regeneration axis). MaxTokensPolicy string `json:"max_tokens_policy"` // ClassifierVersion versions the intrinsic classify() verdict logic (thresholds // + order), so a re-verdict on a resumed checkpoint is a loud --resnapshot, not // a silent flagged↔ok divergence (external-review; symmetric to coverage). ClassifierVersion string `json:"classifier_version"` PipelineCore string `json:"pipeline_core"` // Defaults влияют на maxTokens, а тот входит в request-hash: без них // правка max_output_ratio молча инвалидировала бы все чекпоинты в // обход snapshot-гейта (находка ревью). MaxOutputRatio float64 `json:"max_output_ratio"` MinMaxTokens int `json:"min_max_tokens"` // ContextAssembly — ручки сборки контекста (глоссарий/STM/overlap/TTL, // §3.8). Их правка меняет ВХОД модели, когда память войдёт в msgs; сворачи- // ваем ДО инъекции, чтобы смена budget'а инъекции не промахнулась мимо // resnapshot-гейта (D5.2). Фикс-порядок полей — это content-hash. ContextAssembly contextSnap `json:"context_assembly"` // Segmentation — the WS2 output-token chunk/edit-unit budget + fertility (segmentationSnap). // It changes chunk boundaries → the wire, so a budget/fertility edit is a loud --resnapshot. Segmentation segmentationSnap `json:"segmentation"` // RenderFormatVersion versions the FORMAT of the role-injection renderers not captured by // memoryMatchVersion (a scope/matcher-algorithm version): the editor constraint block's // src→dst layout (WS2 §2в) and the gender-constraint render (WS5). A format change shifts the // injected bytes → the wire, so folding it separately keeps a flip a loud --resnapshot. RenderFormatVersion string `json:"render_format_version"` // LangpackVersion — the loaded language-data pack's content hash (internal/lang, D39.15/16), // folded ONLY when a pack is present (omitempty otherwise, so a no-pack book is byte-stable and // never re-billed for a feature it does not use). The pack feeds the the bank-mining stop bank-miner; a pack DATA // edit changes what it proposes, so folding its version makes a pack edit a loud --resnapshot, // drift-proof by mechanism (the pack's bytes ARE its version). Wire-neutral for existing // checkpoints (the miner produces Source:mined proposals, inert until owner-approved), but folded // per the R1 directive so the run's snapshot durably records which pack version produced it. LangpackVersion string `json:"langpack_version,omitempty"` // MemoryVersion — content-hash детерминированно материализованной инъекти- // руемой памяти (approved-глоссарий+резюме+series-bible). Не bump-счётчик // (D8): забыть пересчитать нельзя, а забытый bump переоткрыл бы класс тихой // переоплаты D5.2. До миграции банка памяти (v2) материализация пуста — // хэш стабилен; когда память войдёт в msgs, поле сменится и resnapshot- // гейт сработает громко. STM сюда НЕ входит (пересобирается из чекпоинтов). MemoryVersion string `json:"memory_version"` // PostcheckGate — the memory-bank post-check gate (E1). When true a post-check // miss flips the chunk to flagged; folding its on/off state makes toggling it a // loud --resnapshot (it changes a checkpoint's RESOLVED disposition), not a silent // re-verdict on resume (same class as coverage/classifier). The post-check // ALGORITHM version is already inside MemoryVersion (memoryMatchVersion). PostcheckGate bool `json:"postcheck_gate"` // Coverage — excision QA-gate config (D12 Q3). It does not touch the wire, // but it determines a checkpoint's RESOLVED verdict, so a gate change must be // a loud --resnapshot, not a silent re-verdict on resume (coverageSnapshot). Coverage coverageSnap `json:"coverage"` // StyleCheckVersion versions the cheap deterministic post-check rules (dialogue-dash, // yofikator, translit-interjection blocklist, 万/億 magnitude gate — cheapgates.go). They // are observability, not wire, but editing a rule shifts the recorded style-flag counts, so // a bump is a loud --resnapshot (same verdict class as ClassifierVersion). The ё-policy is // part of BriefHash (a book field), so a policy change already re-pins via brief_hash. StyleCheckVersion string `json:"style_check_version"` // Sanitizer — the output-sanitizer gate (D30.3). Like Coverage it does not touch the wire // but determines a checkpoint's RESOLVED verdict (a defect flips the chunk to flagged), so a // gate flip / rule edit is a loud --resnapshot. Folded only when enabled (sanitizerSnapshot). Sanitizer sanitizerSnap `json:"sanitizer"` // Banknote — the banknote channel (WS4). Verdict-axis (governs the sliced/derived draft), folded // only when ENABLED (nil→omitempty otherwise), so adding the feature never re-bills an off book. Banknote *banknoteSnap `json:"banknote,omitempty"` Stages []stageSnap `json:"stages"` }{ BriefHash: r.Book.BriefHash(), ChunkerVersion: chunkerVersion, EstimatorVersion: estimatorVersion, MaxTokensPolicy: maxTokensPolicyVersion, ClassifierVersion: classifierVersion, PipelineCore: r.Pipeline.Core, MaxOutputRatio: r.Pipeline.Defaults.MaxOutputRatio, MinMaxTokens: r.Pipeline.Defaults.MinMaxTokens, ContextAssembly: contextSnap{ GlossaryInjection: r.Pipeline.Context.GlossaryInjection, GlossaryTokenBudget: r.Pipeline.Context.GlossaryTokenBudget, CacheTTL: r.Pipeline.Context.CacheTTL, }, Segmentation: r.segmentationSnapshot(), RenderFormatVersion: renderFormatVersion, LangpackVersion: r.packVersion(), MemoryVersion: memVersion, PostcheckGate: r.Pipeline.Gates.Glossary.PostcheckGate, Coverage: r.coverageSnapshot(), StyleCheckVersion: cheapGateVersion, Sanitizer: r.sanitizerSnapshot(), Banknote: r.banknoteSnapshot(), } for _, st := range stages { ss := stageSnap{ Name: st.Name, Role: st.Role, Model: st.Model, PromptVersion: st.PromptVersion, PromptSHA256: r.templates[st.Name].SHA256, Temperature: st.Temperature, Reasoning: st.Reasoning, } // Fold few_shot only where the prompt actually carries a ---FEWSHOT--- block, so // stages without examples stay byte-identical in the snapshot (D38.4). if r.templates[st.Name].FewShot != "" { on := fewShotEnabled(st) ss.FewShot = &on } // Fold the PRIMARY model's wire-affecting inputs (prov-triple, extra_body, resolved capability) // via the shared foldModelWire (D39 слой 7, L8-snapshot-fold-copypaste-tripwire): the same // helper folds the escalate_to fallback below, so a third model axis (channel B / annotator) // reuses it and cannot silently drift the snapshot. primary, perr := r.foldModelWire(st.Model) if perr != nil { return "", "", perr } ss.ProviderTemp, ss.ProviderMaxTok, ss.ProviderModel = primary.provTemp, primary.provMaxTok, primary.provModel ss.ModelExtra = primary.extra ss.Capability = primary.capability // Fold the single-hop fallback model + its wire inputs (Веха 2.5): both are wire-affecting // inputs of an escalated call, so changing them is a loud --resnapshot, not a silent false-hit // on an escalated checkpoint. Same fold as the primary — no copy-paste to drift. if st.EscalateTo != "" { ss.EscalateTo = st.EscalateTo esc, eerr := r.foldModelWire(st.EscalateTo) if eerr != nil { return "", "", eerr } ss.EscalateProviderTemp, ss.EscalateProviderMaxTok, ss.EscalateProviderModel = esc.provTemp, esc.provMaxTok, esc.provModel ss.EscalateExtra = esc.extra ss.EscalateCapability = esc.capability } snap.Stages = append(snap.Stages, ss) } data, err := json.Marshal(snap) if err != nil { return "", "", err } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]), string(data), nil } // modelWireFold is a model's snapshot-folded wire-affecting inputs: its provider-level // temperature/max_tokens/model overrides (local kind, applied AFTER the request-hash), its top-level // extra_body, and its RESOLVED capability (budget-key/temperature-mode/reasoning-control). Extracted // (D39 слой 7, L8-snapshot-fold-copypaste-tripwire) so the primary model, the escalate_to fallback, // and any future third model axis fold the SAME way — the copy-paste the old tripwire warned about. type modelWireFold struct { provTemp float64 provMaxTok int provModel string extra json.RawMessage // nil when the model has no extra_body → omitted (omitempty) capability json.RawMessage // always present (ResolveCapability, the same the client sends) } // foldModelWire renders one model's wire-fold. ResolveCapability matches exactly what the client // sends, and json.Marshal sorts map keys (off_extra_body) → deterministic render. Returns an error // only on a marshal failure (a struct of scalars/known types cannot fail in practice). func (r *Runner) foldModelWire(model string) (modelWireFold, error) { var f modelWireFold if prov, ok := r.Models.Providers[r.Models.Models[model].Provider]; ok { f.provTemp, f.provMaxTok, f.provModel = prov.Temperature, prov.MaxTokens, prov.Model } if extra := r.Models.Models[model].ExtraBody; len(extra) > 0 { raw, err := json.Marshal(extra) if err != nil { return f, err } f.extra = raw } capRaw, err := json.Marshal(r.Models.ResolveCapability(model)) if err != nil { return f, err } f.capability = capRaw return f, nil }