576 lines
32 KiB
Go
576 lines
32 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
|
||
"textmachine/backend/internal/checks"
|
||
"textmachine/backend/internal/config"
|
||
"textmachine/backend/internal/lang"
|
||
"textmachine/backend/internal/membank"
|
||
)
|
||
|
||
// snapshot.go: materializes snapshotID (§3.2/D5.2/D8) — the content-hash of everything that
|
||
// affects the wire bytes OR the resolution of a checkpoint's verdict. Editing any input —
|
||
// a loud --resnapshot (= re-paying for the whole book, D15), never a silent false-hit.
|
||
|
||
// contextSnap freezes the context-assembly knobs (§3.8) inside the snapshot.
|
||
// Fixed field order — it is part of a content hash. WS2 dropped STMDepth/OverlapTokens
|
||
// (dead carryover knobs — §2а), a structural change folded loudly (§8 manifest).
|
||
type contextSnap struct {
|
||
GlossaryInjection string `json:"glossary_injection"`
|
||
GlossaryTokenBudget int `json:"glossary_token_budget"`
|
||
CacheTTL string `json:"cache_ttl"`
|
||
}
|
||
|
||
// segmentationSnap freezes the WS2 output-token chunking budget (draft/edit ceilings + per-pair
|
||
// fertility) inside the snapshot, mirroring coverageSnap's discipline: a budget or fertility edit
|
||
// re-chunks the book (changes chunk boundaries → the wire), so it is a loud --resnapshot (D30.9),
|
||
// never a silent divergence. Unlike coverageSnap it is ALWAYS folded — segmentation always runs and
|
||
// its values are wire-determining. Fixed field order — part of a content hash.
|
||
type segmentationSnap struct {
|
||
DraftBudgetOut int `json:"draft_budget_out"`
|
||
EditCeilingOut int `json:"edit_ceiling_out"`
|
||
FertilityCJK float64 `json:"fertility_cjk"`
|
||
FertilityOther float64 `json:"fertility_other"`
|
||
}
|
||
|
||
func (r *Runner) segmentationSnapshot() segmentationSnap {
|
||
seg := r.Pipeline.Segmentation
|
||
return segmentationSnap{
|
||
DraftBudgetOut: seg.DraftBudgetOut,
|
||
EditCeilingOut: seg.EditCeilingOut,
|
||
FertilityCJK: seg.Fertility.CJK,
|
||
FertilityOther: seg.Fertility.Other,
|
||
}
|
||
}
|
||
|
||
// coverageSnap freezes the excision coverage-gate config inside the snapshot, so
|
||
// enabling the gate OR tuning its thresholds is a loud --resnapshot, never a silent
|
||
// mismatch between an old checkpoint's stored disposition and a changed gate. HONEST
|
||
// COST (D15): the snapshotID is folded into every call's RequestHash (render.go), so a
|
||
// --resnapshot changes EVERY request_hash → every checkpoint misses → the whole book is
|
||
// RE-BILLED, even for byte-identical wire requests whose verdict merely needs
|
||
// re-classifying. There is no free "re-classify from the checkpoint" here: the
|
||
// content-addressed reuse that COULD make an unchanged wire request free (msgsContentHash,
|
||
// render.go) is gated on an UNCHANGED snapshot (stagerun.go), so it never fires across a
|
||
// resnapshot. This all-or-nothing re-pay is ACCEPTED for the static acceptance book (the
|
||
// gate is loud, divergences do not occur) but is exactly why content-addressed checkpoint
|
||
// reuse must be designed before the ongoing/append-chapters mode (D15.2). Only folded when
|
||
// ENABLED: tweaking a disabled gate's bounds must not force a re-pin (the gate produces no
|
||
// verdicts while off). json.Marshal sorts the LenRatio map keys → deterministic render.
|
||
// Mirrors the estimator/max_tokens-policy discipline (render.go) for a NON-wire input that
|
||
// still determines a checkpoint's resolved verdict.
|
||
type coverageSnap struct {
|
||
Enabled bool `json:"enabled"`
|
||
Version string `json:"version,omitempty"`
|
||
LenRatio map[string][]float64 `json:"len_ratio,omitempty"`
|
||
SentCovMin float64 `json:"sent_cov_min,omitempty"`
|
||
MinChunkChars int `json:"min_chunk_chars,omitempty"`
|
||
}
|
||
|
||
// coverageSnapshot renders the gate's snapshot component: {enabled:false} when off
|
||
// (so toggling on is a visible change), the full config + algorithm version when on.
|
||
func (r *Runner) coverageSnapshot() coverageSnap {
|
||
cov := coverageSnap{Enabled: r.Pipeline.Gates.Coverage.Enabled}
|
||
if cov.Enabled {
|
||
cov.Version = checks.CoverageGateVersion
|
||
cov.LenRatio = r.Pipeline.Gates.Coverage.LenRatio
|
||
cov.SentCovMin = r.Pipeline.Gates.Coverage.SentCovMin
|
||
cov.MinChunkChars = r.Pipeline.Gates.Coverage.MinChunkChars
|
||
}
|
||
return cov
|
||
}
|
||
|
||
// sanitizerSnap freezes the output-sanitizer gate (D30.3) inside the snapshot, mirroring
|
||
// coverageSnap: {enabled:false} when off (so toggling on is a visible --resnapshot), the
|
||
// algorithm version when on. It changes a checkpoint's RESOLVED disposition (a defect flips
|
||
// the chunk to flagged), never the wire, so a rule edit is a loud re-pin, not a silent
|
||
// re-verdict on resume. Folded ONLY when enabled — tweaking a disabled gate must not force a
|
||
// re-pin (it produces no verdicts while off). Moves to verdictSnapshotID with D15.2.
|
||
type sanitizerSnap struct {
|
||
Enabled bool `json:"enabled"`
|
||
Version string `json:"version,omitempty"`
|
||
}
|
||
|
||
func (r *Runner) sanitizerSnapshot() sanitizerSnap {
|
||
s := sanitizerSnap{Enabled: r.Pipeline.Gates.Sanitizer.Enabled}
|
||
if s.Enabled {
|
||
s.Version = checks.SanitizerVersion
|
||
}
|
||
return s
|
||
}
|
||
|
||
// banknoteSnap freezes the banknote channel (WS4 §4в, F9) inside the snapshot — the VERDICT-AXIS that
|
||
// governs the resolved draft (the slice + the derived-checkpoint), mirroring sanitizerSnap. parser_version
|
||
// is the split/parse algorithm version: bumping it re-resolves the stripped draft, so it must be a loud
|
||
// --resnapshot even without a prompt edit (§4а point 6). DEVIATION-with-rationale from "mirror
|
||
// sanitizerSnap exactly": it is folded via a POINTER with omitempty (nil when the channel is off) rather
|
||
// than an unconditional {enabled:false} — so ADDING this optional feature does NOT invalidate every
|
||
// banknote-OFF book (incl. the golden fixture), while ENABLING the channel or bumping the parser is still
|
||
// a loud snapshot move (the substantive loudness point 6 requires). A snapshot re-pin is never billed for
|
||
// a feature a book does not use.
|
||
type banknoteSnap struct {
|
||
Enabled bool `json:"enabled"`
|
||
// SliceVersion folds the SLICE algorithm only (banknote.go): it decides where the block is cut, and
|
||
// the cut text is a checkpoint. The PARSE rule rides with the run instead — it produces evidence, not
|
||
// paid bytes, and every run recomputes it from the stored answers.
|
||
SliceVersion string `json:"slice_version,omitempty"`
|
||
// TokenBudget folds bankTokenBudget (FL-1): the extra max_tokens the enabled channel adds to the
|
||
// translator draft (stagerun.go, integration point 5) enters request_hash, but bankSliceVersion
|
||
// versions only the SLICE algorithm — so editing bankMaxLines (or the per-line multiplier)
|
||
// would change the wire without bumping the parser version = a silent re-bill on resume. Folding
|
||
// the resolved budget makes any such edit a loud --resnapshot by mechanism, not discipline.
|
||
TokenBudget int `json:"token_budget,omitempty"`
|
||
}
|
||
|
||
// banknoteSnapshot returns the fold ONLY when the channel is enabled (nil otherwise → omitempty drops it).
|
||
func (r *Runner) banknoteSnapshot() *banknoteSnap {
|
||
if !r.Pipeline.Gates.Banknote.Enabled {
|
||
return nil
|
||
}
|
||
return &banknoteSnap{Enabled: true, SliceVersion: bankSliceVersion, TokenBudget: bankTokenBudget}
|
||
}
|
||
|
||
// repairSnap freezes the addressable-defect repair sub-step (pack-16) inside the snapshot. It follows
|
||
// banknoteSnap's POINTER discipline rather than coverage/sanitizer's `{"enabled":false}` value form, and for
|
||
// the same reason: the field is folded ONLY when the gate is on, so ADDING the feature leaves every existing
|
||
// book's payload byte-identical and re-bills nobody. Everything that can change the repaired bytes is here —
|
||
// the algorithm version, the model and its resolved wire shape (through the SAME foldModelWire the primary
|
||
// and the escalation hop use), the call cap, the class set, and the SHA of every class prompt. The prompts
|
||
// need an explicit fold because a repair prompt is not a stage: the per-stage PromptSHA256 fold does not see
|
||
// it, and msgsContentHash covers only the host stage's messages, so without this a prompt edit would silently
|
||
// reuse checkpoints. BudgetUSD is deliberately NOT folded — a dollar cap is not a wire or verdict input (the
|
||
// escalation budget is not folded either), and folding it would re-bill the book on a ceiling edit.
|
||
type repairSnap struct {
|
||
Enabled bool `json:"enabled"`
|
||
Version string `json:"version"`
|
||
Model string `json:"model"`
|
||
ModelWire json.RawMessage `json:"model_wire,omitempty"`
|
||
MaxCalls int `json:"max_calls_per_unit"`
|
||
Classes []string `json:"classes"`
|
||
PromptSHA256 map[string]string `json:"prompt_sha256"`
|
||
GuardsVersion string `json:"guards_version"`
|
||
}
|
||
|
||
// repairSnapshot returns the fold ONLY when the gate is enabled (nil otherwise → omitempty drops it).
|
||
func (r *Runner) repairSnapshot() (*repairSnap, error) {
|
||
if !r.Pipeline.Gates.Repair.Enabled {
|
||
return nil, nil
|
||
}
|
||
rep := r.Pipeline.Gates.Repair
|
||
s := &repairSnap{
|
||
Enabled: true, Version: repairVersion, Model: rep.Model,
|
||
MaxCalls: rep.MaxCallsPerUnit, GuardsVersion: repairGuardsVersion,
|
||
Classes: r.repairClasses(), PromptSHA256: map[string]string{},
|
||
}
|
||
for cls, tpl := range r.repairTemplates {
|
||
s.PromptSHA256[string(cls)] = tpl.SHA256
|
||
}
|
||
wire, err := r.foldModelWire(rep.Model)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
s.ModelWire = wire.capability
|
||
return s, nil
|
||
}
|
||
|
||
// memoryVersion is the content-hash of the deterministically materialized injected
|
||
// memory (the frozen glossary rows + the normalization/matcher algorithm
|
||
// versions), the memory component of the snapshot (D5.2/D8, F1 CLOSED). It is the
|
||
// Version() of the bank materialized once before the loop, so a change to the
|
||
// glossary — or to the deterministic machinery (trad→simp table, matcher) — fires the
|
||
// resnapshot gate LOUDLY instead of a stale checkpoint re-paying a diverged translation.
|
||
// nil bank (report path / a book with no glossary) → the empty-materialization hash, a
|
||
// stable constant. STM is excluded (rebuilt from checkpoints, §3.2). EVERY row folds
|
||
// whatever its status since pack-20 (D39.42 п.3 — an auto/draft row changes the injection
|
||
// and hence the re-payment, which the per-chunk content_hash alone never reached);
|
||
// the approved-only fold this comment used to describe is gone. See membank.ComputeVersion.
|
||
func (r *Runner) memoryVersion() string {
|
||
if r.memory != nil {
|
||
return r.memory.Version()
|
||
}
|
||
return membank.ComputeVersion(nil, r.Pipeline.Gates.Glossary.PostcheckGate)
|
||
}
|
||
|
||
// wave selects which pipeline wave a per-wave snapshot folds (WS1 §1в).
|
||
type wave int
|
||
|
||
const (
|
||
waveDraft wave = iota // DRAFT wave: translator-role stages + the BASE bank version (excl Source:mined)
|
||
waveEdit // EDIT wave: non-translator stages + the ENRICHED bank version (all approved incl mined)
|
||
)
|
||
|
||
// snapshotID materializes the BOOK-GLOBAL job-context snapshot (§3.2): brief_hash, chunker version,
|
||
// context-assembly knobs, the (enriched) memory version and the FULL stage plan. Payload is a
|
||
// fixed-field-order content hash.
|
||
//
|
||
// ⚠ NO PRODUCTION CALLER: the run pins each job to its own WAVE's snapshot (snapshotIDForWave), and the
|
||
// sequential driver this fold was written for no longer exists. It is test-only, and its five call sites in
|
||
// four test files pin payload INPUTS (a max-tokens floor, a few-shot flip, the banknote gate, the classifier
|
||
// version alongside the local backend tag) rather than the fold itself —
|
||
// nothing compares it against the per-wave folds, and the two do not sum: the draft wave folds the BASE bank
|
||
// version precisely so a mining stop cannot move it.
|
||
func (r *Runner) snapshotID() (id, payload string, err error) {
|
||
rep, err := r.repairSnapshot()
|
||
if err != nil {
|
||
return "", "", err
|
||
}
|
||
return r.buildSnapshotID(r.Pipeline.Stages, r.memoryVersion(), rep)
|
||
}
|
||
|
||
// snapshotIDForWave computes the PER-WAVE snapshot (WS1 §1в) — the one the wave driver pins every job to.
|
||
// The draft wave folds the DRAFT stages (translator role) + the BASE bank version (Source∈{seed,ruby,auto},
|
||
// EXCL mined); the edit wave folds the EDIT stages (non-translator) + the ENRICHED bank version (all
|
||
// approved incl Source:mined). A bank-mining stop's enrichment (adding mined rows) moves ONLY the enriched
|
||
// version → ONLY the edit-wave snapshot, keeping the draft wave's checkpoints valid (a single re-pay). Byte-consistent with snapshotID's building — the SAME payload
|
||
// struct, only the stage subset and the memory version differ.
|
||
func (r *Runner) snapshotIDForWave(w wave) (id, payload string, err error) {
|
||
memVer := r.memoryVersion() // enriched (the edit wave)
|
||
if w == waveDraft {
|
||
memVer = r.baseMemoryVersion()
|
||
}
|
||
// The repair fold rides ONLY the wave that owns the SHIPPING stage — repair runs on the final stage's
|
||
// output, so it can change the resolved bytes of that wave alone. Folding it into both would move the
|
||
// DRAFT-wave snapshot when the gate is flipped, invalidating the paid draft checkpoints and destroying
|
||
// the $0 draft resume the wave split exists to protect (the "re-payment stays ONE" invariant).
|
||
var rep *repairSnap
|
||
if w == r.finalStageWave() {
|
||
var err error
|
||
if rep, err = r.repairSnapshot(); err != nil {
|
||
return "", "", err
|
||
}
|
||
}
|
||
return r.buildSnapshotID(waveStages(r.Pipeline.Stages, w), memVer, rep)
|
||
}
|
||
|
||
// finalStageWave is the wave that owns the SHIPPING (last) stage: the edit wave when the last stage is an editor/other
|
||
// role (the normal 2-stage pipeline — the edit unit ships), the draft wave when the pipeline is draft-only (the draft
|
||
// itself ships). The wave-aware read-models (export/status) use it to compare a stored final-stage row
|
||
// against the correct per-wave snapshot and to know whether the shipping unit is an edit unit or a chunk.
|
||
func (r *Runner) finalStageWave() wave {
|
||
n := len(r.Pipeline.Stages)
|
||
if n > 0 && r.Pipeline.Stages[n-1].Role != roleTranslator {
|
||
return waveEdit
|
||
}
|
||
return waveDraft
|
||
}
|
||
|
||
// finalStageName is the SHIPPING stage's name — the value every read model filters stored rows by, because
|
||
// a unit's verdict and its exported text live on that stage's row. It is not finalStageWave's business: one
|
||
// answers which wave owns the shipping stage, the other what that stage is called, and both call sites had
|
||
// spelled the second one out inline.
|
||
func (r *Runner) finalStageName() string {
|
||
if n := len(r.Pipeline.Stages); n > 0 {
|
||
return r.Pipeline.Stages[n-1].Name
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// waveStages partitions the pipeline stages by role for a wave: the draft wave = the translator-role (draft) stages,
|
||
// the edit wave = every non-translator (editor/other) stage. Order preserved. The prod C1 core is one draft + one
|
||
// edit, so this yields [draft] and [edit] respectively.
|
||
func waveStages(stages []config.Stage, w wave) []config.Stage {
|
||
var out []config.Stage
|
||
for _, st := range stages {
|
||
isDraft := st.Role == roleTranslator
|
||
if (w == waveDraft) == isDraft {
|
||
out = append(out, st)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// baseMemoryVersion is the DRAFT-wave (the draft wave) memory component: the BASE bank version (excl Source:mined).
|
||
// nil bank → the empty-materialization base hash (a stable constant), mirroring memoryVersion().
|
||
func (r *Runner) baseMemoryVersion() string {
|
||
if r.memory != nil {
|
||
return r.memory.BaseVersion()
|
||
}
|
||
return membank.ComputeVersionScoped(nil, r.Pipeline.Gates.Glossary.PostcheckGate, true)
|
||
}
|
||
|
||
// stageSnap is ONE stage as the snapshot folds it. Package-level rather than local to
|
||
// buildSnapshotID so a test can walk its fields by reflection — the shape of what a snapshot
|
||
// promises to notice cannot be asserted over a JSON dump: 12 of its 19 slots are omitempty, and which of
|
||
// them a given book emits depends on the book — on the golden fixture the draft stage carries 3 of those 12
|
||
// (capability, escalate_to, escalate_capability) and omits 9. snapshotPayload's own count is 3 of 21.
|
||
//
|
||
// ⛔ MOVING IT CHANGED NO BYTE. Field set, order and tags are the declaration that was here;
|
||
// encoding/json marshals a named type exactly as it marshalled the anonymous one.
|
||
//
|
||
// ⚠ NOTHING REFLECTS OVER IT YET, deliberately: the axis table that will is the design pack's (D39.224
|
||
// §8(2)), and a second author for it is what that note forbids. What this move bought is that the criterion
|
||
// became possible to write at all.
|
||
type stageSnap struct {
|
||
Name string `json:"name"`
|
||
Role string `json:"role"`
|
||
Model string `json:"model"`
|
||
PromptVersion string `json:"prompt_version"`
|
||
PromptSHA256 string `json:"prompt_sha256"`
|
||
Temperature float64 `json:"temperature"`
|
||
Reasoning string `json:"reasoning"`
|
||
// FewShot folds the stage's few_shot on/off state (D38.5) — but ONLY for a prompt
|
||
// that HAS a ---FEWSHOT--- block (nil/omitted otherwise, so stages without examples
|
||
// keep a byte-identical snapshot). The block IS inside the raw file already folded via
|
||
// PromptSHA256, yet dropping it at render (few_shot:false) leaves PromptSHA256 unchanged
|
||
// while the wire changes, so the resolved flag must be folded to keep a flip a loud
|
||
// --resnapshot rather than a silent false-hit — same discipline as Temperature/Reasoning.
|
||
FewShot *bool `json:"few_shot,omitempty"`
|
||
// ExtraBody of the model changes the request BODY (GLM thinking-off, etc.) —
|
||
// without it a knob edit in models.yaml would silently not invalidate the
|
||
// checkpoints (a review finding). json.Marshal sorts map keys →
|
||
// a deterministic render.
|
||
ModelExtra json.RawMessage `json:"model_extra,omitempty"`
|
||
// Provider-level overrides (local kind overrides the wire
|
||
// temperature/max_tokens AFTER the request-hash is computed) — also affect
|
||
// the actual request; without them a providers.local.max_tokens edit
|
||
// would yield the same snapshot and a false checkpoint-hit on the stand's
|
||
// local path (external-review finding F2).
|
||
ProviderTemp float64 `json:"provider_temp,omitempty"`
|
||
ProviderMaxTok int `json:"provider_max_tok,omitempty"`
|
||
// ProviderModel — the local-kind backend tag the provider swaps onto the wire
|
||
// AFTER the request-hash (the actual model that answers). The MOST impactful
|
||
// local override, yet it was missing here while its weaker temp/max_tok siblings
|
||
// were folded: a local swap 8b→14b mid-book keeps the same snapID and resume
|
||
// serves the old model (external-review). Folded so it is a loud --resnapshot.
|
||
ProviderModel string `json:"provider_model,omitempty"`
|
||
// Capability — the model's resolved wire form (D3.1): budget-key,
|
||
// temperature-mode, reasoning-control. Changes the request BODY (max_tokens
|
||
// vs max_completion_tokens, whether to send temperature, the thinking-
|
||
// disabler), so it must enter the snapshot — otherwise a cap edit silently
|
||
// false-hits the checkpoints (the same D5.2 class as the payload below).
|
||
Capability json.RawMessage `json:"capability,omitempty"`
|
||
// EscalateTo + EscalateCapability — the single-hop fallback model and its
|
||
// resolved wire shape (Milestone 2.5). The fallback's request_hash uses ITS model,
|
||
// and its wire body uses ITS capability — both must be in the snapshot, or
|
||
// changing the escalation model / its caps would silently false-hit an
|
||
// escalated chunk's checkpoint (closes the D5.2 escalation-capability techdebt:
|
||
// stages were folded, escalation models were not, until the cycle landed).
|
||
EscalateTo string `json:"escalate_to,omitempty"`
|
||
EscalateCapability json.RawMessage `json:"escalate_capability,omitempty"`
|
||
// The fallback model's OTHER wire-affecting inputs, mirroring the primary
|
||
// fold above: its top-level extra_body (merged into the escalation wire body)
|
||
// and its provider-level temperature/max_tokens overrides (local kind). Without
|
||
// these, editing the fallback's extra_body / provider knobs would change the
|
||
// escalation wire but leave the snapshot identical → a silent false-hit on a
|
||
// resumed escalated chunk (self-review: the primary path guards this, escalation
|
||
// did not).
|
||
EscalateExtra json.RawMessage `json:"escalate_extra,omitempty"`
|
||
EscalateProviderTemp float64 `json:"escalate_provider_temp,omitempty"`
|
||
EscalateProviderMaxTok int `json:"escalate_provider_max_tok,omitempty"`
|
||
EscalateProviderModel string `json:"escalate_provider_model,omitempty"`
|
||
}
|
||
|
||
// snapshotPayload is the BOOK-GLOBAL job-context fold, as JSON, in a fixed field order (the id is
|
||
// its sha256). Package-level for the same reason as stageSnap: the field set is the contract, and
|
||
// reflection over it is the only way to assert that contract without a book that happens to fill
|
||
// every omitempty slot.
|
||
type snapshotPayload struct {
|
||
BriefHash string `json:"brief_hash"`
|
||
ChunkerVersion string `json:"chunker_version"`
|
||
EstimatorVersion string `json:"estimator_version"`
|
||
// MaxTokensPolicy versions the attempt→max_tokens scaling (Milestone 2): a
|
||
// change to the retry doubling shifts attempt≥1 request_hashes, so it
|
||
// belongs in the snapshot as a loud invalidation (same class as
|
||
// estimator_version, applied to the regeneration axis).
|
||
MaxTokensPolicy string `json:"max_tokens_policy"`
|
||
// ClassifierVersion versions the intrinsic classify() verdict logic (thresholds
|
||
// + order), so a re-verdict on a resumed checkpoint is a loud --resnapshot, not
|
||
// a silent flagged↔ok divergence (external-review; symmetric to coverage).
|
||
ClassifierVersion string `json:"classifier_version"`
|
||
PipelineCore string `json:"pipeline_core"`
|
||
// Defaults affect maxTokens, and that enters the request-hash: without them
|
||
// a max_output_ratio edit would silently invalidate all checkpoints,
|
||
// bypassing the snapshot gate (a review finding).
|
||
MaxOutputRatio float64 `json:"max_output_ratio"`
|
||
MinMaxTokens int `json:"min_max_tokens"`
|
||
// ContextAssembly — the context-assembly knobs (glossary/STM/overlap/TTL,
|
||
// §3.8). Editing them changes the model INPUT once memory enters msgs; we
|
||
// fold BEFORE injection, so an injection-budget change does not miss the
|
||
// resnapshot gate (D5.2). Fixed field order — this is a content hash.
|
||
ContextAssembly contextSnap `json:"context_assembly"`
|
||
// Segmentation — the WS2 output-token chunk/edit-unit budget + fertility (segmentationSnap).
|
||
// It changes chunk boundaries → the wire, so a budget/fertility edit is a loud --resnapshot.
|
||
Segmentation segmentationSnap `json:"segmentation"`
|
||
// RenderFormatVersion versions the FORMAT of the role-injection renderers not captured by
|
||
// memoryMatchVersion (a scope/matcher-algorithm version): the editor constraint block's
|
||
// src→dst layout (WS2 §2в) and the gender-constraint render (WS5). A format change shifts the
|
||
// injected bytes → the wire, so folding it separately keeps a flip a loud --resnapshot.
|
||
RenderFormatVersion string `json:"render_format_version"`
|
||
// LangpackVersion — the loaded language-data pack's content hash (internal/lang, D39.15/16),
|
||
// folded ONLY when a pack is present (omitempty otherwise, so a no-pack book is byte-stable and
|
||
// never re-billed for a feature it does not use). The pack feeds the bank-mining stop's miner; a pack DATA
|
||
// edit changes what it proposes, so folding its version makes a pack edit a loud --resnapshot,
|
||
// drift-proof by mechanism (the pack's bytes ARE its version). Wire-neutral for existing
|
||
// checkpoints (the miner produces Source:mined proposals, inert until owner-approved), but folded
|
||
// per the R1 directive so the run's snapshot durably records which pack version produced it.
|
||
LangpackVersion string `json:"langpack_version,omitempty"`
|
||
// EmbeddedVersion — the content hash of the engine's EMBEDDED language data (internal/lang/data/*:
|
||
// CJK section numerals/terminators feeding chunk boundaries, the Russian glossary/editor injection
|
||
// wire-text, the target-ru sanitizer/checker verdict data). Unlike LangpackVersion it is ALWAYS present
|
||
// (embedded in the binary and used by EVERY book — even a no-pack book still splits 第X章 chapters,
|
||
// injects a Russian block and runs the target checkers), so it folds UNCONDITIONALLY. Its bytes ride
|
||
// the wire (injection.txt) and resolve verdicts (target-ru.txt), yet the plane sat in NO content hash
|
||
// (D39.60 §6.1) — folding it makes an embed-data edit a loud --resnapshot instead of a silent verdict
|
||
// or wire change, drift-proof by mechanism (lang.EmbeddedVersion IS the bytes, like Pack.Version()).
|
||
EmbeddedVersion string `json:"embedded_version"`
|
||
// MemoryVersion — the content-hash of the deterministically materialized injected
|
||
// memory (approved-glossary+summary+series-bible). Not a bump-counter (D8): you
|
||
// cannot forget to recompute it, whereas a forgotten bump would reopen the silent
|
||
// re-payment class D5.2. Until the memory-bank migration (v2) the materialization is
|
||
// empty — the hash is stable; once memory enters msgs, the field changes and the
|
||
// resnapshot gate fires loudly. STM is NOT included here (rebuilt from checkpoints).
|
||
MemoryVersion string `json:"memory_version"`
|
||
// PostcheckGate — the memory-bank post-check gate (E1). When true a post-check
|
||
// miss flips the chunk to flagged; folding its on/off state makes toggling it a
|
||
// loud --resnapshot (it changes a checkpoint's RESOLVED disposition), not a silent
|
||
// re-verdict on resume (same class as coverage/classifier). The post-check
|
||
// ALGORITHM version is already inside MemoryVersion (memoryMatchVersion).
|
||
PostcheckGate bool `json:"postcheck_gate"`
|
||
// Coverage — excision QA-gate config (D12 Q3). It does not touch the wire,
|
||
// but it determines a checkpoint's RESOLVED verdict, so a gate change must be
|
||
// a loud --resnapshot, not a silent re-verdict on resume (coverageSnapshot).
|
||
Coverage coverageSnap `json:"coverage"`
|
||
// StyleCheckVersion versions the cheap deterministic post-check rules (dialogue-dash,
|
||
// yofikator, translit-interjection blocklist, 万/億 magnitude gate — cheapgates.go). They
|
||
// are observability, not wire, but editing a rule shifts the recorded style-flag counts, so
|
||
// a bump is a loud --resnapshot (same verdict class as ClassifierVersion). The ё-policy is
|
||
// part of BriefHash (a book field), so a policy change already re-pins via brief_hash.
|
||
StyleCheckVersion string `json:"style_check_version"`
|
||
// Sanitizer — the output-sanitizer gate (D30.3). Like Coverage it does not touch the wire
|
||
// but determines a checkpoint's RESOLVED verdict (a defect flips the chunk to flagged), so a
|
||
// gate flip / rule edit is a loud --resnapshot. Folded only when enabled (sanitizerSnapshot).
|
||
Sanitizer sanitizerSnap `json:"sanitizer"`
|
||
// Banknote — the banknote channel (WS4). Verdict-axis (governs the sliced/derived draft), folded
|
||
// only when ENABLED (nil→omitempty otherwise), so adding the feature never re-bills an off book.
|
||
Banknote *banknoteSnap `json:"banknote,omitempty"`
|
||
// Repair — the addressable-defect repair sub-step (pack-16). Verdict-axis (it re-points final_hash at
|
||
// a derived checkpoint holding the repaired text), folded only when ENABLED and only into the wave
|
||
// that owns the shipping stage.
|
||
Repair *repairSnap `json:"repair,omitempty"`
|
||
Stages []stageSnap `json:"stages"`
|
||
}
|
||
|
||
// buildSnapshotID materializes the snapshot payload for a given stage subset + memory version (§3.2).
|
||
// Extracted from snapshotID so the book-global snapshot (all stages + enriched version) AND the two
|
||
// per-wave snapshots (WS1 §1в) share one byte-exact builder — a stage subset / memory version is the
|
||
// ONLY axis that differs. Fixed field order (a content hash).
|
||
func (r *Runner) buildSnapshotID(stages []config.Stage, memVersion string, repair *repairSnap) (id, payload string, err error) {
|
||
snap := snapshotPayload{
|
||
BriefHash: r.Book.BriefHash(),
|
||
ChunkerVersion: chunkerVersion,
|
||
EstimatorVersion: estimatorVersion,
|
||
MaxTokensPolicy: maxTokensPolicyVersion,
|
||
ClassifierVersion: classifierVersion,
|
||
PipelineCore: r.Pipeline.Core,
|
||
MaxOutputRatio: r.Pipeline.Defaults.MaxOutputRatio,
|
||
MinMaxTokens: r.Pipeline.Defaults.MinMaxTokens,
|
||
ContextAssembly: contextSnap{
|
||
GlossaryInjection: r.Pipeline.Context.GlossaryInjection,
|
||
GlossaryTokenBudget: r.Pipeline.Context.GlossaryTokenBudget,
|
||
CacheTTL: r.Pipeline.Context.CacheTTL,
|
||
},
|
||
Segmentation: r.segmentationSnapshot(),
|
||
RenderFormatVersion: membank.RenderFormatVersion,
|
||
LangpackVersion: r.packVersion(),
|
||
EmbeddedVersion: lang.EmbeddedVersion(),
|
||
MemoryVersion: memVersion,
|
||
PostcheckGate: r.Pipeline.Gates.Glossary.PostcheckGate,
|
||
Coverage: r.coverageSnapshot(),
|
||
StyleCheckVersion: checks.CheapGateVersion,
|
||
Sanitizer: r.sanitizerSnapshot(),
|
||
Banknote: r.banknoteSnapshot(),
|
||
Repair: repair,
|
||
}
|
||
for _, st := range stages {
|
||
ss := stageSnap{
|
||
// Model is the RESOLVED model — the one this stage will actually call (label routing applied).
|
||
// That is the ASSIGNED snapshot layer for content labels (D39.26 point 1, discharging D20.2-Q3):
|
||
// the label set itself enters no hash, and its whole wire effect rides here, per stage and per
|
||
// wave — so a label that re-routes only the editor moves only the edit-wave snapshot and the paid
|
||
// draft still resumes at $0. With no labels this is st.Model, byte for byte.
|
||
Name: st.Name, Role: st.Role, Model: st.ResolvedModel,
|
||
PromptVersion: st.PromptVersion, PromptSHA256: r.templates[st.Name].SHA256,
|
||
Temperature: st.Temperature, Reasoning: st.Reasoning,
|
||
}
|
||
// Fold few_shot only where the prompt actually carries a ---FEWSHOT--- block, so
|
||
// stages without examples stay byte-identical in the snapshot (D38.4).
|
||
if r.templates[st.Name].FewShot != "" {
|
||
on := fewShotEnabled(st)
|
||
ss.FewShot = &on
|
||
}
|
||
// Fold the PRIMARY model's wire-affecting inputs (prov-triple, extra_body, resolved capability)
|
||
// via the shared foldModelWire (D39 layer 7, L8-snapshot-fold-copypaste-tripwire): the same
|
||
// helper folds the escalate_to fallback below, so a third model axis (channel B / annotator)
|
||
// reuses it and cannot silently drift the snapshot.
|
||
primary, perr := r.foldModelWire(st.ResolvedModel)
|
||
if perr != nil {
|
||
return "", "", perr
|
||
}
|
||
ss.ProviderTemp, ss.ProviderMaxTok, ss.ProviderModel = primary.provTemp, primary.provMaxTok, primary.provModel
|
||
ss.ModelExtra = primary.extra
|
||
ss.Capability = primary.capability
|
||
// Fold the single-hop fallback model + its wire inputs (Milestone 2.5): both are wire-affecting
|
||
// inputs of an escalated call, so changing them is a loud --resnapshot, not a silent false-hit
|
||
// on an escalated checkpoint. Same fold as the primary — no copy-paste to drift.
|
||
// The hop's fold keeps its EXISTING field names, tags, positions and the "only when there is a hop"
|
||
// population rule (D39.26 point 1, sixth byte-identity condition): a labelled run puts the resolved
|
||
// hop in the SAME slots rather than adding a parallel array, so an unlabelled book's payload is
|
||
// unchanged to the byte.
|
||
if st.ResolvedHop != "" {
|
||
ss.EscalateTo = st.ResolvedHop
|
||
esc, eerr := r.foldModelWire(st.ResolvedHop)
|
||
if eerr != nil {
|
||
return "", "", eerr
|
||
}
|
||
ss.EscalateProviderTemp, ss.EscalateProviderMaxTok, ss.EscalateProviderModel = esc.provTemp, esc.provMaxTok, esc.provModel
|
||
ss.EscalateExtra = esc.extra
|
||
ss.EscalateCapability = esc.capability
|
||
}
|
||
snap.Stages = append(snap.Stages, ss)
|
||
}
|
||
data, err := json.Marshal(snap)
|
||
if err != nil {
|
||
return "", "", err
|
||
}
|
||
sum := sha256.Sum256(data)
|
||
return hex.EncodeToString(sum[:]), string(data), nil
|
||
}
|
||
|
||
// modelWireFold is a model's snapshot-folded wire-affecting inputs: its provider-level
|
||
// temperature/max_tokens/model overrides (local kind, applied AFTER the request-hash), its top-level
|
||
// extra_body, and its RESOLVED capability (budget-key/temperature-mode/reasoning-control). Extracted
|
||
// (D39 layer 7, L8-snapshot-fold-copypaste-tripwire) so the primary model, the escalate_to fallback,
|
||
// and any future third model axis fold the SAME way — the copy-paste the old tripwire warned about.
|
||
type modelWireFold struct {
|
||
provTemp float64
|
||
provMaxTok int
|
||
provModel string
|
||
extra json.RawMessage // nil when the model has no extra_body → omitted (omitempty)
|
||
capability json.RawMessage // always present (ResolveCapability, the same the client sends)
|
||
}
|
||
|
||
// foldModelWire renders one model's wire-fold. ResolveCapability matches exactly what the client
|
||
// sends, and json.Marshal sorts map keys (off_extra_body) → deterministic render. Returns an error
|
||
// only on a marshal failure (a struct of scalars/known types cannot fail in practice).
|
||
func (r *Runner) foldModelWire(model string) (modelWireFold, error) {
|
||
var f modelWireFold
|
||
if prov, ok := r.Models.Providers[r.Models.Models[model].Provider]; ok {
|
||
f.provTemp, f.provMaxTok, f.provModel = prov.Temperature, prov.MaxTokens, prov.Model
|
||
}
|
||
if extra := r.Models.Models[model].ExtraBody; len(extra) > 0 {
|
||
raw, err := json.Marshal(extra)
|
||
if err != nil {
|
||
return f, err
|
||
}
|
||
f.extra = raw
|
||
}
|
||
capRaw, err := json.Marshal(r.Models.ResolveCapability(model))
|
||
if err != nil {
|
||
return f, err
|
||
}
|
||
f.capability = capRaw
|
||
return f, nil
|
||
}
|