344 lines
20 KiB
Go
344 lines
20 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
)
|
||
|
||
// snapshot.go: материализация snapshotID (§3.2/D5.2/D8) — контент-хеш всего, что
|
||
// влияет на wire-байты ИЛИ на резолв вердикта чекпоинта. Правка любого входа —
|
||
// громкий --resnapshot (= переоплата книги, D15), никогда тихий false-hit.
|
||
|
||
// contextSnap freezes the context-assembly knobs (§3.8) inside the snapshot.
|
||
// Fixed field order — it is part of a content hash. WS2 dropped STMDepth/OverlapTokens
|
||
// (dead carryover knobs — §2а), a structural change folded loudly (§8 manifest).
|
||
type contextSnap struct {
|
||
GlossaryInjection string `json:"glossary_injection"`
|
||
GlossaryTokenBudget int `json:"glossary_token_budget"`
|
||
CacheTTL string `json:"cache_ttl"`
|
||
}
|
||
|
||
// segmentationSnap freezes the WS2 output-token chunking budget (draft/edit ceilings + per-pair
|
||
// fertility) inside the snapshot, mirroring coverageSnap's discipline: a budget or fertility edit
|
||
// re-chunks the book (changes chunk boundaries → the wire), so it is a loud --resnapshot (D30.9),
|
||
// never a silent divergence. Unlike coverageSnap it is ALWAYS folded — segmentation always runs and
|
||
// its values are wire-determining. Fixed field order — part of a content hash.
|
||
type segmentationSnap struct {
|
||
DraftBudgetOut int `json:"draft_budget_out"`
|
||
EditCeilingOut int `json:"edit_ceiling_out"`
|
||
FertilityCJK float64 `json:"fertility_cjk"`
|
||
FertilityOther float64 `json:"fertility_other"`
|
||
}
|
||
|
||
func (r *Runner) segmentationSnapshot() segmentationSnap {
|
||
seg := r.Pipeline.Segmentation
|
||
return segmentationSnap{
|
||
DraftBudgetOut: seg.DraftBudgetOut,
|
||
EditCeilingOut: seg.EditCeilingOut,
|
||
FertilityCJK: seg.Fertility.CJK,
|
||
FertilityOther: seg.Fertility.Other,
|
||
}
|
||
}
|
||
|
||
// coverageSnap freezes the excision coverage-gate config inside the snapshot, so
|
||
// enabling the gate OR tuning its thresholds is a loud --resnapshot, never a silent
|
||
// mismatch between an old checkpoint's stored disposition and a changed gate. HONEST
|
||
// COST (D15): the snapshotID is folded into every call's RequestHash (render.go), so a
|
||
// --resnapshot changes EVERY request_hash → every checkpoint misses → the whole book is
|
||
// RE-BILLED, even for byte-identical wire requests whose verdict merely needs
|
||
// re-classifying. There is no free "re-classify from the checkpoint" here: the
|
||
// content-addressed reuse that COULD make an unchanged wire request free (msgsContentHash,
|
||
// render.go) is gated on an UNCHANGED snapshot (stagerun.go), so it never fires across a
|
||
// resnapshot. This all-or-nothing re-pay is ACCEPTED for the static acceptance book (the
|
||
// gate is loud, divergences do not occur) but is exactly why content-addressed checkpoint
|
||
// reuse must be designed before the ongoing/append-chapters mode (D15.2). Only folded when
|
||
// ENABLED: tweaking a disabled gate's bounds must not force a re-pin (the gate produces no
|
||
// verdicts while off). json.Marshal sorts the LenRatio map keys → deterministic render.
|
||
// Mirrors the estimator/max_tokens-policy discipline (render.go) for a NON-wire input that
|
||
// still determines a checkpoint's resolved verdict.
|
||
type coverageSnap struct {
|
||
Enabled bool `json:"enabled"`
|
||
Version string `json:"version,omitempty"`
|
||
LenRatio map[string][]float64 `json:"len_ratio,omitempty"`
|
||
SentCovMin float64 `json:"sent_cov_min,omitempty"`
|
||
MinChunkChars int `json:"min_chunk_chars,omitempty"`
|
||
}
|
||
|
||
// coverageSnapshot renders the gate's snapshot component: {enabled:false} when off
|
||
// (so toggling on is a visible change), the full config + algorithm version when on.
|
||
func (r *Runner) coverageSnapshot() coverageSnap {
|
||
cov := coverageSnap{Enabled: r.Pipeline.Gates.Coverage.Enabled}
|
||
if cov.Enabled {
|
||
cov.Version = coverageGateVersion
|
||
cov.LenRatio = r.Pipeline.Gates.Coverage.LenRatio
|
||
cov.SentCovMin = r.Pipeline.Gates.Coverage.SentCovMin
|
||
cov.MinChunkChars = r.Pipeline.Gates.Coverage.MinChunkChars
|
||
}
|
||
return cov
|
||
}
|
||
|
||
// sanitizerSnap freezes the output-sanitizer gate (D30.3) inside the snapshot, mirroring
|
||
// coverageSnap: {enabled:false} when off (so toggling on is a visible --resnapshot), the
|
||
// algorithm version when on. It changes a checkpoint's RESOLVED disposition (a defect flips
|
||
// the chunk to flagged), never the wire, so a rule edit is a loud re-pin, not a silent
|
||
// re-verdict on resume. Folded ONLY when enabled — tweaking a disabled gate must not force a
|
||
// re-pin (it produces no verdicts while off). Moves to verdictSnapshotID with D15.2.
|
||
type sanitizerSnap struct {
|
||
Enabled bool `json:"enabled"`
|
||
Version string `json:"version,omitempty"`
|
||
}
|
||
|
||
func (r *Runner) sanitizerSnapshot() sanitizerSnap {
|
||
s := sanitizerSnap{Enabled: r.Pipeline.Gates.Sanitizer.Enabled}
|
||
if s.Enabled {
|
||
s.Version = sanitizerVersion
|
||
}
|
||
return s
|
||
}
|
||
|
||
// memoryVersion is the content-hash of the deterministically materialized injected
|
||
// memory (the frozen APPROVED glossary rows + the normalization/matcher algorithm
|
||
// versions), the memory component of the snapshot (D5.2/D8, F1 CLOSED). It is the
|
||
// Version() of the bank materialized once before the loop, so a change to the approved
|
||
// glossary — or to the deterministic machinery (trad→simp table, matcher) — fires the
|
||
// resnapshot gate LOUDLY instead of a stale checkpoint re-paying a diverged translation.
|
||
// nil bank (report path / a book with no glossary) → the empty-materialization hash, a
|
||
// stable constant. STM is excluded (rebuilt from checkpoints, §3.2). Auto/draft rows are
|
||
// excluded per D8 (their injected-content changes are caught at the per-chunk
|
||
// content_hash level; a future autopopulation milestone revisits this).
|
||
func (r *Runner) memoryVersion() string {
|
||
if r.memory != nil {
|
||
return r.memory.Version()
|
||
}
|
||
return computeMemoryVersion(nil, r.Pipeline.Gates.Glossary.PostcheckGate)
|
||
}
|
||
|
||
// snapshotID materializes the job-context snapshot (§3.2): brief_hash, chunker
|
||
// version, context-assembly knobs, the memory version and the full stage plan
|
||
// (model, prompt version + CONTENT hash, sampling, resolved capability).
|
||
// Payload is rendered with a fixed field order — the id is a content hash, so
|
||
// identical context re-upserts idempotently.
|
||
func (r *Runner) snapshotID() (id, payload string, err error) {
|
||
type stageSnap struct {
|
||
Name string `json:"name"`
|
||
Role string `json:"role"`
|
||
Model string `json:"model"`
|
||
PromptVersion string `json:"prompt_version"`
|
||
PromptSHA256 string `json:"prompt_sha256"`
|
||
Temperature float64 `json:"temperature"`
|
||
Reasoning string `json:"reasoning"`
|
||
// FewShot folds the stage's few_shot on/off state (D38.4) — but ONLY for a prompt
|
||
// that HAS a ---FEWSHOT--- block (nil/omitted otherwise, so stages without examples
|
||
// keep a byte-identical snapshot). The block IS inside the raw file already folded via
|
||
// PromptSHA256, yet dropping it at render (few_shot:false) leaves PromptSHA256 unchanged
|
||
// while the wire changes, so the resolved flag must be folded to keep a flip a loud
|
||
// --resnapshot rather than a silent false-hit — same discipline as Temperature/Reasoning.
|
||
FewShot *bool `json:"few_shot,omitempty"`
|
||
// ExtraBody модели меняет ТЕЛО запроса (GLM thinking-off и т.п.) —
|
||
// без него правка ручки в models.yaml молча не инвалидировала бы
|
||
// чекпоинты (находка ревью). json.Marshal карты сортирует ключи →
|
||
// детерминированный рендер.
|
||
ModelExtra json.RawMessage `json:"model_extra,omitempty"`
|
||
// Провайдер-уровневые оверрайды (local kind переопределяет wire
|
||
// temperature/max_tokens ПОСЛЕ вычисления request-hash) — тоже влияют
|
||
// на фактический запрос; без них правка providers.local.max_tokens
|
||
// давала бы тот же snapshot и ложный checkpoint-hit на стендовом
|
||
// local-пути (находка внешнего ревью F2).
|
||
ProviderTemp float64 `json:"provider_temp,omitempty"`
|
||
ProviderMaxTok int `json:"provider_max_tok,omitempty"`
|
||
// ProviderModel — the local-kind backend tag the provider swaps onto the wire
|
||
// AFTER the request-hash (the actual model that answers). The MOST impactful
|
||
// local override, yet it was missing here while its weaker temp/max_tok siblings
|
||
// were folded: a local swap 8b→14b mid-book keeps the same snapID and resume
|
||
// serves the old model (external-review). Folded so it is a loud --resnapshot.
|
||
ProviderModel string `json:"provider_model,omitempty"`
|
||
// Capability — резолвнутая wire-форма модели (D3.1): budget-ключ,
|
||
// temperature-режим, reasoning-контроль. Меняет ТЕЛО запроса (max_tokens
|
||
// vs max_completion_tokens, отправлять ли temperature, thinking-выключа-
|
||
// тель), поэтому обязана входить в снапшот — иначе правка каппы молча
|
||
// false-хитит чекпоинты (тот же класс D5.2, что и payload ниже).
|
||
Capability json.RawMessage `json:"capability,omitempty"`
|
||
// EscalateTo + EscalateCapability — the single-hop fallback model and its
|
||
// resolved wire shape (Веха 2.5). The fallback's request_hash uses ITS model,
|
||
// and its wire body uses ITS capability — both must be in the snapshot, or
|
||
// changing the escalation model / its caps would silently false-hit an
|
||
// escalated chunk's checkpoint (closes the D5.2 escalation-capability techdebt:
|
||
// stages were folded, escalation models were not, until the cycle landed).
|
||
EscalateTo string `json:"escalate_to,omitempty"`
|
||
EscalateCapability json.RawMessage `json:"escalate_capability,omitempty"`
|
||
// The fallback model's OTHER wire-affecting inputs, mirroring the primary
|
||
// fold above: its top-level extra_body (merged into the escalation wire body)
|
||
// and its provider-level temperature/max_tokens overrides (local kind). Without
|
||
// these, editing the fallback's extra_body / provider knobs would change the
|
||
// escalation wire but leave the snapshot identical → a silent false-hit on a
|
||
// resumed escalated chunk (self-review: the primary path guards this, escalation
|
||
// did not).
|
||
EscalateExtra json.RawMessage `json:"escalate_extra,omitempty"`
|
||
EscalateProviderTemp float64 `json:"escalate_provider_temp,omitempty"`
|
||
EscalateProviderMaxTok int `json:"escalate_provider_max_tok,omitempty"`
|
||
EscalateProviderModel string `json:"escalate_provider_model,omitempty"`
|
||
}
|
||
snap := struct {
|
||
BriefHash string `json:"brief_hash"`
|
||
ChunkerVersion string `json:"chunker_version"`
|
||
EstimatorVersion string `json:"estimator_version"`
|
||
// MaxTokensPolicy versions the attempt→max_tokens scaling (Веха 2): a
|
||
// change to the retry doubling shifts attempt≥1 request_hashes, so it
|
||
// belongs in the snapshot as a loud invalidation (same class as
|
||
// estimator_version, applied to the regeneration axis).
|
||
MaxTokensPolicy string `json:"max_tokens_policy"`
|
||
// ClassifierVersion versions the intrinsic classify() verdict logic (thresholds
|
||
// + order), so a re-verdict on a resumed checkpoint is a loud --resnapshot, not
|
||
// a silent flagged↔ok divergence (external-review; symmetric to coverage).
|
||
ClassifierVersion string `json:"classifier_version"`
|
||
PipelineCore string `json:"pipeline_core"`
|
||
// Defaults влияют на maxTokens, а тот входит в request-hash: без них
|
||
// правка max_output_ratio молча инвалидировала бы все чекпоинты в
|
||
// обход snapshot-гейта (находка ревью).
|
||
MaxOutputRatio float64 `json:"max_output_ratio"`
|
||
MinMaxTokens int `json:"min_max_tokens"`
|
||
// ContextAssembly — ручки сборки контекста (глоссарий/STM/overlap/TTL,
|
||
// §3.8). Их правка меняет ВХОД модели, когда память войдёт в msgs; сворачи-
|
||
// ваем ДО инъекции, чтобы смена budget'а инъекции не промахнулась мимо
|
||
// resnapshot-гейта (D5.2). Фикс-порядок полей — это content-hash.
|
||
ContextAssembly contextSnap `json:"context_assembly"`
|
||
// Segmentation — the WS2 output-token chunk/edit-unit budget + fertility (segmentationSnap).
|
||
// It changes chunk boundaries → the wire, so a budget/fertility edit is a loud --resnapshot.
|
||
Segmentation segmentationSnap `json:"segmentation"`
|
||
// RenderFormatVersion versions the FORMAT of the role-injection renderers not captured by
|
||
// memoryMatchVersion (a scope/matcher-algorithm version): the editor constraint block's
|
||
// src→dst layout (WS2 §2в) and the gender-constraint render (WS5). A format change shifts the
|
||
// injected bytes → the wire, so folding it separately keeps a flip a loud --resnapshot.
|
||
RenderFormatVersion string `json:"render_format_version"`
|
||
// MemoryVersion — content-hash детерминированно материализованной инъекти-
|
||
// руемой памяти (approved-глоссарий+резюме+series-bible). Не bump-счётчик
|
||
// (D8): забыть пересчитать нельзя, а забытый bump переоткрыл бы класс тихой
|
||
// переоплаты D5.2. До миграции банка памяти (v2) материализация пуста —
|
||
// хэш стабилен; когда память войдёт в msgs, поле сменится и resnapshot-
|
||
// гейт сработает громко. STM сюда НЕ входит (пересобирается из чекпоинтов).
|
||
MemoryVersion string `json:"memory_version"`
|
||
// PostcheckGate — the memory-bank post-check gate (E1). When true a post-check
|
||
// miss flips the chunk to flagged; folding its on/off state makes toggling it a
|
||
// loud --resnapshot (it changes a checkpoint's RESOLVED disposition), not a silent
|
||
// re-verdict on resume (same class as coverage/classifier). The post-check
|
||
// ALGORITHM version is already inside MemoryVersion (memoryMatchVersion).
|
||
PostcheckGate bool `json:"postcheck_gate"`
|
||
// Coverage — excision QA-gate config (D12 Q3). It does not touch the wire,
|
||
// but it determines a checkpoint's RESOLVED verdict, so a gate change must be
|
||
// a loud --resnapshot, not a silent re-verdict on resume (coverageSnapshot).
|
||
Coverage coverageSnap `json:"coverage"`
|
||
// StyleCheckVersion versions the cheap deterministic post-check rules (dialogue-dash,
|
||
// yofikator, translit-interjection blocklist, 万/億 magnitude gate — cheapgates.go). They
|
||
// are observability, not wire, but editing a rule shifts the recorded style-flag counts, so
|
||
// a bump is a loud --resnapshot (same verdict class as ClassifierVersion). The ё-policy is
|
||
// part of BriefHash (a book field), so a policy change already re-pins via brief_hash.
|
||
StyleCheckVersion string `json:"style_check_version"`
|
||
// Sanitizer — the output-sanitizer gate (D30.3). Like Coverage it does not touch the wire
|
||
// but determines a checkpoint's RESOLVED verdict (a defect flips the chunk to flagged), so a
|
||
// gate flip / rule edit is a loud --resnapshot. Folded only when enabled (sanitizerSnapshot).
|
||
Sanitizer sanitizerSnap `json:"sanitizer"`
|
||
Stages []stageSnap `json:"stages"`
|
||
}{
|
||
BriefHash: r.Book.BriefHash(),
|
||
ChunkerVersion: chunkerVersion,
|
||
EstimatorVersion: estimatorVersion,
|
||
MaxTokensPolicy: maxTokensPolicyVersion,
|
||
ClassifierVersion: classifierVersion,
|
||
PipelineCore: r.Pipeline.Core,
|
||
MaxOutputRatio: r.Pipeline.Defaults.MaxOutputRatio,
|
||
MinMaxTokens: r.Pipeline.Defaults.MinMaxTokens,
|
||
ContextAssembly: contextSnap{
|
||
GlossaryInjection: r.Pipeline.Context.GlossaryInjection,
|
||
GlossaryTokenBudget: r.Pipeline.Context.GlossaryTokenBudget,
|
||
CacheTTL: r.Pipeline.Context.CacheTTL,
|
||
},
|
||
Segmentation: r.segmentationSnapshot(),
|
||
RenderFormatVersion: renderFormatVersion,
|
||
MemoryVersion: r.memoryVersion(),
|
||
PostcheckGate: r.Pipeline.Gates.Glossary.PostcheckGate,
|
||
Coverage: r.coverageSnapshot(),
|
||
StyleCheckVersion: cheapGateVersion,
|
||
Sanitizer: r.sanitizerSnapshot(),
|
||
}
|
||
for _, st := range r.Pipeline.Stages {
|
||
ss := stageSnap{
|
||
Name: st.Name, Role: st.Role, Model: st.Model,
|
||
PromptVersion: st.PromptVersion, PromptSHA256: r.templates[st.Name].SHA256,
|
||
Temperature: st.Temperature, Reasoning: st.Reasoning,
|
||
}
|
||
// Fold few_shot only where the prompt actually carries a ---FEWSHOT--- block, so
|
||
// stages without examples stay byte-identical in the snapshot (D38.4).
|
||
if r.templates[st.Name].FewShot != "" {
|
||
on := fewShotEnabled(st)
|
||
ss.FewShot = &on
|
||
}
|
||
// Fold the PRIMARY model's wire-affecting inputs (prov-triple, extra_body, resolved capability)
|
||
// via the shared foldModelWire (D39 слой 7, L8-snapshot-fold-copypaste-tripwire): the same
|
||
// helper folds the escalate_to fallback below, so a third model axis (channel B / annotator)
|
||
// reuses it and cannot silently drift the snapshot.
|
||
primary, perr := r.foldModelWire(st.Model)
|
||
if perr != nil {
|
||
return "", "", perr
|
||
}
|
||
ss.ProviderTemp, ss.ProviderMaxTok, ss.ProviderModel = primary.provTemp, primary.provMaxTok, primary.provModel
|
||
ss.ModelExtra = primary.extra
|
||
ss.Capability = primary.capability
|
||
// Fold the single-hop fallback model + its wire inputs (Веха 2.5): both are wire-affecting
|
||
// inputs of an escalated call, so changing them is a loud --resnapshot, not a silent false-hit
|
||
// on an escalated checkpoint. Same fold as the primary — no copy-paste to drift.
|
||
if st.EscalateTo != "" {
|
||
ss.EscalateTo = st.EscalateTo
|
||
esc, eerr := r.foldModelWire(st.EscalateTo)
|
||
if eerr != nil {
|
||
return "", "", eerr
|
||
}
|
||
ss.EscalateProviderTemp, ss.EscalateProviderMaxTok, ss.EscalateProviderModel = esc.provTemp, esc.provMaxTok, esc.provModel
|
||
ss.EscalateExtra = esc.extra
|
||
ss.EscalateCapability = esc.capability
|
||
}
|
||
snap.Stages = append(snap.Stages, ss)
|
||
}
|
||
data, err := json.Marshal(snap)
|
||
if err != nil {
|
||
return "", "", err
|
||
}
|
||
sum := sha256.Sum256(data)
|
||
return hex.EncodeToString(sum[:]), string(data), nil
|
||
}
|
||
|
||
// modelWireFold is a model's snapshot-folded wire-affecting inputs: its provider-level
|
||
// temperature/max_tokens/model overrides (local kind, applied AFTER the request-hash), its top-level
|
||
// extra_body, and its RESOLVED capability (budget-key/temperature-mode/reasoning-control). Extracted
|
||
// (D39 слой 7, L8-snapshot-fold-copypaste-tripwire) so the primary model, the escalate_to fallback,
|
||
// and any future third model axis fold the SAME way — the copy-paste the old tripwire warned about.
|
||
type modelWireFold struct {
|
||
provTemp float64
|
||
provMaxTok int
|
||
provModel string
|
||
extra json.RawMessage // nil when the model has no extra_body → omitted (omitempty)
|
||
capability json.RawMessage // always present (ResolveCapability, the same the client sends)
|
||
}
|
||
|
||
// foldModelWire renders one model's wire-fold. ResolveCapability matches exactly what the client
|
||
// sends, and json.Marshal sorts map keys (off_extra_body) → deterministic render. Returns an error
|
||
// only on a marshal failure (a struct of scalars/known types cannot fail in practice).
|
||
func (r *Runner) foldModelWire(model string) (modelWireFold, error) {
|
||
var f modelWireFold
|
||
if prov, ok := r.Models.Providers[r.Models.Models[model].Provider]; ok {
|
||
f.provTemp, f.provMaxTok, f.provModel = prov.Temperature, prov.MaxTokens, prov.Model
|
||
}
|
||
if extra := r.Models.Models[model].ExtraBody; len(extra) > 0 {
|
||
raw, err := json.Marshal(extra)
|
||
if err != nil {
|
||
return f, err
|
||
}
|
||
f.extra = raw
|
||
}
|
||
capRaw, err := json.Marshal(r.Models.ResolveCapability(model))
|
||
if err != nil {
|
||
return f, err
|
||
}
|
||
f.capability = capRaw
|
||
return f, nil
|
||
}
|