286 lines
17 KiB
Go
286 lines
17 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
)
|
||
|
||
// snapshot.go: материализация snapshotID (§3.2/D5.2/D8) — контент-хеш всего, что
|
||
// влияет на wire-байты ИЛИ на резолв вердикта чекпоинта. Правка любого входа —
|
||
// громкий --resnapshot (= переоплата книги, D15), никогда тихий false-hit.
|
||
|
||
// contextSnap freezes the context-assembly knobs (§3.8) inside the snapshot.
|
||
// Fixed field order — it is part of a content hash.
|
||
type contextSnap struct {
|
||
GlossaryInjection string `json:"glossary_injection"`
|
||
GlossaryTokenBudget int `json:"glossary_token_budget"`
|
||
STMDepth int `json:"stm_depth"`
|
||
OverlapTokens int `json:"overlap_tokens"`
|
||
CacheTTL string `json:"cache_ttl"`
|
||
}
|
||
|
||
// coverageSnap freezes the excision coverage-gate config inside the snapshot, so
|
||
// enabling the gate OR tuning its thresholds is a loud --resnapshot, never a silent
|
||
// mismatch between an old checkpoint's stored disposition and a changed gate. HONEST
|
||
// COST (D15): the snapshotID is folded into every call's RequestHash (render.go), so a
|
||
// --resnapshot changes EVERY request_hash → every checkpoint misses → the whole book is
|
||
// RE-BILLED, even for byte-identical wire requests whose verdict merely needs
|
||
// re-classifying. There is no free "re-classify from the checkpoint" here: the
|
||
// content-addressed reuse that COULD make an unchanged wire request free (msgsContentHash,
|
||
// render.go) is gated on an UNCHANGED snapshot (stagerun.go), so it never fires across a
|
||
// resnapshot. This all-or-nothing re-pay is ACCEPTED for the static acceptance book (the
|
||
// gate is loud, divergences do not occur) but is exactly why content-addressed checkpoint
|
||
// reuse must be designed before the ongoing/append-chapters mode (D15.2). Only folded when
|
||
// ENABLED: tweaking a disabled gate's bounds must not force a re-pin (the gate produces no
|
||
// verdicts while off). json.Marshal sorts the LenRatio map keys → deterministic render.
|
||
// Mirrors the estimator/max_tokens-policy discipline (render.go) for a NON-wire input that
|
||
// still determines a checkpoint's resolved verdict.
|
||
type coverageSnap struct {
|
||
Enabled bool `json:"enabled"`
|
||
Version string `json:"version,omitempty"`
|
||
LenRatio map[string][]float64 `json:"len_ratio,omitempty"`
|
||
SentCovMin float64 `json:"sent_cov_min,omitempty"`
|
||
MinChunkChars int `json:"min_chunk_chars,omitempty"`
|
||
}
|
||
|
||
// coverageSnapshot renders the gate's snapshot component: {enabled:false} when off
|
||
// (so toggling on is a visible change), the full config + algorithm version when on.
|
||
func (r *Runner) coverageSnapshot() coverageSnap {
|
||
cov := coverageSnap{Enabled: r.Pipeline.Gates.Coverage.Enabled}
|
||
if cov.Enabled {
|
||
cov.Version = coverageGateVersion
|
||
cov.LenRatio = r.Pipeline.Gates.Coverage.LenRatio
|
||
cov.SentCovMin = r.Pipeline.Gates.Coverage.SentCovMin
|
||
cov.MinChunkChars = r.Pipeline.Gates.Coverage.MinChunkChars
|
||
}
|
||
return cov
|
||
}
|
||
|
||
// sanitizerSnap freezes the output-sanitizer gate (D30.3) inside the snapshot, mirroring
|
||
// coverageSnap: {enabled:false} when off (so toggling on is a visible --resnapshot), the
|
||
// algorithm version when on. It changes a checkpoint's RESOLVED disposition (a defect flips
|
||
// the chunk to flagged), never the wire, so a rule edit is a loud re-pin, not a silent
|
||
// re-verdict on resume. Folded ONLY when enabled — tweaking a disabled gate must not force a
|
||
// re-pin (it produces no verdicts while off). Moves to verdictSnapshotID with D15.2.
|
||
type sanitizerSnap struct {
|
||
Enabled bool `json:"enabled"`
|
||
Version string `json:"version,omitempty"`
|
||
}
|
||
|
||
func (r *Runner) sanitizerSnapshot() sanitizerSnap {
|
||
s := sanitizerSnap{Enabled: r.Pipeline.Gates.Sanitizer.Enabled}
|
||
if s.Enabled {
|
||
s.Version = sanitizerVersion
|
||
}
|
||
return s
|
||
}
|
||
|
||
// memoryVersion is the content-hash of the deterministically materialized injected
|
||
// memory (the frozen APPROVED glossary rows + the normalization/matcher algorithm
|
||
// versions), the memory component of the snapshot (D5.2/D8, F1 CLOSED). It is the
|
||
// Version() of the bank materialized once before the loop, so a change to the approved
|
||
// glossary — or to the deterministic machinery (trad→simp table, matcher) — fires the
|
||
// resnapshot gate LOUDLY instead of a stale checkpoint re-paying a diverged translation.
|
||
// nil bank (report path / a book with no glossary) → the empty-materialization hash, a
|
||
// stable constant. STM is excluded (rebuilt from checkpoints, §3.2). Auto/draft rows are
|
||
// excluded per D8 (their injected-content changes are caught at the per-chunk
|
||
// content_hash level; a future autopopulation milestone revisits this).
|
||
func (r *Runner) memoryVersion() string {
|
||
if r.memory != nil {
|
||
return r.memory.Version()
|
||
}
|
||
return computeMemoryVersion(nil, r.Pipeline.Gates.Glossary.PostcheckGate)
|
||
}
|
||
|
||
// snapshotID materializes the job-context snapshot (§3.2): brief_hash, chunker
|
||
// version, context-assembly knobs, the memory version and the full stage plan
|
||
// (model, prompt version + CONTENT hash, sampling, resolved capability).
|
||
// Payload is rendered with a fixed field order — the id is a content hash, so
|
||
// identical context re-upserts idempotently.
|
||
func (r *Runner) snapshotID() (id, payload string, err error) {
|
||
type stageSnap struct {
|
||
Name string `json:"name"`
|
||
Role string `json:"role"`
|
||
Model string `json:"model"`
|
||
PromptVersion string `json:"prompt_version"`
|
||
PromptSHA256 string `json:"prompt_sha256"`
|
||
Temperature float64 `json:"temperature"`
|
||
Reasoning string `json:"reasoning"`
|
||
// ExtraBody модели меняет ТЕЛО запроса (GLM thinking-off и т.п.) —
|
||
// без него правка ручки в models.yaml молча не инвалидировала бы
|
||
// чекпоинты (находка ревью). json.Marshal карты сортирует ключи →
|
||
// детерминированный рендер.
|
||
ModelExtra json.RawMessage `json:"model_extra,omitempty"`
|
||
// Провайдер-уровневые оверрайды (local kind переопределяет wire
|
||
// temperature/max_tokens ПОСЛЕ вычисления request-hash) — тоже влияют
|
||
// на фактический запрос; без них правка providers.local.max_tokens
|
||
// давала бы тот же snapshot и ложный checkpoint-hit на стендовом
|
||
// local-пути (находка внешнего ревью F2).
|
||
ProviderTemp float64 `json:"provider_temp,omitempty"`
|
||
ProviderMaxTok int `json:"provider_max_tok,omitempty"`
|
||
// ProviderModel — the local-kind backend tag the provider swaps onto the wire
|
||
// AFTER the request-hash (the actual model that answers). The MOST impactful
|
||
// local override, yet it was missing here while its weaker temp/max_tok siblings
|
||
// were folded: a local swap 8b→14b mid-book keeps the same snapID and resume
|
||
// serves the old model (external-review). Folded so it is a loud --resnapshot.
|
||
ProviderModel string `json:"provider_model,omitempty"`
|
||
// Capability — резолвнутая wire-форма модели (D3.1): budget-ключ,
|
||
// temperature-режим, reasoning-контроль. Меняет ТЕЛО запроса (max_tokens
|
||
// vs max_completion_tokens, отправлять ли temperature, thinking-выключа-
|
||
// тель), поэтому обязана входить в снапшот — иначе правка каппы молча
|
||
// false-хитит чекпоинты (тот же класс D5.2, что и payload ниже).
|
||
Capability json.RawMessage `json:"capability,omitempty"`
|
||
// EscalateTo + EscalateCapability — the single-hop fallback model and its
|
||
// resolved wire shape (Веха 2.5). The fallback's request_hash uses ITS model,
|
||
// and its wire body uses ITS capability — both must be in the snapshot, or
|
||
// changing the escalation model / its caps would silently false-hit an
|
||
// escalated chunk's checkpoint (closes the D5.2 escalation-capability techdebt:
|
||
// stages were folded, escalation models were not, until the cycle landed).
|
||
EscalateTo string `json:"escalate_to,omitempty"`
|
||
EscalateCapability json.RawMessage `json:"escalate_capability,omitempty"`
|
||
// The fallback model's OTHER wire-affecting inputs, mirroring the primary
|
||
// fold above: its top-level extra_body (merged into the escalation wire body)
|
||
// and its provider-level temperature/max_tokens overrides (local kind). Without
|
||
// these, editing the fallback's extra_body / provider knobs would change the
|
||
// escalation wire but leave the snapshot identical → a silent false-hit on a
|
||
// resumed escalated chunk (self-review: the primary path guards this, escalation
|
||
// did not).
|
||
EscalateExtra json.RawMessage `json:"escalate_extra,omitempty"`
|
||
EscalateProviderTemp float64 `json:"escalate_provider_temp,omitempty"`
|
||
EscalateProviderMaxTok int `json:"escalate_provider_max_tok,omitempty"`
|
||
EscalateProviderModel string `json:"escalate_provider_model,omitempty"`
|
||
}
|
||
snap := struct {
|
||
BriefHash string `json:"brief_hash"`
|
||
ChunkerVersion string `json:"chunker_version"`
|
||
EstimatorVersion string `json:"estimator_version"`
|
||
// MaxTokensPolicy versions the attempt→max_tokens scaling (Веха 2): a
|
||
// change to the retry doubling shifts attempt≥1 request_hashes, so it
|
||
// belongs in the snapshot as a loud invalidation (same class as
|
||
// estimator_version, applied to the regeneration axis).
|
||
MaxTokensPolicy string `json:"max_tokens_policy"`
|
||
// ClassifierVersion versions the intrinsic classify() verdict logic (thresholds
|
||
// + order), so a re-verdict on a resumed checkpoint is a loud --resnapshot, not
|
||
// a silent flagged↔ok divergence (external-review; symmetric to coverage).
|
||
ClassifierVersion string `json:"classifier_version"`
|
||
PipelineCore string `json:"pipeline_core"`
|
||
// Defaults влияют на maxTokens, а тот входит в request-hash: без них
|
||
// правка max_output_ratio молча инвалидировала бы все чекпоинты в
|
||
// обход snapshot-гейта (находка ревью).
|
||
MaxOutputRatio float64 `json:"max_output_ratio"`
|
||
MinMaxTokens int `json:"min_max_tokens"`
|
||
// ContextAssembly — ручки сборки контекста (глоссарий/STM/overlap/TTL,
|
||
// §3.8). Их правка меняет ВХОД модели, когда память войдёт в msgs; сворачи-
|
||
// ваем ДО инъекции, чтобы смена budget'а инъекции не промахнулась мимо
|
||
// resnapshot-гейта (D5.2). Фикс-порядок полей — это content-hash.
|
||
ContextAssembly contextSnap `json:"context_assembly"`
|
||
// MemoryVersion — content-hash детерминированно материализованной инъекти-
|
||
// руемой памяти (approved-глоссарий+резюме+series-bible). Не bump-счётчик
|
||
// (D8): забыть пересчитать нельзя, а забытый bump переоткрыл бы класс тихой
|
||
// переоплаты D5.2. До миграции банка памяти (v2) материализация пуста —
|
||
// хэш стабилен; когда память войдёт в msgs, поле сменится и resnapshot-
|
||
// гейт сработает громко. STM сюда НЕ входит (пересобирается из чекпоинтов).
|
||
MemoryVersion string `json:"memory_version"`
|
||
// PostcheckGate — the memory-bank post-check gate (E1). When true a post-check
|
||
// miss flips the chunk to flagged; folding its on/off state makes toggling it a
|
||
// loud --resnapshot (it changes a checkpoint's RESOLVED disposition), not a silent
|
||
// re-verdict on resume (same class as coverage/classifier). The post-check
|
||
// ALGORITHM version is already inside MemoryVersion (memoryMatchVersion).
|
||
PostcheckGate bool `json:"postcheck_gate"`
|
||
// Coverage — excision QA-gate config (D12 Q3). It does not touch the wire,
|
||
// but it determines a checkpoint's RESOLVED verdict, so a gate change must be
|
||
// a loud --resnapshot, not a silent re-verdict on resume (coverageSnapshot).
|
||
Coverage coverageSnap `json:"coverage"`
|
||
// StyleCheckVersion versions the cheap deterministic post-check rules (dialogue-dash,
|
||
// yofikator, translit-interjection blocklist, 万/億 magnitude gate — cheapgates.go). They
|
||
// are observability, not wire, but editing a rule shifts the recorded style-flag counts, so
|
||
// a bump is a loud --resnapshot (same verdict class as ClassifierVersion). The ё-policy is
|
||
// part of BriefHash (a book field), so a policy change already re-pins via brief_hash.
|
||
StyleCheckVersion string `json:"style_check_version"`
|
||
// Sanitizer — the output-sanitizer gate (D30.3). Like Coverage it does not touch the wire
|
||
// but determines a checkpoint's RESOLVED verdict (a defect flips the chunk to flagged), so a
|
||
// gate flip / rule edit is a loud --resnapshot. Folded only when enabled (sanitizerSnapshot).
|
||
Sanitizer sanitizerSnap `json:"sanitizer"`
|
||
Stages []stageSnap `json:"stages"`
|
||
}{
|
||
BriefHash: r.Book.BriefHash(),
|
||
ChunkerVersion: chunkerVersion,
|
||
EstimatorVersion: estimatorVersion,
|
||
MaxTokensPolicy: maxTokensPolicyVersion,
|
||
ClassifierVersion: classifierVersion,
|
||
PipelineCore: r.Pipeline.Core,
|
||
MaxOutputRatio: r.Pipeline.Defaults.MaxOutputRatio,
|
||
MinMaxTokens: r.Pipeline.Defaults.MinMaxTokens,
|
||
ContextAssembly: contextSnap{
|
||
GlossaryInjection: r.Pipeline.Context.GlossaryInjection,
|
||
GlossaryTokenBudget: r.Pipeline.Context.GlossaryTokenBudget,
|
||
STMDepth: r.Pipeline.Context.STMDepth,
|
||
OverlapTokens: r.Pipeline.Context.OverlapTokens,
|
||
CacheTTL: r.Pipeline.Context.CacheTTL,
|
||
},
|
||
MemoryVersion: r.memoryVersion(),
|
||
PostcheckGate: r.Pipeline.Gates.Glossary.PostcheckGate,
|
||
Coverage: r.coverageSnapshot(),
|
||
StyleCheckVersion: cheapGateVersion,
|
||
Sanitizer: r.sanitizerSnapshot(),
|
||
}
|
||
for _, st := range r.Pipeline.Stages {
|
||
ss := stageSnap{
|
||
Name: st.Name, Role: st.Role, Model: st.Model,
|
||
PromptVersion: st.PromptVersion, PromptSHA256: r.templates[st.Name].SHA256,
|
||
Temperature: st.Temperature, Reasoning: st.Reasoning,
|
||
}
|
||
if prov, ok := r.Models.Providers[r.Models.Models[st.Model].Provider]; ok {
|
||
ss.ProviderTemp, ss.ProviderMaxTok, ss.ProviderModel = prov.Temperature, prov.MaxTokens, prov.Model
|
||
}
|
||
if extra := r.Models.Models[st.Model].ExtraBody; len(extra) > 0 {
|
||
raw, merr := json.Marshal(extra)
|
||
if merr != nil {
|
||
return "", "", merr
|
||
}
|
||
ss.ModelExtra = raw
|
||
}
|
||
// Резолвнутая каппа — тем же ResolveCapability, что и у клиента, поэтому
|
||
// хэшируемое всегда совпадает с отправляемым. json.Marshal сортирует
|
||
// ключи карт (off_extra_body) → детерминированный рендер.
|
||
capRaw, cerr := json.Marshal(r.Models.ResolveCapability(st.Model))
|
||
if cerr != nil {
|
||
return "", "", cerr
|
||
}
|
||
ss.Capability = capRaw
|
||
// Fold the single-hop fallback model + its resolved capability (Веха 2.5):
|
||
// both are wire-affecting inputs of an escalated call, so changing them is a
|
||
// loud --resnapshot, not a silent false-hit on an escalated checkpoint.
|
||
// TRIPWIRE (пакет №4): этот блок — зеркало primary-фолда выше (2×, ниже порога
|
||
// извлечения ≥3). Когда канал B/annotator добавит ТРЕТЬЮ модельную ось стадии —
|
||
// сначала извлечь общий foldModelWire(model) (prov-тройка, extra, capability),
|
||
// потом добавлять ось: копипаста здесь тише всего расходится и портит
|
||
// идентичность снапшота (false-hit/false-miss на эскалированных чекпоинтах).
|
||
if st.EscalateTo != "" {
|
||
ss.EscalateTo = st.EscalateTo
|
||
escCap, eerr := json.Marshal(r.Models.ResolveCapability(st.EscalateTo))
|
||
if eerr != nil {
|
||
return "", "", eerr
|
||
}
|
||
ss.EscalateCapability = escCap
|
||
if prov, ok := r.Models.Providers[r.Models.Models[st.EscalateTo].Provider]; ok {
|
||
ss.EscalateProviderTemp, ss.EscalateProviderMaxTok, ss.EscalateProviderModel = prov.Temperature, prov.MaxTokens, prov.Model
|
||
}
|
||
if extra := r.Models.Models[st.EscalateTo].ExtraBody; len(extra) > 0 {
|
||
raw, merr := json.Marshal(extra)
|
||
if merr != nil {
|
||
return "", "", merr
|
||
}
|
||
ss.EscalateExtra = raw
|
||
}
|
||
}
|
||
snap.Stages = append(snap.Stages, ss)
|
||
}
|
||
data, err := json.Marshal(snap)
|
||
if err != nil {
|
||
return "", "", err
|
||
}
|
||
sum := sha256.Sum256(data)
|
||
return hex.EncodeToString(sum[:]), string(data), nil
|
||
}
|