Apply self-review fixes: fold fallback wire config into the snapshot, make escalation resume-idempotent and ceiling-tolerant, strip think in the coverage gate, gate only the translator
This commit is contained in:
Claude (backend session)
parent
a98af88b12
commit
ed4f4ff5a7
9 changed files with 330 additions and 49 deletions
|
|
@ -62,23 +62,25 @@ func TestCheckKeysOnlyUsedNonLocalModels(t *testing.T) {
|
|||
if err := m.CheckKeys(cloudPipe); err != nil {
|
||||
t.Fatalf("present key must pass: %v", err)
|
||||
}
|
||||
// Эскалационные ключи НЕ требуются, пока гейты выключены (эскалация
|
||||
// недостижима — Фаза 0): local-стадия + облачная цепочка при gates off.
|
||||
// Эскалационные ключи НЕ требуются, пока эскалация выключена (budget_usd=0,
|
||||
// opt-in). Включённый coverage-гейт НЕ делает эскалацию достижимой — гейт
|
||||
// моделей не зовёт (self-review: иначе включение гейта блокирует валидный
|
||||
// deepseek+glm прогон из-за отсутствующего Kimi-ключа заглушки-цепочки).
|
||||
t.Setenv("TEST_CLOUD_KEY", "")
|
||||
escOffPipe := &Pipeline{
|
||||
Stages: []Stage{{Model: "local-model"}},
|
||||
Escal: Escalation{Chains: map[string][]string{"default": {"cloud-model"}}},
|
||||
Gates: Gates{Coverage: CoverageGate{Enabled: true}}, // gate ON, budget 0
|
||||
}
|
||||
if err := m.CheckKeys(escOffPipe); err != nil {
|
||||
t.Fatalf("escalation keys must not be required while gates are off: %v", err)
|
||||
t.Fatalf("enabling the coverage gate must NOT require escalation keys at budget 0: %v", err)
|
||||
}
|
||||
// С включённым гейтом эскалация достижима → ключ цепочки обязателен.
|
||||
// При budget_usd>0 эскалация достижима → ключ цепочки обязателен.
|
||||
escOnPipe := &Pipeline{
|
||||
Stages: []Stage{{Model: "local-model"}},
|
||||
Escal: Escalation{Chains: map[string][]string{"default": {"cloud-model"}}},
|
||||
Gates: Gates{Coverage: CoverageGate{Enabled: true}},
|
||||
Escal: Escalation{Chains: map[string][]string{"default": {"cloud-model"}}, BudgetUSD: 1.0},
|
||||
}
|
||||
if err := m.CheckKeys(escOnPipe); err == nil {
|
||||
t.Fatal("escalation chain model without a key must fail-fast when gates are enabled")
|
||||
t.Fatal("escalation chain model without a key must fail-fast when budget_usd>0")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -398,17 +398,18 @@ func (m *Models) CheckKeys(pipe *Pipeline) error {
|
|||
for _, st := range pipe.Stages {
|
||||
needed[st.Model] = struct{}{}
|
||||
}
|
||||
if pipe.gatesEnabled() {
|
||||
// Escalation models — named chains (шаг 7) and single-hop fallbacks (Веха 2.5) —
|
||||
// are reachable ONLY once an escalation budget is set (opt-in), NOT when a gate is
|
||||
// enabled: the coverage gate is a deterministic runner-side check that calls no
|
||||
// model, so enabling it must not demand escalation keys (self-review finding —
|
||||
// otherwise turning on the gate blocks a valid deepseek+glm run on a missing Kimi
|
||||
// key from an unused chain stub).
|
||||
if pipe.Escal.BudgetUSD > 0 {
|
||||
for _, chain := range pipe.Escal.Chains {
|
||||
for _, mdl := range chain {
|
||||
needed[mdl] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Single-hop fallback models are reachable only once an escalation budget is set
|
||||
// (opt-in); preflight their keys then, so an escalation does not 401 mid-book
|
||||
// after the primary already flagged.
|
||||
if pipe.Escal.BudgetUSD > 0 {
|
||||
for _, st := range pipe.Stages {
|
||||
if st.EscalateTo != "" {
|
||||
needed[st.EscalateTo] = struct{}{}
|
||||
|
|
|
|||
|
|
@ -242,17 +242,19 @@ func LoadPipeline(path string, models *Models) (*Pipeline, error) {
|
|||
// Coverage QA-gate thresholds (шаг 6): an ENABLED gate must be able to flag
|
||||
// something, otherwise it is a silent no-op that passes every chunk (against Р7).
|
||||
if cov := p.Gates.Coverage; cov.Enabled {
|
||||
if cov.SentCovMin < 0 || cov.SentCovMin > 1 {
|
||||
bad("gates.coverage.sent_cov_min must be within [0,1], got %v", cov.SentCovMin)
|
||||
// sent_cov_min must be a LIVE threshold in (0,1]: it is the pair-INDEPENDENT
|
||||
// half of the gate, so requiring it guarantees the gate can never silently pass
|
||||
// every chunk for a language pair that lacks a len_ratio corridor (self-review:
|
||||
// the old "both empty" check missed the per-pair gap — a non-empty len_ratio map
|
||||
// that just lacks the book's pair, with sent_cov_min=0, passed everything).
|
||||
if cov.SentCovMin <= 0 || cov.SentCovMin > 1 {
|
||||
bad("gates.coverage.sent_cov_min must be within (0, 1] when the gate is enabled, got %v (else the gate silently passes any language pair without a len_ratio corridor)", cov.SentCovMin)
|
||||
}
|
||||
for pair, b := range cov.LenRatio {
|
||||
if len(b) != 2 || b[0] <= 0 || b[1] < b[0] {
|
||||
bad("gates.coverage.len_ratio_bounds[%q] must be [low, high] with 0 < low <= high, got %v", pair, b)
|
||||
}
|
||||
}
|
||||
if cov.SentCovMin <= 0 && len(cov.LenRatio) == 0 {
|
||||
bad("gates.coverage.enabled is true but neither sent_cov_min nor len_ratio_bounds is set — the gate would silently pass every chunk")
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||||
|
|
|
|||
|
|
@ -55,6 +55,20 @@ func isCJKSentenceTerminator(r rune) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// isOracleSpace matches Python's whitespace notion used by the oracle — `re.\s`
|
||||
// (splitting) and `str.isspace()`/`.strip()` (trimming & the non-space count). It is
|
||||
// unicode.IsSpace PLUS the C0 information separators U+001C–U+001F, which Python
|
||||
// classifies as whitespace but Go's unicode.IsSpace (Unicode White_Space) does not.
|
||||
// These survive EPUB/PDF extraction, so matching Python here is required for the
|
||||
// bit-parity contract (D12 Q1), not cosmetic (self-review finding).
|
||||
func isOracleSpace(r rune) bool {
|
||||
return unicode.IsSpace(r) || (r >= 0x1C && r <= 0x1F)
|
||||
}
|
||||
|
||||
// trimOracleSpace is strings.TrimSpace with the oracle's whitespace set (mirrors the
|
||||
// Python `.strip()` in split_sentences).
|
||||
func trimOracleSpace(s string) string { return strings.TrimFunc(s, isOracleSpace) }
|
||||
|
||||
// splitSentences ports refusal_bench.py::split_sentences 1:1. Go's RE2 has no
|
||||
// lookbehind, so the regex `(?<=[.!?…。！？])\s+|(?<=[。！？])` is reproduced by a manual
|
||||
// left-to-right scan (branch-1 before branch-2, matching Python's alternation order):
|
||||
|
|
@ -69,7 +83,7 @@ func splitSentences(text string) []string {
|
|||
runes := []rune(text)
|
||||
var segs []string
|
||||
emit := func(a, b int) {
|
||||
if s := strings.TrimSpace(string(runes[a:b])); s != "" {
|
||||
if s := trimOracleSpace(string(runes[a:b])); s != "" {
|
||||
segs = append(segs, s)
|
||||
}
|
||||
}
|
||||
|
|
@ -77,10 +91,10 @@ func splitSentences(text string) []string {
|
|||
for i := 1; i < len(runes); i++ {
|
||||
prev := runes[i-1]
|
||||
// Branch 1: any terminator, then a whitespace run (the run is consumed).
|
||||
if isSentenceTerminator(prev) && unicode.IsSpace(runes[i]) {
|
||||
if isSentenceTerminator(prev) && isOracleSpace(runes[i]) {
|
||||
emit(segStart, i)
|
||||
j := i
|
||||
for j < len(runes) && unicode.IsSpace(runes[j]) {
|
||||
for j < len(runes) && isOracleSpace(runes[j]) {
|
||||
j++
|
||||
}
|
||||
segStart = j
|
||||
|
|
@ -103,7 +117,7 @@ func splitSentences(text string) []string {
|
|||
func nonSpaceCount(s string) int {
|
||||
n := 0
|
||||
for _, r := range s {
|
||||
if !unicode.IsSpace(r) {
|
||||
if !isOracleSpace(r) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
|
|
@ -132,6 +146,11 @@ type coverageResult struct {
|
|||
// 500–2500-char fragments and short chunks' ratios are noise (§3.7). Pure &
|
||||
// deterministic. Returns excision_suspect (lower-bound only) or ok.
|
||||
func coverageCheck(cfg config.CoverageGate, src, out, srcLang, dstLang string) coverageResult {
|
||||
// Strip a leaked <think>…</think> block BEFORE measuring, exactly as classify()
|
||||
// and the Python oracle do (refusal_bench: re.sub then .strip()). Without this a
|
||||
// reasoning block leaked into content inflates the sentence/char counts and MASKS
|
||||
// a real excision — a false negative, and a break of oracle parity (self-review).
|
||||
out = trimOracleSpace(stripThink(out))
|
||||
srcChars := nonSpaceCount(src)
|
||||
if srcChars < cfg.MinChunkChars {
|
||||
return coverageResult{cls: classification{reasonOK, ""}, applied: false}
|
||||
|
|
|
|||
|
|
@ -47,6 +47,9 @@ var oracleSplit = []struct {
|
|||
{" leading and trailing. ", []string{"leading and trailing."}},
|
||||
{"第一。\n\n第二。", []string{"第一。", "第二。"}},
|
||||
{"Mr. Smith went. He left.", []string{"Mr.", "Smith went.", "He left."}}, // naive over-split (known)
|
||||
// U+001F (unit separator) is whitespace to Python (\s / str.isspace) but NOT to
|
||||
// Go's unicode.IsSpace — isOracleSpace bridges the gap for oracle parity.
|
||||
{"A. B.", []string{"A.", "B."}},
|
||||
}
|
||||
|
||||
func TestSplitSentencesOracleParity(t *testing.T) {
|
||||
|
|
@ -93,6 +96,32 @@ func boevoyGate(minChars int) config.CoverageGate {
|
|||
}
|
||||
}
|
||||
|
||||
// TestOracleSpaceControlChars pins isOracleSpace to the Python oracle on the C0
|
||||
// information separators U+001C–U+001F, which Python treats as whitespace (\s /
|
||||
// str.isspace) but Go's unicode.IsSpace does not — a real bit-parity gap (self-review).
|
||||
func TestOracleSpaceControlChars(t *testing.T) {
|
||||
us := " " // U+001F unit separator (survives EPUB/PDF extraction)
|
||||
if got := splitSentences("A." + us + "B."); len(got) != 2 || got[0] != "A." || got[1] != "B." {
|
||||
t.Fatalf("U+001F after a terminator must split like the oracle, got %#v", got)
|
||||
}
|
||||
if got := nonSpaceCount("A" + us + "B"); got != 2 {
|
||||
t.Fatalf("U+001F must count as whitespace (oracle parity), got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoverageCheckStripsThink guards that a leaked <think>…</think> block is stripped
|
||||
// BEFORE measuring (parity with classify() and the oracle), so a reasoning block does
|
||||
// not inflate the counts and MASK a real excision (self-review finding). Mutation:
|
||||
// drop the stripThink in coverageCheck and this goes green-to-red.
|
||||
func TestCoverageCheckStripsThink(t *testing.T) {
|
||||
src := strings.Repeat("这是一个相当长的测试句子。", 20) // 20 sentences
|
||||
leaked := "<think>" + strings.Repeat("Долгое рассуждение о выборе слова. ", 40) + "</think>Короткий огрызок."
|
||||
res := coverageCheck(boevoyGate(0), src, leaked, "zh", "ru")
|
||||
if res.cls.Reason != FlagExcisionSuspect {
|
||||
t.Fatalf("a <think> block must be stripped so the excision behind it is still caught, got %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageCheckExcision(t *testing.T) {
|
||||
// Mirrors the three classify_output cases captured from the live oracle.
|
||||
zhExcised := coverageCheck(boevoyGate(0), strings.Repeat("这是一个测试。", 20), "Короткий перевод.", "zh", "ru")
|
||||
|
|
|
|||
|
|
@ -29,6 +29,17 @@ import (
|
|||
// retryable length/empty subset, up to the regenerate cap, then flag).
|
||||
// Escalation and the real linguistic chunker are later steps of Фаза 1.
|
||||
|
||||
// roleTranslator is the stage role whose output is a direct source→target
|
||||
// translation — the only role the excision coverage-gate is meaningful on.
|
||||
const roleTranslator = "translator"
|
||||
|
||||
// errReserveCeiling is wrapped into the error runAttempt returns when a book/day USD
|
||||
// ceiling denies a reservation. The PRIMARY path propagates it (the book durably
|
||||
// pauses and resumes once the ceiling is raised, D4); the OPTIONAL escalation hop
|
||||
// catches it and degrades to keeping the primary flag instead of aborting the whole
|
||||
// book on every run (self-review finding).
|
||||
var errReserveCeiling = errors.New("reserve ceiling reached")
|
||||
|
||||
// Runner executes a book's pipeline.
|
||||
type Runner struct {
|
||||
Book *config.Book
|
||||
|
|
@ -168,11 +179,20 @@ func (r *Runner) coverageSnapshot() coverageSnap {
|
|||
// gate part under the same coverage config (folded into the snapshot, so a gate change
|
||||
// re-pins loudly). The gate compares the output against the ORIGINAL source (ch.Text)
|
||||
// at EVERY stage, so excision introduced by the draft OR the editor is caught.
|
||||
func (r *Runner) classifyOutput(source, output, finish string) classification {
|
||||
func (r *Runner) classifyOutput(role, source, output, finish string) classification {
|
||||
cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang})
|
||||
if !cls.ok() || !r.Pipeline.Gates.Coverage.Enabled {
|
||||
return cls
|
||||
}
|
||||
// The coverage gate compares the output against the ORIGINAL source, which is a
|
||||
// source→target translation only for the TRANSLATOR role. A monolingual editor
|
||||
// legitimately restructures/merges sentences, so gating ITS output against the
|
||||
// source with the translation corridor false-flags a correct edit as excision
|
||||
// (self-review finding). Editor/other-role fidelity is a Phase-2 concern (a
|
||||
// bilingual judge over the draft, §04 mode 2), not this excision gate.
|
||||
if role != roleTranslator {
|
||||
return cls
|
||||
}
|
||||
if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() {
|
||||
return cov.cls
|
||||
}
|
||||
|
|
@ -183,8 +203,10 @@ func (r *Runner) classifyOutput(source, output, finish string) classification {
|
|||
// fallback draft: escalation is OPT-IN via escalation.budget_usd (0 = disabled, the
|
||||
// boevoy default — a stage's escalate_to is inert until a premium budget is set), and
|
||||
// capped at that budget summed over the book's escalation checkpoints (money-path
|
||||
// durable, resume-safe). A soft cap: the hop already in flight may cross it slightly,
|
||||
// but the NEXT chunk's escalation is then denied.
|
||||
// durable, resume-safe). A PRE-HOP soft cap: the gate admits a hop while spent <
|
||||
// budget and does NOT pre-estimate the hop's own cost, so a single hop may overshoot
|
||||
// the budget by up to its full cost; the NEXT chunk's escalation is then denied. Size
|
||||
// the budget with that worst case in mind — it bounds TOTAL escalation, not per-hop.
|
||||
func (r *Runner) escalationBudgetRemains() (bool, error) {
|
||||
budget := r.Pipeline.Escal.BudgetUSD
|
||||
if budget <= 0 {
|
||||
|
|
@ -252,6 +274,16 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
|||
// stages were folded, escalation models were not, until the cycle landed).
|
||||
EscalateTo string `json:"escalate_to,omitempty"`
|
||||
EscalateCapability json.RawMessage `json:"escalate_capability,omitempty"`
|
||||
// The fallback model's OTHER wire-affecting inputs, mirroring the primary
|
||||
// fold above: its top-level extra_body (merged into the escalation wire body)
|
||||
// and its provider-level temperature/max_tokens overrides (local kind). Without
|
||||
// these, editing the fallback's extra_body / provider knobs would change the
|
||||
// escalation wire but leave the snapshot identical → a silent false-hit on a
|
||||
// resumed escalated chunk (self-review: the primary path guards this, escalation
|
||||
// did not).
|
||||
EscalateExtra json.RawMessage `json:"escalate_extra,omitempty"`
|
||||
EscalateProviderTemp float64 `json:"escalate_provider_temp,omitempty"`
|
||||
EscalateProviderMaxTok int `json:"escalate_provider_max_tok,omitempty"`
|
||||
}
|
||||
snap := struct {
|
||||
BriefHash string `json:"brief_hash"`
|
||||
|
|
@ -337,6 +369,16 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
|||
return "", "", eerr
|
||||
}
|
||||
ss.EscalateCapability = escCap
|
||||
if prov, ok := r.Models.Providers[r.Models.Models[st.EscalateTo].Provider]; ok {
|
||||
ss.EscalateProviderTemp, ss.EscalateProviderMaxTok = prov.Temperature, prov.MaxTokens
|
||||
}
|
||||
if extra := r.Models.Models[st.EscalateTo].ExtraBody; len(extra) > 0 {
|
||||
raw, merr := json.Marshal(extra)
|
||||
if merr != nil {
|
||||
return "", "", merr
|
||||
}
|
||||
ss.EscalateExtra = raw
|
||||
}
|
||||
}
|
||||
snap.Stages = append(snap.Stages, ss)
|
||||
}
|
||||
|
|
@ -629,28 +671,50 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
|||
// reset on the fallback (the LiteLLM #19985 retry×fallback blow-up guard).
|
||||
escalated, escModel := false, ""
|
||||
if last.cls.Reason.escalatable() && st.EscalateTo != "" {
|
||||
remains, err := r.escalationBudgetRemains()
|
||||
// Idempotency (self-review): if the fallback hop already happened its
|
||||
// checkpoint exists and was already paid — REPLAY it for free regardless of the
|
||||
// budget, so a crash AFTER the hop settled but BEFORE chunk_status was written
|
||||
// re-serves it on resume rather than discarding a paid, successful translation
|
||||
// and flipping the verdict OK→flagged. Only a FRESH hop is budget-gated. The
|
||||
// hash mirrors runAttempt's for (model=EscalateTo, attempt=0, maxTokens=base).
|
||||
fbHash := RequestHash(r.Book.BookID, ch.Chapter, ch.ChunkIdx, 0, st.Name, st.Role, st.EscalateTo,
|
||||
st.Temperature, st.Reasoning, false, baseMaxTokens, snapID, msgs)
|
||||
fbExists, err := r.Store.GetCheckpoint(fbHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if remains {
|
||||
fb, err := r.runAttempt(ctx, st, st.EscalateTo, snapID, ch, job, 0, baseMaxTokens, msgs, true)
|
||||
if err != nil {
|
||||
mayHop := fbExists != nil
|
||||
if !mayHop {
|
||||
if mayHop, err = r.escalationBudgetRemains(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cumCost += fb.cumCost
|
||||
runCost += fb.runCost
|
||||
anyFresh = anyFresh || fb.freshCall
|
||||
escalated, escModel = true, st.EscalateTo
|
||||
r.Log.WarnContext(ctx, "stage escalated to a fallback model",
|
||||
"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx,
|
||||
"primary_reason", string(last.cls.Reason), "fallback", st.EscalateTo,
|
||||
"fallback_disposition", string(fb.cls.Reason.disposition()))
|
||||
if fb.cls.ok() {
|
||||
last = fb // the fallback draft passed the re-gate → it is authoritative
|
||||
}
|
||||
if mayHop {
|
||||
fb, err := r.runAttempt(ctx, st, st.EscalateTo, snapID, ch, job, 0, baseMaxTokens, msgs, true)
|
||||
if err != nil {
|
||||
// An OPTIONAL hop that trips a USD ceiling must NOT abort the whole book
|
||||
// (and re-abort on every resume): the chunk is already flagged, so keep
|
||||
// the primary flag and continue. Any other infra error still aborts.
|
||||
if !errors.Is(err, errReserveCeiling) {
|
||||
return nil, err
|
||||
}
|
||||
r.Log.WarnContext(ctx, "escalation hop denied by a USD ceiling; keeping the primary flag",
|
||||
"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "reason", string(last.cls.Reason))
|
||||
} else {
|
||||
cumCost += fb.cumCost
|
||||
runCost += fb.runCost
|
||||
anyFresh = anyFresh || fb.freshCall
|
||||
escalated, escModel = true, st.EscalateTo
|
||||
r.Log.WarnContext(ctx, "stage escalated to a fallback model",
|
||||
"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx,
|
||||
"primary_reason", string(last.cls.Reason), "fallback", st.EscalateTo,
|
||||
"fallback_disposition", string(fb.cls.Reason.disposition()))
|
||||
if fb.cls.ok() {
|
||||
last = fb // the fallback draft passed the re-gate → it is authoritative
|
||||
}
|
||||
// else: the fallback also failed → keep the primary flag (last unchanged);
|
||||
// the fallback call is billed and counted, the chunk stays flagged (1 hop).
|
||||
}
|
||||
// else: the fallback also failed → keep the primary flag (last unchanged);
|
||||
// the fallback call is billed and counted, the chunk stays flagged (1 hop).
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -732,7 +796,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
|
|||
att.finish = cp.FinishReason
|
||||
att.modelActual = cp.ModelActual
|
||||
att.cumCost = cp.CostUSD
|
||||
att.cls = r.classifyOutput(ch.Text, cp.ResponseText, cp.FinishReason)
|
||||
att.cls = r.classifyOutput(st.Role, ch.Text, cp.ResponseText, cp.FinishReason)
|
||||
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
||||
Stage: st.Name, Role: st.Role, ModelRequested: model, ModelActual: cp.ModelActual,
|
||||
|
|
@ -763,10 +827,11 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
|
|||
switch verdict {
|
||||
case store.ReserveDeniedBook:
|
||||
// Ceiling is a hard, book-wide stop (not a per-chunk flag): the job stays
|
||||
// 'pending' and resume continues once the ceiling is raised.
|
||||
return att, fmt.Errorf("pipeline: book USD ceiling reached (%.2f$) — raise ceilings.book_usd or stop", r.Book.Ceilings.BookUSD)
|
||||
// 'pending' and resume continues once the ceiling is raised. Wrapped so an
|
||||
// optional escalation hop can degrade instead of aborting the book.
|
||||
return att, fmt.Errorf("pipeline: book USD ceiling reached (%.2f$) — raise ceilings.book_usd or stop: %w", r.Book.Ceilings.BookUSD, errReserveCeiling)
|
||||
case store.ReserveDeniedDay:
|
||||
return att, fmt.Errorf("pipeline: daily USD ceiling reached (%.2f$)", r.Book.Ceilings.DayUSD)
|
||||
return att, fmt.Errorf("pipeline: daily USD ceiling reached (%.2f$): %w", r.Book.Ceilings.DayUSD, errReserveCeiling)
|
||||
}
|
||||
|
||||
client, err := r.client(model)
|
||||
|
|
@ -878,7 +943,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
|
|||
// non-empty truncated length draft is now classified (flagged/retried), never
|
||||
// silently passed downstream as OK; an empty completion is flagged too, not a
|
||||
// run-crash.
|
||||
att.cls = r.classifyOutput(ch.Text, resp.Text, resp.FinishReason)
|
||||
att.cls = r.classifyOutput(st.Role, ch.Text, resp.Text, resp.FinishReason)
|
||||
|
||||
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
||||
|
|
@ -926,6 +991,12 @@ func (r *Runner) resumeFromChunkStatus(ctx context.Context, st config.Stage, ch
|
|||
sr.Model = cp.ModelActual
|
||||
sr.FinishReason = cp.FinishReason
|
||||
sr.Text = cp.ResponseText
|
||||
// Preserve escalation attribution across resume (self-review): a chunk resolved
|
||||
// via a fallback checkpoint is still an escalation, so a resumed run reports it.
|
||||
if cp.Escalation {
|
||||
sr.Escalated = true
|
||||
sr.EscalationModel = cp.ModelRequested
|
||||
}
|
||||
}
|
||||
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
||||
|
|
|
|||
|
|
@ -1171,6 +1171,153 @@ func TestRunnerEscalationResumeServesFallback(t *testing.T) {
|
|||
if res.Chunks[0].FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
||||
t.Fatalf("resume text differs: %q", res.Chunks[0].FinalText)
|
||||
}
|
||||
// Escalation attribution survives resume (self-review [11/14]).
|
||||
if d := res.Chunks[0].Stages[0]; !d.Escalated || d.EscalationModel != "fake-fallback" {
|
||||
t.Fatalf("resume must preserve escalation attribution, got %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the TRANSLATOR stage is coverage-gated (self-review [6]): a monolingual editor
|
||||
// that legitimately merges/tightens sentences (low sent_cov vs the CJK source) must
|
||||
// NOT be flagged as excision. Mutation: gate every stage and the short edit flags.
|
||||
func TestRunnerCoverageGateOnlyTranslator(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isEditBody(body) {
|
||||
return "Одно.", "stop" // a very short "polish" — would flag if the editor were gated
|
||||
}
|
||||
return strings.Repeat("Это предложение перевода. ", 30), "stop" // faithful translator draft
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||||
source: strings.Repeat("これは長い文である。", 30), regenerate: 0, gatesYAML: coverageGateBlock,
|
||||
})
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
res, err := r.TranslateBook(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Flagged != 0 || res.ExitCode() != 0 || res.Chunks[0].Disposition != DispOK {
|
||||
t.Fatalf("a short monolingual edit must NOT be excision-flagged (editor is not gated), got %+v", res.Chunks[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A successful, already-PAID escalation must be resume-idempotent even when the budget
|
||||
// no longer admits a fresh hop (self-review [2/9/13]): a crash between the fallback
|
||||
// settle and the chunk_status write must REPLAY the paid checkpoint, not discard it and
|
||||
// flip the chunk OK→flagged. Mutation: drop the fbExists check and the chunk flips.
|
||||
func TestRunnerEscalationResumeIdempotentUnderBudget(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, echoOrClean)
|
||||
defer srv.Close()
|
||||
// budget < one hop cost: after the first hop, EscalationSpentUSD ≥ budget, so a
|
||||
// FRESH hop would be denied on resume — the paid hop must replay regardless.
|
||||
bookPath := setupEscalationProject(t, srv.URL, 0.001, false, "")
|
||||
ctx := context.Background()
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
res1, err := r1.TranslateBook(ctx)
|
||||
if err != nil || res1.Chunks[0].Disposition != DispOK {
|
||||
t.Fatalf("run1 must escalate to OK, got %+v (err %v)", res1.Chunks[0], err)
|
||||
}
|
||||
dbPath := r1.Book.ProjectDB
|
||||
r1.Close()
|
||||
callsAfterRun1 := rec.count()
|
||||
|
||||
// Simulate the crash window: chunk_status lost, checkpoints (incl. escalation) survive.
|
||||
db, err := sql.Open("sqlite", "file:"+dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(`DELETE FROM chunk_status WHERE stage = 'draft'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Close()
|
||||
|
||||
r2 := newRunner(t, bookPath)
|
||||
defer r2.Close()
|
||||
res2, err := r2.TranslateBook(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.count() != callsAfterRun1 {
|
||||
t.Fatalf("resume must REPLAY the paid escalation checkpoint, not re-call: calls %d -> %d", callsAfterRun1, rec.count())
|
||||
}
|
||||
if res2.Chunks[0].Disposition != DispOK {
|
||||
t.Fatalf("resume must re-serve the escalated OK chunk, not flip to flagged (budget must not orphan a paid hop), got %+v", res2.Chunks[0])
|
||||
}
|
||||
}
|
||||
|
||||
// An OPTIONAL escalation hop that trips the book USD ceiling must degrade to keeping
|
||||
// the primary flag, NOT abort the whole book (and re-abort on every resume) — self-
|
||||
// review [10]. Mutation: propagate the ceiling error and TranslateBook returns non-nil.
|
||||
func TestRunnerEscalationCeilingDegrades(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, echoOrClean)
|
||||
defer srv.Close()
|
||||
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||||
|
||||
// Lower the book ceiling so the primary draft call fits but the escalation hop does not.
|
||||
bp := filepath.Join(filepath.Dir(bookPath), "book.yaml")
|
||||
raw, err := os.ReadFile(bp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFile(t, bp, strings.Replace(string(raw), "book_usd: 5.0", "book_usd: 0.0025", 1))
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
res, err := r.TranslateBook(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("an escalation-hop ceiling denial must NOT abort the book, got: %v", err)
|
||||
}
|
||||
ch := res.Chunks[0]
|
||||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagCJKArtifact {
|
||||
t.Fatalf("the chunk must keep its primary flag when escalation is ceiling-denied, got %+v", ch)
|
||||
}
|
||||
if ch.Stages[0].Escalated {
|
||||
t.Fatalf("a ceiling-denied hop must not be recorded as an escalation, got %+v", ch.Stages[0])
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback model's top-level extra_body is folded into the snapshot (self-review
|
||||
// [1]): editing it changes the escalation wire, so resume must fail loud until
|
||||
// --resnapshot, never silently serve a stale escalated checkpoint.
|
||||
func TestRunnerEscalationFoldsFallbackExtraBody(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, echoOrClean)
|
||||
defer srv.Close()
|
||||
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||||
ctx := context.Background()
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r1.Close()
|
||||
|
||||
// Add extra_body to the FALLBACK model only — it changes the escalation wire body.
|
||||
mp := filepath.Join(filepath.Dir(bookPath), "models.yaml")
|
||||
raw, err := os.ReadFile(mp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
patched := strings.Replace(string(raw),
|
||||
" fake-fallback:\n provider: fake",
|
||||
" fake-fallback:\n provider: fake\n extra_body: { top_p: 0.5 }", 1)
|
||||
if patched == string(raw) {
|
||||
t.Fatal("failed to inject extra_body into fake-fallback")
|
||||
}
|
||||
writeFile(t, mp, patched)
|
||||
|
||||
r2 := newRunner(t, bookPath)
|
||||
_, err = r2.TranslateBook(ctx)
|
||||
r2.Close()
|
||||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||||
t.Fatalf("editing the fallback's extra_body must fail loud mentioning --resnapshot, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback model is folded into the snapshot: removing escalate_to changes the
|
||||
|
|
|
|||
|
|
@ -178,10 +178,10 @@ func (s *Store) GetCheckpoint(requestHash string) (*Checkpoint, error) {
|
|||
cp := Checkpoint{RequestHash: requestHash}
|
||||
err := s.r.QueryRowContext(ctx, `
|
||||
SELECT job_id, chunk_idx, attempt, stage, role, model_requested, model_actual,
|
||||
response_text, usage_json, cost_usd, finish_reason, provider_request_id
|
||||
response_text, usage_json, cost_usd, finish_reason, provider_request_id, escalation
|
||||
FROM checkpoints WHERE request_hash = ?`, requestHash).Scan(
|
||||
&cp.JobID, &cp.ChunkIdx, &cp.Attempt, &cp.Stage, &cp.Role, &cp.ModelRequested, &cp.ModelActual,
|
||||
&cp.ResponseText, &cp.UsageJSON, &cp.CostUSD, &cp.FinishReason, &cp.ProviderRequestID)
|
||||
&cp.ResponseText, &cp.UsageJSON, &cp.CostUSD, &cp.FinishReason, &cp.ProviderRequestID, &cp.Escalation)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -340,7 +340,15 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор
|
|||
|
||||
**Live-валидация (1 целевой вызов deepseek, ~$0.00004):** фрагмент Лу Синя «祝福» (эхо-репро-семья) → `cjk_share=0.00`, `verdict=ok`, `finish=stop`, `reasoning_tokens=0`, чистый русский перевод. **DeepSeek-мина подтверждённо обезврежена на реальном проводе** — `reasoning:"off"` no-op держит thinking ON → перевод, не эхо. Адаптер читает `content`, слаг = `deepseek-v4-flash` (канонизация чистая). grok закомментирован в наборе (гигиена clean-prod xAI — включать осознанно).
|
||||
|
||||
**Веха 2.5 завершена (A+B+C+D + §2-гейт).** Коммиты: `2c1c38d` (эхо-мина гейт), `2acfdf2` (coverage-гейт), `d97f832` (Kimi base_url), `659cce0` (single-hop эскалация), + этот. Дальше — агентское селфревью на вехе.
|
||||
**Веха 2.5 (A+B+C+D + §2-гейт) реализована.** Коммиты: `2c1c38d` (эхо-мина гейт), `2acfdf2` (coverage-гейт), `d97f832` (Kimi base_url), `659cce0` (single-hop эскалация), `a844765` (live-harness). Дальше — агентское селфревью на вехе.
|
||||
|
||||
### 2026-07-05 — Веха 2.5: агентское адверсариальное селфревью + фиксы
|
||||
|
||||
Многоагентный Workflow (6 дименсий: детерминизм / деньги / корректность coverage / корректность эскалации / эхо-мина+конфиг / нейтральность+хаки → независимая верификация КАЖДОЙ находки CONFIRMED/REFUTED с конкретным сценарием инпут→неверный-выход). 25 агентов, **14 подтверждённых** (5 опровергнуто). Все исправлены; ключевые фиксы мутационно-проверены (сломай → тест падает).
|
||||
|
||||
**Детерминизм/snapshot:** [1/3 HIGH] `escalate_to`-модель шлёт `extra_body`/provider-оверрайды на провод, но они НЕ были в snapshot → тихий false-hit на resume (примари-путь гейтил, эскалация нет) → свёрнуты `EscalateExtra`/`EscalateProviderTemp/MaxTok`. **Эскалация:** [2/9/13 HIGH] бюджет-гейт неидемпотентен — краш между settle фолбэка и chunk_status выбрасывал уже-оплаченный успешный фолбэк и флипал OK→flagged → **checkpoint-first replay** (существующий фолбэк-чекпоинт реиграется бесплатно, минуя бюджет-гейт); [10] эскалация, пробившая book-потолок, роняла всю книгу → `errReserveCeiling`-деградация (оставить примари-флаг, книга продолжает); [11/14] телеметрия эскалации терялась на resume → `GetCheckpoint` отдаёт флаг escalation, `resumeFromChunkStatus` восстанавливает. **Coverage:** [5 HIGH] гейт НЕ стрипал `<think>` (в отличие от `classify`/оракула) → утёкший reasoning маскировал вырезание → `stripThink` в `coverageCheck`; [6] гейт на КАЖДОЙ стадии ложно флагал легитимную монолингвальную правку → **только translator-роль** (верность editor — Ф2, билингв-судья §04); [7] `sent_cov_min=0`+нет коридора пары → молча пропускал всё → требуем `sent_cov_min∈(0,1]` при enabled; [8] Go `unicode.IsSpace` расходился с оракулом на U+001C–001F → `isOracleSpace` (паритет). **Конфиг:** [12] включение coverage-гейта требовало ключи недостижимых chain-заглушек → префлайт ключей эскалации отвязан от `gatesEnabled` → `budget_usd>0`; [4] честный doc soft-cap. Регресс-тесты добавлены; полный набор зелёный `-race`, `tmctl report` $0, live-harness компилируется под `-tags live`.
|
||||
|
||||
**Веха 2.5 сдана на внешнее ревью.** Открытые для владельца/оркестратора: боевой флип `gates.coverage.enabled` (после валидации precision), цена `deepseek-v4-pro` (в /models слаг есть, цены нет), формула `escalation.budget_usd`, F3-idempotency-key + `tmctl status` (D12 — отдельные задачи).
|
||||
|
||||
## Полигон
|
||||
(секция параллельной сессии — записи добавлять сюда)
|
||||
|
|
@ -457,3 +465,5 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор
|
|||
> **[СЕССИЯ ЗАКРЫТА, 04.07]** Выходы: research/13 (+§«Честные слабые места»), architecture/06, `eval/memory_hotpath.py` (спека), `eval/retrieval_bench.py` (эмбеддинги). Эксп-05 снят как низкосигнальный. **Перед кодом реализующей сессии — прочитать research/13 §«Честные слабые места»:** F1 подан как решённый, но там развилка (snapshot материализует байты vs store версионирует глоссарий); post-check в русской морфологии — несущий, но невалидированный (померить до доверия как гейту); «пусто=безопасно» — размен на ложный-пропуск, который тоже тихий; ставка на детерминированный no-LLM путь — cost-driven, не evidence-driven (DelTA валидирует LLM-в-цикле). Гигиена: фоновых процессов нет, ollama полигона не тронут, `eval/.venv` дополнен (torch-cpu/sentence-transformers/openai/dotenv — переиспользуемо).
|
||||
|
||||
> **[ДОБАВЛЕНО после закрытия] Локалка × банк памяти — экстракция терминов, ВЕРИФИЦИРОВАННЫЙ бенчмарк (дыра G1)** → [experiments/06-local-extraction.md](experiments/06-local-extraction.md), `eval/extract_bench.py`. 13 моделей × 12 кейсов (zh/ja/en + эдж: катакана-запад, буддийские имена, алиасы), запросы через `refusal_bench.call_provider`+`providers.json` (квирки верны: deepseek thinking-ON+max_tokens 8000 — прошлый пустой был МОЯ ошибка бюджета, не свойство модели), **каждый ответ health-верифицирован** (пустой/echo не засчитан как recall 0). Итог: **(1) СПОТ сущностей решён у всех, локаль вкл.** (recall ~0.98 на всех языках, 0 галлюцинаций). **(2) РЕНДЕР dst — разрыв с ЯЗЫКОВЫМ градиентом у локали:** en 0.82 / ja 0.53 / **zh 0.26** (阿Q→«Ах-Чу», 赵太爷→«Тяо Тай-Я» — Палладий-мусор → **эмпирика для B6**); облако-топ zh 0.75/ja 0.98/en 1.0. **(3) бо́льшая локаль не помогает** (30b render 0.55≈8b, ×3 медленнее). **(4) deepseek thinking помогает рендеру** (0.85 vs 0.58 off; на экстракции thinking-off здоров — echo бьёт перевод, не JSON). Рендер-value: **gemini-2.5-flash** (0.92/1.4с). **Дизайн-следствие (уточняет Р4/G1):** спот=дешёвая 8b (все языки); рендер dst язык-зависим — zh обязательно облако/человек + таблица Палладия (B6), en локаль потянет. ruadapt ненадёжен (5/12 health).
|
||||
|
||||
> **[В ОЧЕРЕДИ, 05.07] Исследование «Адаптивный/обучающийся слой памяти»** — промт `docs/ADAPTIVE_MEMORY_RESEARCH_PROMPT.md` (подготовлен сессией валидации по запросу владельца). Вопрос: стоит ли гибрид «детерминированный банк + слой, обучающийся по ходу книги», и как смёржить под контур (локаль-first, дешёвое API ок, флагман без логитов → kNN-MT/frontier-LoRA заблокированы; обучаемое — только на локали, роль support, не переводчик). Центр — арбитраж «кто побеждает» (логика + VRAM 8ГБ). БАР: обязан бить eval-валидированный детерминированный базис на zh/ja→ru, иначе не мёржить. Тайминг: Фаза 2+, целить в реальные дыры (после in-house eval банка). Запускает владелец отдельной сессией.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue