Apply self-review fixes: fold fallback wire config into the snapshot, make escalation resume-idempotent and ceiling-tolerant, strip think in the coverage gate, gate only the translator
This commit is contained in:
parent
a98af88b12
commit
ed4f4ff5a7
9 changed files with 330 additions and 49 deletions
|
|
@ -62,23 +62,25 @@ func TestCheckKeysOnlyUsedNonLocalModels(t *testing.T) {
|
||||||
if err := m.CheckKeys(cloudPipe); err != nil {
|
if err := m.CheckKeys(cloudPipe); err != nil {
|
||||||
t.Fatalf("present key must pass: %v", err)
|
t.Fatalf("present key must pass: %v", err)
|
||||||
}
|
}
|
||||||
// Эскалационные ключи НЕ требуются, пока гейты выключены (эскалация
|
// Эскалационные ключи НЕ требуются, пока эскалация выключена (budget_usd=0,
|
||||||
// недостижима — Фаза 0): local-стадия + облачная цепочка при gates off.
|
// opt-in). Включённый coverage-гейт НЕ делает эскалацию достижимой — гейт
|
||||||
|
// моделей не зовёт (self-review: иначе включение гейта блокирует валидный
|
||||||
|
// deepseek+glm прогон из-за отсутствующего Kimi-ключа заглушки-цепочки).
|
||||||
t.Setenv("TEST_CLOUD_KEY", "")
|
t.Setenv("TEST_CLOUD_KEY", "")
|
||||||
escOffPipe := &Pipeline{
|
escOffPipe := &Pipeline{
|
||||||
Stages: []Stage{{Model: "local-model"}},
|
Stages: []Stage{{Model: "local-model"}},
|
||||||
Escal: Escalation{Chains: map[string][]string{"default": {"cloud-model"}}},
|
Escal: Escalation{Chains: map[string][]string{"default": {"cloud-model"}}},
|
||||||
|
Gates: Gates{Coverage: CoverageGate{Enabled: true}}, // gate ON, budget 0
|
||||||
}
|
}
|
||||||
if err := m.CheckKeys(escOffPipe); err != nil {
|
if err := m.CheckKeys(escOffPipe); err != nil {
|
||||||
t.Fatalf("escalation keys must not be required while gates are off: %v", err)
|
t.Fatalf("enabling the coverage gate must NOT require escalation keys at budget 0: %v", err)
|
||||||
}
|
}
|
||||||
// С включённым гейтом эскалация достижима → ключ цепочки обязателен.
|
// При budget_usd>0 эскалация достижима → ключ цепочки обязателен.
|
||||||
escOnPipe := &Pipeline{
|
escOnPipe := &Pipeline{
|
||||||
Stages: []Stage{{Model: "local-model"}},
|
Stages: []Stage{{Model: "local-model"}},
|
||||||
Escal: Escalation{Chains: map[string][]string{"default": {"cloud-model"}}},
|
Escal: Escalation{Chains: map[string][]string{"default": {"cloud-model"}}, BudgetUSD: 1.0},
|
||||||
Gates: Gates{Coverage: CoverageGate{Enabled: true}},
|
|
||||||
}
|
}
|
||||||
if err := m.CheckKeys(escOnPipe); err == nil {
|
if err := m.CheckKeys(escOnPipe); err == nil {
|
||||||
t.Fatal("escalation chain model without a key must fail-fast when gates are enabled")
|
t.Fatal("escalation chain model without a key must fail-fast when budget_usd>0")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -398,17 +398,18 @@ func (m *Models) CheckKeys(pipe *Pipeline) error {
|
||||||
for _, st := range pipe.Stages {
|
for _, st := range pipe.Stages {
|
||||||
needed[st.Model] = struct{}{}
|
needed[st.Model] = struct{}{}
|
||||||
}
|
}
|
||||||
if pipe.gatesEnabled() {
|
// Escalation models — named chains (шаг 7) and single-hop fallbacks (Веха 2.5) —
|
||||||
|
// are reachable ONLY once an escalation budget is set (opt-in), NOT when a gate is
|
||||||
|
// enabled: the coverage gate is a deterministic runner-side check that calls no
|
||||||
|
// model, so enabling it must not demand escalation keys (self-review finding —
|
||||||
|
// otherwise turning on the gate blocks a valid deepseek+glm run on a missing Kimi
|
||||||
|
// key from an unused chain stub).
|
||||||
|
if pipe.Escal.BudgetUSD > 0 {
|
||||||
for _, chain := range pipe.Escal.Chains {
|
for _, chain := range pipe.Escal.Chains {
|
||||||
for _, mdl := range chain {
|
for _, mdl := range chain {
|
||||||
needed[mdl] = struct{}{}
|
needed[mdl] = struct{}{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
// Single-hop fallback models are reachable only once an escalation budget is set
|
|
||||||
// (opt-in); preflight their keys then, so an escalation does not 401 mid-book
|
|
||||||
// after the primary already flagged.
|
|
||||||
if pipe.Escal.BudgetUSD > 0 {
|
|
||||||
for _, st := range pipe.Stages {
|
for _, st := range pipe.Stages {
|
||||||
if st.EscalateTo != "" {
|
if st.EscalateTo != "" {
|
||||||
needed[st.EscalateTo] = struct{}{}
|
needed[st.EscalateTo] = struct{}{}
|
||||||
|
|
|
||||||
|
|
@ -242,17 +242,19 @@ func LoadPipeline(path string, models *Models) (*Pipeline, error) {
|
||||||
// Coverage QA-gate thresholds (шаг 6): an ENABLED gate must be able to flag
|
// Coverage QA-gate thresholds (шаг 6): an ENABLED gate must be able to flag
|
||||||
// something, otherwise it is a silent no-op that passes every chunk (against Р7).
|
// something, otherwise it is a silent no-op that passes every chunk (against Р7).
|
||||||
if cov := p.Gates.Coverage; cov.Enabled {
|
if cov := p.Gates.Coverage; cov.Enabled {
|
||||||
if cov.SentCovMin < 0 || cov.SentCovMin > 1 {
|
// sent_cov_min must be a LIVE threshold in (0,1]: it is the pair-INDEPENDENT
|
||||||
bad("gates.coverage.sent_cov_min must be within [0,1], got %v", cov.SentCovMin)
|
// half of the gate, so requiring it guarantees the gate can never silently pass
|
||||||
|
// every chunk for a language pair that lacks a len_ratio corridor (self-review:
|
||||||
|
// the old "both empty" check missed the per-pair gap — a non-empty len_ratio map
|
||||||
|
// that just lacks the book's pair, with sent_cov_min=0, passed everything).
|
||||||
|
if cov.SentCovMin <= 0 || cov.SentCovMin > 1 {
|
||||||
|
bad("gates.coverage.sent_cov_min must be within (0, 1] when the gate is enabled, got %v (else the gate silently passes any language pair without a len_ratio corridor)", cov.SentCovMin)
|
||||||
}
|
}
|
||||||
for pair, b := range cov.LenRatio {
|
for pair, b := range cov.LenRatio {
|
||||||
if len(b) != 2 || b[0] <= 0 || b[1] < b[0] {
|
if len(b) != 2 || b[0] <= 0 || b[1] < b[0] {
|
||||||
bad("gates.coverage.len_ratio_bounds[%q] must be [low, high] with 0 < low <= high, got %v", pair, b)
|
bad("gates.coverage.len_ratio_bounds[%q] must be [low, high] with 0 < low <= high, got %v", pair, b)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if cov.SentCovMin <= 0 && len(cov.LenRatio) == 0 {
|
|
||||||
bad("gates.coverage.enabled is true but neither sent_cov_min nor len_ratio_bounds is set — the gate would silently pass every chunk")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if len(problems) > 0 {
|
if len(problems) > 0 {
|
||||||
return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - "))
|
return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,20 @@ func isCJKSentenceTerminator(r rune) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isOracleSpace matches Python's whitespace notion used by the oracle — `re.\s`
|
||||||
|
// (splitting) and `str.isspace()`/`.strip()` (trimming & the non-space count). It is
|
||||||
|
// unicode.IsSpace PLUS the C0 information separators U+001C–U+001F, which Python
|
||||||
|
// classifies as whitespace but Go's unicode.IsSpace (Unicode White_Space) does not.
|
||||||
|
// These survive EPUB/PDF extraction, so matching Python here is required for the
|
||||||
|
// bit-parity contract (D12 Q1), not cosmetic (self-review finding).
|
||||||
|
func isOracleSpace(r rune) bool {
|
||||||
|
return unicode.IsSpace(r) || (r >= 0x1C && r <= 0x1F)
|
||||||
|
}
|
||||||
|
|
||||||
|
// trimOracleSpace is strings.TrimSpace with the oracle's whitespace set (mirrors the
|
||||||
|
// Python `.strip()` in split_sentences).
|
||||||
|
func trimOracleSpace(s string) string { return strings.TrimFunc(s, isOracleSpace) }
|
||||||
|
|
||||||
// splitSentences ports refusal_bench.py::split_sentences 1:1. Go's RE2 has no
|
// splitSentences ports refusal_bench.py::split_sentences 1:1. Go's RE2 has no
|
||||||
// lookbehind, so the regex `(?<=[.!?…。!?])\s+|(?<=[。!?])` is reproduced by a manual
|
// lookbehind, so the regex `(?<=[.!?…。!?])\s+|(?<=[。!?])` is reproduced by a manual
|
||||||
// left-to-right scan (branch-1 before branch-2, matching Python's alternation order):
|
// left-to-right scan (branch-1 before branch-2, matching Python's alternation order):
|
||||||
|
|
@ -69,7 +83,7 @@ func splitSentences(text string) []string {
|
||||||
runes := []rune(text)
|
runes := []rune(text)
|
||||||
var segs []string
|
var segs []string
|
||||||
emit := func(a, b int) {
|
emit := func(a, b int) {
|
||||||
if s := strings.TrimSpace(string(runes[a:b])); s != "" {
|
if s := trimOracleSpace(string(runes[a:b])); s != "" {
|
||||||
segs = append(segs, s)
|
segs = append(segs, s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -77,10 +91,10 @@ func splitSentences(text string) []string {
|
||||||
for i := 1; i < len(runes); i++ {
|
for i := 1; i < len(runes); i++ {
|
||||||
prev := runes[i-1]
|
prev := runes[i-1]
|
||||||
// Branch 1: any terminator, then a whitespace run (the run is consumed).
|
// Branch 1: any terminator, then a whitespace run (the run is consumed).
|
||||||
if isSentenceTerminator(prev) && unicode.IsSpace(runes[i]) {
|
if isSentenceTerminator(prev) && isOracleSpace(runes[i]) {
|
||||||
emit(segStart, i)
|
emit(segStart, i)
|
||||||
j := i
|
j := i
|
||||||
for j < len(runes) && unicode.IsSpace(runes[j]) {
|
for j < len(runes) && isOracleSpace(runes[j]) {
|
||||||
j++
|
j++
|
||||||
}
|
}
|
||||||
segStart = j
|
segStart = j
|
||||||
|
|
@ -103,7 +117,7 @@ func splitSentences(text string) []string {
|
||||||
func nonSpaceCount(s string) int {
|
func nonSpaceCount(s string) int {
|
||||||
n := 0
|
n := 0
|
||||||
for _, r := range s {
|
for _, r := range s {
|
||||||
if !unicode.IsSpace(r) {
|
if !isOracleSpace(r) {
|
||||||
n++
|
n++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -132,6 +146,11 @@ type coverageResult struct {
|
||||||
// 500–2500-char fragments and short chunks' ratios are noise (§3.7). Pure &
|
// 500–2500-char fragments and short chunks' ratios are noise (§3.7). Pure &
|
||||||
// deterministic. Returns excision_suspect (lower-bound only) or ok.
|
// deterministic. Returns excision_suspect (lower-bound only) or ok.
|
||||||
func coverageCheck(cfg config.CoverageGate, src, out, srcLang, dstLang string) coverageResult {
|
func coverageCheck(cfg config.CoverageGate, src, out, srcLang, dstLang string) coverageResult {
|
||||||
|
// Strip a leaked <think>…</think> block BEFORE measuring, exactly as classify()
|
||||||
|
// and the Python oracle do (refusal_bench: re.sub then .strip()). Without this a
|
||||||
|
// reasoning block leaked into content inflates the sentence/char counts and MASKS
|
||||||
|
// a real excision — a false negative, and a break of oracle parity (self-review).
|
||||||
|
out = trimOracleSpace(stripThink(out))
|
||||||
srcChars := nonSpaceCount(src)
|
srcChars := nonSpaceCount(src)
|
||||||
if srcChars < cfg.MinChunkChars {
|
if srcChars < cfg.MinChunkChars {
|
||||||
return coverageResult{cls: classification{reasonOK, ""}, applied: false}
|
return coverageResult{cls: classification{reasonOK, ""}, applied: false}
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,9 @@ var oracleSplit = []struct {
|
||||||
{" leading and trailing. ", []string{"leading and trailing."}},
|
{" leading and trailing. ", []string{"leading and trailing."}},
|
||||||
{"第一。\n\n第二。", []string{"第一。", "第二。"}},
|
{"第一。\n\n第二。", []string{"第一。", "第二。"}},
|
||||||
{"Mr. Smith went. He left.", []string{"Mr.", "Smith went.", "He left."}}, // naive over-split (known)
|
{"Mr. Smith went. He left.", []string{"Mr.", "Smith went.", "He left."}}, // naive over-split (known)
|
||||||
|
// U+001F (unit separator) is whitespace to Python (\s / str.isspace) but NOT to
|
||||||
|
// Go's unicode.IsSpace — isOracleSpace bridges the gap for oracle parity.
|
||||||
|
{"A.B.", []string{"A.", "B."}},
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSplitSentencesOracleParity(t *testing.T) {
|
func TestSplitSentencesOracleParity(t *testing.T) {
|
||||||
|
|
@ -93,6 +96,32 @@ func boevoyGate(minChars int) config.CoverageGate {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestOracleSpaceControlChars pins isOracleSpace to the Python oracle on the C0
|
||||||
|
// information separators U+001C–U+001F, which Python treats as whitespace (\s /
|
||||||
|
// str.isspace) but Go's unicode.IsSpace does not — a real bit-parity gap (self-review).
|
||||||
|
func TestOracleSpaceControlChars(t *testing.T) {
|
||||||
|
us := "" // U+001F unit separator (survives EPUB/PDF extraction)
|
||||||
|
if got := splitSentences("A." + us + "B."); len(got) != 2 || got[0] != "A." || got[1] != "B." {
|
||||||
|
t.Fatalf("U+001F after a terminator must split like the oracle, got %#v", got)
|
||||||
|
}
|
||||||
|
if got := nonSpaceCount("A" + us + "B"); got != 2 {
|
||||||
|
t.Fatalf("U+001F must count as whitespace (oracle parity), got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCoverageCheckStripsThink guards that a leaked <think>…</think> block is stripped
|
||||||
|
// BEFORE measuring (parity with classify() and the oracle), so a reasoning block does
|
||||||
|
// not inflate the counts and MASK a real excision (self-review finding). Mutation:
|
||||||
|
// drop the stripThink in coverageCheck and this goes green-to-red.
|
||||||
|
func TestCoverageCheckStripsThink(t *testing.T) {
|
||||||
|
src := strings.Repeat("这是一个相当长的测试句子。", 20) // 20 sentences
|
||||||
|
leaked := "<think>" + strings.Repeat("Долгое рассуждение о выборе слова. ", 40) + "</think>Короткий огрызок."
|
||||||
|
res := coverageCheck(boevoyGate(0), src, leaked, "zh", "ru")
|
||||||
|
if res.cls.Reason != FlagExcisionSuspect {
|
||||||
|
t.Fatalf("a <think> block must be stripped so the excision behind it is still caught, got %+v", res)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCoverageCheckExcision(t *testing.T) {
|
func TestCoverageCheckExcision(t *testing.T) {
|
||||||
// Mirrors the three classify_output cases captured from the live oracle.
|
// Mirrors the three classify_output cases captured from the live oracle.
|
||||||
zhExcised := coverageCheck(boevoyGate(0), strings.Repeat("这是一个测试。", 20), "Короткий перевод.", "zh", "ru")
|
zhExcised := coverageCheck(boevoyGate(0), strings.Repeat("这是一个测试。", 20), "Короткий перевод.", "zh", "ru")
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,17 @@ import (
|
||||||
// retryable length/empty subset, up to the regenerate cap, then flag).
|
// retryable length/empty subset, up to the regenerate cap, then flag).
|
||||||
// Escalation and the real linguistic chunker are later steps of Фаза 1.
|
// Escalation and the real linguistic chunker are later steps of Фаза 1.
|
||||||
|
|
||||||
|
// roleTranslator is the stage role whose output is a direct source→target
|
||||||
|
// translation — the only role the excision coverage-gate is meaningful on.
|
||||||
|
const roleTranslator = "translator"
|
||||||
|
|
||||||
|
// errReserveCeiling is wrapped into the error runAttempt returns when a book/day USD
|
||||||
|
// ceiling denies a reservation. The PRIMARY path propagates it (the book durably
|
||||||
|
// pauses and resumes once the ceiling is raised, D4); the OPTIONAL escalation hop
|
||||||
|
// catches it and degrades to keeping the primary flag instead of aborting the whole
|
||||||
|
// book on every run (self-review finding).
|
||||||
|
var errReserveCeiling = errors.New("reserve ceiling reached")
|
||||||
|
|
||||||
// Runner executes a book's pipeline.
|
// Runner executes a book's pipeline.
|
||||||
type Runner struct {
|
type Runner struct {
|
||||||
Book *config.Book
|
Book *config.Book
|
||||||
|
|
@ -168,11 +179,20 @@ func (r *Runner) coverageSnapshot() coverageSnap {
|
||||||
// gate part under the same coverage config (folded into the snapshot, so a gate change
|
// gate part under the same coverage config (folded into the snapshot, so a gate change
|
||||||
// re-pins loudly). The gate compares the output against the ORIGINAL source (ch.Text)
|
// re-pins loudly). The gate compares the output against the ORIGINAL source (ch.Text)
|
||||||
// at EVERY stage, so excision introduced by the draft OR the editor is caught.
|
// at EVERY stage, so excision introduced by the draft OR the editor is caught.
|
||||||
func (r *Runner) classifyOutput(source, output, finish string) classification {
|
func (r *Runner) classifyOutput(role, source, output, finish string) classification {
|
||||||
cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang})
|
cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang})
|
||||||
if !cls.ok() || !r.Pipeline.Gates.Coverage.Enabled {
|
if !cls.ok() || !r.Pipeline.Gates.Coverage.Enabled {
|
||||||
return cls
|
return cls
|
||||||
}
|
}
|
||||||
|
// The coverage gate compares the output against the ORIGINAL source, which is a
|
||||||
|
// source→target translation only for the TRANSLATOR role. A monolingual editor
|
||||||
|
// legitimately restructures/merges sentences, so gating ITS output against the
|
||||||
|
// source with the translation corridor false-flags a correct edit as excision
|
||||||
|
// (self-review finding). Editor/other-role fidelity is a Phase-2 concern (a
|
||||||
|
// bilingual judge over the draft, §04 mode 2), not this excision gate.
|
||||||
|
if role != roleTranslator {
|
||||||
|
return cls
|
||||||
|
}
|
||||||
if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() {
|
if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() {
|
||||||
return cov.cls
|
return cov.cls
|
||||||
}
|
}
|
||||||
|
|
@ -183,8 +203,10 @@ func (r *Runner) classifyOutput(source, output, finish string) classification {
|
||||||
// fallback draft: escalation is OPT-IN via escalation.budget_usd (0 = disabled, the
|
// fallback draft: escalation is OPT-IN via escalation.budget_usd (0 = disabled, the
|
||||||
// boevoy default — a stage's escalate_to is inert until a premium budget is set), and
|
// boevoy default — a stage's escalate_to is inert until a premium budget is set), and
|
||||||
// capped at that budget summed over the book's escalation checkpoints (money-path
|
// capped at that budget summed over the book's escalation checkpoints (money-path
|
||||||
// durable, resume-safe). A soft cap: the hop already in flight may cross it slightly,
|
// durable, resume-safe). A PRE-HOP soft cap: the gate admits a hop while spent <
|
||||||
// but the NEXT chunk's escalation is then denied.
|
// budget and does NOT pre-estimate the hop's own cost, so a single hop may overshoot
|
||||||
|
// the budget by up to its full cost; the NEXT chunk's escalation is then denied. Size
|
||||||
|
// the budget with that worst case in mind — it bounds TOTAL escalation, not per-hop.
|
||||||
func (r *Runner) escalationBudgetRemains() (bool, error) {
|
func (r *Runner) escalationBudgetRemains() (bool, error) {
|
||||||
budget := r.Pipeline.Escal.BudgetUSD
|
budget := r.Pipeline.Escal.BudgetUSD
|
||||||
if budget <= 0 {
|
if budget <= 0 {
|
||||||
|
|
@ -252,6 +274,16 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
||||||
// stages were folded, escalation models were not, until the cycle landed).
|
// stages were folded, escalation models were not, until the cycle landed).
|
||||||
EscalateTo string `json:"escalate_to,omitempty"`
|
EscalateTo string `json:"escalate_to,omitempty"`
|
||||||
EscalateCapability json.RawMessage `json:"escalate_capability,omitempty"`
|
EscalateCapability json.RawMessage `json:"escalate_capability,omitempty"`
|
||||||
|
// The fallback model's OTHER wire-affecting inputs, mirroring the primary
|
||||||
|
// fold above: its top-level extra_body (merged into the escalation wire body)
|
||||||
|
// and its provider-level temperature/max_tokens overrides (local kind). Without
|
||||||
|
// these, editing the fallback's extra_body / provider knobs would change the
|
||||||
|
// escalation wire but leave the snapshot identical → a silent false-hit on a
|
||||||
|
// resumed escalated chunk (self-review: the primary path guards this, escalation
|
||||||
|
// did not).
|
||||||
|
EscalateExtra json.RawMessage `json:"escalate_extra,omitempty"`
|
||||||
|
EscalateProviderTemp float64 `json:"escalate_provider_temp,omitempty"`
|
||||||
|
EscalateProviderMaxTok int `json:"escalate_provider_max_tok,omitempty"`
|
||||||
}
|
}
|
||||||
snap := struct {
|
snap := struct {
|
||||||
BriefHash string `json:"brief_hash"`
|
BriefHash string `json:"brief_hash"`
|
||||||
|
|
@ -337,6 +369,16 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
||||||
return "", "", eerr
|
return "", "", eerr
|
||||||
}
|
}
|
||||||
ss.EscalateCapability = escCap
|
ss.EscalateCapability = escCap
|
||||||
|
if prov, ok := r.Models.Providers[r.Models.Models[st.EscalateTo].Provider]; ok {
|
||||||
|
ss.EscalateProviderTemp, ss.EscalateProviderMaxTok = prov.Temperature, prov.MaxTokens
|
||||||
|
}
|
||||||
|
if extra := r.Models.Models[st.EscalateTo].ExtraBody; len(extra) > 0 {
|
||||||
|
raw, merr := json.Marshal(extra)
|
||||||
|
if merr != nil {
|
||||||
|
return "", "", merr
|
||||||
|
}
|
||||||
|
ss.EscalateExtra = raw
|
||||||
|
}
|
||||||
}
|
}
|
||||||
snap.Stages = append(snap.Stages, ss)
|
snap.Stages = append(snap.Stages, ss)
|
||||||
}
|
}
|
||||||
|
|
@ -629,28 +671,50 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
||||||
// reset on the fallback (the LiteLLM #19985 retry×fallback blow-up guard).
|
// reset on the fallback (the LiteLLM #19985 retry×fallback blow-up guard).
|
||||||
escalated, escModel := false, ""
|
escalated, escModel := false, ""
|
||||||
if last.cls.Reason.escalatable() && st.EscalateTo != "" {
|
if last.cls.Reason.escalatable() && st.EscalateTo != "" {
|
||||||
remains, err := r.escalationBudgetRemains()
|
// Idempotency (self-review): if the fallback hop already happened its
|
||||||
|
// checkpoint exists and was already paid — REPLAY it for free regardless of the
|
||||||
|
// budget, so a crash AFTER the hop settled but BEFORE chunk_status was written
|
||||||
|
// re-serves it on resume rather than discarding a paid, successful translation
|
||||||
|
// and flipping the verdict OK→flagged. Only a FRESH hop is budget-gated. The
|
||||||
|
// hash mirrors runAttempt's for (model=EscalateTo, attempt=0, maxTokens=base).
|
||||||
|
fbHash := RequestHash(r.Book.BookID, ch.Chapter, ch.ChunkIdx, 0, st.Name, st.Role, st.EscalateTo,
|
||||||
|
st.Temperature, st.Reasoning, false, baseMaxTokens, snapID, msgs)
|
||||||
|
fbExists, err := r.Store.GetCheckpoint(fbHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if remains {
|
mayHop := fbExists != nil
|
||||||
fb, err := r.runAttempt(ctx, st, st.EscalateTo, snapID, ch, job, 0, baseMaxTokens, msgs, true)
|
if !mayHop {
|
||||||
if err != nil {
|
if mayHop, err = r.escalationBudgetRemains(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
cumCost += fb.cumCost
|
}
|
||||||
runCost += fb.runCost
|
if mayHop {
|
||||||
anyFresh = anyFresh || fb.freshCall
|
fb, err := r.runAttempt(ctx, st, st.EscalateTo, snapID, ch, job, 0, baseMaxTokens, msgs, true)
|
||||||
escalated, escModel = true, st.EscalateTo
|
if err != nil {
|
||||||
r.Log.WarnContext(ctx, "stage escalated to a fallback model",
|
// An OPTIONAL hop that trips a USD ceiling must NOT abort the whole book
|
||||||
"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx,
|
// (and re-abort on every resume): the chunk is already flagged, so keep
|
||||||
"primary_reason", string(last.cls.Reason), "fallback", st.EscalateTo,
|
// the primary flag and continue. Any other infra error still aborts.
|
||||||
"fallback_disposition", string(fb.cls.Reason.disposition()))
|
if !errors.Is(err, errReserveCeiling) {
|
||||||
if fb.cls.ok() {
|
return nil, err
|
||||||
last = fb // the fallback draft passed the re-gate → it is authoritative
|
}
|
||||||
|
r.Log.WarnContext(ctx, "escalation hop denied by a USD ceiling; keeping the primary flag",
|
||||||
|
"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "reason", string(last.cls.Reason))
|
||||||
|
} else {
|
||||||
|
cumCost += fb.cumCost
|
||||||
|
runCost += fb.runCost
|
||||||
|
anyFresh = anyFresh || fb.freshCall
|
||||||
|
escalated, escModel = true, st.EscalateTo
|
||||||
|
r.Log.WarnContext(ctx, "stage escalated to a fallback model",
|
||||||
|
"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx,
|
||||||
|
"primary_reason", string(last.cls.Reason), "fallback", st.EscalateTo,
|
||||||
|
"fallback_disposition", string(fb.cls.Reason.disposition()))
|
||||||
|
if fb.cls.ok() {
|
||||||
|
last = fb // the fallback draft passed the re-gate → it is authoritative
|
||||||
|
}
|
||||||
|
// else: the fallback also failed → keep the primary flag (last unchanged);
|
||||||
|
// the fallback call is billed and counted, the chunk stays flagged (1 hop).
|
||||||
}
|
}
|
||||||
// else: the fallback also failed → keep the primary flag (last unchanged);
|
|
||||||
// the fallback call is billed and counted, the chunk stays flagged (1 hop).
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -732,7 +796,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
|
||||||
att.finish = cp.FinishReason
|
att.finish = cp.FinishReason
|
||||||
att.modelActual = cp.ModelActual
|
att.modelActual = cp.ModelActual
|
||||||
att.cumCost = cp.CostUSD
|
att.cumCost = cp.CostUSD
|
||||||
att.cls = r.classifyOutput(ch.Text, cp.ResponseText, cp.FinishReason)
|
att.cls = r.classifyOutput(st.Role, ch.Text, cp.ResponseText, cp.FinishReason)
|
||||||
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
||||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
||||||
Stage: st.Name, Role: st.Role, ModelRequested: model, ModelActual: cp.ModelActual,
|
Stage: st.Name, Role: st.Role, ModelRequested: model, ModelActual: cp.ModelActual,
|
||||||
|
|
@ -763,10 +827,11 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
|
||||||
switch verdict {
|
switch verdict {
|
||||||
case store.ReserveDeniedBook:
|
case store.ReserveDeniedBook:
|
||||||
// Ceiling is a hard, book-wide stop (not a per-chunk flag): the job stays
|
// Ceiling is a hard, book-wide stop (not a per-chunk flag): the job stays
|
||||||
// 'pending' and resume continues once the ceiling is raised.
|
// 'pending' and resume continues once the ceiling is raised. Wrapped so an
|
||||||
return att, fmt.Errorf("pipeline: book USD ceiling reached (%.2f$) — raise ceilings.book_usd or stop", r.Book.Ceilings.BookUSD)
|
// optional escalation hop can degrade instead of aborting the book.
|
||||||
|
return att, fmt.Errorf("pipeline: book USD ceiling reached (%.2f$) — raise ceilings.book_usd or stop: %w", r.Book.Ceilings.BookUSD, errReserveCeiling)
|
||||||
case store.ReserveDeniedDay:
|
case store.ReserveDeniedDay:
|
||||||
return att, fmt.Errorf("pipeline: daily USD ceiling reached (%.2f$)", r.Book.Ceilings.DayUSD)
|
return att, fmt.Errorf("pipeline: daily USD ceiling reached (%.2f$): %w", r.Book.Ceilings.DayUSD, errReserveCeiling)
|
||||||
}
|
}
|
||||||
|
|
||||||
client, err := r.client(model)
|
client, err := r.client(model)
|
||||||
|
|
@ -878,7 +943,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
|
||||||
// non-empty truncated length draft is now classified (flagged/retried), never
|
// non-empty truncated length draft is now classified (flagged/retried), never
|
||||||
// silently passed downstream as OK; an empty completion is flagged too, not a
|
// silently passed downstream as OK; an empty completion is flagged too, not a
|
||||||
// run-crash.
|
// run-crash.
|
||||||
att.cls = r.classifyOutput(ch.Text, resp.Text, resp.FinishReason)
|
att.cls = r.classifyOutput(st.Role, ch.Text, resp.Text, resp.FinishReason)
|
||||||
|
|
||||||
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
||||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
||||||
|
|
@ -926,6 +991,12 @@ func (r *Runner) resumeFromChunkStatus(ctx context.Context, st config.Stage, ch
|
||||||
sr.Model = cp.ModelActual
|
sr.Model = cp.ModelActual
|
||||||
sr.FinishReason = cp.FinishReason
|
sr.FinishReason = cp.FinishReason
|
||||||
sr.Text = cp.ResponseText
|
sr.Text = cp.ResponseText
|
||||||
|
// Preserve escalation attribution across resume (self-review): a chunk resolved
|
||||||
|
// via a fallback checkpoint is still an escalation, so a resumed run reports it.
|
||||||
|
if cp.Escalation {
|
||||||
|
sr.Escalated = true
|
||||||
|
sr.EscalationModel = cp.ModelRequested
|
||||||
|
}
|
||||||
}
|
}
|
||||||
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
||||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
||||||
|
|
|
||||||
|
|
@ -1171,6 +1171,153 @@ func TestRunnerEscalationResumeServesFallback(t *testing.T) {
|
||||||
if res.Chunks[0].FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
if res.Chunks[0].FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
||||||
t.Fatalf("resume text differs: %q", res.Chunks[0].FinalText)
|
t.Fatalf("resume text differs: %q", res.Chunks[0].FinalText)
|
||||||
}
|
}
|
||||||
|
// Escalation attribution survives resume (self-review [11/14]).
|
||||||
|
if d := res.Chunks[0].Stages[0]; !d.Escalated || d.EscalationModel != "fake-fallback" {
|
||||||
|
t.Fatalf("resume must preserve escalation attribution, got %+v", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only the TRANSLATOR stage is coverage-gated (self-review [6]): a monolingual editor
|
||||||
|
// that legitimately merges/tightens sentences (low sent_cov vs the CJK source) must
|
||||||
|
// NOT be flagged as excision. Mutation: gate every stage and the short edit flags.
|
||||||
|
func TestRunnerCoverageGateOnlyTranslator(t *testing.T) {
|
||||||
|
rec := &reqRec{}
|
||||||
|
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||||
|
if isEditBody(body) {
|
||||||
|
return "Одно.", "stop" // a very short "polish" — would flag if the editor were gated
|
||||||
|
}
|
||||||
|
return strings.Repeat("Это предложение перевода. ", 30), "stop" // faithful translator draft
|
||||||
|
})
|
||||||
|
defer srv.Close()
|
||||||
|
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||||||
|
source: strings.Repeat("これは長い文である。", 30), regenerate: 0, gatesYAML: coverageGateBlock,
|
||||||
|
})
|
||||||
|
|
||||||
|
r := newRunner(t, bookPath)
|
||||||
|
defer r.Close()
|
||||||
|
res, err := r.TranslateBook(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Flagged != 0 || res.ExitCode() != 0 || res.Chunks[0].Disposition != DispOK {
|
||||||
|
t.Fatalf("a short monolingual edit must NOT be excision-flagged (editor is not gated), got %+v", res.Chunks[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A successful, already-PAID escalation must be resume-idempotent even when the budget
|
||||||
|
// no longer admits a fresh hop (self-review [2/9/13]): a crash between the fallback
|
||||||
|
// settle and the chunk_status write must REPLAY the paid checkpoint, not discard it and
|
||||||
|
// flip the chunk OK→flagged. Mutation: drop the fbExists check and the chunk flips.
|
||||||
|
func TestRunnerEscalationResumeIdempotentUnderBudget(t *testing.T) {
|
||||||
|
rec := &reqRec{}
|
||||||
|
srv := newJSONProvider(rec, echoOrClean)
|
||||||
|
defer srv.Close()
|
||||||
|
// budget < one hop cost: after the first hop, EscalationSpentUSD ≥ budget, so a
|
||||||
|
// FRESH hop would be denied on resume — the paid hop must replay regardless.
|
||||||
|
bookPath := setupEscalationProject(t, srv.URL, 0.001, false, "")
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
r1 := newRunner(t, bookPath)
|
||||||
|
res1, err := r1.TranslateBook(ctx)
|
||||||
|
if err != nil || res1.Chunks[0].Disposition != DispOK {
|
||||||
|
t.Fatalf("run1 must escalate to OK, got %+v (err %v)", res1.Chunks[0], err)
|
||||||
|
}
|
||||||
|
dbPath := r1.Book.ProjectDB
|
||||||
|
r1.Close()
|
||||||
|
callsAfterRun1 := rec.count()
|
||||||
|
|
||||||
|
// Simulate the crash window: chunk_status lost, checkpoints (incl. escalation) survive.
|
||||||
|
db, err := sql.Open("sqlite", "file:"+dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := db.Exec(`DELETE FROM chunk_status WHERE stage = 'draft'`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
db.Close()
|
||||||
|
|
||||||
|
r2 := newRunner(t, bookPath)
|
||||||
|
defer r2.Close()
|
||||||
|
res2, err := r2.TranslateBook(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rec.count() != callsAfterRun1 {
|
||||||
|
t.Fatalf("resume must REPLAY the paid escalation checkpoint, not re-call: calls %d -> %d", callsAfterRun1, rec.count())
|
||||||
|
}
|
||||||
|
if res2.Chunks[0].Disposition != DispOK {
|
||||||
|
t.Fatalf("resume must re-serve the escalated OK chunk, not flip to flagged (budget must not orphan a paid hop), got %+v", res2.Chunks[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An OPTIONAL escalation hop that trips the book USD ceiling must degrade to keeping
|
||||||
|
// the primary flag, NOT abort the whole book (and re-abort on every resume) — self-
|
||||||
|
// review [10]. Mutation: propagate the ceiling error and TranslateBook returns non-nil.
|
||||||
|
func TestRunnerEscalationCeilingDegrades(t *testing.T) {
|
||||||
|
rec := &reqRec{}
|
||||||
|
srv := newJSONProvider(rec, echoOrClean)
|
||||||
|
defer srv.Close()
|
||||||
|
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||||||
|
|
||||||
|
// Lower the book ceiling so the primary draft call fits but the escalation hop does not.
|
||||||
|
bp := filepath.Join(filepath.Dir(bookPath), "book.yaml")
|
||||||
|
raw, err := os.ReadFile(bp)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writeFile(t, bp, strings.Replace(string(raw), "book_usd: 5.0", "book_usd: 0.0025", 1))
|
||||||
|
|
||||||
|
r := newRunner(t, bookPath)
|
||||||
|
defer r.Close()
|
||||||
|
res, err := r.TranslateBook(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("an escalation-hop ceiling denial must NOT abort the book, got: %v", err)
|
||||||
|
}
|
||||||
|
ch := res.Chunks[0]
|
||||||
|
if ch.Disposition != DispFlagged || ch.FlagReason != FlagCJKArtifact {
|
||||||
|
t.Fatalf("the chunk must keep its primary flag when escalation is ceiling-denied, got %+v", ch)
|
||||||
|
}
|
||||||
|
if ch.Stages[0].Escalated {
|
||||||
|
t.Fatalf("a ceiling-denied hop must not be recorded as an escalation, got %+v", ch.Stages[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The fallback model's top-level extra_body is folded into the snapshot (self-review
|
||||||
|
// [1]): editing it changes the escalation wire, so resume must fail loud until
|
||||||
|
// --resnapshot, never silently serve a stale escalated checkpoint.
|
||||||
|
func TestRunnerEscalationFoldsFallbackExtraBody(t *testing.T) {
|
||||||
|
rec := &reqRec{}
|
||||||
|
srv := newJSONProvider(rec, echoOrClean)
|
||||||
|
defer srv.Close()
|
||||||
|
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
r1 := newRunner(t, bookPath)
|
||||||
|
if _, err := r1.TranslateBook(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
r1.Close()
|
||||||
|
|
||||||
|
// Add extra_body to the FALLBACK model only — it changes the escalation wire body.
|
||||||
|
mp := filepath.Join(filepath.Dir(bookPath), "models.yaml")
|
||||||
|
raw, err := os.ReadFile(mp)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
patched := strings.Replace(string(raw),
|
||||||
|
" fake-fallback:\n provider: fake",
|
||||||
|
" fake-fallback:\n provider: fake\n extra_body: { top_p: 0.5 }", 1)
|
||||||
|
if patched == string(raw) {
|
||||||
|
t.Fatal("failed to inject extra_body into fake-fallback")
|
||||||
|
}
|
||||||
|
writeFile(t, mp, patched)
|
||||||
|
|
||||||
|
r2 := newRunner(t, bookPath)
|
||||||
|
_, err = r2.TranslateBook(ctx)
|
||||||
|
r2.Close()
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||||||
|
t.Fatalf("editing the fallback's extra_body must fail loud mentioning --resnapshot, got: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The fallback model is folded into the snapshot: removing escalate_to changes the
|
// The fallback model is folded into the snapshot: removing escalate_to changes the
|
||||||
|
|
|
||||||
|
|
@ -178,10 +178,10 @@ func (s *Store) GetCheckpoint(requestHash string) (*Checkpoint, error) {
|
||||||
cp := Checkpoint{RequestHash: requestHash}
|
cp := Checkpoint{RequestHash: requestHash}
|
||||||
err := s.r.QueryRowContext(ctx, `
|
err := s.r.QueryRowContext(ctx, `
|
||||||
SELECT job_id, chunk_idx, attempt, stage, role, model_requested, model_actual,
|
SELECT job_id, chunk_idx, attempt, stage, role, model_requested, model_actual,
|
||||||
response_text, usage_json, cost_usd, finish_reason, provider_request_id
|
response_text, usage_json, cost_usd, finish_reason, provider_request_id, escalation
|
||||||
FROM checkpoints WHERE request_hash = ?`, requestHash).Scan(
|
FROM checkpoints WHERE request_hash = ?`, requestHash).Scan(
|
||||||
&cp.JobID, &cp.ChunkIdx, &cp.Attempt, &cp.Stage, &cp.Role, &cp.ModelRequested, &cp.ModelActual,
|
&cp.JobID, &cp.ChunkIdx, &cp.Attempt, &cp.Stage, &cp.Role, &cp.ModelRequested, &cp.ModelActual,
|
||||||
&cp.ResponseText, &cp.UsageJSON, &cp.CostUSD, &cp.FinishReason, &cp.ProviderRequestID)
|
&cp.ResponseText, &cp.UsageJSON, &cp.CostUSD, &cp.FinishReason, &cp.ProviderRequestID, &cp.Escalation)
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -340,7 +340,15 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор
|
||||||
|
|
||||||
**Live-валидация (1 целевой вызов deepseek, ~$0.00004):** фрагмент Лу Синя «祝福» (эхо-репро-семья) → `cjk_share=0.00`, `verdict=ok`, `finish=stop`, `reasoning_tokens=0`, чистый русский перевод. **DeepSeek-мина подтверждённо обезврежена на реальном проводе** — `reasoning:"off"` no-op держит thinking ON → перевод, не эхо. Адаптер читает `content`, слаг = `deepseek-v4-flash` (канонизация чистая). grok закомментирован в наборе (гигиена clean-prod xAI — включать осознанно).
|
**Live-валидация (1 целевой вызов deepseek, ~$0.00004):** фрагмент Лу Синя «祝福» (эхо-репро-семья) → `cjk_share=0.00`, `verdict=ok`, `finish=stop`, `reasoning_tokens=0`, чистый русский перевод. **DeepSeek-мина подтверждённо обезврежена на реальном проводе** — `reasoning:"off"` no-op держит thinking ON → перевод, не эхо. Адаптер читает `content`, слаг = `deepseek-v4-flash` (канонизация чистая). grok закомментирован в наборе (гигиена clean-prod xAI — включать осознанно).
|
||||||
|
|
||||||
**Веха 2.5 завершена (A+B+C+D + §2-гейт).** Коммиты: `2c1c38d` (эхо-мина гейт), `2acfdf2` (coverage-гейт), `d97f832` (Kimi base_url), `659cce0` (single-hop эскалация), + этот. Дальше — агентское селфревью на вехе.
|
**Веха 2.5 (A+B+C+D + §2-гейт) реализована.** Коммиты: `2c1c38d` (эхо-мина гейт), `2acfdf2` (coverage-гейт), `d97f832` (Kimi base_url), `659cce0` (single-hop эскалация), `a844765` (live-harness). Дальше — агентское селфревью на вехе.
|
||||||
|
|
||||||
|
### 2026-07-05 — Веха 2.5: агентское адверсариальное селфревью + фиксы
|
||||||
|
|
||||||
|
Многоагентный Workflow (6 дименсий: детерминизм / деньги / корректность coverage / корректность эскалации / эхо-мина+конфиг / нейтральность+хаки → независимая верификация КАЖДОЙ находки CONFIRMED/REFUTED с конкретным сценарием инпут→неверный-выход). 25 агентов, **14 подтверждённых** (5 опровергнуто). Все исправлены; ключевые фиксы мутационно-проверены (сломай → тест падает).
|
||||||
|
|
||||||
|
**Детерминизм/snapshot:** [1/3 HIGH] `escalate_to`-модель шлёт `extra_body`/provider-оверрайды на провод, но они НЕ были в snapshot → тихий false-hit на resume (примари-путь гейтил, эскалация нет) → свёрнуты `EscalateExtra`/`EscalateProviderTemp/MaxTok`. **Эскалация:** [2/9/13 HIGH] бюджет-гейт неидемпотентен — краш между settle фолбэка и chunk_status выбрасывал уже-оплаченный успешный фолбэк и флипал OK→flagged → **checkpoint-first replay** (существующий фолбэк-чекпоинт реиграется бесплатно, минуя бюджет-гейт); [10] эскалация, пробившая book-потолок, роняла всю книгу → `errReserveCeiling`-деградация (оставить примари-флаг, книга продолжает); [11/14] телеметрия эскалации терялась на resume → `GetCheckpoint` отдаёт флаг escalation, `resumeFromChunkStatus` восстанавливает. **Coverage:** [5 HIGH] гейт НЕ стрипал `<think>` (в отличие от `classify`/оракула) → утёкший reasoning маскировал вырезание → `stripThink` в `coverageCheck`; [6] гейт на КАЖДОЙ стадии ложно флагал легитимную монолингвальную правку → **только translator-роль** (верность editor — Ф2, билингв-судья §04); [7] `sent_cov_min=0`+нет коридора пары → молча пропускал всё → требуем `sent_cov_min∈(0,1]` при enabled; [8] Go `unicode.IsSpace` расходился с оракулом на U+001C–001F → `isOracleSpace` (паритет). **Конфиг:** [12] включение coverage-гейта требовало ключи недостижимых chain-заглушек → префлайт ключей эскалации отвязан от `gatesEnabled` → `budget_usd>0`; [4] честный doc soft-cap. Регресс-тесты добавлены; полный набор зелёный `-race`, `tmctl report` $0, live-harness компилируется под `-tags live`.
|
||||||
|
|
||||||
|
**Веха 2.5 сдана на внешнее ревью.** Открытые для владельца/оркестратора: боевой флип `gates.coverage.enabled` (после валидации precision), цена `deepseek-v4-pro` (в /models слаг есть, цены нет), формула `escalation.budget_usd`, F3-idempotency-key + `tmctl status` (D12 — отдельные задачи).
|
||||||
|
|
||||||
## Полигон
|
## Полигон
|
||||||
(секция параллельной сессии — записи добавлять сюда)
|
(секция параллельной сессии — записи добавлять сюда)
|
||||||
|
|
@ -457,3 +465,5 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор
|
||||||
> **[СЕССИЯ ЗАКРЫТА, 04.07]** Выходы: research/13 (+§«Честные слабые места»), architecture/06, `eval/memory_hotpath.py` (спека), `eval/retrieval_bench.py` (эмбеддинги). Эксп-05 снят как низкосигнальный. **Перед кодом реализующей сессии — прочитать research/13 §«Честные слабые места»:** F1 подан как решённый, но там развилка (snapshot материализует байты vs store версионирует глоссарий); post-check в русской морфологии — несущий, но невалидированный (померить до доверия как гейту); «пусто=безопасно» — размен на ложный-пропуск, который тоже тихий; ставка на детерминированный no-LLM путь — cost-driven, не evidence-driven (DelTA валидирует LLM-в-цикле). Гигиена: фоновых процессов нет, ollama полигона не тронут, `eval/.venv` дополнен (torch-cpu/sentence-transformers/openai/dotenv — переиспользуемо).
|
> **[СЕССИЯ ЗАКРЫТА, 04.07]** Выходы: research/13 (+§«Честные слабые места»), architecture/06, `eval/memory_hotpath.py` (спека), `eval/retrieval_bench.py` (эмбеддинги). Эксп-05 снят как низкосигнальный. **Перед кодом реализующей сессии — прочитать research/13 §«Честные слабые места»:** F1 подан как решённый, но там развилка (snapshot материализует байты vs store версионирует глоссарий); post-check в русской морфологии — несущий, но невалидированный (померить до доверия как гейту); «пусто=безопасно» — размен на ложный-пропуск, который тоже тихий; ставка на детерминированный no-LLM путь — cost-driven, не evidence-driven (DelTA валидирует LLM-в-цикле). Гигиена: фоновых процессов нет, ollama полигона не тронут, `eval/.venv` дополнен (torch-cpu/sentence-transformers/openai/dotenv — переиспользуемо).
|
||||||
|
|
||||||
> **[ДОБАВЛЕНО после закрытия] Локалка × банк памяти — экстракция терминов, ВЕРИФИЦИРОВАННЫЙ бенчмарк (дыра G1)** → [experiments/06-local-extraction.md](experiments/06-local-extraction.md), `eval/extract_bench.py`. 13 моделей × 12 кейсов (zh/ja/en + эдж: катакана-запад, буддийские имена, алиасы), запросы через `refusal_bench.call_provider`+`providers.json` (квирки верны: deepseek thinking-ON+max_tokens 8000 — прошлый пустой был МОЯ ошибка бюджета, не свойство модели), **каждый ответ health-верифицирован** (пустой/echo не засчитан как recall 0). Итог: **(1) СПОТ сущностей решён у всех, локаль вкл.** (recall ~0.98 на всех языках, 0 галлюцинаций). **(2) РЕНДЕР dst — разрыв с ЯЗЫКОВЫМ градиентом у локали:** en 0.82 / ja 0.53 / **zh 0.26** (阿Q→«Ах-Чу», 赵太爷→«Тяо Тай-Я» — Палладий-мусор → **эмпирика для B6**); облако-топ zh 0.75/ja 0.98/en 1.0. **(3) бо́льшая локаль не помогает** (30b render 0.55≈8b, ×3 медленнее). **(4) deepseek thinking помогает рендеру** (0.85 vs 0.58 off; на экстракции thinking-off здоров — echo бьёт перевод, не JSON). Рендер-value: **gemini-2.5-flash** (0.92/1.4с). **Дизайн-следствие (уточняет Р4/G1):** спот=дешёвая 8b (все языки); рендер dst язык-зависим — zh обязательно облако/человек + таблица Палладия (B6), en локаль потянет. ruadapt ненадёжен (5/12 health).
|
> **[ДОБАВЛЕНО после закрытия] Локалка × банк памяти — экстракция терминов, ВЕРИФИЦИРОВАННЫЙ бенчмарк (дыра G1)** → [experiments/06-local-extraction.md](experiments/06-local-extraction.md), `eval/extract_bench.py`. 13 моделей × 12 кейсов (zh/ja/en + эдж: катакана-запад, буддийские имена, алиасы), запросы через `refusal_bench.call_provider`+`providers.json` (квирки верны: deepseek thinking-ON+max_tokens 8000 — прошлый пустой был МОЯ ошибка бюджета, не свойство модели), **каждый ответ health-верифицирован** (пустой/echo не засчитан как recall 0). Итог: **(1) СПОТ сущностей решён у всех, локаль вкл.** (recall ~0.98 на всех языках, 0 галлюцинаций). **(2) РЕНДЕР dst — разрыв с ЯЗЫКОВЫМ градиентом у локали:** en 0.82 / ja 0.53 / **zh 0.26** (阿Q→«Ах-Чу», 赵太爷→«Тяо Тай-Я» — Палладий-мусор → **эмпирика для B6**); облако-топ zh 0.75/ja 0.98/en 1.0. **(3) бо́льшая локаль не помогает** (30b render 0.55≈8b, ×3 медленнее). **(4) deepseek thinking помогает рендеру** (0.85 vs 0.58 off; на экстракции thinking-off здоров — echo бьёт перевод, не JSON). Рендер-value: **gemini-2.5-flash** (0.92/1.4с). **Дизайн-следствие (уточняет Р4/G1):** спот=дешёвая 8b (все языки); рендер dst язык-зависим — zh обязательно облако/человек + таблица Палладия (B6), en локаль потянет. ruadapt ненадёжен (5/12 health).
|
||||||
|
|
||||||
|
> **[В ОЧЕРЕДИ, 05.07] Исследование «Адаптивный/обучающийся слой памяти»** — промт `docs/ADAPTIVE_MEMORY_RESEARCH_PROMPT.md` (подготовлен сессией валидации по запросу владельца). Вопрос: стоит ли гибрид «детерминированный банк + слой, обучающийся по ходу книги», и как смёржить под контур (локаль-first, дешёвое API ок, флагман без логитов → kNN-MT/frontier-LoRA заблокированы; обучаемое — только на локали, роль support, не переводчик). Центр — арбитраж «кто побеждает» (логика + VRAM 8ГБ). БАР: обязан бить eval-валидированный детерминированный базис на zh/ja→ru, иначе не мёржить. Тайминг: Фаза 2+, целить в реальные дыры (после in-house eval банка). Запускает владелец отдельной сессией.
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue