54 lines
3 KiB
Go
54 lines
3 KiB
Go
package config
|
||
|
||
import (
|
||
"path/filepath"
|
||
"testing"
|
||
)
|
||
|
||
// echoregen_shipping_test.go pins ONE data decision: every shipping pipeline turns the echo
|
||
// re-generation ON before the escalation hop (`retries.regenerate_echo_before_escalate: 1`).
|
||
//
|
||
// WHY IT IS PINNED IN A TEST AND NOT LEFT TO THE FILES. The knob's default in Go stays 0 on purpose —
|
||
// a re-generation is the right answer only where the provider's echo is STOCHASTIC per call (D39.61,
|
||
// deepseek-v4-flash 0731), and on a provider whose echo is DETERMINISTIC it buys a guaranteed re-refusal
|
||
// (00-provider-quirks.md still records that shape for deepseek-chat). So "is a re-generation worth it"
|
||
// is a property of the MODEL, which by canon lives in DATA and not in a Go zero value. That makes the
|
||
// files the only carrier of the decision — and an un-pinned data decision is one a later config edit
|
||
// reverts silently, on the money path, without anything going red.
|
||
//
|
||
// The number is not taste. Re-derived from the cold run's own ledger (coldrun-v16, read-only): an
|
||
// escalation hop on deepseek-v4-pro cost a mean of $0.02066090 over 5 hops ($0.10330452 = 23.7% of the
|
||
// run) while a SUCCESSFUL flash draft cost a mean of $0.00382851 over 15 calls — the hop is 5.40× the
|
||
// call it replaces, and five of twenty fresh flash drafts echoed.
|
||
//
|
||
// Mutation this catches: drop the key from any shipping pipeline → RED, naming the file.
|
||
func TestShippingPipelinesRegenerateEchoBeforeEscalating(t *testing.T) {
|
||
m, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
|
||
if err != nil {
|
||
t.Fatalf("load the shipping models.yaml: %v", err)
|
||
}
|
||
// Every pipeline that ships a deepseek-v4-flash translator — the two boevoy cores and the two
|
||
// editor swap-arms. The arms are included deliberately: they exist to isolate the EDITOR, so a draft
|
||
// policy that differed between an arm and its baseline would put a second variable in the comparison.
|
||
for _, pf := range []string{
|
||
"pipeline-c1.yaml", "pipeline-c2.yaml",
|
||
"pipeline-arm-glm.yaml", "pipeline-arm-mistral.yaml",
|
||
} {
|
||
p, err := LoadPipeline(filepath.Join("..", "..", "configs", pf), m, "zh-ru", nil)
|
||
if err != nil {
|
||
t.Fatalf("load %s: %v", pf, err)
|
||
}
|
||
if got := p.Retries.RegenerateEchoBeforeEscalate; got != 1 {
|
||
t.Errorf("%s: retries.regenerate_echo_before_escalate = %d, want 1 — without it a stochastic "+
|
||
"echo goes STRAIGHT to the escalation hop, which the cold run measured at 5.40× the cost of "+
|
||
"the same-model call that recovers it (D39.61 + coldrun-v16 ledger)", pf, got)
|
||
}
|
||
// The sibling budget is asserted too, because the echo budget is a threshold on the SAME attempt
|
||
// counter: if the content-regeneration budget ever grows, a chunk that first flagged `length`
|
||
// arrives at the echo check already past the echo budget and silently stops being re-generated.
|
||
if got := p.Retries.RegenerateBeforeEscalate; got != 1 {
|
||
t.Errorf("%s: retries.regenerate_before_escalate = %d, want 1 — the two budgets share the attempt "+
|
||
"axis, and this test's premise is that they are equal", pf, got)
|
||
}
|
||
}
|
||
}
|