339 lines
13 KiB
Go
339 lines
13 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// runner_memory_test.go: the шаг-4 acceptance — the memory bank v2 mechanism end-to-end
|
||
// on a seeded glossary against the mock provider (injection → translate → post-check →
|
||
// retrieval-state → resume $0 → resnapshot on an approved change). Per §8, acceptance is
|
||
// "the mechanism is correct on a seeded glossary", not "≥98% on a real book".
|
||
|
||
const suzukiSeed = `
|
||
terms:
|
||
- src: 鈴木
|
||
dst: Судзуки
|
||
type: name
|
||
status: approved
|
||
decl: { invariant: true, forms: ["Судзуки"] }
|
||
`
|
||
|
||
// suzukiSource is one ja chunk that names 鈴木 (so the glossary fires).
|
||
const suzukiSource = "鈴木は静かな図書館へ行った。"
|
||
|
||
type wireMsg struct {
|
||
Role string
|
||
Content string
|
||
}
|
||
|
||
func bodyMessages(t *testing.T, body string) []wireMsg {
|
||
t.Helper()
|
||
var req struct {
|
||
Messages []wireMsg `json:"messages"`
|
||
}
|
||
if err := json.Unmarshal([]byte(body), &req); err != nil {
|
||
t.Fatalf("bad request body: %v\n%s", err, body)
|
||
}
|
||
return req.Messages
|
||
}
|
||
|
||
// translatorBody returns the recorded draft(translator) request body.
|
||
func translatorBody(t *testing.T, rec *reqRec) string {
|
||
t.Helper()
|
||
for _, b := range rec.all() {
|
||
if !isEditBody(b) {
|
||
return b
|
||
}
|
||
}
|
||
t.Fatal("no translator request recorded")
|
||
return ""
|
||
}
|
||
|
||
func TestRunnerMemoryInjectionAndPostcheck(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД про Судзуки.", "stop"
|
||
}
|
||
return "Судзуки пошёл в тихую библиотеку.", "stop" // renders the approved dst
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
res, err := r.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res.Flagged != 0 {
|
||
t.Fatalf("flagger default must not flag a correct render: flagged=%d", res.Flagged)
|
||
}
|
||
|
||
// The TRANSLATOR request carries the injection as its own system message between the
|
||
// stable prefix and the user chunk; the dst appears only via injection (not in the ja
|
||
// source), so its presence proves the injection landed on the wire.
|
||
tb := translatorBody(t, rec)
|
||
msgs := bodyMessages(t, tb)
|
||
if len(msgs) != 3 {
|
||
t.Fatalf("translator must have 3 messages (system, injection, user), got %d: %+v", len(msgs), msgs)
|
||
}
|
||
if msgs[1].Role != "system" || !strings.Contains(msgs[1].Content, "Судзуки") || !strings.Contains(msgs[1].Content, "ГЛОССАРИЙ") {
|
||
t.Errorf("injection message wrong: %+v", msgs[1])
|
||
}
|
||
if strings.Contains(msgs[2].Content, "ГЛОССАРИЙ") {
|
||
t.Errorf("glossary leaked into the user (ch.Text) message: %q", msgs[2].Content)
|
||
}
|
||
// D30.1/WS2 §2в: the BILINGUAL editor gets the CONFIRMED canon as a src→dst MAPPING under its
|
||
// OWN header (КАНОНИЧЕСКИЕ), binding each canonical form to its source term (homonym-safe) — NOT
|
||
// the translator's ГЛОССАРИЙ header, and never an AMBIGUOUS row.
|
||
editorSaw := false
|
||
for _, b := range rec.all() {
|
||
if !isEditBody(b) {
|
||
continue
|
||
}
|
||
em := bodyMessages(t, b)
|
||
if len(em) != 3 || em[1].Role != "system" {
|
||
t.Fatalf("editor must have 3 messages (system, dst-constraints, user), got %d: %+v", len(em), em)
|
||
}
|
||
if !strings.Contains(em[1].Content, "КАНОНИЧЕСКИЕ ПЕРЕВОДЫ") || !strings.Contains(em[1].Content, "鈴木 → «Судзуки»") {
|
||
t.Errorf("editor src→dst constraint block missing/wrong: %q", em[1].Content)
|
||
}
|
||
if strings.Contains(em[1].Content, "ГЛОССАРИЙ") {
|
||
t.Errorf("editor block must use its own КАНОНИЧЕСКИЕ header, not the translator ГЛОССАРИЙ: %q", em[1].Content)
|
||
}
|
||
editorSaw = true
|
||
}
|
||
if !editorSaw {
|
||
t.Error("editor stage recorded no request")
|
||
}
|
||
|
||
// retrieval-state: one exact hit, post-check ran with 0 misses.
|
||
rs, err := r.Store.GetRetrievalState("test-book", 1, 0)
|
||
if err != nil || rs == nil {
|
||
t.Fatalf("retrieval-state missing: %v", err)
|
||
}
|
||
if rs.NExactHits != 1 || rs.NPostcheckMiss != 0 {
|
||
t.Errorf("retrieval-state wrong: exact=%d miss=%d", rs.NExactHits, rs.NPostcheckMiss)
|
||
}
|
||
callsRun1 := rec.count()
|
||
r.Close()
|
||
|
||
// Resume: no new provider calls, $0 (the deterministic memory path re-derives for free).
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != callsRun1 || res2.TotalUSD != 0 {
|
||
t.Fatalf("resume must be $0 with no new calls: calls=%d (run1=%d) usd=%v", rec.count(), callsRun1, res2.TotalUSD)
|
||
}
|
||
// retrieval-state re-derived identically.
|
||
rs2, _ := r2.Store.GetRetrievalState("test-book", 1, 0)
|
||
if rs2 == nil || rs2.NExactHits != 1 || rs2.NPostcheckMiss != 0 {
|
||
t.Errorf("retrieval-state not stable on resume: %+v", rs2)
|
||
}
|
||
}
|
||
|
||
func TestRunnerMemoryFlaggerRecordsMissWithoutFlagging(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ.", "stop"
|
||
}
|
||
return "Некто пошёл в библиотеку.", "stop" // DROPS the approved name → a post-check miss
|
||
})
|
||
defer srv.Close()
|
||
// Default flagger mode (postcheckGate: false).
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// Flagger mode: the chunk is NOT flagged (translation still exported), but the miss IS
|
||
// recorded (silent degradation converted to a loud, visible record).
|
||
if res.Flagged != 0 {
|
||
t.Errorf("flagger mode must not flag the chunk: flagged=%d", res.Flagged)
|
||
}
|
||
rs, _ := r.Store.GetRetrievalState("test-book", 1, 0)
|
||
if rs == nil || rs.NPostcheckMiss != 1 {
|
||
t.Fatalf("post-check miss not recorded: %+v", rs)
|
||
}
|
||
if !strings.Contains(rs.PostcheckDetail, "鈴木") {
|
||
t.Errorf("post-check detail should name the missed src: %q", rs.PostcheckDetail)
|
||
}
|
||
}
|
||
|
||
func TestRunnerMemoryPostcheckGateFlags(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ.", "stop"
|
||
}
|
||
return "Некто пошёл в библиотеку.", "stop" // drops the name
|
||
})
|
||
defer srv.Close()
|
||
// Opt-in hard gate.
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed, postcheckGate: true})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res.Flagged != 1 {
|
||
t.Fatalf("gate mode must flag the missed chunk: flagged=%d", res.Flagged)
|
||
}
|
||
if res.Chunks[0].FlagReason != FlagGlossaryMiss {
|
||
t.Errorf("flag reason = %q, want glossary_miss", res.Chunks[0].FlagReason)
|
||
}
|
||
// The post-check runs on the FINAL exported output, so both stages ran (each ok in
|
||
// chunk_status), but the CHUNK is flagged and its text is not exported.
|
||
if res.Chunks[0].FinalText != "" {
|
||
t.Errorf("a glossary-flagged chunk must not export text, got %q", res.Chunks[0].FinalText)
|
||
}
|
||
var editorRan bool
|
||
for _, st := range res.Chunks[0].Stages {
|
||
if st.Role == "editor" && st.Disposition == DispOK {
|
||
editorRan = true
|
||
}
|
||
}
|
||
if !editorRan {
|
||
t.Error("editor stage should have run (post-check validates the final/editor output, not the draft)")
|
||
}
|
||
}
|
||
|
||
// TestRunnerGateIgnoresAmbiguousMiss covers external-review major #1: with the hard gate
|
||
// ON, a model that renders the APPROVED term but drops an unverified DRAFT candidate must
|
||
// NOT have its (correct) translation discarded — only a CONFIRMED miss flags.
|
||
const suzukiPlusDraftSeed = `
|
||
terms:
|
||
- src: 鈴木
|
||
dst: Судзуки
|
||
status: approved
|
||
decl: { invariant: true, forms: ["Судзуки"] }
|
||
- src: 田中
|
||
dst: Танака
|
||
status: draft
|
||
decl: { invariant: true, forms: ["Танака"] }
|
||
`
|
||
|
||
func TestRunnerGateIgnoresAmbiguousMiss(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ про Судзуки.", "stop"
|
||
}
|
||
// Renders the APPROVED name, DROPS the draft candidate (its right to reject it).
|
||
return "Судзуки пошёл в библиотеку, а спутник задержался.", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1,
|
||
source: "鈴木と田中が図書館へ行った。", glossarySeed: suzukiPlusDraftSeed, postcheckGate: true})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// The chunk is NOT flagged: the only miss (田中/Танака) is an AMBIGUOUS candidate.
|
||
if res.Flagged != 0 {
|
||
t.Fatalf("gate flipped on an ambiguous-only miss (inverts the contract): flagged=%d", res.Flagged)
|
||
}
|
||
rs, _ := r.Store.GetRetrievalState("test-book", 1, 0)
|
||
if rs == nil || rs.NPostcheckMiss != 0 {
|
||
t.Fatalf("confirmed-miss count should be 0 (only an ambiguous miss occurred): %+v", rs)
|
||
}
|
||
// The ambiguous miss is still visible in the detail (A2 forced-post-check observability).
|
||
if rs.PostcheckDetail == "" || !strings.Contains(rs.PostcheckDetail, "田中") {
|
||
t.Errorf("ambiguous miss should be recorded in the detail: %q", rs.PostcheckDetail)
|
||
}
|
||
}
|
||
|
||
func TestRunnerMemoryResnapshotOnApprovedChange(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ про Судзуки.", "stop"
|
||
}
|
||
return "Судзуки пошёл в библиотеку.", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
callsAfterRun1 := rec.count()
|
||
|
||
// Change the APPROVED dst in the seed → the frozen materialization changes → the
|
||
// snapshot changes → a resume without --resnapshot must fail loud (F1: no silent
|
||
// divergent re-pay).
|
||
seedPath := filepath.Join(filepath.Dir(bookPath), "glossary-seed.yaml")
|
||
if err := os.WriteFile(seedPath, []byte(strings.Replace(suzukiSeed, "Судзуки", "Сузуки", -1)), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r2 := newRunner(t, bookPath)
|
||
_, err := r2.TranslateBook(ctx)
|
||
r2.Close()
|
||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||
t.Fatalf("changed approved glossary must fail loud mentioning --resnapshot, got: %v", err)
|
||
}
|
||
if rec.count() != callsAfterRun1 {
|
||
t.Fatalf("denied resume must not call the provider: calls=%d (after run1=%d)", rec.count(), callsAfterRun1)
|
||
}
|
||
|
||
// With --resnapshot the book re-translates under the new approved glossary.
|
||
r3 := newRunner(t, bookPath)
|
||
defer r3.Close()
|
||
r3.Resnapshot = true
|
||
if _, err := r3.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() <= callsAfterRun1 {
|
||
t.Fatal("resnapshot run must re-call the provider for the re-translation")
|
||
}
|
||
}
|
||
|
||
// TestEditorPromptIsBilingual pins the D30.1 flip (D1 monolingual → BILINGUAL editor,
|
||
// supersede D17.в): exp12 proved the monolingual editor is both passive and structurally
|
||
// blind to draft distortions (it cannot know «ударил головой» is a 磕头 kowtow without the
|
||
// source). The default editor.md now sees BOTH the source ({{text}}) and the draft
|
||
// ({{draft}}). The monolingual variant is PRESERVED as prompts/editor-mono.md (the D13.1
|
||
// confirming pilot arm) — it must NOT reference {{text}}.
|
||
func TestEditorPromptIsBilingual(t *testing.T) {
|
||
raw, err := os.ReadFile(filepath.Join("..", "..", "prompts", "editor.md"))
|
||
if err != nil {
|
||
t.Fatalf("read editor.md: %v", err)
|
||
}
|
||
if !strings.Contains(string(raw), "{{text}}") {
|
||
t.Error("editor.md must reference {{text}} (D30.1: the editor is now BILINGUAL — it sees the source)")
|
||
}
|
||
if !strings.Contains(string(raw), "{{draft}}") {
|
||
t.Error("editor.md must reference {{draft}} (it edits the draft)")
|
||
}
|
||
mono, err := os.ReadFile(filepath.Join("..", "..", "prompts", "editor-mono.md"))
|
||
if err != nil {
|
||
t.Fatalf("read editor-mono.md: %v (D30.1: the monolingual variant must be preserved for the D13.1 pilot arm)", err)
|
||
}
|
||
if strings.Contains(string(mono), "{{text}}") {
|
||
t.Error("editor-mono.md must NOT reference {{text}} (it is the monolingual variant)")
|
||
}
|
||
if !strings.Contains(string(mono), "{{draft}}") {
|
||
t.Error("editor-mono.md must reference {{draft}}")
|
||
}
|
||
}
|