textmachine/backend/internal/pipeline/contourpreflight_test.go

486 lines
22 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package pipeline
import (
"bytes"
"errors"
"io"
"log/slog"
"os"
"path/filepath"
"strings"
"testing"
"textmachine/backend/internal/store"
)
// contourpreflight_test.go: the WRITE-path half of "a configured bank contour cannot silently do nothing"
// (backlog row 140). The loader judges the KEY; whether the artifact is on this machine, and whether the
// book has a langpack at all, are facts only the runner has — and both decide whether a single bank-role
// call can ever happen.
//
// ⚠ BOTH GUARDS WERE UNPINNED AFTER BEING BUILT, which the acceptance found by planting: deleting either
// refusal left the full config, pipeline and tmctl suites green. The cost of the hole is precise: the run
// opens the contrast artifact at the bank-mining stop, which is AFTER the whole draft wave is bought, so a
// deployment that shipped the configs and forgot the artifact pays for a wave to be told.
// TestTheContourRefusesAWriteRunItCannotExecute covers both halves and, as importantly, the NEGATIVE: a $0
// read must stay possible on exactly the book a paid run refuses. A guard that also blocked `status` would
// be the D20.4 mistake — a book must stay inspectable on a host that cannot run it.
func TestTheContourRefusesAWriteRunItCannotExecute(t *testing.T) {
for _, tc := range []struct {
name string
mutate func(t *testing.T, bookPath string)
want []string
}{
{
// The artifact the deployment provides and git does not.
name: "the contrast artifact is not on this machine",
mutate: func(t *testing.T, bookPath string) {
if err := os.Remove(filepath.Join(filepath.Dir(bookPath), "mining-contrast.txt")); err != nil {
t.Fatal(err)
}
},
want: []string{"mining.contrast_path", "not readable", "draft wave"},
},
{
// The OTHER condition of the same early return, and the one a config loader structurally cannot
// see: a book with no langpack gets a nil pack and the stop returns before any candidate exists.
name: "the book has no langpack to mine with",
mutate: func(t *testing.T, bookPath string) {
raw, err := os.ReadFile(bookPath)
if err != nil {
t.Fatal(err)
}
var kept []string
for _, ln := range strings.Split(string(raw), "\n") {
if !strings.HasPrefix(ln, "langpack_root:") {
kept = append(kept, ln)
}
}
if err := os.WriteFile(bookPath, []byte(strings.Join(kept, "\n")), 0o644); err != nil {
t.Fatal(err)
}
},
want: []string{"cannot mine a bank", "langpack_root", "never executed"},
},
} {
t.Run(tc.name, func(t *testing.T) {
srv := newJSONProvider(&reqRec{}, draftEdit)
defer srv.Close()
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
// Premise: BEFORE the mutation the same book opens for writing. Without this the test would
// pass on a fixture that was broken for some entirely different reason.
r, err := NewRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatalf("premise broken: the intact fixture must open for writing: %v", err)
}
r.Close()
tc.mutate(t, bookPath)
if _, err := NewRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil))); err == nil {
t.Fatal("a run that cannot execute one line of its configured contour must refuse BEFORE it buys a wave")
} else {
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal must name the cure, missing %q: %v", want, err)
}
}
var refusal *Refusal
if !errors.As(err, &refusal) || refusal.Class != RefusalBadConfig {
t.Errorf("a preflight refusal is a CONFIG refusal, not a crash: %T %v", err, err)
}
}
// ⛔ AND THE $0 READ STILL WORKS. The same book, the same defect, the read-only door.
ro, err := NewReadOnlyRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatalf("a book must stay inspectable on a host that cannot run it (D20.4): %v", err)
}
ro.Close()
})
}
}
// TestTheBanknoteChannelSaysWhenNobodyWillReadIt is the sibling of the refusals above, and it is a WARNING
// because it is a different fault: the run is correct and merely wasteful. The gate's own job — slicing the
// ⟦TM-BANK-v1⟧ block off the draft so it never reaches the editor or the export — is done either way; what
// is bought and unread sits upstream of it, in the pair's prompt asking the translator for a term table on
// every call. Its only consumer is the bank-mining stop, and a book with no langpack or no contrast artifact
// returns from that stop before reading anything.
//
// Found by planting: the class the two refusals cover has this neighbour, and neither guard keys on it.
func TestTheBanknoteChannelSaysWhenNobodyWillReadIt(t *testing.T) {
srv := newJSONProvider(&reqRec{}, draftEdit)
defer srv.Close()
// The banknote gate ON, and NO mining inputs — the shape the fixtures of the banknote channel take.
bookPath := setupProjectOpts(t, srv.URL, projectOpts{banknote: true})
var log bytes.Buffer
r, err := NewRunner(bookPath, slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{Level: slog.LevelInfo})))
if err != nil {
t.Fatalf("this configuration is legal and must still open — it is wasteful, not broken: %v", err)
}
defer r.Close()
out := log.String()
for _, want := range []string{"banknote channel is ON", "billed", "mining.contrast_path"} {
if !strings.Contains(out, want) {
t.Errorf("the warning must name what is bought and what would fix it, missing %q:\n%s", want, out)
}
}
// ⛔ AND IT MUST NOT FIRE ON A BOOK THAT CAN READ THEM. A warning that cries on the healthy configuration
// is a warning an operator learns to ignore, which is worse than none.
var quiet bytes.Buffer
full := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})
r2, err := NewRunner(full, slog.New(slog.NewTextHandler(&quiet, &slog.HandlerOptions{Level: slog.LevelInfo})))
if err != nil {
t.Fatal(err)
}
defer r2.Close()
if strings.Contains(quiet.String(), "banknote channel is ON") {
t.Errorf("a book that CAN reach the bank-mining stop must not be warned:\n%s", quiet.String())
}
}
// TestEveryPromptLoadPathRefusesAnUnrenderableTemplate is the load gate asserted PER PATH, and the reason it
// exists is a planting result rather than a worry: every one of the four paths could be routed around the
// check without a single test noticing. The gate was pinned only as a unit property of LoadPromptTemplate —
// "the check stands once and covers all of them" rested on the funnel's SHAPE, and a shape is not a
// guarantee. Four paths load a prompt (a stage, a repair class, and each bank role) and each is a place a
// future edit could read a file directly.
//
// The cost of the hole is the one the gate exists to remove: the refusal returns to Render, mid-run, after
// the wave that stage sits behind has been bought.
func TestEveryPromptLoadPathRefusesAnUnrenderableTemplate(t *testing.T) {
const bad = "Роль {{chapter_title}}.\n---USER---\n{{text}}"
for _, tc := range []struct {
name string
setup func(t *testing.T, providerURL string) string
file []string // path segments under the book's directory
}{
// The stage fixture points its stages at `prompts/<role>.md` through prompt_override, which is the
// second of the two ways a stage prompt is resolved and therefore the one worth exercising here.
{"a stage prompt", func(t *testing.T, u string) string {
return setupProjectOpts(t, u, projectOpts{})
}, []string{"prompts", "translator.md"}},
{"the terminologist's prompt", func(t *testing.T, u string) string {
return setupMiningStopProject(t, u, miningStopOpts{terminology: true})
}, []string{"prompts", "zh-ru", "terminologist.md"}},
{"the classifier's prompt", func(t *testing.T, u string) string {
return setupMiningStopProject(t, u, miningStopOpts{terminology: true, classify: true})
}, []string{"prompts", "zh-ru", "classifier.md"}},
{"a repair class prompt", setupRepairProject, []string{"prompts", "zh-ru", "repair", "dc1_fractional.md"}},
} {
t.Run(tc.name, func(t *testing.T) {
srv := newJSONProvider(&reqRec{}, draftEdit)
defer srv.Close()
bookPath := tc.setup(t, srv.URL)
// Premise: the intact fixture opens. Without it a refusal proves nothing about the placeholder.
r, err := NewRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatalf("premise broken: the intact fixture must open: %v", err)
}
r.Close()
path := filepath.Join(append([]string{filepath.Dir(bookPath)}, tc.file...)...)
if _, err := os.Stat(path); err != nil {
t.Fatalf("the fixture must actually lay down %s: %v", strings.Join(tc.file, "/"), err)
}
if err := os.WriteFile(path, []byte(bad), 0o644); err != nil {
t.Fatal(err)
}
_, err = NewRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err == nil {
t.Fatal("a template the engine cannot render must be refused at LOAD, on THIS path, before the run buys anything")
}
for _, want := range []string{"{{chapter_title}}", tc.file[len(tc.file)-1]} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal must name the marker and the file, missing %q: %v", want, err)
}
}
})
}
}
// TestTheBanknoteWarningKeysOnEachConditionSeparately closes a WB41-class hole in the test above: its
// positive fixture has NEITHER mining input and its negative has BOTH, so every one of the guard's parts
// could be deleted and the pair still passed. Planted separately, three of them survived the whole package —
// dropping the pack half of the disjunction, dropping the contrast half, and dropping `forWrite` entirely.
//
// The guard is `forWrite && banknote.Enabled && (pack == nil || contrast == "")`. A disjunction pinned only
// at "neither" and "both" is pinned at neither of its arms.
func TestTheBanknoteWarningKeysOnEachConditionSeparately(t *testing.T) {
packRoot, err := filepath.Abs(filepath.Join("..", "..", "configs", "langpacks"))
if err != nil {
t.Fatal(err)
}
open := func(t *testing.T, bookPath string, readOnly bool) string {
t.Helper()
var log bytes.Buffer
h := slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{Level: slog.LevelInfo}))
var r *Runner
var oerr error
if readOnly {
r, oerr = NewReadOnlyRunner(bookPath, h)
} else {
r, oerr = NewRunner(bookPath, h)
}
if oerr != nil {
t.Fatalf("this configuration is legal and must open — it is wasteful, not broken: %v", oerr)
}
r.Close()
return log.String()
}
// ARM 1 — a pack, but no contrast artifact. The bank-mining stop returns before reading anything, so
// the banknote tokens are bought and unread; the pack half of the disjunction must not be what decides.
t.Run("a pack but no contrast artifact", func(t *testing.T) {
srv := newJSONProvider(&reqRec{}, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{banknote: true})
raw, rerr := os.ReadFile(bookPath)
if rerr != nil {
t.Fatal(rerr)
}
body := strings.Replace(string(raw), "source_lang: ja", "source_lang: zh\nlangpack_root: "+packRoot, 1)
if !strings.Contains(body, "langpack_root:") {
t.Fatalf("premise broken: the fixture's book.yaml no longer carries the anchor this test edits:\n%s", raw)
}
writeFile(t, bookPath, body)
if out := open(t, bookPath, false); !strings.Contains(out, "banknote channel is ON") {
t.Errorf("a book with a pack and NO contrast artifact still cannot reach the stop, and still pays "+
"for the term table on every draft call — it must be warned:\n%s", out)
}
})
// ARM 2 — a contrast artifact, but no pack. The mirror image, and the one a config loader structurally
// cannot see: the pack is book data, not pipeline data.
t.Run("a contrast artifact but no pack", func(t *testing.T) {
srv := newJSONProvider(&reqRec{}, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{banknote: true,
gatesYAML: "\nmining:\n contrast_path: mining-contrast.txt\n"})
dir := filepath.Dir(bookPath)
writeFile(t, filepath.Join(dir, "mining-contrast.txt"), miningContrastData)
if out := open(t, bookPath, false); !strings.Contains(out, "banknote channel is ON") {
t.Errorf("a book with a contrast artifact and NO langpack cannot mine either, and must be warned:\n%s", out)
}
})
// ARM 3 — `forWrite`. A $0 read path buys nothing, so warning there teaches an operator to ignore the
// line on the run where it matters. Planted, dropping the guard entirely stayed green.
t.Run("a read-only open is quiet", func(t *testing.T) {
srv := newJSONProvider(&reqRec{}, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{banknote: true})
// Premise: this very book WOULD be warned on a write open — otherwise the silence below proves nothing.
if out := open(t, bookPath, false); !strings.Contains(out, "banknote channel is ON") {
t.Fatalf("premise broken: the write open must warn for this fixture:\n%s", out)
}
if out := open(t, bookPath, true); strings.Contains(out, "banknote channel is ON") {
t.Errorf("a read-only open spends nothing, so it must not warn about what a write run would buy:\n%s", out)
}
})
}
// retargetToASilentTarget turns a fixture book into a book of a pair the ENGINE is complete for and the
// injection TEXTS are not: source zh (a real langpack), target `xx` (no rows in lang/data/injection.txt).
// The point is that everything else passes — the pair has a prompt pack, the langpack resolves, both
// refusals beside this one stay quiet — so what the guard under test refuses is its own subject and not a
// neighbour's fixture defect.
//
// The synthetic pair's langpack is the real zh source directory and a `zh-xx` pair directory that IS the
// real `zh-ru` one: the pack loader requires the transliteration files for a source that declares no
// manifest, and copying rather than inventing them keeps the fixture about the target's TEXTS.
func retargetToASilentTarget(t *testing.T, bookPath string) {
t.Helper()
dir := filepath.Dir(bookPath)
realPacks, err := filepath.Abs(filepath.Join("..", "..", "configs", "langpacks"))
if err != nil {
t.Fatal(err)
}
packRoot := filepath.Join(dir, "langpacks-xx")
if err := os.MkdirAll(packRoot, 0o755); err != nil {
t.Fatal(err)
}
for _, l := range [][2]string{{"zh", "zh"}, {"zh-ru", "zh-xx"}} {
if err := os.Symlink(filepath.Join(realPacks, l[0]), filepath.Join(packRoot, l[1])); err != nil {
t.Fatal(err)
}
}
// The pair's prompt pack, by the same convention production takes (`<prompts root>/<pair>/<role>.md`).
writeFile(t, filepath.Join(dir, "pairs", "zh-xx.yaml"), "pair: zh-xx\nprompts_root: ../prompts\n")
writeFile(t, filepath.Join(dir, "prompts", "zh-xx", "terminologist.md"),
"Ты — терминолог {{source_lang}}→{{target_lang}}.\n---USER---\nТермины книги:\n\n{{text}}")
writeFile(t, filepath.Join(dir, "prompts", "zh-xx", "classifier.md"),
"Ты — классификатор {{source_lang}}→{{target_lang}}.\n---USER---\nКлассы терминов:\n\n{{text}}")
raw, err := os.ReadFile(bookPath)
if err != nil {
t.Fatal(err)
}
body := string(raw)
for _, sub := range [][2]string{{"target_lang: ru", "target_lang: xx"}} {
if !strings.Contains(body, sub[0]) {
t.Fatalf("premise broken: the fixture book no longer carries %q, so this helper would leave a ru book and every assertion below would be vacuous:\n%s", sub[0], body)
}
body = strings.Replace(body, sub[0], sub[1], 1)
}
var out []string
replaced := false
for _, ln := range strings.Split(body, "\n") {
if strings.HasPrefix(ln, "langpack_root:") {
ln, replaced = "langpack_root: "+packRoot, true
}
out = append(out, ln)
}
if !replaced {
out = append(out, "langpack_root: "+packRoot)
}
writeFile(t, bookPath, strings.Join(out, "\n"))
}
// TestABankItsTargetCannotStateRefusesBeforeItIsBought is the third member of the contour preflight, and
// the one whose silence lives in the TARGET-LANGUAGE DATA rather than in the book's files: both memory
// blocks are gated on the target's rows in lang/data/injection.txt, and lang.InjectionTextsFor answers for
// an unknown target with a ZERO VALUE and no error. A book of such a target buys every bank role it has,
// stops for a signature, and injects zero bytes into every call — nothing red, nothing logged.
//
// ⛔ THE ARMS ARE SEPARATE SUBTESTS ON PURPOSE (the WB57-class lesson beside this file): the condition is a
// disjunction over the four doors bank rows come through, and a disjunction pinned only at «all» and at
// «none» is pinned at neither of its arms. Each arm below is the ONLY door open in its fixture.
func TestABankItsTargetCannotStateRefusesBeforeItIsBought(t *testing.T) {
// A bank row an earlier run could have left in the store: the fourth door, and the only one that is not
// in any config file.
storedRow := func(t *testing.T, bookPath string) {
t.Helper()
st, err := store.Open(filepath.Join(filepath.Dir(bookPath), "test-book.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
if err := st.ReplaceGlossary("test-book", []store.GlossaryEntry{{
BookID: "test-book", Src: "方源", Dst: "Фан Юань", Type: "name",
Status: "auto", Source: "auto", SinceCh: 1, UntilCh: 0,
}}); err != nil {
t.Fatal(err)
}
}
for _, tc := range []struct {
name string
opts miningStopOpts
// door opens the ONE door this arm is about, after the fixture is built.
door func(t *testing.T, bookPath string)
// stripGates removes the gate block the mining-stop fixture writes, so the arm's door is the only one.
stripGates bool
}{
{
name: "the terminology gate is on",
opts: miningStopOpts{terminology: true},
},
{
name: "only the banknote gate is on",
opts: miningStopOpts{}, // the helper's gates block carries banknote alone
},
{
name: "only a glossary seed, both gates off",
opts: miningStopOpts{glossarySeed: "terms:\n - src: 方源\n dst: Фан Юань\n type: name\n status: approved\n"},
stripGates: true,
},
{
name: "only rows an earlier run left in the store",
opts: miningStopOpts{},
door: storedRow,
stripGates: true,
},
{
// The owner's own signed rows, the door that is neither a gate nor the seed: the delta is
// DERIVED beside book.yaml, so it is not declared anywhere a config check could see.
name: "only the owner's mined-delta",
opts: miningStopOpts{},
door: func(t *testing.T, bookPath string) {
t.Helper()
writeFile(t, filepath.Join(filepath.Dir(bookPath), "test-book.mined-delta.yaml"),
"terms:\n - src: 方源\n dst: Фан Юань\n type: name\n status: approved\n")
},
stripGates: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
srv := newJSONProvider(&reqRec{}, draftEdit)
defer srv.Close()
bookPath := setupMiningStopProject(t, srv.URL, tc.opts)
if tc.stripGates {
stripGateBlock(t, bookPath)
}
// PREMISE, and it is the whole reason this fixture is built from the mining-stop helper: with a
// target the engine HAS texts for, this very book opens for writing. Without it the refusal below
// could be any of the three guards in this file, or the pack loader, or the prompt convention.
r, err := NewRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatalf("premise broken: the same book with a target that HAS injection texts must open: %v", err)
}
r.Close()
if tc.door != nil {
tc.door(t, bookPath)
}
retargetToASilentTarget(t, bookPath)
_, err = NewRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err == nil {
t.Fatal("a run whose bank cannot reach a single request must refuse BEFORE it buys the roles that fill it")
}
// The message has to carry the target, BOTH missing rows (a dropped arm of roleInjectionKeys
// shows up here and nowhere else), the file to edit, and the price of editing it.
for _, want := range []string{"xx", "glossary_header", "editor_header", "injection.txt", "resnapshot", "ZERO BYTES"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal must name the target, the missing rows and the cure, missing %q: %v", want, err)
}
}
var refusal *Refusal
if !errors.As(err, &refusal) || refusal.Class != RefusalBadConfig {
t.Errorf("a preflight refusal is a CONFIG refusal, not a crash: %T %v", err, err)
}
// ⛔ AND THE $0 READ STILL WORKS — the D20.4 line the two guards above hold.
ro, err := NewReadOnlyRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatalf("a book must stay inspectable on a host that cannot run it (D20.4): %v", err)
}
ro.Close()
})
}
// ⛔ AND THE NEGATIVE THE GUARD LIVES OR DIES BY: a target with no injection texts and NO bank at all is
// a legal book. Nothing is silent there — there is no bank to say — and refusing it would block a run
// that works. This arm is what stops the condition from being «the target has no texts».
t.Run("no bank at all is not refused", func(t *testing.T) {
srv := newJSONProvider(&reqRec{}, draftEdit)
defer srv.Close()
bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{})
stripGateBlock(t, bookPath)
retargetToASilentTarget(t, bookPath)
r, err := NewRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatalf("a book with no bank of any kind must still run into a target the engine has no texts for: %v", err)
}
r.Close()
})
}
// stripGateBlock removes the `gates:` block the mining-stop fixture authors, leaving the book's OTHER bank
// doors as the only ones open. It fails loudly when there is nothing to strip: a helper that quietly did
// nothing would turn every arm below it into a test of the gate arm.
func stripGateBlock(t *testing.T, bookPath string) {
t.Helper()
dir := filepath.Dir(bookPath)
raw, err := os.ReadFile(filepath.Join(dir, "pipeline.yaml"))
if err != nil {
t.Fatal(err)
}
body := string(raw)
i := strings.Index(body, "gates:")
if i < 0 {
t.Fatalf("premise broken: the fixture pipeline.yaml carries no `gates:` block to strip:\n%s", body)
}
writeFile(t, filepath.Join(dir, "pipeline.yaml"), body[:i])
}