147 lines
6.5 KiB
Go
147 lines
6.5 KiB
Go
package config
|
||
|
||
import (
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// A named variant resolves from the BOOK's pair: `<prompts root>/<pair>/<role>-<variant>.md`.
|
||
func TestPromptVariantResolvesInsideTheBooksPair(t *testing.T) {
|
||
dir := promptProject(t, "zh-ru", "en-ru")
|
||
for _, pair := range []string{"zh-ru", "en-ru"} {
|
||
writeTmp(t, filepath.Join(dir, "prompts", pair, "translator-banknote.md"), "sys "+pair+"\n---USER---\n{{text}}")
|
||
}
|
||
cfg := draftOnlyConfig(t, dir, " prompt_variant: banknote\n")
|
||
for _, pair := range []string{"zh-ru", "en-ru"} {
|
||
p, err := LoadPipeline(cfg, miniModels(t), pair, nil)
|
||
if err != nil {
|
||
t.Fatalf("pair %q must load: %v", pair, err)
|
||
}
|
||
want := filepath.Join(dir, "prompts", pair, "translator-banknote.md")
|
||
if p.Stages[0].PromptPath != want {
|
||
t.Errorf("pair %q resolved %q, want %q", pair, p.Stages[0].PromptPath, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// A missing variant stops the load naming pair, role and variant; a fall-through to the plain role
|
||
// prompt would be a stage running a prompt nobody chose.
|
||
func TestPromptVariantMissingFileFailsLoudNamingPairAndVariant(t *testing.T) {
|
||
dir := promptProject(t, "zh-ru")
|
||
cfg := draftOnlyConfig(t, dir, " prompt_variant: nosuch\n")
|
||
_, err := LoadPipeline(cfg, miniModels(t), "zh-ru", nil)
|
||
if err == nil {
|
||
t.Fatal("a missing variant file must fail loud")
|
||
}
|
||
// `variant %q` verbatim, not the bare token: "nosuch" is also a substring of the PRINTED PATH, so the
|
||
// variant half of this assertion would be satisfied by the path even if the message never named a variant.
|
||
for _, want := range []string{"zh-ru", "translator", `variant "nosuch"`, filepath.Join("prompts", "zh-ru", "translator-nosuch.md")} {
|
||
if !strings.Contains(err.Error(), want) {
|
||
t.Errorf("error must name %q, got: %v", want, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Both keys set is a config error, not a precedence puzzle: precedence would make the loser silently dead.
|
||
func TestPromptVariantAndOverrideAreMutuallyExclusive(t *testing.T) {
|
||
dir := promptProject(t, "zh-ru")
|
||
writeTmp(t, filepath.Join(dir, "prompts", "zh-ru", "translator-banknote.md"), "sys\n---USER---\n{{text}}")
|
||
own := writeTmp(t, filepath.Join(dir, "own.md"), "sys\n---USER---\n{{text}}")
|
||
cfg := draftOnlyConfig(t, dir, " prompt_variant: banknote\n prompt_override: "+own+"\n")
|
||
_, err := LoadPipeline(cfg, miniModels(t), "zh-ru", nil)
|
||
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
||
t.Errorf("setting both keys must fail loud, got: %v", err)
|
||
}
|
||
}
|
||
|
||
// A separator or `..` would make this key a path — which is what prompt_override is for.
|
||
func TestPromptVariantMustBeABareName(t *testing.T) {
|
||
dir := promptProject(t, "zh-ru")
|
||
for _, bad := range []string{"../zh-ru/translator-banknote", "zh-ru/banknote", `sub\banknote`, " banknote"} {
|
||
cfg := draftOnlyConfig(t, dir, " prompt_variant: '"+bad+"'\n")
|
||
_, err := LoadPipeline(cfg, miniModels(t), "zh-ru", nil)
|
||
if err == nil || !strings.Contains(err.Error(), "bare name") {
|
||
t.Errorf("prompt_variant %q must be refused as not-a-bare-name, got: %v", bad, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestEveryShippingConfigResolvesInsideItsOwnPair: no shipping config may resolve a stage prompt outside
|
||
// the book's own pair directory. Until a second prompt pack existed the property was held by ABSENCE —
|
||
// other pairs were refused on the stages whose files were missing — so it was never asserted.
|
||
//
|
||
// ⛔ THE PREDICATE IS THE PARENT DIRECTORY, NOT A SUBSTRING, and the accounting is per (config, pair).
|
||
// A `strings.Contains(path, "/en-ru/")` test passes on `…/prompts/en-ru/../zh-ru/translator.md`, and a
|
||
// per-PAIR counter cannot tell three configs from four — a config that stops loading removes itself from
|
||
// the gate in silence. Both were live holes in the first version of this pin.
|
||
func TestEveryShippingConfigResolvesInsideItsOwnPair(t *testing.T) {
|
||
configsDir := filepath.Join("..", "..", "configs")
|
||
promptsDir := filepath.Join("..", "..", "prompts")
|
||
models, err := LoadModels(filepath.Join(configsDir, "models.yaml"))
|
||
if err != nil {
|
||
t.Fatalf("load models: %v", err)
|
||
}
|
||
cfgs, err := filepath.Glob(filepath.Join(configsDir, "pipeline-*.yaml"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
sort.Strings(cfgs)
|
||
if len(cfgs) < 4 {
|
||
t.Fatalf("premise broken: found %d shipping configs in %s, expected at least the four this repo ships", len(cfgs), configsDir)
|
||
}
|
||
ents, err := os.ReadDir(promptsDir)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var pairs []string
|
||
for _, e := range ents {
|
||
if e.IsDir() {
|
||
pairs = append(pairs, e.Name())
|
||
}
|
||
}
|
||
sort.Strings(pairs)
|
||
if len(pairs) < 2 {
|
||
t.Skipf("only %d prompt pack(s) in %s — the cross-pair property needs two to be observable", len(pairs), promptsDir)
|
||
}
|
||
|
||
// inPairDir answers the question the pin is about: is this file's PARENT the pair's directory. The path
|
||
// is cleaned first, so a `..` segment cannot smuggle another pair's file past a substring match.
|
||
inPairDir := func(p, pair string) bool {
|
||
return filepath.Base(filepath.Dir(filepath.Clean(p))) == pair
|
||
}
|
||
checked := 0
|
||
for _, cfg := range cfgs {
|
||
for _, pair := range pairs {
|
||
p, lerr := LoadPipeline(cfg, models, pair, nil)
|
||
if lerr != nil {
|
||
// NOT skipped: a shipping config that cannot serve a pair whose pack this repo carries is
|
||
// either a half-landed pack or a config that quietly left the gate. Both must be seen.
|
||
t.Errorf("%s + pair %q: does not load (%v) — a shipping config must serve every pair whose prompt pack this repo carries, or this gate stops covering it in silence",
|
||
filepath.Base(cfg), pair, lerr)
|
||
continue
|
||
}
|
||
checked++
|
||
for _, st := range p.Stages {
|
||
if !inPairDir(st.PromptPath, pair) {
|
||
t.Errorf("%s + pair %q: stage %q (role %q) resolved %q — a prompt OUTSIDE the book's own pair directory %q; a shipping config must never hand a book another pair's conventions",
|
||
filepath.Base(cfg), pair, st.Name, st.Role, st.PromptPath, pair)
|
||
}
|
||
}
|
||
for _, g := range []struct{ what, path string }{
|
||
{"gates.terminology", p.Gates.Terminology.PromptPath},
|
||
{"gates.terminology classifier", p.Gates.Terminology.ClassifyPromptPath},
|
||
} {
|
||
if g.path != "" && !inPairDir(g.path, pair) {
|
||
t.Errorf("%s + pair %q: %s prompt resolved %q — outside the pair directory %q", filepath.Base(cfg), pair, g.what, g.path, pair)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if want := len(cfgs) * len(pairs); checked != want {
|
||
t.Errorf("checked %d of %d (config, pair) combinations — the shortfall is reported above; a silent one would read as full coverage", checked, want)
|
||
}
|
||
t.Logf("checked %d (config, pair) combinations: %d configs × %d pairs", checked, len(cfgs), len(pairs))
|
||
}
|