89 lines
3.4 KiB
Go
89 lines
3.4 KiB
Go
package config
|
||
|
||
import (
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// prompt_lint_test.go: a data lint over the SHIPPED prompt packs of every pair. It exists because of a
|
||
// cold-run measurement, and it lints the class rather than the instance.
|
||
//
|
||
// What happened: `translator-banknote.md` spelled the wire format as
|
||
// `исходный_термин<TAB>предлагаемый_перевод<TAB>тип`, and the model wrote the six characters `<TAB>`
|
||
// instead of a tab. Every line of that block then had one field instead of three, so the whole densest
|
||
// block of the run was rejected — 20 of 135 lines, including a term the owner had signed by hand.
|
||
//
|
||
// The root cause is not the parser and not that model: it is a prompt that shows a control character as a
|
||
// word a generator can copy. The fix that generalises is therefore here, over the DATA, and over every
|
||
// pair — a ja-ru or en-ru pack authored by copying the zh-ru one would otherwise inherit the same defect
|
||
// silently, and the engine cannot see it (a prompt is opaque bytes to it).
|
||
func TestPromptPacksNameNoControlCharacterPlaceholders(t *testing.T) {
|
||
// Tokens that name a control character instead of being one. A prompt template uses {{...}} for
|
||
// substitution, so an angle-bracketed token is never legitimate content here.
|
||
banned := []string{"<tab>", "<newline>", "<nl>", "<cr>", "<lf>", "<space>", "<таб>", "<табуляция>"}
|
||
|
||
root := filepath.Join("..", "..", "prompts")
|
||
seen := 0
|
||
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if info.IsDir() || !strings.HasSuffix(path, ".md") {
|
||
return nil
|
||
}
|
||
b, rerr := os.ReadFile(path)
|
||
if rerr != nil {
|
||
return rerr
|
||
}
|
||
seen++
|
||
low := strings.ToLower(string(b))
|
||
for _, tok := range banned {
|
||
if strings.Contains(low, tok) {
|
||
t.Errorf("%s writes %q as a literal token: a model copies it verbatim and the field split then sees one field instead of three (measured: a whole banknote block lost). Put the real character in the example line instead.", path, tok)
|
||
}
|
||
}
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if seen == 0 {
|
||
t.Fatal("the lint walked no prompt files — a moved prompts root would make this test silently vacuous")
|
||
}
|
||
t.Logf("linted %d prompt files across every pair pack", seen)
|
||
}
|
||
|
||
// TestBanknotePromptShowsARealTab is the positive half: the banknote prompt of every pair that ships one
|
||
// must demonstrate the delimiter by USING it. Checking only the absence of the bad token would pass on a
|
||
// prompt that describes the format in words and shows no example at all — which is how the ambiguity
|
||
// came back the first time.
|
||
func TestBanknotePromptShowsARealTab(t *testing.T) {
|
||
root := filepath.Join("..", "..", "prompts")
|
||
pairs, err := os.ReadDir(root)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
checked := 0
|
||
for _, p := range pairs {
|
||
if !p.IsDir() {
|
||
continue
|
||
}
|
||
path := filepath.Join(root, p.Name(), "translator-banknote.md")
|
||
b, err := os.ReadFile(path)
|
||
if os.IsNotExist(err) {
|
||
continue // a pair without the channel ships no such prompt — legitimate
|
||
}
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
checked++
|
||
if !strings.Contains(string(b), "\t") {
|
||
t.Errorf("%s asks for a tab-delimited table but contains no tab character — the format has to be SHOWN, not named", path)
|
||
}
|
||
}
|
||
if checked == 0 {
|
||
t.Skip("no pair ships a banknote prompt")
|
||
}
|
||
}
|