262 lines
11 KiB
Go
262 lines
11 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/config"
|
||
"textmachine/backend/internal/llm"
|
||
"textmachine/backend/internal/text"
|
||
)
|
||
|
||
func testBook() *config.Book {
|
||
return &config.Book{
|
||
BookID: "b", Title: "T", SourceLang: "ja", TargetLang: "ru",
|
||
Audience: "взрослые", Venuti: 0.6,
|
||
Honorifics: "keep", Transcription: "polivanov", Footnotes: "minimal",
|
||
}
|
||
}
|
||
|
||
func writeTemplate(t *testing.T, content string) *PromptTemplate {
|
||
t.Helper()
|
||
path := filepath.Join(t.TempDir(), "tpl.md")
|
||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tpl, err := LoadPromptTemplate(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return tpl
|
||
}
|
||
|
||
// Invariant §3.1: render is a pure function; two renders are byte-for-byte identical.
|
||
// Without this the request-hash drifts between runs, resume re-translates and
|
||
// re-pays for the chapter, and the DeepSeek byte-prefix cache misses.
|
||
func TestRenderIsDeterministic(t *testing.T) {
|
||
tpl := writeTemplate(t, "Перевод с {{source_lang}} на {{target_lang}}, аудитория {{audience}}, venuti {{venuti}}.\n---USER---\nТекст: {{text}}\nЧерновик: {{draft}}")
|
||
v := RenderVars{Book: testBook(), Text: "исходник", Draft: "черновик"}
|
||
|
||
m1, err := Messages(tpl, v)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for i := 0; i < 100; i++ {
|
||
m2, err := Messages(tpl, v)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(m1) != len(m2) {
|
||
t.Fatal("length differs")
|
||
}
|
||
for j := range m1 {
|
||
if m1[j] != m2[j] {
|
||
t.Fatalf("render %d differs at message %d:\n%+v\nvs\n%+v", i, j, m1[j], m2[j])
|
||
}
|
||
}
|
||
}
|
||
h1 := RequestHash(baseRequest(m1))
|
||
h2 := RequestHash(baseRequest(m1))
|
||
if h1 != h2 {
|
||
t.Fatal("request hash is not stable")
|
||
}
|
||
}
|
||
|
||
// baseRequest is the reference call every request-hash fixture varies ONE field of — the shape the
|
||
// Request struct exists for: a variant now names the field it changes instead of moving a value between
|
||
// two of thirteen positional arguments.
|
||
func baseRequest(msgs []llm.Message) Request {
|
||
return Request{BookID: "b", Chapter: 1, ChunkIdx: 0, Attempt: 0, Stage: "draft", Role: "translator",
|
||
Model: "m", Temperature: 0.3, Reasoning: "off", JSONOnly: false, MaxTokens: 1024, SnapshotID: "snap",
|
||
Messages: msgs}
|
||
}
|
||
|
||
func TestRequestHashSensitivity(t *testing.T) {
|
||
msgs := []llm.Message{{Role: "system", Content: "s", CacheBoundary: true}, {Role: "user", Content: "u"}}
|
||
base := RequestHash(baseRequest(msgs))
|
||
|
||
vary := func(f func(*Request)) string {
|
||
req := baseRequest(msgs)
|
||
f(&req)
|
||
return RequestHash(req)
|
||
}
|
||
variants := map[string]string{
|
||
"model": vary(func(q *Request) { q.Model = "m2" }),
|
||
"temp": vary(func(q *Request) { q.Temperature = 0.4 }),
|
||
"snapshot": vary(func(q *Request) { q.SnapshotID = "snap2" }),
|
||
"maxTokens": vary(func(q *Request) { q.MaxTokens = 2048 }),
|
||
"chunk": vary(func(q *Request) { q.ChunkIdx = 1 }),
|
||
"content": vary(func(q *Request) {
|
||
q.Messages = []llm.Message{{Role: "system", Content: "s", CacheBoundary: true}, {Role: "user", Content: "u2"}}
|
||
}),
|
||
}
|
||
for name, h := range variants {
|
||
if h == base {
|
||
t.Errorf("hash must change when %s changes", name)
|
||
}
|
||
}
|
||
|
||
// Concatenation with a separator does not merge: ("ab","c") != ("a","bc").
|
||
a := RequestHash(baseRequest([]llm.Message{{Role: "user", Content: "ab"}, {Role: "user", Content: "c"}}))
|
||
b := RequestHash(baseRequest([]llm.Message{{Role: "user", Content: "a"}, {Role: "user", Content: "bc"}}))
|
||
if a == b {
|
||
t.Fatal("field separator is not injective")
|
||
}
|
||
}
|
||
|
||
// TestRenderRejectsUnknownPlaceholder — ⚠ THE REFUSAL MOVED EARLIER, and that is the ordered change, not a
|
||
// weakening: it used to arrive from Render, which runs per stage per chunk MID-RUN, so a prompt pack
|
||
// carrying a retired placeholder failed on the first call of that stage — for an editor prompt, after the
|
||
// whole draft wave had been bought (measured on the `{{genre}}` retirement: one chunk drafted for
|
||
// $0.001820, the edit wave started, and only then the refusal). LoadPromptTemplate now scans the canonical
|
||
// text, so the same refusal costs nothing. The test asserts BOTH: the gate at load, and that Render itself
|
||
// still refuses — a gate is defence in depth, never a replacement for the check it fronts.
|
||
func TestRenderRejectsUnknownPlaceholder(t *testing.T) {
|
||
path := filepath.Join(t.TempDir(), "tpl.md")
|
||
if err := os.WriteFile(path, []byte("Привет {{nonexistent}}\n---USER---\n{{text}}"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
_, err := LoadPromptTemplate(path)
|
||
if err == nil {
|
||
t.Fatal("a template the engine cannot render must be refused at LOAD, before the run buys anything")
|
||
}
|
||
// The refusal names the marker AND what the engine does accept: an author who has to guess the closed
|
||
// set will guess wrong.
|
||
for _, want := range []string{"{{nonexistent}}", "{{text}}", "{{draft}}"} {
|
||
if !strings.Contains(err.Error(), want) {
|
||
t.Errorf("the refusal must name %q: %v", want, err)
|
||
}
|
||
}
|
||
// A retired placeholder is the live case: 27 prompt files under eval/ still carry {{genre}}, and a pair
|
||
// pack copied from one of them is exactly what this gate is for.
|
||
if err := CheckPlaceholders("Жанр: {{genre}}."); err == nil {
|
||
t.Error("a retired placeholder must be refused by name")
|
||
}
|
||
// Render's own guard is unchanged.
|
||
if _, err := Render("Привет {{nonexistent}}", RenderVars{Book: testBook()}); err == nil {
|
||
t.Fatal("Render must still refuse an unknown placeholder — the load gate does not replace it")
|
||
}
|
||
// And every placeholder the engine DOES substitute passes, so the gate cannot be a blanket refusal.
|
||
if err := CheckPlaceholders("{{book_id}} {{title}} {{source_lang}} {{target_lang}} {{audience}} {{venuti}} {{honorifics}} {{transcription}} {{footnotes}} {{text}} {{draft}}"); err != nil {
|
||
t.Errorf("the closed set must pass whole: %v", err)
|
||
}
|
||
}
|
||
|
||
// TestTheValidatorAndTheRendererAcceptTheSameSet is the structural half, and it exists because the pair was
|
||
// pinned only BY EXAMPLE. `{{genre}}` and `{{nonexistent}}` are refused by both, so a validator that blessed
|
||
// some THIRD name the renderer refuses — a superset — passed the whole tree: planted with
|
||
// `{{chapter_title}}`, nothing went red. Two functions reading one table is a good arrangement and not a
|
||
// guarantee; this asserts the property instead of the arrangement.
|
||
//
|
||
// The direction that matters is the SUPERSET: a validator laxer than the renderer re-opens exactly the hole
|
||
// the load-time gate was built to close — the refusal comes back mid-run, after a wave is bought. The
|
||
// subset direction is asserted too, since it would refuse prompts that work.
|
||
func TestTheValidatorAndTheRendererAcceptTheSameSet(t *testing.T) {
|
||
blessed := placeholderNames()
|
||
if len(blessed) == 0 {
|
||
t.Fatal("the closed set is empty — every assertion below would hold vacuously")
|
||
}
|
||
for _, marker := range blessed {
|
||
// Anything the VALIDATOR accepts, the RENDERER must substitute. A name here that Render refuses is
|
||
// the superset hole.
|
||
if err := CheckPlaceholders(marker); err != nil {
|
||
t.Fatalf("placeholderNames lists %s and the validator refuses it: %v", marker, err)
|
||
}
|
||
if _, err := Render(marker, RenderVars{Book: testBook()}); err != nil {
|
||
t.Errorf("the validator blesses %s but the renderer refuses it — a prompt that passes the load gate would fail MID-RUN, after a wave is bought: %v", marker, err)
|
||
}
|
||
}
|
||
// And nothing outside the set passes either half. The names are chosen to be plausible rather than
|
||
// absurd: a real pack author reaches for exactly these.
|
||
for _, marker := range []string{"{{chapter_title}}", "{{genre}}", "{{book}}", "{{Text}}"} {
|
||
if err := CheckPlaceholders(marker); err == nil {
|
||
t.Errorf("the validator blesses %s, which the renderer does not substitute", marker)
|
||
}
|
||
if _, err := Render(marker, RenderVars{Book: testBook()}); err == nil {
|
||
t.Errorf("the renderer substituted %s, which is outside the closed set", marker)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Literal «{{…}}» in the source/draft are just text: values
|
||
// are inserted verbatim and not re-scanned (single-pass render).
|
||
func TestRenderValuesAreNotRescanned(t *testing.T) {
|
||
tpl := writeTemplate(t, "Система.\n---USER---\nТекст: {{text}}\nЧерновик: {{draft}}")
|
||
v := RenderVars{
|
||
Book: testBook(),
|
||
Text: "герой сказал {{draft}} и ушёл", // literal braces in source
|
||
Draft: "примечание {{TN: сноска}} осталось", // literal braces in prior LLM output
|
||
}
|
||
msgs, err := Messages(tpl, v)
|
||
if err != nil {
|
||
t.Fatalf("literal braces inside values must not fail: %v", err)
|
||
}
|
||
user := msgs[1].Content
|
||
for _, want := range []string{"герой сказал {{draft}} и ушёл", "примечание {{TN: сноска}} осталось"} {
|
||
if !strings.Contains(user, want) {
|
||
t.Fatalf("value %q must survive verbatim, got:\n%s", want, user)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestTemplateRequiresUserSeparator(t *testing.T) {
|
||
path := filepath.Join(t.TempDir(), "bad.md")
|
||
if err := os.WriteFile(path, []byte("только системная часть"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := LoadPromptTemplate(path); err == nil {
|
||
t.Fatal("template without ---USER--- must be rejected")
|
||
}
|
||
}
|
||
|
||
func TestNormalizeSource(t *testing.T) {
|
||
// BOM, CRLF/CR and decomposed Unicode form → canonical; hash is stable across
|
||
// OSes/editors.
|
||
crlf := "\uFEFFпервая\r\nвторая\rтретья"
|
||
if got := text.NormalizeSource(crlf); got != "первая\nвторая\nтретья" {
|
||
t.Fatalf("BOM/CRLF/CR not normalized: %q", got)
|
||
}
|
||
// Explicitly build NFD (e + U+0301) and NFC (U+00E9) so the forms definitely differ.
|
||
decomposed := "cafe\u0301"
|
||
precomposed := "caf\u00e9"
|
||
if decomposed == precomposed {
|
||
t.Fatal("test setup: the two forms must differ before normalization")
|
||
}
|
||
if text.NormalizeSource(decomposed) != text.NormalizeSource(precomposed) {
|
||
t.Fatal("NFD and NFC forms must normalize to the same string")
|
||
}
|
||
// Different representations of the same text → one request-hash.
|
||
tpl := writeTemplate(t, "S.\n---USER---\n{{text}}")
|
||
m1, _ := Messages(tpl, RenderVars{Book: testBook(), Text: text.NormalizeSource(decomposed)})
|
||
m2, _ := Messages(tpl, RenderVars{Book: testBook(), Text: text.NormalizeSource(precomposed)})
|
||
h1 := RequestHash(baseRequest(m1))
|
||
h2 := RequestHash(baseRequest(m2))
|
||
if h1 != h2 {
|
||
t.Fatal("normalized equivalent sources must yield the same request hash")
|
||
}
|
||
}
|
||
|
||
// Length-prefix eliminates the NUL collision: content with \x00 does not shift field
|
||
// boundaries and does not fake extra messages.
|
||
func TestRequestHashNulRobust(t *testing.T) {
|
||
a := RequestHash(baseRequest([]llm.Message{{Role: "user", Content: "a\x00b"}}))
|
||
bb := RequestHash(baseRequest([]llm.Message{{Role: "user", Content: "a"}, {Role: "\x00b", Content: ""}}))
|
||
if a == bb {
|
||
t.Fatal("NUL byte in content must not collide with an extra message boundary")
|
||
}
|
||
}
|
||
|
||
func TestEstimateTokens(t *testing.T) {
|
||
// Order of magnitude per the eval calibration: an ideograph ≈1 token, Cyrillic ≈3
|
||
// characters per token. The estimate is for reserves, not for billing.
|
||
ja := EstimateTokens("図書館は静寂に包まれていた") // 13 CJK runes
|
||
if ja < 10 || ja > 20 {
|
||
t.Errorf("ja estimate out of range: %d", ja)
|
||
}
|
||
ru := EstimateTokens("Библиотека была окутана тишиной") // ~30 non-whitespace
|
||
if ru < 8 || ru > 16 {
|
||
t.Errorf("ru estimate out of range: %d", ru)
|
||
}
|
||
}
|