107 lines
4.1 KiB
Text
107 lines
4.1 KiB
Text
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"textmachine/backend/internal/llm"
|
|
)
|
|
|
|
// This is a request-capture probe, not a model-quality test. It executes the
|
|
// actual book runner and current production prompt files with an in-memory
|
|
// provider. Every response is synthetic and no network client is called.
|
|
type contextAuditClient struct{ requests []llm.LLMRequest }
|
|
|
|
func (c *contextAuditClient) Complete(_ context.Context, r llm.LLMRequest) (*llm.LLMResponse, error) {
|
|
c.requests = append(c.requests, r)
|
|
body, _ := json.Marshal(r.Messages)
|
|
output := "Наконец согласился."
|
|
if strings.Contains(string(body), "女孩") {
|
|
output = "Девушка одна сидела в комнате и обдумывала просьбу."
|
|
}
|
|
if strings.Contains(string(body), "男孩") {
|
|
output = "Юноша один сидел в комнате и обдумывал просьбу."
|
|
}
|
|
return &llm.LLMResponse{Text: output, Model: r.Model, FinishReason: llm.FinishStop,
|
|
Usage: llm.Usage{PromptTokens: 100, CompletionTokens: 20}}, nil
|
|
}
|
|
|
|
func TestAuditContextAcrossEditUnits(t *testing.T) {
|
|
current := "终于答应了。"
|
|
prefixes := []string{"女孩独自坐在房里，反复考虑那个请求。", "男孩独自坐在房里，反复考虑那个请求。"}
|
|
var captures [2][]llm.LLMRequest
|
|
for variant, prefix := range prefixes {
|
|
bookPath := setupProjectOpts(t, "http://127.0.0.1:1", projectOpts{
|
|
source: prefix + "\f" + current, banknote: true, waveWorkers: 1})
|
|
projectDir := filepath.Dir(bookPath)
|
|
bookBytes, err := os.ReadFile(bookPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeFile(t, bookPath, strings.Replace(string(bookBytes), "source_lang: ja", "source_lang: zh", 1))
|
|
for _, pair := range [][2]string{{"translator-banknote.md", "translator.md"}, {"editor.md", "editor.md"}} {
|
|
data, err := os.ReadFile(filepath.Join("..", "..", "prompts", "zh-ru", pair[0]))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeFile(t, filepath.Join(projectDir, "prompts", pair[1]), string(data))
|
|
}
|
|
r := newRunner(t, bookPath)
|
|
chunks, err := r.bookChunks()
|
|
if err != nil {
|
|
r.Close()
|
|
t.Fatal(err)
|
|
}
|
|
units := buildEditUnits(chunks)
|
|
if len(units) != 2 || len(units[0].Members) != 1 || len(units[1].Members) != 1 ||
|
|
units[0].Chapter == units[1].Chapter || units[1].sourceText() != current {
|
|
r.Close()
|
|
t.Fatalf("fixture did not cross an edit-unit boundary: chunks=%+v units=%+v", chunks, units)
|
|
}
|
|
fake := &contextAuditClient{}
|
|
r.clients["fake-model"] = fake
|
|
result, err := r.TranslateBook(context.Background())
|
|
r.Close()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(fake.requests) != 4 || result.Flagged != 0 {
|
|
t.Fatalf("want 2 draft+2 edit calls, clean result; calls=%d result=%+v", len(fake.requests), result)
|
|
}
|
|
captures[variant] = fake.requests
|
|
t.Logf("variant=%d: 2 chapters / 2 draft chunks / 2 edit units / %d in-memory provider calls; previous=%s", variant, len(fake.requests), prefix)
|
|
}
|
|
var currentByVariant [2][]llm.LLMRequest
|
|
var priorByVariant [2][]llm.LLMRequest
|
|
for variant, reqs := range captures {
|
|
for _, req := range reqs {
|
|
b, _ := json.Marshal(req.Messages)
|
|
if strings.Contains(string(b), current) {
|
|
currentByVariant[variant] = append(currentByVariant[variant], req)
|
|
} else {
|
|
priorByVariant[variant] = append(priorByVariant[variant], req)
|
|
}
|
|
}
|
|
if len(currentByVariant[variant]) != 2 || len(priorByVariant[variant]) != 2 {
|
|
t.Fatal("missing draft/editor captures")
|
|
}
|
|
}
|
|
for i, role := range []string{"draft", "editor"} {
|
|
a, _ := json.Marshal(currentByVariant[0][i])
|
|
b, _ := json.Marshal(currentByVariant[1][i])
|
|
if string(a) != string(b) {
|
|
t.Fatalf("current %s request DID change with preceding context: A=%s B=%s", role, a, b)
|
|
}
|
|
pa, _ := json.Marshal(priorByVariant[0][i])
|
|
pb, _ := json.Marshal(priorByVariant[1][i])
|
|
if string(pa) == string(pb) {
|
|
t.Fatalf("prior %s requests should differ", role)
|
|
}
|
|
t.Logf("current %s request is byte-identical: bytes=%d sha256=%x; prior request differs", role, len(a), sha256.Sum256(a))
|
|
}
|
|
}
|