171 lines
9.2 KiB
Go
171 lines
9.2 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/lang"
|
||
"textmachine/backend/internal/llm"
|
||
)
|
||
|
||
// classify: ORDER is the contract (D2.1/D2.4) — refusal/echo/deterministic finish
|
||
// signals are decided BEFORE length/empty, so a truncated refusal is not turned
|
||
// into a paid length retry.
|
||
func TestClassifyOrder(t *testing.T) {
|
||
const ru = "ru"
|
||
cases := []struct {
|
||
name string
|
||
in classifyInput
|
||
wantReason FlagReason
|
||
}{
|
||
{"clean stop", classifyInput{Source: "静かな朝。", Output: "Тихое утро.", Finish: llm.FinishStop, TargetLang: ru}, reasonOK},
|
||
{"empty at stop", classifyInput{Source: "静かな朝。", Output: " ", Finish: llm.FinishStop, TargetLang: ru}, FlagEmpty},
|
||
{"empty at length (thinking ate budget)", classifyInput{Source: "静かな朝。", Output: "", Finish: llm.FinishLength, TargetLang: ru}, FlagEmpty},
|
||
{"genuine length cut", classifyInput{Source: strings.Repeat("текст ", 50), Output: "Начало предложения обрывается на", Finish: llm.FinishLength, TargetLang: ru}, FlagLength},
|
||
{"length but repetition loop", classifyInput{Source: "x", Output: strings.Repeat("ло ", 60), Finish: llm.FinishLength, TargetLang: ru}, FlagLoopDegenerate},
|
||
{"truncated soft refusal (NOT length)", classifyInput{Source: "静かな朝、彼女は本を開いた。長い一日の始まりだった。", Output: "Я не могу помочь с этим запросом.", Finish: llm.FinishLength, TargetLang: ru}, FlagSoftRefusal},
|
||
{"english refusal short", classifyInput{Source: "長い長い長い長い文章です。", Output: "I'm sorry, but I cannot assist with that.", Finish: llm.FinishStop, TargetLang: ru}, FlagSoftRefusal},
|
||
{"cjk echo (untranslated)", classifyInput{Source: "彼は言った。", Output: "彼は言った。とても静かな朝だった。", Finish: llm.FinishStop, TargetLang: ru, SourceScripts: lang.LangScripts("ja")}, FlagCJKArtifact},
|
||
{"provider content_filter finish", classifyInput{Source: "x", Output: "partial", Finish: llm.FinishContentFilter, TargetLang: ru}, FlagContentFilter},
|
||
{"provider refusal finish", classifyInput{Source: "x", Output: "", Finish: llm.FinishRefusal, TargetLang: ru}, FlagHardRefusal},
|
||
{"resumed decode checkpoint", classifyInput{Source: "x", Output: "", Finish: decodeErrorFinish, TargetLang: ru}, FlagDecodeError},
|
||
}
|
||
for _, tc := range cases {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
got := classify(tc.in)
|
||
if got.Reason != tc.wantReason {
|
||
t.Fatalf("classify reason = %q, want %q (detail %q)", got.Reason, tc.wantReason, got.Detail)
|
||
}
|
||
if (got.Reason == reasonOK) != got.ok() {
|
||
t.Fatalf("ok() inconsistent for reason %q", got.Reason)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// A refusal that is NOT anomalously short (a long faithful translation that
|
||
// merely quotes a refusal-like phrase as dialogue) must NOT flag.
|
||
func TestClassifyLongOutputWithRefusalPhraseIsOK(t *testing.T) {
|
||
src := strings.Repeat("短", 100) // 100 CJK runes → threshold max(400, 50) = 400
|
||
out := "«Я не могу помочь тебе», — тихо сказал он, отводя взгляд. " + strings.Repeat("Слова растворялись в утреннем тумане, и никто их не услышал. ", 12)
|
||
got := classify(classifyInput{Source: src, Output: out, Finish: llm.FinishStop, TargetLang: "ru"})
|
||
if !got.ok() {
|
||
t.Fatalf("a long translation quoting a refusal phrase must be ok, got %q", got.Reason)
|
||
}
|
||
}
|
||
|
||
// TestClassifyBilingualTableIsNotAnEcho pins the one exemption from the CJK-echo rule, which the live
|
||
// probe of 26.07 forced: the terminologist answers `<source term><TAB><rendering>` per line, so HALF its
|
||
// reply is CJK by specification. Left unexempted, every healthy call of that role landed in request_log as
|
||
// ok=0/degraded=cjk_artifact — the exact signal that is supposed to mean "the provider misbehaved". The
|
||
// exemption is narrow: refusals, empties and truncations must still be caught on the same reply.
|
||
func TestClassifyBilingualTableIsNotAnEcho(t *testing.T) {
|
||
table := "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао"
|
||
zh := lang.LangScripts("zh")
|
||
if got := classify(classifyInput{Output: table, Finish: llm.FinishStop, TargetLang: "ru", SourceScripts: zh}); got.Reason != FlagCJKArtifact {
|
||
t.Fatalf("without the exemption a term table reads as an echo (that is the rule this test exempts), got %q", got.Reason)
|
||
}
|
||
got := classify(classifyInput{Output: table, Finish: llm.FinishStop, TargetLang: "ru", SourceScripts: zh, SourceEchoExpected: true})
|
||
if !got.ok() {
|
||
t.Fatalf("a term table is the specified format, not an echo: %q (%s)", got.Reason, got.Detail)
|
||
}
|
||
// The exemption must not swallow the checks that still matter on this wire.
|
||
if e := classify(classifyInput{Output: "", Finish: llm.FinishStop, TargetLang: "ru", SourceEchoExpected: true}); e.Reason != FlagEmpty {
|
||
t.Fatalf("an empty terminology reply must still be caught, got %q", e.Reason)
|
||
}
|
||
if rf := classify(classifyInput{Output: "", Finish: llm.FinishRefusal, TargetLang: "ru", SourceEchoExpected: true}); rf.Reason != FlagHardRefusal {
|
||
t.Fatalf("a refusal must still be caught, got %q", rf.Reason)
|
||
}
|
||
}
|
||
|
||
// Determinism: identical input yields the identical verdict (pure function).
|
||
func TestClassifyDeterministic(t *testing.T) {
|
||
in := classifyInput{Source: "彼は静かに歩いた。", Output: strings.Repeat("эхо ", 40), Finish: llm.FinishLength, TargetLang: "ru"}
|
||
a, b := classify(in), classify(in)
|
||
if a != b {
|
||
t.Fatalf("classify is not deterministic: %+v vs %+v", a, b)
|
||
}
|
||
}
|
||
|
||
func TestRetryableSubset(t *testing.T) {
|
||
retry := map[FlagReason]bool{FlagLength: true, FlagEmpty: true}
|
||
all := []FlagReason{
|
||
reasonOK, FlagLength, FlagEmpty, FlagLoopDegenerate, FlagHardRefusal, FlagSoftRefusal,
|
||
FlagContentFilter, FlagCJKArtifact, FlagDecodeError, FlagCoverageFail, FlagExcisionSuspect,
|
||
FlagHardBlock, FlagUpstreamNotOK,
|
||
}
|
||
for _, r := range all {
|
||
if got, want := r.retryable(), retry[r]; got != want {
|
||
t.Fatalf("%q.retryable() = %v, want %v (only length/empty are retryable)", r, got, want)
|
||
}
|
||
}
|
||
if reasonOK.disposition() != DispOK {
|
||
t.Fatal("reasonOK must resolve to DispOK")
|
||
}
|
||
if FlagLength.disposition() != DispFlagged {
|
||
t.Fatal("a flag reason must resolve to DispFlagged")
|
||
}
|
||
}
|
||
|
||
// maxTokensForAttempt is pure and doubles per attempt (D2.3), which is what puts
|
||
// a DIFFERENT max_tokens (and thus request_hash) on each regeneration.
|
||
func TestMaxTokensForAttempt(t *testing.T) {
|
||
base := 2048
|
||
for attempt, want := range map[int]int{0: base, 1: 2 * base, 2: 4 * base, 3: 8 * base} {
|
||
if got := maxTokensForAttempt(base, attempt); got != want {
|
||
t.Fatalf("maxTokensForAttempt(%d, %d) = %d, want %d", base, attempt, got, want)
|
||
}
|
||
}
|
||
// Overflow guard: an absurd attempt is clamped, never a negative shift.
|
||
if got := maxTokensForAttempt(base, 999); got <= 0 {
|
||
t.Fatalf("maxTokensForAttempt overflow guard failed: %d", got)
|
||
}
|
||
}
|
||
|
||
func TestDegenerateLoop(t *testing.T) {
|
||
if !degenerateLoop(strings.Repeat("одно и то же ", 40)) {
|
||
t.Fatal("a repeated phrase must be a degenerate loop")
|
||
}
|
||
healthy := "Она открыла книгу и начала читать первую главу, где юный герой впервые покидал родную деревню навстречу далёким горам и неизвестной судьбе, полной опасностей."
|
||
if degenerateLoop(healthy) {
|
||
t.Fatal("healthy prose must NOT be a degenerate loop")
|
||
}
|
||
if degenerateLoop("короткий обрыв на пол") {
|
||
t.Fatal("a short length cut is not enough evidence for a loop")
|
||
}
|
||
}
|
||
|
||
func TestIsRefusalShortVsLong(t *testing.T) {
|
||
if !isRefusal("Извините, я не могу перевести это.", "abc") {
|
||
t.Fatal("a short refusal must be detected")
|
||
}
|
||
long := strings.Repeat("Это обычный длинный перевод без всякого отказа. ", 30)
|
||
if isRefusal(long, "abc") {
|
||
t.Fatal("a long non-refusal must not be flagged")
|
||
}
|
||
}
|
||
|
||
func TestSourceScriptShareAndTarget(t *testing.T) {
|
||
jaScripts := lang.LangScripts("ja")
|
||
if s := sourceScriptShare("彼は言った", jaScripts); s < 0.9 {
|
||
t.Fatalf("all-CJK text share = %.2f, want ~1", s)
|
||
}
|
||
if s := sourceScriptShare("Полностью русский текст", jaScripts); s > 0.01 {
|
||
t.Fatalf("russian text share = %.2f, want ~0", s)
|
||
}
|
||
// The echo detector now measures the DECLARED source script: a ko (Hangul) echo is no longer a blind spot.
|
||
if s := sourceScriptShare("그는말했다", lang.LangScripts("ko")); s < 0.9 {
|
||
t.Fatalf("all-Hangul text share under a ko source = %.2f, want ~1 (the fixed live ko blind spot)", s)
|
||
}
|
||
if s := sourceScriptShare("그는말했다", jaScripts); s > 0.01 {
|
||
t.Fatalf("Hangul under a ja source share = %.2f, want ~0 (ja scripts do not include Hangul)", s)
|
||
}
|
||
if !isCJKTarget("ja") || !isCJKTarget("ZH") || isCJKTarget("ru") {
|
||
t.Fatal("isCJKTarget classification wrong")
|
||
}
|
||
// Into a CJK target, source-script output is expected — no echo flag.
|
||
got := classify(classifyInput{Source: "Утро.", Output: "静かな朝。", Finish: llm.FinishStop, TargetLang: "ja", SourceScripts: lang.LangScripts("ru")})
|
||
if got.Reason == FlagCJKArtifact {
|
||
t.Fatal("CJK output must not be an echo artifact when the target IS CJK")
|
||
}
|
||
}
|