230 lines
9.6 KiB
Go
230 lines
9.6 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"testing"
|
||
)
|
||
|
||
// --- dialogue-dash --------------------------------------------------------------
|
||
|
||
func TestLintDialogueDash(t *testing.T) {
|
||
cases := []struct {
|
||
name string
|
||
text string
|
||
want int
|
||
}{
|
||
{"em-dash correct", "— Привет, — сказал он.\n— И тебе.", 0},
|
||
{"hyphen dialogue", "- Привет, — сказал он.\n- Пока.", 2},
|
||
{"en-dash dialogue", "– Привет, мир.", 1},
|
||
{"straight quote dialogue", "\"Привет\", — сказал он.", 1},
|
||
{"hyphenated word not dialogue", "Он сделал это как-то по-своему.\n-то не считается.", 0},
|
||
{"list-like dash not dialogue", "-1 балл\n-2 очка", 0},
|
||
{"mixing em-dash and chevrons", "— Реплика первая.\n«Реплика вторая», — подумал он.", 1},
|
||
{"chevron only no mixing", "«Цитата из книги».\nОбычный текст.", 0},
|
||
{"chevron term in prose not flagged", "Это была «Война и мир».", 0},
|
||
// D20.4 FP1: a chevron CITATION/definition at line start (« … » — prose, NO attribution comma)
|
||
// mixed with em-dash dialogue must NOT count as style mixing.
|
||
{"chevron citation at line start not mixing", "— Реплика.\n«Мастер Гу» — так его называли.", 0},
|
||
{"chevron title alone at line start not mixing", "— Реплика.\n«Война и мир» стояла на полке.", 0},
|
||
// Self-review: an em-dash + space + digit is a scene break, NOT dialogue — must not count.
|
||
{"em-dash scene break not dialogue", "— 15 минут спустя он вернулся.", 0},
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
n, _ := lintDialogueDash(c.text)
|
||
if n != c.want {
|
||
t.Errorf("lintDialogueDash(%q) = %d, want %d", c.text, n, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// --- yofikator ------------------------------------------------------------------
|
||
|
||
func TestLintYofikation(t *testing.T) {
|
||
cases := []struct {
|
||
name string
|
||
text string
|
||
policy string
|
||
want int
|
||
}{
|
||
{"name both spellings auto", "Пётр вошёл. Потом Петр ушёл.", "auto", 1},
|
||
{"name consistent auto", "Пётр вошёл и Пётр ушёл.", "auto", 0},
|
||
{"homograph vse not flagged", "Все ушли. И всё пропало.", "auto", 0},
|
||
{"no yo at all auto", "Все ели мед.", "auto", 0},
|
||
{"all-e flags any yo", "Пётр ел мёд.", "all-e", 2},
|
||
{"all-e clean", "Петр ел мед.", "all-e", 0},
|
||
{"artem both spellings", "Артём и Артем — один человек.", "auto", 1},
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
n, _ := lintYofikation(c.text, c.policy)
|
||
if n != c.want {
|
||
t.Errorf("lintYofikation(%q, %q) = %d, want %d", c.text, c.policy, n, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// --- translit interjections -----------------------------------------------------
|
||
|
||
func TestLintTranslitInterjections(t *testing.T) {
|
||
empty := map[string]bool{}
|
||
cases := []struct {
|
||
name string
|
||
text string
|
||
allow map[string]bool
|
||
want int
|
||
}{
|
||
{"ara-ara and hmf", "Ара-ара, — протянула она. Хмф!", empty, 2},
|
||
{"nani", "Нани?! — вскрикнул он.", empty, 1},
|
||
{"inside word not flagged", "У него сильный характер и добрая душа.", empty, 0},
|
||
{"allowlisted maa", "Маа, что же делать.", map[string]bool{"маа": true}, 0},
|
||
{"non-allowlisted maa fires", "Маа, что же делать.", empty, 1},
|
||
// Self-review majors: «ара» (macaw noun / name) and «уму» (dative of «ум») were removed from
|
||
// the blocklist — they must NOT flag ordinary prose.
|
||
{"bare ara is a macaw not flagged", "Красный ара сидел на ветке.", empty, 0},
|
||
{"umu dative not flagged", "Это уму непостижимо.", empty, 0},
|
||
{"clean prose", "Он молча кивнул и ушёл.", empty, 0},
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
n, _ := lintTranslitInterjections(c.text, c.allow)
|
||
if n != c.want {
|
||
t.Errorf("lintTranslitInterjections(%q) = %d, want %d", c.text, n, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// --- CJK number parser ----------------------------------------------------------
|
||
|
||
func TestParseCJKNumber(t *testing.T) {
|
||
cases := []struct {
|
||
s string
|
||
want int64
|
||
ok bool
|
||
}{
|
||
{"三万", 30000, true},
|
||
{"三千万", 30000000, true},
|
||
{"一億二千万", 120000000, true},
|
||
{"十万", 100000, true},
|
||
{"30万", 300000, true},
|
||
{"五億", 500000000, true},
|
||
{"一兆", 1000000000000, true},
|
||
{"三", 0, false}, // no big marker
|
||
{"三千", 0, false}, // no big marker (千 is a small unit)
|
||
{"", 0, false},
|
||
}
|
||
for _, c := range cases {
|
||
v, ok := parseCJKNumber(c.s)
|
||
if ok != c.ok || (ok && v != c.want) {
|
||
t.Errorf("parseCJKNumber(%q) = (%d,%v), want (%d,%v)", c.s, v, ok, c.want, c.ok)
|
||
}
|
||
}
|
||
}
|
||
|
||
// --- number magnitude gate ------------------------------------------------------
|
||
|
||
func TestLintNumberMagnitude(t *testing.T) {
|
||
cases := []struct {
|
||
name string
|
||
source string
|
||
final string
|
||
want int
|
||
}{
|
||
{"30k mistranslated as millions", "他有三万石粮食。", "У него было три миллиона мешков зерна.", 1},
|
||
{"30k correct as tens of thousands", "他有三万石粮食。", "У него было тридцать тысяч мешков зерна.", 0},
|
||
{"300 million correct", "损失三億元。", "Убытки составили триста миллионов юаней.", 0},
|
||
{"hundred million as thousand wrong", "军队三億人。", "Армия в три тысячи человек.", 1},
|
||
{"wan as myriad not a number", "世间万物皆有灵。", "Все вещи в мире имеют душу.", 0},
|
||
// Self-review major: CJK idioms carrying 万/億 but no digit coefficient are NOT magnitudes.
|
||
{"wan-yi idiom", "万一出事了怎么办。", "Что если что-то случится?", 0},
|
||
{"qianwan idiom", "你千万要小心。", "Ты обязательно будь осторожен.", 0},
|
||
{"wanfen idiom", "他万分感激。", "Он был крайне благодарен.", 0},
|
||
{"no magnitude at all", "他走进房间。", "Он вошёл в комнату.", 0},
|
||
{"arabic output matches", "三万里。", "30000 ли.", 0},
|
||
{"no output number silent", "三万大军。", "Огромная армия.", 0},
|
||
// D20.4 FP3: a magnitude rephrased as prose + a STRAY unrelated output integer (a year) must
|
||
// NOT flag — the year is not evidence the 万 was mistranslated (bare integers only confirm).
|
||
{"rephrased magnitude plus stray year not flagged", "三万大军压境，时值1990年。", "Несметное войско подступило; шёл 1990 год.", 0},
|
||
// D20.4 FP2: 万 inside a NAME (digit-before-万 makes it parse as a magnitude) + an unrelated
|
||
// output number must NOT flag.
|
||
{"wan in a name plus stray year not flagged", "他叫赵三万，生于1985年。", "Его звали Чжао Саньвань, родился в 1985 году.", 0},
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
n, _ := lintNumberMagnitude(c.source, c.final)
|
||
if n != c.want {
|
||
t.Errorf("lintNumberMagnitude(%q → %q) = %d, want %d", c.source, c.final, n, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// --- combined ------------------------------------------------------------------
|
||
|
||
func TestRunCheapGatesCombined(t *testing.T) {
|
||
src := "他有三万石粮食。"
|
||
final := "- Ара-ара, — у Пётр было три миллиона мешков. Потом Петр ушёл."
|
||
cfg := cheapGateConfig{yoPolicy: "auto", allowlist: map[string]bool{}}
|
||
r := runCheapGates(src, final, cfg)
|
||
if r.DialogueDash == 0 {
|
||
t.Error("expected a dialogue-dash flag (hyphen-led line)")
|
||
}
|
||
if r.TranslitInterj == 0 {
|
||
t.Error("expected a translit-interjection flag (ара-ара)")
|
||
}
|
||
if r.YoInconsistent == 0 {
|
||
t.Error("expected a yofikation flag (Пётр/Петр)")
|
||
}
|
||
if r.NumberMagnitude == 0 {
|
||
t.Error("expected a number-magnitude flag (三万 → миллиона)")
|
||
}
|
||
if r.total() < 4 {
|
||
t.Errorf("expected all four gates to fire, total=%d detail=%v", r.total(), r.Detail)
|
||
}
|
||
}
|
||
|
||
// TestCheapGatesSurfaceInStatus is the end-to-end wiring test: a chunk whose FINAL text has a style
|
||
// issue (a hyphen-led dialogue line) records a style flag in retrieval_state and surfaces it in the
|
||
// status projection — WITHOUT flagging the chunk (observability, never a disposition).
|
||
func TestCheapGatesSurfaceInStatus(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "- Привет, — сказал он.", "stop" // hyphen dialogue → dialogue-dash flag
|
||
}
|
||
return "ЧЕРНОВИК ПЕРЕВОДА", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "テキスト。", regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res.Flagged != 0 {
|
||
t.Fatalf("a style flag must not flag the chunk (observability only), got flagged=%d", res.Flagged)
|
||
}
|
||
rs, err := r.Store.GetRetrievalState("test-book", 1, 0)
|
||
if err != nil || rs == nil {
|
||
t.Fatalf("no retrieval state persisted: %v", err)
|
||
}
|
||
if rs.NStyleFlags == 0 {
|
||
t.Fatalf("style flag not persisted in retrieval_state: %+v", rs)
|
||
}
|
||
rep, err := r.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.StyleFlags == 0 {
|
||
t.Fatal("status must surface the book-wide style-flag count")
|
||
}
|
||
if rep.Done != 1 || rep.Flagged != 0 {
|
||
t.Fatalf("the chunk must be done, not flagged: done=%d flagged=%d", rep.Done, rep.Flagged)
|
||
}
|
||
}
|