101 lines
4.3 KiB
Go
101 lines
4.3 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// banknote_test.go pins the WS4 §4(д) parser invariants (byte-faithful port of exp16 banknote.py).
|
||
// The 95-line corpus pin (0 parse_fail / 0 truncated over 14 saved raw outputs) is the domain of the
|
||
// $0 reference eval/design11/ws4_banknote_verify.py; these unit tests cover the parser CONTRACT.
|
||
|
||
func TestSplitBanknoteNoSeparatorIsAllClean(t *testing.T) {
|
||
out := "Судзуки шёл по коридору. \n"
|
||
clean, block := splitBanknote(out)
|
||
if clean != "Судзуки шёл по коридору." || block != "" {
|
||
t.Fatalf("no-SEP: clean=%q block=%q, want the whole right-trimmed output and empty block", clean, block)
|
||
}
|
||
}
|
||
|
||
func TestSplitBanknoteSlicesBlock(t *testing.T) {
|
||
out := "Перевод... последнее предложение.\n\n" + bankSeparator + "\n方源\tФан Юань\tname\n蛊师\tгу-мастер\ttitle"
|
||
clean, block := splitBanknote(out)
|
||
if !strings.HasSuffix(clean, "последнее предложение.") {
|
||
t.Fatalf("clean tail wrong: %q", clean)
|
||
}
|
||
if strings.Contains(clean, bankSeparator) || strings.Contains(clean, "方源") {
|
||
t.Fatalf("the banknote block leaked into the clean translation: %q", clean)
|
||
}
|
||
ents, flags := parseBanknote(block, false)
|
||
if len(ents) != 2 || flags.ParseFail || flags.Truncated || flags.NLines != 2 {
|
||
t.Fatalf("parse = %+v flags=%+v, want 2 clean entries", ents, flags)
|
||
}
|
||
if ents[0].Src != "方源" || ents[0].Dst != "Фан Юань" || ents[0].Type != "name" {
|
||
t.Fatalf("entry[0] = %+v", ents[0])
|
||
}
|
||
if ents[1].Type != "title" {
|
||
t.Fatalf("entry[1] type = %q, want title", ents[1].Type)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteTolerantFieldSplit(t *testing.T) {
|
||
// tab, ≥2 spaces, and pipe are all accepted delimiters (a model that emits spaces instead of a
|
||
// real TAB still parses). type falls back to "term" when absent or unknown.
|
||
block := "方源\tФан Юань\tname\n蛊师 гу-мастер title\n青茅山 | гора Цинмао | place\n古月\tГу Юэ"
|
||
ents, flags := parseBanknote(block, false)
|
||
if flags.ParseFail {
|
||
t.Fatalf("tolerant split should not fail: %+v", flags)
|
||
}
|
||
if len(ents) != 4 {
|
||
t.Fatalf("want 4 entries across tab/space/pipe delimiters, got %d: %+v", len(ents), ents)
|
||
}
|
||
if ents[3].Type != "term" { // no type field → default term
|
||
t.Fatalf("missing type must default to term, got %q", ents[3].Type)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteNonHanSrcIsBad(t *testing.T) {
|
||
// A zh→ru channel line whose src has no Han ideograph is malformed → parse_fail.
|
||
block := "方源\tФан Юань\tname\nRoseanne\tРозанна\tname"
|
||
ents, flags := parseBanknote(block, false)
|
||
if !flags.ParseFail {
|
||
t.Fatalf("a non-Han src must set parse_fail")
|
||
}
|
||
if len(ents) != 1 || ents[0].Src != "方源" {
|
||
t.Fatalf("only the Han-src line should survive, got %+v", ents)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteTruncationTolerated(t *testing.T) {
|
||
// The LAST line cut by generation length is tolerated (banknote_truncated), not a parse fail —
|
||
// but ONLY under truncated_generation; the same short line otherwise IS a parse fail.
|
||
block := "方源\tФан Юань\tname\n蛊" // last line has no dst
|
||
entsT, flagsT := parseBanknote(block, true)
|
||
if !flagsT.Truncated || flagsT.ParseFail || len(entsT) != 1 {
|
||
t.Fatalf("truncated=true: want 1 entry + truncated flag + no parse_fail, got %+v %+v", entsT, flagsT)
|
||
}
|
||
entsF, flagsF := parseBanknote(block, false)
|
||
if flagsF.Truncated || !flagsF.ParseFail || len(entsF) != 1 {
|
||
t.Fatalf("truncated=false: the short last line must be a parse_fail, got %+v %+v", entsF, flagsF)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteEmptyBlock(t *testing.T) {
|
||
ents, flags := parseBanknote("", false)
|
||
if len(ents) != 0 || flags.ParseFail || flags.Truncated || flags.NLines != 0 {
|
||
t.Fatalf("empty block must yield no entries and clean flags, got %+v %+v", ents, flags)
|
||
}
|
||
}
|
||
|
||
func TestBankSeparatorIsNotANoteWord(t *testing.T) {
|
||
// The separator must not be a trailing-note word the sanitizer reserves (§B3-1) — the two
|
||
// channels must never collide.
|
||
for _, note := range []string{"Примечание", "Сноска", "Комментарий", "Note", "TN"} {
|
||
if strings.Contains(bankSeparator, note) {
|
||
t.Fatalf("separator %q collides with the reserved note-word %q", bankSeparator, note)
|
||
}
|
||
}
|
||
if !strings.HasPrefix(bankSeparator, "⟦") {
|
||
t.Fatalf("separator lost its unusual delimiter: %q", bankSeparator)
|
||
}
|
||
}
|