textmachine/backend/cmd/tmctl/exportgap_cli_test.go

382 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package main
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"textmachine/backend/internal/pipeline"
)
// exportgap_cli_test.go is the §3.2 proof, and it is deliberately end-to-end through the REAL
// binary: the claim under test is about the bytes a human reads out of `tmctl export`, and only
// the binary produces those bytes. A unit-level assertion on a struct would prove the projection
// and leave the shipped text unexamined — which is exactly where the defect lived.
//
// THE DEFECT: a c-lite member drop ships the editor's text over the unit's CLEAN members and
// leaves the flagged member's text out entirely. The reader got a seamless concatenation with a
// chunk-sized hole in it, under a banner that said «leak cleaned, verify» — the wording for a
// COSMETIC sanitizer strip, which had not happened. So the marker was not missing; it was wrong,
// and a wrong marker is worse than none, because it answers the reader's question falsely.
// gapProvider drives one unit into the c-lite state: the member paragraph carrying echoMarker
// comes back as untranslated CJK (classify → cjk_artifact → the member's draft flags and is
// dropped from the edit), the other translates, and the editor returns its own text.
func gapProvider(t *testing.T, echoMarker string) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
text := "ЧЕРНОВИК ПЕРЕВОДА"
switch {
case strings.Contains(string(body), "Черновик перевода для редактуры"):
text = "Отредактированный текст уцелевшей части."
case strings.Contains(string(body), echoMarker):
text = "这是完全没有翻译的中文内容。"
}
tb, _ := json.Marshal(text)
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":"stop"}],
"usage":{"prompt_tokens":100,"completion_tokens":50}}`, tb)
}))
t.Cleanup(srv.Close)
return srv
}
// setupGapProject is setupCLIProject plus an EDIT stage (units need members to lose one) and a
// two-paragraph source, the second of which the provider echoes.
func setupGapProject(t *testing.T, providerURL, echoMarker string) string {
t.Helper()
dir := t.TempDir()
writeCLIFile(t, filepath.Join(dir, "prompts", "translator.md"),
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
writeCLIFile(t, filepath.Join(dir, "prompts", "editor.md"),
"Редактируй перевод.\n---USER---\nИсходник: {{text}}\nЧерновик перевода для редактуры: {{draft}}")
writeCLIFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
prices_checked: %q
default_model: fake-model
providers:
fake:
kind: openai
base_url: %q
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
models:
fake-model:
provider: fake
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
`, time.Now().UTC().Format("2006-01-02"), providerURL))
writeCLIFile(t, filepath.Join(dir, "pipeline.yaml"), `
core: C1
version: 1
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
retries: { regenerate_before_escalate: 0 }
context: { glossary_injection: selective, glossary_token_budget: 800 }
stages:
- { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-cli, temperature: 0.3, reasoning: "off" }
- { name: edit, role: editor, model: fake-model, prompt_override: prompts/editor.md, prompt_version: v-cli, temperature: 0.4, reasoning: "off" }
`)
writeCLIFile(t, filepath.Join(dir, "source.txt"),
strings.Repeat("文", 1400)+"。\n\n"+strings.Repeat(echoMarker, 1100)+"。")
writeCLIFile(t, filepath.Join(dir, "book.yaml"), `
book_id: gap-book
title: Тест
source_lang: zh
target_lang: ru
genre: ранобэ
audience: тест
venuti: 0.5
honorifics: keep
transcription: palladius
footnotes: minimal
pipeline: pipeline.yaml
models: models.yaml
source_file: source.txt
ceilings: { book_usd: 1.0, day_usd: 2.0 }
`)
return filepath.Join(dir, "book.yaml")
}
// setupStrippedDraftOnlyProject builds a DRAFT-ONLY book with the output sanitizer on, whose single
// chunk comes back with a leading markdown header. The strip removes the «### » artifact and ships the
// whole prose: the chunk is flagged sanitizer_stripped and loses NO reader-visible text.
func setupStrippedDraftOnlyProject(t *testing.T, providerURL string) string {
t.Helper()
cfg := setupCLIProject(t, providerURL)
writeCLIFile(t, filepath.Join(filepath.Dir(cfg), "pipeline.yaml"), `
core: C1
version: 1
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
retries: { regenerate_before_escalate: 0 }
context: { glossary_injection: selective, glossary_token_budget: 800 }
stages:
- { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-cli, temperature: 0.3, reasoning: "off" }
gates:
sanitizer:
enabled: true
`)
return cfg
}
// TestADraftOnlyStrippedChunkIsNotCalledIncomplete is the ANTI-SCOPE guard, and it is here because an
// adversarial pass found the change failing it: a draft-only pipeline makes every chunk a SINGLETON
// unit whose own draft row is its final row, so the member-drop rule counted the chunk's own flag as a
// lost member. The reader was then told a fragment was missing from a chunk whose text is entirely
// present — a marker that misinforms, which the order forbids as explicitly as the silence it replaces.
func TestADraftOnlyStrippedChunkIsNotCalledIncomplete(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.ReadAll(r.Body)
tb, _ := json.Marshal("### Глава 7\n\nСудзуки открыл седьмую дверь и замер на пороге.")
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":"stop"}],
"usage":{"prompt_tokens":100,"completion_tokens":50}}`, tb)
}))
t.Cleanup(srv.Close)
bookPath := setupStrippedDraftOnlyProject(t, srv.URL)
report := runTmctl(t, bookPath, "translate")
shipped := runTmctl(t, bookPath, "export", "--plaintext")
t.Logf("TRANSLATE:\n%s\nEXPORT:\n%s", report, shipped)
var doc pipeline.BookExport
if err := json.Unmarshal([]byte(runTmctl(t, bookPath, "export")), &doc); err != nil {
t.Fatal(err)
}
if len(doc.Chunks) != 1 || doc.Chunks[0].FlagReason != string(pipeline.FlagSanitizerStripped) {
t.Fatalf("fixture must produce ONE cosmetically stripped chunk, got %+v", doc.Chunks)
}
if doc.Chunks[0].DroppedMembers != 0 {
t.Fatalf("a draft-only unit has no members to drop, got dropped_members=%d", doc.Chunks[0].DroppedMembers)
}
// The whole prose is there — nothing was lost, so nothing may claim it was.
if !strings.Contains(shipped, "Судзуки открыл седьмую дверь и замер на пороге.") {
t.Fatalf("the stripped chunk must still ship its prose:\n%s", shipped)
}
for _, out := range []string{shipped, report} {
if strings.Contains(out, "TEXT MISSING") || strings.Contains(out, "INCOMPLETE") {
t.Fatalf("nothing was lost; the gap marker must not fire:\n%s", out)
}
}
// And the TRUE banner is restored, not shadowed by the incomplete branch.
if !strings.Contains(shipped, "(leak cleaned, verify)") {
t.Fatalf("a real cosmetic strip must keep its own banner:\n%s", shipped)
}
if !strings.Contains(shipped, "(of them incomplete=0)") {
t.Fatalf("the summary must not count it as incomplete:\n%s", shipped)
}
}
// runTmctl runs the real binary and returns its stdout, tolerating the documented non-error exit
// codes (2 = completed with flags — which is precisely the run this test needs).
func runTmctl(t *testing.T, bookPath string, args ...string) string {
t.Helper()
cmd := exec.Command(buildTmctl(t), append(args, "--config", bookPath)...)
var out, errb bytes.Buffer
cmd.Stdout, cmd.Stderr = &out, &errb
err := cmd.Run()
if code := exitCodeOf(t, err); code != 0 && code != 2 {
t.Fatalf("tmctl %v exited %d\nstdout:\n%s\nstderr:\n%s", args, code, out.String(), errb.String())
}
return out.String()
}
// TestExportPlaintextMarksTheGapInTheShippedText is the deliverable: on a real c-lite run, the
// text a human reads out of `tmctl export --plaintext` says a piece is missing, and does NOT
// claim a cosmetic clean-up that never happened.
func TestExportPlaintextMarksTheGapInTheShippedText(t *testing.T) {
const echoMarker = "禁"
srv := gapProvider(t, echoMarker)
bookPath := setupGapProject(t, srv.URL, echoMarker)
runTmctl(t, bookPath, "translate")
shipped := runTmctl(t, bookPath, "export", "--plaintext")
t.Logf("SHIPPED TEXT:\n%s", shipped)
// The state really is the one under test: an ok edit over the clean member, with a member lost.
var doc pipeline.BookExport
if err := json.Unmarshal([]byte(runTmctl(t, bookPath, "export")), &doc); err != nil {
t.Fatalf("export --json must round-trip: %v", err)
}
if len(doc.Chunks) != 1 {
t.Fatalf("fixture must produce ONE unit, got %d — the c-lite state was not reached", len(doc.Chunks))
}
u := doc.Chunks[0]
if u.Disposition != string(pipeline.DispFlagged) || u.FinalText == "" || u.DroppedMembers != 1 {
t.Fatalf("fixture must produce a flagged unit that SHIPS text with 1 dropped member, got %+v", u)
}
// 1. The reader is told, in the text stream, that a piece is missing.
if !strings.Contains(shipped, "TEXT MISSING") {
t.Fatalf("the shipped text does not tell the reader a fragment is missing:\n%s", shipped)
}
if !strings.Contains(shipped, "1 source fragment of this unit could not be translated") {
t.Fatalf("the marker must say HOW MUCH is missing:\n%s", shipped)
}
// 2. And it is NOT told the false thing it used to be told.
if strings.Contains(shipped, "leak cleaned") {
t.Fatalf("a member drop must not be reported as a cosmetic sanitizer clean-up:\n%s", shipped)
}
// 3. The marker sits with the prose, above the text it qualifies — not only in a header far
// above, which a reader scrolling a concatenation passes once and never sees again.
iMark, iText := strings.Index(shipped, "TEXT MISSING"), strings.Index(shipped, "Отредактированный текст")
if iMark < 0 || iText < 0 || iMark > iText {
t.Fatalf("the marker must precede the incomplete text (marker@%d, text@%d):\n%s", iMark, iText, shipped)
}
// 4. The surviving text still ships in full — the marker informs, it does not withhold.
if !strings.Contains(shipped, "Отредактированный текст уцелевшей части.") {
t.Fatalf("the paid, clean remainder must still ship:\n%s", shipped)
}
// 5. The summary counts it as incomplete rather than folding it into `exported`.
if !strings.Contains(shipped, "(of them incomplete=1)") {
t.Fatalf("the export summary must count the incomplete unit:\n%s", shipped)
}
}
// TestTranslateMarksTheGapInTheShippedText: the same run's own report is the OTHER surface a human
// reads, and it carried the same false wording. One state, both renderers.
func TestTranslateMarksTheGapInTheShippedText(t *testing.T) {
const echoMarker = "禁"
srv := gapProvider(t, echoMarker)
bookPath := setupGapProject(t, srv.URL, echoMarker)
report := runTmctl(t, bookPath, "translate")
t.Logf("TRANSLATE REPORT:\n%s", report)
if !strings.Contains(report, "TEXT MISSING") {
t.Fatalf("translate must tell the reader a fragment is missing:\n%s", report)
}
if strings.Contains(report, "leak cleaned") {
t.Fatalf("translate must not report a member drop as a cosmetic clean-up:\n%s", report)
}
if !strings.Contains(report, "Отредактированный текст уцелевшей части.") {
t.Fatalf("the clean remainder must still be printed:\n%s", report)
}
}
// TestExportPlaintextStateMatrix pins all four unit states on ONE document, so the branches cannot
// drift apart: only the member-drop unit is called incomplete, only a real sanitizer strip is
// called a cleaned leak, a withheld unit is neither, and the header counts each honestly.
func TestExportPlaintextStateMatrix(t *testing.T) {
doc := &pipeline.BookExport{BookID: "b", TotalUnits: 8, PendingUnits: 1, Chunks: []pipeline.ChunkExport{
{Chapter: 1, ChunkIdx: 0, Disposition: "ok", FinalText: "Чистый текст."},
{Chapter: 2, ChunkIdx: 0, Disposition: "flagged", FlagReason: "sanitizer_stripped", FinalText: "Очищенный текст."},
{Chapter: 3, ChunkIdx: 0, Disposition: "flagged", FlagReason: "cjk_artifact", FinalText: "Уцелевшая часть.", DroppedMembers: 2, DroppedReason: "cjk_artifact"},
{Chapter: 4, ChunkIdx: 0, Disposition: "flagged", FlagReason: "glossary_miss", FinalText: ""},
{Chapter: 5, ChunkIdx: 0, Disposition: "pending"},
// EVERY member dropped: the unit ships NOTHING. It carries a drop count all the same, and the
// marker must NOT fire — «a fragment is missing from the text below» over an empty body points
// at text that does not exist. This case is why both call sites guard on a non-empty text.
{Chapter: 6, ChunkIdx: 0, Disposition: "flagged", FlagReason: "cjk_artifact", FinalText: "", DroppedMembers: 3, DroppedReason: "cjk_artifact"},
// A flagged unit with the sanitizer's reason but NO text. The engine does not produce one today
// (classifyOutput hands back a non-empty strip or a different reason), but the branch order must
// not be the thing standing between that and a banner announcing a clean-up of nothing.
{Chapter: 7, ChunkIdx: 0, Disposition: "flagged", FlagReason: "sanitizer_stripped", FinalText: ""},
// THE INTERSECTION, and the one an adversarial pass caught: the EDIT flagged on its own account
// (a cosmetic strip) AND a member dropped. The unit's FlagReason is the strip's; the cause of
// the HOLE is the member's. Printing the unit's reason as the cause of the loss would tell the
// reader a clean-up ate a chunk of the book — the same false claim, moved into the prose.
{Chapter: 8, ChunkIdx: 0, Disposition: "flagged", FlagReason: "sanitizer_stripped",
FinalText: "Почищенная уцелевшая часть.", DroppedMembers: 1, DroppedReason: "untranslated_echo"},
}}
var b bytes.Buffer
if err := renderExport(&b, doc, true); err != nil {
t.Fatal(err)
}
out := b.String()
for _, want := range []string{
// exported counts only units that actually shipped text: 8 total 1 pending 3 withheld = 4,
// of which 2 are incomplete (a SUBSET of exported, not a fourth part of the total).
"# export b — total units=8, exported=4 (of them incomplete=2), withheld=3, pending=1",
"=== CHAPTER 1 CHUNK 0 ===\nЧистый текст.",
// A REAL cosmetic strip keeps the wording that is true of it, and gains no gap marker.
"=== CHAPTER 2 CHUNK 0 — flagged (sanitizer_stripped) (leak cleaned, verify) ===\nОчищенный текст.",
// The member drop is named, counted and marked in the text stream — plural form included.
"=== CHAPTER 3 CHUNK 0 — flagged (cjk_artifact) (INCOMPLETE) ===\n" +
"[⚠ TEXT MISSING — 2 source fragments of this unit could not be translated and are NOT in the text below: cjk_artifact]\n" +
"Уцелевшая часть.",
"=== CHAPTER 4 CHUNK 0 — flagged (glossary_miss) (not translated, flagged for a human) ===",
"=== CHAPTER 6 CHUNK 0 — flagged (cjk_artifact) (not translated, flagged for a human) ===",
"=== CHAPTER 7 CHUNK 0 — flagged (sanitizer_stripped) (not translated, flagged for a human) ===",
// The banner keeps the unit's verdict; the MARKER names the hole's own cause, not the strip's.
"=== CHAPTER 8 CHUNK 0 — flagged (sanitizer_stripped) (INCOMPLETE) ===\n" +
"[⚠ TEXT MISSING — 1 source fragment of this unit could not be translated and is NOT in the text below: untranslated_echo]\n" +
"Почищенная уцелевшая часть.",
"=== CHAPTER 5 CHUNK 0 — pending (not yet translated) ===",
} {
if !strings.Contains(out, want) {
t.Fatalf("plaintext export must contain\n%q\ngot:\n%s", want, out)
}
}
// ANTI-SCOPE: a unit that lost nothing must never carry the marker — a marker on complete text
// misinforms the reader exactly as the old wording did, in the other direction.
head := out[:strings.Index(out, "=== CHAPTER 3")]
if strings.Contains(head, "TEXT MISSING") {
t.Fatalf("the gap marker leaked onto a unit that lost nothing:\n%s", head)
}
if n := strings.Count(out, "TEXT MISSING"); n != 2 {
t.Fatalf("exactly two units ship INCOMPLETE text, got %d markers:\n%s", n, out)
}
// The strip's reason must never be printed as the CAUSE of a hole.
if strings.Contains(out, "could not be translated and is NOT in the text below: sanitizer_stripped") {
t.Fatalf("the marker named the unit's own flag reason as the cause of the loss:\n%s", out)
}
// The wholly-withheld unit (ch.6: every member dropped, nothing shipped) must not have been dressed
// as merely incomplete — scoped to ITS block, since ch.8 legitimately carries both.
ch6 := out[strings.Index(out, "=== CHAPTER 6"):strings.Index(out, "=== CHAPTER 7")]
if strings.Contains(ch6, "TEXT MISSING") || strings.Contains(ch6, "INCOMPLETE") {
t.Fatalf("a unit that ships NOTHING must not claim a fragment is missing below it:\n%s", ch6)
}
}
// TestTranslateStateMatrix is renderExport's matrix for the OTHER human surface. Its twin existed and
// this one did not, so an adversarial pass could revert renderTranslate's two guards — the ones the
// code itself calls LOAD-BEARING — and watch the whole cmd/tmctl suite stay green. A guard nothing
// pins is a comment.
func TestTranslateStateMatrix(t *testing.T) {
res := &pipeline.BookResult{BookID: "b", Flagged: 5, Chunks: []pipeline.ChunkOutcome{
{Chapter: 1, Disposition: pipeline.DispOK, FinalText: "Чистый текст."},
{Chapter: 2, Disposition: pipeline.DispFlagged, FlagReason: pipeline.FlagSanitizerStripped, FinalText: "Очищенный текст."},
{Chapter: 3, Disposition: pipeline.DispFlagged, FlagReason: "cjk_artifact", FinalText: "Уцелевшая часть.",
DroppedMembers: 2, DroppedReason: "cjk_artifact"},
// Every member dropped: nothing ships. The marker must not point at text that is not there.
{Chapter: 4, Disposition: pipeline.DispFlagged, FlagReason: "cjk_artifact", FinalText: "",
DroppedMembers: 3, DroppedReason: "cjk_artifact"},
// The sanitizer's reason with NO text: the branch order must not print a clean-up of nothing.
{Chapter: 5, Disposition: pipeline.DispFlagged, FlagReason: pipeline.FlagSanitizerStripped, FinalText: ""},
// The intersection: the EDIT flagged for its own reason AND a member dropped. The marker names
// the hole's cause, never the strip's.
{Chapter: 6, Disposition: pipeline.DispFlagged, FlagReason: pipeline.FlagSanitizerStripped,
FinalText: "Почищенная уцелевшая часть.", DroppedMembers: 1, DroppedReason: "untranslated_echo"},
}}
var b bytes.Buffer
err := renderTranslate(&b, res, func() (float64, float64, error) { return 0, 0, nil })
var flagged *pipeline.CompletedWithFlags
if !errors.As(err, &flagged) {
t.Fatalf("a flagged run must return the exit-2 sentinel, got %v", err)
}
out := b.String()
for _, want := range []string{
"=== CHAPTER 1 CHUNK 0 — ok ===\nЧистый текст.",
"[FLAG sanitizer_stripped — leak cleaned, exported cleaned, verify] ↓\nОчищенный текст.",
"[FLAG cjk_artifact] ↓\n[⚠ TEXT MISSING — 2 source fragments of this unit could not be translated and are NOT in the text below: cjk_artifact]\nУцелевшая часть.",
"=== CHAPTER 4 CHUNK 0 — flagged(cjk_artifact) ===\n[FLAG cjk_artifact] chunk not translated",
"=== CHAPTER 5 CHUNK 0 — flagged(sanitizer_stripped) ===\n[FLAG sanitizer_stripped] chunk not translated",
"[⚠ TEXT MISSING — 1 source fragment of this unit could not be translated and is NOT in the text below: untranslated_echo]\nПочищенная уцелевшая часть.",
} {
if !strings.Contains(out, want) {
t.Fatalf("translate must contain\n%q\ngot:\n%s", want, out)
}
}
if n := strings.Count(out, "TEXT MISSING"); n != 2 {
t.Fatalf("exactly two units ship INCOMPLETE text, got %d markers:\n%s", n, out)
}
if strings.Contains(out, "could not be translated and is NOT in the text below: sanitizer_stripped") {
t.Fatalf("the marker named the unit's own flag reason as the cause of the loss:\n%s", out)
}
}