textmachine/backend/internal/pipeline/secondpair_test.go

package pipeline

import (
	"errors"
	"io"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"textmachine/backend/internal/lang"
)

// secondpair_test.go: the money-path half of "a second language pair is DATA" (backlog row 347). The
// engine's generality is not decided by whether Go branches on a language code — it does not — but by
// whether a pair's data can be REACHED at all. Until this pack the answer for a Latin source was no: the
// bank contour refuses a write run whose book cannot load a langpack, a langpack with no manifest declares
// EVERY channel, and every channel's files are the zh family's morphology schema. A pair that cannot ship
// the Chinese tables therefore could not ship a pack, and could not run with the contour its product goal
// requires.
//
// ⚠ THE FIXTURE USES THE SHIPPED DATA, not a synthetic copy, and that is the point: what is pinned is that
// `configs/langpacks/en/manifest.txt` + `configs/langpacks/en-ru/` are ENOUGH, on the real files, to open
// the same write path a paid run opens. A synthetic pack would pin the loader and say nothing about
// whether the pair ships.

// setupSecondPairProject builds an en→ru project whose contour is configured exactly as production's is
// (mining artifact + banknote + terminologist), and points it at packRoot. Returns the book.yaml path.
func setupSecondPairProject(t *testing.T, packRoot string) string {
	return setupPairProject(t, packRoot, "en", "", "")
}

// setupPairProject is the same fixture for an arbitrary source language, an optional brief value for the
// book and an optional brief VOCABULARY for the pair — the two halves whose disagreement is the subject.
//
// ⚠ The pair file is written HERE and not taken from configs/: the pair layer resolves beside the RUN
// CONFIG, and this fixture's run config is in a temp dir. What the shipped files declare is pinned where
// they live (config.TestTheShippedPairsDeclareTheirBriefVocabulary); what is pinned here is the ENGINE's
// behaviour when a pair declares, which no amount of reading the data file can show.
func setupPairProject(t *testing.T, packRoot, srcLang, transcription, pairBrief string) string {
	t.Helper()
	gates := "\nmining:\n  contrast_path: mining-contrast.txt\n" +
		"gates:\n  banknote:\n    enabled: true\n" +
		"  terminology:\n    enabled: true\n    model: fake-model\n    budget_usd: 1\n    target_script: Cyrillic\n"
	bookPath := setupProjectOpts(t, "http://127.0.0.1:1", projectOpts{
		source:    "The Marchwarden rode north at dawn.",
		gatesYAML: gates,
	})
	dir := filepath.Dir(bookPath)
	writeFile(t, filepath.Join(dir, "mining-contrast.txt"), miningContrastData)
	// The terminologist prompt resolves by the ordinary `<prompts root>/<pair>/<role>.md` convention, so a
	// fixture for a pair must lay out that pair's pack — the same path production takes.
	pair := srcLang + "-ru"
	writeFile(t, filepath.Join(dir, "pairs", pair+".yaml"), "pair: "+pair+"\nprompts_root: ../prompts\n"+pairBrief)
	writeFile(t, filepath.Join(dir, "prompts", pair, "terminologist.md"),
		"Ты — терминолог {{source_lang}}→{{target_lang}}.\n---USER---\nТермины книги:\n\n{{text}}")

	raw, err := os.ReadFile(bookPath)
	if err != nil {
		t.Fatal(err)
	}
	body := strings.Replace(string(raw), "source_lang: ja", "source_lang: "+srcLang+"\nlangpack_root: "+packRoot, 1)
	// The fixture book carries `transcription: polivanov`; a test that wants the operator template's value
	// (or none at all) says so, and the replacement is asserted rather than hoped for.
	if transcription != "polivanov" {
		before := body
		line := "transcription: " + transcription
		if transcription == "" {
			line = "# transcription: (unstated)"
		}
		body = strings.Replace(body, "transcription: polivanov", line, 1)
		if body == before {
			t.Fatal("premise broken: the fixture book no longer carries `transcription: polivanov`, so this helper silently left the brief as it found it")
		}
	}
	if !strings.Contains(body, "langpack_root:") {
		t.Fatal("premise broken: the fixture book no longer says `source_lang: ja`, so this helper produced a book with no langpack and every assertion below would be vacuous")
	}
	writeFile(t, bookPath, body)
	return bookPath
}

// TestTheSecondPairOpensTheWritePathOnShippedData is the one that decides whether the English book can be
// bought at all, and it carries its own PREMISE: the same tree with the source manifest taken away must
// still refuse, naming a Chinese table. Without that half the test would pass on a tree where nothing was
// ever required, and "the pair loads" would be a statement about the fixture rather than about the pack.
func TestTheSecondPairOpensTheWritePathOnShippedData(t *testing.T) {
	shipped, err := filepath.Abs("../../configs/langpacks")
	if err != nil {
		t.Fatal(err)
	}
	bookPath := setupSecondPairProject(t, shipped)
	r, err := NewRunner(bookPath, slog.New(slog.NewTextHandler(io.Discard, nil)))
	if err != nil {
		t.Fatalf("an en→ru book must open the WRITE path on the shipped data — this is the refusal the pair pack exists to lift: %v", err)
	}
	defer r.Close()
	if r.pack == nil {
		t.Fatal("the pair directory exists, so the pack must be loaded: a nil pack is what makes the contour refuse")
	}
	if r.pack.HasMorphologyChannel() || r.pack.HasChannel("transliteration") {
		t.Errorf("the en source declares NO channel; claiming one would mean the miner runs on tables nobody authored: %+v", r.pack)
	}

	// The PREMISE, and the reason this fixture is not vacuous: without the manifest the very same pair
	// directory demands the zh morphology schema, which is the state row 347 describes.
	stripped := t.TempDir()
	copyPackTree(t, shipped, stripped)
	if err := os.Remove(filepath.Join(stripped, "en", "manifest.txt")); err != nil {
		t.Fatal(err)
	}
	without := setupSecondPairProject(t, stripped)
	if _, err := NewRunner(without, slog.New(slog.NewTextHandler(io.Discard, nil))); err == nil {
		t.Fatal("premise broken: with no manifest an `en` pack must demand the Chinese morphology tables and fail loud — if it does not, this test proves nothing about the manifest")
	} else if !strings.Contains(err.Error(), "surnames-single.txt") {
		t.Errorf("the refusal must name the file it wanted, so the reader learns WHICH schema was demanded: %v", err)
	}
}

// TestTheSecondPairSizesItsOwnTerminologyCalls pins the OTHER half of the same data reachability, and it is
// the half that costs money rather than refusing loudly: the sizing knobs are counted in RUNES, so a pair
// that states nothing runs a Latin source on numbers measured on Han. The ladder is explicit config → the
// pair's own data → the engine default, and the middle rung is the one that did not exist for batch_runes.
func TestTheSecondPairSizesItsOwnTerminologyCalls(t *testing.T) {
	shipped, err := filepath.Abs("../../configs/langpacks")
	if err != nil {
		t.Fatal(err)
	}
	r, err := NewRunner(setupSecondPairProject(t, shipped), slog.New(slog.NewTextHandler(io.Discard, nil)))
	if err != nil {
		t.Fatal(err)
	}
	defer r.Close()

	batch, per, width := r.terminologyOpts()
	// The values themselves live in the data and are argued there; what is pinned here is that they REACH
	// the engine. Asserting they merely differ from the defaults would pass on a pack that shipped noise.
	if batch == terminologyDefaultBatchRunes {
		t.Errorf("the pair's batch bound never reached the engine: still the engine default %d", batch)
	}
	if want := r.pack.Terminology; want == nil || batch != want.BatchRunes || width != want.KWICWidth || per != want.KWICPerTerm {
		t.Errorf("the engine must size the call from the pair's own file, got batch=%d per=%d width=%d for %+v", batch, per, width, want)
	}
	// ⚠ WHAT THE LINES ABOVE CANNOT SAY. A key whose pair value EQUALS the engine default is invisible to
	// them: agreement proves nothing about which rung supplied it. kwic_per_term is such a key by design (a
	// count of contexts does not scale with the script), so the rung is exercised HERE, on values no
	// default can be confused with — without this the test would report for four knobs and measure three.
	if r.pack.Terminology.KWICPerTerm != terminologyDefaultKWICPer {
		t.Fatalf("premise broken: kwic_per_term (%d) no longer equals the engine default (%d), so the check below is testing something else",
			r.pack.Terminology.KWICPerTerm, terminologyDefaultKWICPer)
	}
	r.pack.Terminology = &lang.TerminologySizing{KWICPerTerm: 7, KWICWidth: 123, BasisWidth: 321, BatchRunes: 4567}
	if b, p, w := r.terminologyOpts(); b != 4567 || p != 7 || w != 123 {
		t.Errorf("every sizing knob must take the pair rung, got batch=%d per=%d width=%d", b, p, w)
	}
	if got := r.basisWidth(); got != 321 {
		t.Errorf("the evidence width must take the pair rung, got %d", got)
	}
	// An explicit run-config value still wins: the pair states what is true of the pair, a run may state
	// what is true of the run, and the order between them is the whole reason terminologyOpts exists.
	r.Pipeline.Gates.Terminology.BatchRunes = 777
	if batch, _, _ := r.terminologyOpts(); batch != 777 {
		t.Errorf("an explicit run-config batch_runes must override the pair datum, got %d", batch)
	}
}

// copyPackTree copies a langpack root (its per-language subdirectories and their files) into dst. Only
// regular files one level down are copied — that is the whole layout a pack has.
func copyPackTree(t *testing.T, src, dst string) {
	t.Helper()
	dirs, err := os.ReadDir(src)
	if err != nil {
		t.Fatal(err)
	}
	for _, d := range dirs {
		if !d.IsDir() {
			continue
		}
		if err := os.MkdirAll(filepath.Join(dst, d.Name()), 0o755); err != nil {
			t.Fatal(err)
		}
		files, err := os.ReadDir(filepath.Join(src, d.Name()))
		if err != nil {
			t.Fatal(err)
		}
		for _, f := range files {
			if f.IsDir() {
				continue
			}
			b, err := os.ReadFile(filepath.Join(src, d.Name(), f.Name()))
			if err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(filepath.Join(dst, d.Name(), f.Name()), b, 0o644); err != nil {
				t.Fatal(err)
			}
		}
	}
}

// TestTheBriefPolicyOfAPairRefusesAnotherPairsName is the §4.4 pin, and it is written around the message
// rather than around silence: the operator's ONE book template renders `transcription: palladius` into
// every book of every pair, so the fixture is that exact book — the one where the refusal MUST sound.
//
// ⛔ A TEST THAT ONLY ASSERTED "a good book opens" would be vacuous here: it passes identically on an
// engine that judges nothing at all, which is the state this pin ends.
func TestTheBriefPolicyOfAPairRefusesAnotherPairsName(t *testing.T) {
	shipped, err := filepath.Abs("../../configs/langpacks")
	if err != nil {
		t.Fatal(err)
	}
	quiet := slog.New(slog.NewTextHandler(io.Discard, nil))
	const vocab = "brief:\n  transcription:\n    default: practical\n    allowed: [practical]\n"

	// THE REFUSAL. The Chinese system, handed to an English book by a template that knows no pairs.
	bad := setupPairProject(t, shipped, "en", "palladius", vocab)
	_, err = NewRunner(bad, quiet)
	if err == nil {
		t.Fatal("a paid run whose brief names another pair's transcription system must refuse BEFORE its first cent — the value is rendered into the prompt as an instruction for the whole book")
	}
	for _, want := range []string{"palladius", "en-ru", "practical"} {
		if !strings.Contains(err.Error(), want) {
			t.Errorf("the refusal must name the value, the pair and what the pair DOES know, missing %q: %v", want, err)
		}
	}
	var refusal *Refusal
	if !errors.As(err, &refusal) || refusal.Class != RefusalBadConfig {
		t.Errorf("a brief the pair cannot honour is a CONFIG refusal, not a crash: %T %v", err, err)
	}
	// …and the same book stays READABLE at $0 (D20.4): a book somebody already paid for must not become
	// uninspectable because its brief is wrong.
	ro, err := NewReadOnlyRunner(bad, quiet)
	if err != nil {
		t.Fatalf("the $0 projections must stay open on a book the paid path refuses: %v", err)
	}
	ro.Close()

	// THE DEFAULT. A book that names nothing takes the pair's own name — and takes it INTO the book, so it
	// enters brief_hash and a later edit of the pair's default is a loud re-snapshot rather than a silent
	// change of every request.
	r, err := NewRunner(setupPairProject(t, shipped, "en", "", vocab), quiet)
	if err != nil {
		t.Fatalf("a book that states no transcription must open and take the pair's default: %v", err)
	}
	defer r.Close()
	if r.Book.Transcription != "practical" {
		t.Errorf("the pair's default must be settled into the book (it is what brief_hash folds), got %q", r.Book.Transcription)
	}

	// THE PREMISE OF BOTH: the same book, the same value, a pair that declares NOTHING — accepted, exactly
	// as every pair was before this key existed. Without this half the two assertions above could not tell
	// "the vocabulary is read" from "this value happens to be refused by something else".
	kept, err := NewRunner(setupPairProject(t, shipped, "en", "palladius", ""), quiet)
	if err != nil {
		t.Fatalf("a pair that declares no vocabulary must judge nothing — the value is a free string as it always was: %v", err)
	}
	defer kept.Close()
	if kept.Book.Transcription != "palladius" {
		t.Errorf("an undeclared pair must leave the book's value untouched, got %q", kept.Book.Transcription)
	}
}

// TestBothPathsSettleTheBriefTheSameWay is the property the pair default silently depends on: the default
// is written INTO the book, the book's brief_hash folds into every snapshot, and the $0 projections
// re-render that snapshot to tell an operator what a run would cost and what it already bought. If only
// the paid path applied the default, status/export/report would compute a different snapshot than the run
// they describe — and nothing else in the suite asks.
func TestBothPathsSettleTheBriefTheSameWay(t *testing.T) {
	shipped, err := filepath.Abs("../../configs/langpacks")
	if err != nil {
		t.Fatal(err)
	}
	quiet := slog.New(slog.NewTextHandler(io.Discard, nil))
	const vocab = "brief:\n  transcription:\n    default: practical\n    allowed: [practical]\n"
	book := setupPairProject(t, shipped, "en", "", vocab)

	w, err := NewRunner(book, quiet)
	if err != nil {
		t.Fatal(err)
	}
	writeHash, writeValue := w.Book.BriefHash(), w.Book.Transcription
	w.Close()

	ro, err := NewReadOnlyRunner(book, quiet)
	if err != nil {
		t.Fatal(err)
	}
	defer ro.Close()
	if writeValue != "practical" {
		t.Fatalf("premise broken: the write path did not settle the default (%q), so the two paths could agree by both doing nothing", writeValue)
	}
	if got := ro.Book.Transcription; got != writeValue {
		t.Errorf("the $0 path settled %q where the paid path settled %q", got, writeValue)
	}
	if got := ro.Book.BriefHash(); got != writeHash {
		t.Errorf("the two paths hash the brief differently (%s vs %s) — every projection would then describe a snapshot the run never used", got[:12], writeHash[:12])
	}
}