textmachine/backend/internal/pipeline/classifiervocab_test.go

package pipeline

import (
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"textmachine/backend/internal/terminology"
)

// classifiervocab_test.go: the classifier's answer vocabulary is a CLOSED set of engine identifiers, and
// the only place a model is ever told what they are is the pair's own prompt. Nothing checked that the
// prompt says them. Measured: a prompts/ja-ru/classifier.md whose classes read 人名 · 地名 · 称号 · 用語
// passes the entire battery — every off-vocabulary reply is counted and dropped with a warning, never
// refused — so the pair works, silently mistyped, and the project's default review question («will a pair
// that is not in the repository work without editing Go?») gets the answer «yes, and wrongly».
//
// ⛔ IT LINTS THE CANONICAL FORM, comments stripped, and that is not a detail. Every one of these
// identifiers also appears in the zh-ru prompt's HTML header, so a lint that grepped the raw file would
// pass a pack whose instructions to the model are in a different vocabulary entirely. The negative fixture
// below is built with exactly that shape, so a regression to reading raw bytes fails this test.
//
// ⚠ WHAT THIS LINT CANNOT DO, said here rather than left for a reader to discover: it proves the
// identifiers are ON THE WIRE, not that the model is told to ANSWER in them. A pack that glosses its own
// classes bilingually — «人名 (name)» — and then instructs the model to reply in its own language passes
// this and is exactly as broken as the pack the row measured. Closing THAT needs the engine to refuse an
// off-vocabulary reply instead of counting and dropping it (terminologist.go, Log.WarnContext), which is a
// change of paid-run behaviour and is not this pack's to make. What is closed here is the measured case:
// a pack ported by translating everything, in which the identifiers appear nowhere the model can see.

// promptPlaceholder matches a `{{name}}` slot. The engine substitutes these before the call (render.go), so
// their NAMES are not text the model is instructed with.
var promptPlaceholder = regexp.MustCompile(`\{\{[^}]*\}\}`)

// classifierVocabulary is what a classifier prompt has to name: the class identifiers the parser keeps and
// the gender words it keeps. Read off the engine's own sets, so a class added in Go joins the lint by
// being added rather than by anyone remembering this file.
func classifierVocabulary() []string {
	return append(terminology.TypeNames(terminology.Types), terminology.TypeNames(terminology.Genders)...)
}

// classifierVocabularyGaps names the identifiers a prompt never says WHERE THE CLASSIFIER PUTS IT ON THE
// WIRE: the canonical form (comments stripped, which is also what the snapshot folds), System and User
// parts only.
//
// ⛔ THE FEW-SHOT BLOCK IS DELIBERATELY EXCLUDED, and reading it was this lint's own first bug. The
// classifier renders through MessagesWithInjection (render.go), which takes tpl.System — NOT
// SystemFor(fewShotOn) — so a ---FEWSHOT--- block never reaches this role's wire at all. Linting it made
// a pack whose entire English vocabulary sat in the few-shot block read as healthy while the model was
// shown Japanese.
func classifierVocabularyGaps(t *testing.T, path string) []string {
	t.Helper()
	tpl, err := LoadPromptTemplate(path)
	if err != nil {
		t.Fatalf("load %s in its canonical form: %v", path, err)
	}
	// ⛔ PLACEHOLDERS ARE REMOVED BEFORE THE SEARCH. `{{title}}` is the BOOK's title, substituted before the
	// call, and it satisfied a word-boundary search for the class `title` — so a pack that localised the
	// class everywhere the model reads it still counted as naming it. What must be searched is the prompt's
	// own words, not the names of the values poured into it.
	shown := promptPlaceholder.ReplaceAllString(strings.Join([]string{tpl.System, tpl.User}, "\n"), " ")
	var missing []string
	for _, w := range classifierVocabulary() {
		// Word boundaries, not substrings: `term` sits inside `terminology`, and an en-ru pack that only
		// discussed terminology would otherwise read as though it had named the class.
		if !regexp.MustCompile(`\b` + regexp.QuoteMeta(w) + `\b`).MatchString(shown) {
			missing = append(missing, w)
		}
	}
	return missing
}

func TestEveryClassifierPromptSpeaksTheEnginesVocabulary(t *testing.T) {
	packs, err := filepath.Glob(filepath.Join("..", "..", "prompts", "*", "classifier.md"))
	if err != nil {
		t.Fatal(err)
	}
	if len(packs) == 0 {
		t.Fatal("the repository has no prompts/<pair>/classifier.md at all — this lint would be enforcing nothing")
	}
	// ⛔ AND THE OTHER HALF OF THE SAME CONTROL. This lint pins ONE direction — every word the engine keeps
	// is named in the prompt — so it shrinks with its own subject: drop a class from terminology.Types and
	// the assertion about that class disappears with it, silently (measured: removing `title` leaves the
	// whole battery at 19 ok / 0 FAIL). At the limit an empty vocabulary makes this test AND both of its
	// negative fixtures vacuous, since they compare len(missing) against len(vocabulary) — 0 != 0.
	if len(classifierVocabulary()) == 0 {
		t.Fatal("terminology.Types and terminology.Genders are both empty — the lint, and the two negative cases below, would all be comparing nothing with nothing")
	}
	for _, p := range packs {
		if missing := classifierVocabularyGaps(t, p); len(missing) > 0 {
			t.Errorf("%s never names %v where the model can see it. The engine KEEPS only these identifiers "+
				"(terminology.Types and terminology.Genders) and counts-and-drops every other answer, so a pack "+
				"that answers in its own words buys a paid pass and changes nothing.", p, missing)
		}
	}
	t.Logf("classifier prompts linted: %d; vocabulary asserted: %v", len(packs), classifierVocabulary())

	// THE NEGATIVE CASE, IN THE SAME RUN. A pack authored the way the row describes — the zh-ru file
	// translated, its classes and genders now in the pair's own language, the English identifiers left
	// behind in the header comment where they do the model no good. A lint reading raw bytes passes this.
	localized := filepath.Join(t.TempDir(), "classifier.md")
	writeFile(t, localized,
		"<!-- ported from zh-ru: name / place / title / term; male | female | neuter | none -->\n"+
			"用語を分類してください。クラス: 人名 · 地名 · 称号 · 用語。性別: 男性 · 女性 · 中性 · 不明。\n"+
			"---USER---\n用語:\n\n{{text}}\n")
	missing := classifierVocabularyGaps(t, localized)
	if len(missing) != len(classifierVocabulary()) {
		t.Fatalf("a fully localized pack was reported to be missing only %v of %v — the lint is reading the "+
			"prompt's COMMENTS, where the identifiers still are, instead of the form the model is shown",
			missing, classifierVocabulary())
	}

	// THE SECOND NEGATIVE CASE, and the more likely one: a pack that keeps the English identifiers but puts
	// them in the ---FEWSHOT--- block. That block is real prompt text, it is NOT a comment, and it still
	// never reaches this role's wire — MessagesWithInjection renders tpl.System. A lint that read the whole
	// template would pass this pack while the model saw only Japanese.
	fewShotOnly := filepath.Join(t.TempDir(), "classifier.md")
	writeFile(t, fewShotOnly,
		"用語を分類してください。クラス: 人名 · 地名 · 称号 · 用語。性別: 男性 · 女性 · 中性 · 不明。\n"+
			"---FEWSHOT---\nAnswer with one of: name, place, title, term; gender male, female, neuter, none.\n"+
			"---USER---\n用語:\n\n{{text}}\n")
	if missing := classifierVocabularyGaps(t, fewShotOnly); len(missing) != len(classifierVocabulary()) {
		t.Fatalf("a pack whose vocabulary lives only in ---FEWSHOT--- was reported to be missing only %v of %v "+
			"— that block never reaches the classifier's wire, so linting it passes a pack the model cannot read",
			missing, classifierVocabulary())
	}
}