diff --git a/backend/internal/chunk/headingparity_test.go b/backend/internal/chunk/headingparity_test.go new file mode 100644 index 00000000..8674fb8a --- /dev/null +++ b/backend/internal/chunk/headingparity_test.go @@ -0,0 +1,192 @@ +package chunk + +import ( + "fmt" + "strings" + "testing" + + "textmachine/backend/internal/lang" +) + +// headingparity_test.go holds the two halves of a question nobody had asked the code: does the chapter +// heading rule mean the same thing on both sides of the ingest→chunker seam? The rule is spread over three +// predicates in two files — the ingest's length guard (chapterHeaderMaxRunes), the chunker's numeral guard +// (a zero section is not a section) and the shared separator class — and each half was pinned only by its +// own tests, which cannot see a disagreement between them. +// +// The first test is the GUARANTEE, the second its named exception. Neither works alone: a pin of a known +// defect is a second description of the bug, and it leaves nothing behind once the defect is fixed. + +// headingParityBody is chapter prose, long enough to be real and short enough to keep the fixture cheap. +func headingParityBody() string { return "\n" + strings.Repeat("古月方源站在洞口。", 30) } + +// TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders is the live guarantee: on the header shapes a book +// actually uses, the ingest's predicate and the chunker's parse answer the same, and a book cut by the +// ingest comes out of the chunker with a deterministic title on EVERY chapter and no source marker left in +// the text the model reads. Green today, and it must stay green after the divergence below is closed. +func TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders(t *testing.T) { + st := lang.DefaultCJKStructure() + rule := zhRuChapterRule() + // ⛔ THE NUMBERS ARE NOT CONSECUTIVE, ON PURPOSE. A file numbered 一·二·三 makes the parsed header number + // equal the chapter's ORDINAL, and every assertion below would then hold just as well for a title + // rendered from the counter as for one rendered from the header — the two quantities coincide and the + // whole class of «the title came from the wrong place» goes invisible. Here chapter 2 must be «Глава 3». + headers := []struct { + line string + num int + }{ + {"第一章 起始", 1}, + {"第三章 继续", 3}, + {"第七章:正常标题", 7}, + } + var src strings.Builder + for _, h := range headers { + fmt.Fprintf(&src, "%s%s\n\n", h.line, headingParityBody()) + } + doc, err := IngestEncoded(writeTempTXT(t, src.String()), "", "zh", st) + if err != nil { + t.Fatal(err) + } + if len(doc.Chapters) != len(headers) { + t.Fatalf("the ingest cut %d chapters out of %d headers", len(doc.Chapters), len(headers)) + } + unit := detectChapterUnit(strings.Split(src.String(), "\n"), st) + if unit == 0 { + t.Fatal("test premise broken: the ingest detected no chapter unit, so its predicate was never asked") + } + + // The two predicates on the same lines. The chunker's NUMBER is asserted too: agreeing that a line is a + // header while reading a different number out of it is the same defect one layer down. + for _, h := range headers { + byIngest := isChapterHeader(h.line, unit, st) + n, _, byChunker := matchHeaderLine(h.line, rule) + if !byIngest || !byChunker || n != h.num { + t.Fatalf("%q: the ingest says header=%v; the chunker says header=%v number=%d — want a header on both sides and number %d", + h.line, byIngest, byChunker, n, h.num) + } + } + + chunks := SplitChunks(doc.Chapters, testSeg(), rule, testAbbrevs()) + titled := 0 + for _, c := range chunks { + if c.ChunkIdx != 0 { + continue + } + titled++ + // Guarded, not indexed straight into: the whole point of this fixture is that a chapter's NUMBER and + // its position are different quantities, so an index built from one to look up the other has to say + // so when it goes out of range instead of panicking. + if c.Chapter < 1 || c.Chapter > len(headers) { + t.Fatalf("the cut produced chapter %d and the fixture has %d headers — the chapter counter no longer indexes them", + c.Chapter, len(headers)) + } + // The title is the SOURCE HEADER's number, not the chapter's position in the file. On this fixture + // the two differ from chapter 2 on, which is what makes the assertion say anything. + if want := fmt.Sprintf("Глава %d", headers[c.Chapter-1].num); c.Heading != want { + t.Fatalf("chapter %d (header %q) opens with heading %q, want %q — the title must come from the header's own number, not from the chapter counter", + c.Chapter, headers[c.Chapter-1].line, c.Heading, want) + } + if _, opensWithMarker := st.MatchMarker(c.Text); opensWithMarker { + t.Fatalf("chapter %d still opens with its source marker, so the raw header reaches the model: %q", + c.Chapter, firstRunes(c.Text, 16)) + } + } + if titled != len(headers) { + t.Fatalf("%d chapters opened, want %d", titled, len(headers)) + } +} + +// TestIngestAndChunkerDivergeOnAZeroPrologueAndALongHeader pins a DIVERGENCE that exists today (backlog +// row 346), not a guarantee. Two header shapes are cut by the ingest and refused by the chunker, so the +// header line survives into the text the model translates and the chapter numbers shift under the reader: +// +// - «第零章», a prologue: parseSectionNumeral refuses a zero section, the ingest's regex accepts it; +// - a header past chapterHeaderMaxRunes: the length guard exists only in the ingest. +// +// ⛔ THE FIX IS NOT IN SCOPE HERE AND IT IS NOT $0: it changes the TEXT of a chunk, which bumps +// chunkerVersion, which moves both waves' snapshots and re-bills the book. It lands in the re-cut window. +// So what is asserted is TODAY's shape and the test is green — a red test cannot be left in main, and a +// skip would hide the whole class rather than name one case of it. When the fix lands this test fails on +// purpose, and every failure message says what to do about it. +func TestIngestAndChunkerDivergeOnAZeroPrologueAndALongHeader(t *testing.T) { + const closed = "the divergence backlog row 346 pins has CLOSED — that is the fix landing, not a regression. Delete this test and fold the case into TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders" + st := lang.DefaultCJKStructure() + rule := zhRuChapterRule() + src := "第零章:序幕" + headingParityBody() + "\n\n第一章 起始" + headingParityBody() + "\n\n第二章 继续" + headingParityBody() + "\n" + + doc, err := IngestEncoded(writeTempTXT(t, src), "", "zh", st) + if err != nil { + t.Fatal(err) + } + // (1) The ingest cuts the prologue as a chapter of its own. + if len(doc.Chapters) != 3 { + // NOT the `closed` message: this is the test's PREMISE, and it breaks when the INGEST stops cutting + // the prologue as its own chapter — the opposite side of the seam from the fix this test waits for. + // Telling a reader to delete the test would send them to fold a case nobody has closed. + t.Fatalf("premise broken: the ingest cut %d chapters, want 3 (a prologue and two) — the ingest's own header predicate changed, and the divergence below has not been judged either way", len(doc.Chapters)) + } + chunks := SplitChunks(doc.Chapters, testSeg(), rule, testAbbrevs()) + // Guarded rather than indexed straight into: this test is BUILT to fail one day, and whoever reads that + // failure has to get the message below, not an index panic. + if len(chunks) < 3 { + t.Fatalf("the cut produced %d chunks for 3 chapters — %s", len(chunks), closed) + } + // (2) The chunker does not recognise the prologue's header, so the chapter opens untitled … + if chunks[0].Heading != "" { + t.Fatalf("the prologue now carries the title %q — %s", chunks[0].Heading, closed) + } + // (3) … and its raw CJK marker travels to the model inside the text it translates. + if !strings.HasPrefix(chunks[0].Text, "第零章") { + t.Fatalf("the prologue's source marker no longer reaches the model (%q) — %s", firstRunes(chunks[0].Text, 16), closed) + } + // (4) The half a reader sees, and it is NOT a divergence pin: a chapter's ORDINAL and the number in its + // title are two different quantities, and the prologue makes them differ from chapter 2 on — the cut + // numbers 第一章 as chapter 2 while the template renders its own «Глава 1». Correct on both sides, and it + // survives the fix in (2): recognising 第零章 gives it a title, it does not renumber anything. Asserted + // here because this is the file where the two numbers are measurably apart, and the re-cut design has to + // say which of them addresses a chapter (chunk_status is keyed on the ordinal; a reader sees the title). + var second *Chunk + for i := range chunks { + if chunks[i].Chapter == 2 && chunks[i].ChunkIdx == 0 { + second = &chunks[i] + } + } + if second == nil { + t.Fatalf("the cut has no chapter 2 — %s", closed) + } + if second.Heading != "Глава 1" { + t.Fatalf("chapter 2 (第一章) is titled %q, want «Глава 1»: the ordinal and the title's number came apart differently than measured — this is the addressing question, NOT the row-346 divergence, so read it before assuming a fix landed", second.Heading) + } + + // The length half, with its own CONTROL. ⚠ The header must carry a separator after its unit rune + // («第四章:…»): without one BOTH sides refuse it for an unrelated reason and the divergence does not + // reproduce at any length at all. + unit := detectChapterUnit(strings.Split(src, "\n"), st) + atCeiling := "第四章:" + strings.Repeat("标", chapterHeaderMaxRunes-4) + pastCeiling := "第四章:" + strings.Repeat("标", chapterHeaderMaxRunes-3) + if len([]rune(atCeiling)) != chapterHeaderMaxRunes || len([]rune(pastCeiling)) != chapterHeaderMaxRunes+1 { + t.Fatalf("the test built the wrong lengths: %d and %d runes around a ceiling of %d", + len([]rune(atCeiling)), len([]rune(pastCeiling)), chapterHeaderMaxRunes) + } + // CONTROL: at exactly the ceiling both halves still say "header". It is what makes the disagreement + // below a disagreement about LENGTH rather than about the shape of this fixture. + _, _, chunkerAt := matchHeaderLine(atCeiling, rule) + if ingestAt := isChapterHeader(atCeiling, unit, st); !ingestAt || !chunkerAt { + t.Fatalf("control broken: at exactly %d runes the ingest says %v and the chunker says %v — they must still agree there", + chapterHeaderMaxRunes, ingestAt, chunkerAt) + } + n, _, chunkerPast := matchHeaderLine(pastCeiling, rule) + if ingestPast := isChapterHeader(pastCeiling, unit, st); ingestPast || !chunkerPast || n != 4 { + t.Fatalf("at %d runes the ingest says header=%v and the chunker says header=%v number=%d; today the ingest refuses it and the chunker reads chapter 4. %s", + chapterHeaderMaxRunes+1, ingestPast, chunkerPast, n, closed) + } +} + +// firstRunes keeps a failure message readable when the text it quotes is a whole chapter. +func firstRunes(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/backend/internal/config/armparity_test.go b/backend/internal/config/armparity_test.go new file mode 100644 index 00000000..80614b23 --- /dev/null +++ b/backend/internal/config/armparity_test.go @@ -0,0 +1,192 @@ +package config + +import ( + "fmt" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// armparity_test.go: an editor swap-arm exists to isolate ONE variable, and it can only do that if every +// other key is the production config's. Three arm headers claimed exactly that and were false by two dozen +// keys — the whole bank contour among them (b8154cd gave pipeline-c1.yaml sixty-four lines of it and the +// arms a single prompt_version), while the test that called itself the guard of «only the editor differs» +// compared three fields. The cost is money, not tidiness, and it runs the OTHER way: with no +// mining.contrast_path the arm's own book-level price is zero (priceprojection.go, bookOnceUSD returns 0 +// when the terminology gate is off or the contrast path is empty), so a deploy flipped to an arm quietly +// bought a cheaper book than the one it reported — the bank contour was configured nowhere and priced +// nowhere, and nothing said so. +// +// ⛔ THE FORM IS THE POINT. This compares the WHOLE loaded configs and exempts the editor stage, instead of +// listing the keys that must match. A list is what drifted: it can only check what somebody remembered to +// put in it, and a key ADDED to c1 is invisible to it by construction. + +func TestAnArmIsTheProductionConfigWithADifferentEditor(t *testing.T) { + m, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml")) + if err != nil { + t.Fatalf("load the shipping models.yaml: %v", err) + } + // ⛔ THE ARMS ARE DISCOVERED, NOT LISTED. A list here would be the very thing the header refuses one + // paragraph up: a new pipeline-arm-*.yaml would join the repository outside every gate, and the file + // carrying the money defect (an arm with no mining.contrast_path) would be the one nobody loaded. + arms, err := filepath.Glob(filepath.Join("..", "..", "configs", "pipeline-arm-*.yaml")) + if err != nil { + t.Fatal(err) + } + if len(arms) == 0 { + t.Fatal("no configs/pipeline-arm-*.yaml in the repository — this gate would be enforcing nothing") + } + load := func(t *testing.T, file string) *Pipeline { + t.Helper() + p, lerr := LoadPipeline(file, m, "zh-ru", nil) + if lerr != nil { + // ⚠ t is a PARAMETER, not the enclosing test's: a Fatalf on the outer t from inside a subtest + // kills the parent, and every arm after this one goes unchecked. + t.Fatalf("load %s: %v", file, lerr) + } + return p + } + base := load(t, filepath.Join("..", "..", "configs", "pipeline-c1.yaml")) + baseEditor := editorOf(t, base) + t.Logf("arms discovered: %d %v; production editor: %s", len(arms), armNames(arms), baseEditor.Model) + for _, file := range arms { + t.Run(filepath.Base(file), func(t *testing.T) { + arm := load(t, file) + editor := editorOf(t, arm).Model + // An arm whose editor is the production editor isolates nothing, and a file that isolates + // nothing is not an arm — it is a second copy of the production config that has to be kept in + // step forever. Said as a gate, because it is a state the interim editor can drift INTO + // (D39.22 calls the current one interim): the day c1 adopts an arm's editor, that arm has to be + // retired or re-pointed, deliberately. + if baseEditor.Model == editor { + t.Fatalf("the production editor is already %q, so this file isolates no variable: retire it or point it at a different editor", editor) + } + // The exemptions are NAMED in the log, so a green run is never read as "every key was compared". + // reasoning_max_tokens is exempt only where the loader forces it — today that is one model of the + // nine in models.yaml (grok-4.3, the only `additive`), and no arm runs it. + exempt := "model, resolved model, escalate_to, few_shot" + if m.providerReasoning(editorOf(t, arm).Model) == "additive" { + exempt += ", reasoning_max_tokens (additive billing forces it; nothing compares its VALUE)" + } + t.Logf("%s: editor %q; exempt from the comparison: %s", filepath.Base(file), editor, exempt) + for _, d := range pipelineFieldDiffs(withoutEditorModel(t, m, base), withoutEditorModel(t, m, arm)) { + t.Errorf("%s is not pipeline-c1.yaml with a different editor — %s", filepath.Base(file), d) + } + }) + } +} + +// armNames renders the discovered files for a log line. +func armNames(paths []string) []string { + out := make([]string, 0, len(paths)) + for _, p := range paths { + out = append(out, filepath.Base(p)) + } + return out +} + +// editorOf returns the pipeline's editor stage, failing loudly when there is none. +func editorOf(t *testing.T, p *Pipeline) *Stage { + t.Helper() + for i := range p.Stages { + if p.Stages[i].Role == "editor" { + return &p.Stages[i] + } + } + t.Fatal("the config has no editor stage") + return nil +} + +// withoutEditorModel is the config with the editor's MODEL blanked — the model, what it resolves to, and the +// knobs the model choice FORCES: the few-shot policy (a reasoning editor drops the hand examples, a +// non-reasoning one keeps them; TestSwapArmConfigs pins which is which), and reasoning_max_tokens on an +// ADDITIVE-billing provider, where LoadPipeline refuses the stage without a buffer. Everything else — +// temperature, the reasoning knob, the prompt and its label — is COMPARED: chosen, not forced, and each is a +// second variable the arm would isolate without saying so. +// +// ⛔ THE CONDITION IS THE POINT, and blanking unconditionally traded one hole for another. On a subset-billed +// editor reasoning_max_tokens is optional and unchecked, yet it sizes the spend estimate — so an arm could +// carry a buffer production does not have and stay green. Exempt exactly what the other rule forces. +// +// ⛔ Blanking the WHOLE stage was this gate's own first hole: an arm could then differ from production on +// temperature and reasoning too and stay green, isolating three variables while its header claimed one. +// +// The stage SLOT stays in place, so a stage list of a different length or order still differs. The Stages +// slice is copied because a Pipeline value shares its backing array with the original. +func withoutEditorModel(t *testing.T, m *Models, p *Pipeline) Pipeline { + t.Helper() + out := *p + out.Stages = append([]Stage(nil), p.Stages...) + found := false + for i := range out.Stages { + if out.Stages[i].Role != "editor" { + continue + } + // Read the model from p: out's copy is blanked two lines below. + if m.providerReasoning(p.Stages[i].Model) == "additive" { + out.Stages[i].ReasoningMaxTokens = 0 + } + out.Stages[i].Model, out.Stages[i].ResolvedModel = "", "" + out.Stages[i].EscalateTo, out.Stages[i].ResolvedHop = "", "" + out.Stages[i].FewShot = nil + found = true + } + if !found { + t.Fatal("no editor stage: this comparison would exempt nothing, and the two files would not be an arm and its base") + } + return out +} + +// pipelineFieldDiffs names the top-level fields of two loaded pipelines that differ, with both values. +// Top-level is deliberate: it points a reader at the BLOCK (Gates, Mining, Stages), and the two YAML files +// are then a three-line diff apart — where a recursive differ would be a second, hand-written model of a +// struct that already knows how to compare itself. +// renderField prints a field for the failure message with POINTERS DEREFERENCED. Stage.FewShot is a *bool +// — a plain %+v prints its address, which is the one thing an operator cannot compare between two configs, +// and few-shot is exactly the key an arm is allowed to differ on for a documented reason. +func renderField(v reflect.Value) string { + switch v.Kind() { + case reflect.Pointer: + if v.IsNil() { + return "" + } + return "&" + renderField(v.Elem()) + case reflect.Slice, reflect.Array: + parts := make([]string, 0, v.Len()) + for i := 0; i < v.Len(); i++ { + parts = append(parts, renderField(v.Index(i))) + } + return "[" + strings.Join(parts, " ") + "]" + case reflect.Struct: + parts := make([]string, 0, v.NumField()) + for i := 0; i < v.NumField(); i++ { + if !v.Field(i).CanInterface() { + continue + } + parts = append(parts, v.Type().Field(i).Name+":"+renderField(v.Field(i))) + } + return "{" + strings.Join(parts, " ") + "}" + default: + return fmt.Sprintf("%+v", v.Interface()) + } +} + +func pipelineFieldDiffs(a, b Pipeline) []string { + va, vb := reflect.ValueOf(a), reflect.ValueOf(b) + var out []string + for i := 0; i < va.NumField(); i++ { + // CanInterface, because the first unexported field added to Pipeline would otherwise turn this gate + // into a panic instead of a message — and an unexported field is invisible to this comparison in + // any case, which is worth saying out loud rather than crashing over. + if !va.Field(i).CanInterface() { + continue + } + if reflect.DeepEqual(va.Field(i).Interface(), vb.Field(i).Interface()) { + continue + } + out = append(out, fmt.Sprintf("%s differs:\n c1 = %s\n arm = %s", + va.Type().Field(i).Name, renderField(va.Field(i)), renderField(vb.Field(i)))) + } + return out +} diff --git a/backend/internal/pipeline/classifiervocab_test.go b/backend/internal/pipeline/classifiervocab_test.go new file mode 100644 index 00000000..47e617f2 --- /dev/null +++ b/backend/internal/pipeline/classifiervocab_test.go @@ -0,0 +1,128 @@ +package pipeline + +import ( + "path/filepath" + "regexp" + "strings" + "testing" + + "textmachine/backend/internal/terminology" +) + +// classifiervocab_test.go: the classifier's answer vocabulary is a CLOSED set of engine identifiers, and +// the only place a model is ever told what they are is the pair's own prompt. Nothing checked that the +// prompt says them. Measured: a prompts/ja-ru/classifier.md whose classes read 人名 · 地名 · 称号 · 用語 +// passes the entire battery — every off-vocabulary reply is counted and dropped with a warning, never +// refused — so the pair works, silently mistyped, and the project's default review question («will a pair +// that is not in the repository work without editing Go?») gets the answer «yes, and wrongly». +// +// ⛔ IT LINTS THE CANONICAL FORM, comments stripped, and that is not a detail. Every one of these +// identifiers also appears in the zh-ru prompt's HTML header, so a lint that grepped the raw file would +// pass a pack whose instructions to the model are in a different vocabulary entirely. The negative fixture +// below is built with exactly that shape, so a regression to reading raw bytes fails this test. +// +// ⚠ WHAT THIS LINT CANNOT DO, said here rather than left for a reader to discover: it proves the +// identifiers are ON THE WIRE, not that the model is told to ANSWER in them. A pack that glosses its own +// classes bilingually — «人名 (name)» — and then instructs the model to reply in its own language passes +// this and is exactly as broken as the pack the row measured. Closing THAT needs the engine to refuse an +// off-vocabulary reply instead of counting and dropping it (terminologist.go, Log.WarnContext), which is a +// change of paid-run behaviour and is not this pack's to make. What is closed here is the measured case: +// a pack ported by translating everything, in which the identifiers appear nowhere the model can see. + +// promptPlaceholder matches a `{{name}}` slot. The engine substitutes these before the call (render.go), so +// their NAMES are not text the model is instructed with. +var promptPlaceholder = regexp.MustCompile(`\{\{[^}]*\}\}`) + +// classifierVocabulary is what a classifier prompt has to name: the class identifiers the parser keeps and +// the gender words it keeps. Read off the engine's own sets, so a class added in Go joins the lint by +// being added rather than by anyone remembering this file. +func classifierVocabulary() []string { + return append(terminology.TypeNames(terminology.Types), terminology.TypeNames(terminology.Genders)...) +} + +// classifierVocabularyGaps names the identifiers a prompt never says WHERE THE CLASSIFIER PUTS IT ON THE +// WIRE: the canonical form (comments stripped, which is also what the snapshot folds), System and User +// parts only. +// +// ⛔ THE FEW-SHOT BLOCK IS DELIBERATELY EXCLUDED, and reading it was this lint's own first bug. The +// classifier renders through MessagesWithInjection (render.go), which takes tpl.System — NOT +// SystemFor(fewShotOn) — so a ---FEWSHOT--- block never reaches this role's wire at all. Linting it made +// a pack whose entire English vocabulary sat in the few-shot block read as healthy while the model was +// shown Japanese. +func classifierVocabularyGaps(t *testing.T, path string) []string { + t.Helper() + tpl, err := LoadPromptTemplate(path) + if err != nil { + t.Fatalf("load %s in its canonical form: %v", path, err) + } + // ⛔ PLACEHOLDERS ARE REMOVED BEFORE THE SEARCH. `{{title}}` is the BOOK's title, substituted before the + // call, and it satisfied a word-boundary search for the class `title` — so a pack that localised the + // class everywhere the model reads it still counted as naming it. What must be searched is the prompt's + // own words, not the names of the values poured into it. + shown := promptPlaceholder.ReplaceAllString(strings.Join([]string{tpl.System, tpl.User}, "\n"), " ") + var missing []string + for _, w := range classifierVocabulary() { + // Word boundaries, not substrings: `term` sits inside `terminology`, and an en-ru pack that only + // discussed terminology would otherwise read as though it had named the class. + if !regexp.MustCompile(`\b` + regexp.QuoteMeta(w) + `\b`).MatchString(shown) { + missing = append(missing, w) + } + } + return missing +} + +func TestEveryClassifierPromptSpeaksTheEnginesVocabulary(t *testing.T) { + packs, err := filepath.Glob(filepath.Join("..", "..", "prompts", "*", "classifier.md")) + if err != nil { + t.Fatal(err) + } + if len(packs) == 0 { + t.Fatal("the repository has no prompts//classifier.md at all — this lint would be enforcing nothing") + } + // ⛔ AND THE OTHER HALF OF THE SAME CONTROL. This lint pins ONE direction — every word the engine keeps + // is named in the prompt — so it shrinks with its own subject: drop a class from terminology.Types and + // the assertion about that class disappears with it, silently (measured: removing `title` leaves the + // whole battery at 19 ok / 0 FAIL). At the limit an empty vocabulary makes this test AND both of its + // negative fixtures vacuous, since they compare len(missing) against len(vocabulary) — 0 != 0. + if len(classifierVocabulary()) == 0 { + t.Fatal("terminology.Types and terminology.Genders are both empty — the lint, and the two negative cases below, would all be comparing nothing with nothing") + } + for _, p := range packs { + if missing := classifierVocabularyGaps(t, p); len(missing) > 0 { + t.Errorf("%s never names %v where the model can see it. The engine KEEPS only these identifiers "+ + "(terminology.Types and terminology.Genders) and counts-and-drops every other answer, so a pack "+ + "that answers in its own words buys a paid pass and changes nothing.", p, missing) + } + } + t.Logf("classifier prompts linted: %d; vocabulary asserted: %v", len(packs), classifierVocabulary()) + + // THE NEGATIVE CASE, IN THE SAME RUN. A pack authored the way the row describes — the zh-ru file + // translated, its classes and genders now in the pair's own language, the English identifiers left + // behind in the header comment where they do the model no good. A lint reading raw bytes passes this. + localized := filepath.Join(t.TempDir(), "classifier.md") + writeFile(t, localized, + "\n"+ + "用語を分類してください。クラス: 人名 · 地名 · 称号 · 用語。性別: 男性 · 女性 · 中性 · 不明。\n"+ + "---USER---\n用語:\n\n{{text}}\n") + missing := classifierVocabularyGaps(t, localized) + if len(missing) != len(classifierVocabulary()) { + t.Fatalf("a fully localized pack was reported to be missing only %v of %v — the lint is reading the "+ + "prompt's COMMENTS, where the identifiers still are, instead of the form the model is shown", + missing, classifierVocabulary()) + } + + // THE SECOND NEGATIVE CASE, and the more likely one: a pack that keeps the English identifiers but puts + // them in the ---FEWSHOT--- block. That block is real prompt text, it is NOT a comment, and it still + // never reaches this role's wire — MessagesWithInjection renders tpl.System. A lint that read the whole + // template would pass this pack while the model saw only Japanese. + fewShotOnly := filepath.Join(t.TempDir(), "classifier.md") + writeFile(t, fewShotOnly, + "用語を分類してください。クラス: 人名 · 地名 · 称号 · 用語。性別: 男性 · 女性 · 中性 · 不明。\n"+ + "---FEWSHOT---\nAnswer with one of: name, place, title, term; gender male, female, neuter, none.\n"+ + "---USER---\n用語:\n\n{{text}}\n") + if missing := classifierVocabularyGaps(t, fewShotOnly); len(missing) != len(classifierVocabulary()) { + t.Fatalf("a pack whose vocabulary lives only in ---FEWSHOT--- was reported to be missing only %v of %v "+ + "— that block never reaches the classifier's wire, so linting it passes a pack the model cannot read", + missing, classifierVocabulary()) + } +}