Add the three test files the engine zone's new gates live in, which the previous landing left untracked
This commit is contained in:
parent
7ffec51c6d
commit
4188ba5ff3
3 changed files with 512 additions and 0 deletions
192
backend/internal/chunk/headingparity_test.go
Normal file
192
backend/internal/chunk/headingparity_test.go
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
package chunk
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/lang"
|
||||
)
|
||||
|
||||
// headingparity_test.go holds the two halves of a question nobody had asked the code: does the chapter
|
||||
// heading rule mean the same thing on both sides of the ingest→chunker seam? The rule is spread over three
|
||||
// predicates in two files — the ingest's length guard (chapterHeaderMaxRunes), the chunker's numeral guard
|
||||
// (a zero section is not a section) and the shared separator class — and each half was pinned only by its
|
||||
// own tests, which cannot see a disagreement between them.
|
||||
//
|
||||
// The first test is the GUARANTEE, the second its named exception. Neither works alone: a pin of a known
|
||||
// defect is a second description of the bug, and it leaves nothing behind once the defect is fixed.
|
||||
|
||||
// headingParityBody is chapter prose, long enough to be real and short enough to keep the fixture cheap.
|
||||
func headingParityBody() string { return "\n" + strings.Repeat("古月方源站在洞口。", 30) }
|
||||
|
||||
// TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders is the live guarantee: on the header shapes a book
|
||||
// actually uses, the ingest's predicate and the chunker's parse answer the same, and a book cut by the
|
||||
// ingest comes out of the chunker with a deterministic title on EVERY chapter and no source marker left in
|
||||
// the text the model reads. Green today, and it must stay green after the divergence below is closed.
|
||||
func TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders(t *testing.T) {
|
||||
st := lang.DefaultCJKStructure()
|
||||
rule := zhRuChapterRule()
|
||||
// ⛔ THE NUMBERS ARE NOT CONSECUTIVE, ON PURPOSE. A file numbered 一·二·三 makes the parsed header number
|
||||
// equal the chapter's ORDINAL, and every assertion below would then hold just as well for a title
|
||||
// rendered from the counter as for one rendered from the header — the two quantities coincide and the
|
||||
// whole class of «the title came from the wrong place» goes invisible. Here chapter 2 must be «Глава 3».
|
||||
headers := []struct {
|
||||
line string
|
||||
num int
|
||||
}{
|
||||
{"第一章 起始", 1},
|
||||
{"第三章 继续", 3},
|
||||
{"第七章：正常标题", 7},
|
||||
}
|
||||
var src strings.Builder
|
||||
for _, h := range headers {
|
||||
fmt.Fprintf(&src, "%s%s\n\n", h.line, headingParityBody())
|
||||
}
|
||||
doc, err := IngestEncoded(writeTempTXT(t, src.String()), "", "zh", st)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(doc.Chapters) != len(headers) {
|
||||
t.Fatalf("the ingest cut %d chapters out of %d headers", len(doc.Chapters), len(headers))
|
||||
}
|
||||
unit := detectChapterUnit(strings.Split(src.String(), "\n"), st)
|
||||
if unit == 0 {
|
||||
t.Fatal("test premise broken: the ingest detected no chapter unit, so its predicate was never asked")
|
||||
}
|
||||
|
||||
// The two predicates on the same lines. The chunker's NUMBER is asserted too: agreeing that a line is a
|
||||
// header while reading a different number out of it is the same defect one layer down.
|
||||
for _, h := range headers {
|
||||
byIngest := isChapterHeader(h.line, unit, st)
|
||||
n, _, byChunker := matchHeaderLine(h.line, rule)
|
||||
if !byIngest || !byChunker || n != h.num {
|
||||
t.Fatalf("%q: the ingest says header=%v; the chunker says header=%v number=%d — want a header on both sides and number %d",
|
||||
h.line, byIngest, byChunker, n, h.num)
|
||||
}
|
||||
}
|
||||
|
||||
chunks := SplitChunks(doc.Chapters, testSeg(), rule, testAbbrevs())
|
||||
titled := 0
|
||||
for _, c := range chunks {
|
||||
if c.ChunkIdx != 0 {
|
||||
continue
|
||||
}
|
||||
titled++
|
||||
// Guarded, not indexed straight into: the whole point of this fixture is that a chapter's NUMBER and
|
||||
// its position are different quantities, so an index built from one to look up the other has to say
|
||||
// so when it goes out of range instead of panicking.
|
||||
if c.Chapter < 1 || c.Chapter > len(headers) {
|
||||
t.Fatalf("the cut produced chapter %d and the fixture has %d headers — the chapter counter no longer indexes them",
|
||||
c.Chapter, len(headers))
|
||||
}
|
||||
// The title is the SOURCE HEADER's number, not the chapter's position in the file. On this fixture
|
||||
// the two differ from chapter 2 on, which is what makes the assertion say anything.
|
||||
if want := fmt.Sprintf("Глава %d", headers[c.Chapter-1].num); c.Heading != want {
|
||||
t.Fatalf("chapter %d (header %q) opens with heading %q, want %q — the title must come from the header's own number, not from the chapter counter",
|
||||
c.Chapter, headers[c.Chapter-1].line, c.Heading, want)
|
||||
}
|
||||
if _, opensWithMarker := st.MatchMarker(c.Text); opensWithMarker {
|
||||
t.Fatalf("chapter %d still opens with its source marker, so the raw header reaches the model: %q",
|
||||
c.Chapter, firstRunes(c.Text, 16))
|
||||
}
|
||||
}
|
||||
if titled != len(headers) {
|
||||
t.Fatalf("%d chapters opened, want %d", titled, len(headers))
|
||||
}
|
||||
}
|
||||
|
||||
// TestIngestAndChunkerDivergeOnAZeroPrologueAndALongHeader pins a DIVERGENCE that exists today (backlog
|
||||
// row 346), not a guarantee. Two header shapes are cut by the ingest and refused by the chunker, so the
|
||||
// header line survives into the text the model translates and the chapter numbers shift under the reader:
|
||||
//
|
||||
// - «第零章», a prologue: parseSectionNumeral refuses a zero section, the ingest's regex accepts it;
|
||||
// - a header past chapterHeaderMaxRunes: the length guard exists only in the ingest.
|
||||
//
|
||||
// ⛔ THE FIX IS NOT IN SCOPE HERE AND IT IS NOT $0: it changes the TEXT of a chunk, which bumps
|
||||
// chunkerVersion, which moves both waves' snapshots and re-bills the book. It lands in the re-cut window.
|
||||
// So what is asserted is TODAY's shape and the test is green — a red test cannot be left in main, and a
|
||||
// skip would hide the whole class rather than name one case of it. When the fix lands this test fails on
|
||||
// purpose, and every failure message says what to do about it.
|
||||
func TestIngestAndChunkerDivergeOnAZeroPrologueAndALongHeader(t *testing.T) {
|
||||
const closed = "the divergence backlog row 346 pins has CLOSED — that is the fix landing, not a regression. Delete this test and fold the case into TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders"
|
||||
st := lang.DefaultCJKStructure()
|
||||
rule := zhRuChapterRule()
|
||||
src := "第零章：序幕" + headingParityBody() + "\n\n第一章 起始" + headingParityBody() + "\n\n第二章 继续" + headingParityBody() + "\n"
|
||||
|
||||
doc, err := IngestEncoded(writeTempTXT(t, src), "", "zh", st)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// (1) The ingest cuts the prologue as a chapter of its own.
|
||||
if len(doc.Chapters) != 3 {
|
||||
// NOT the `closed` message: this is the test's PREMISE, and it breaks when the INGEST stops cutting
|
||||
// the prologue as its own chapter — the opposite side of the seam from the fix this test waits for.
|
||||
// Telling a reader to delete the test would send them to fold a case nobody has closed.
|
||||
t.Fatalf("premise broken: the ingest cut %d chapters, want 3 (a prologue and two) — the ingest's own header predicate changed, and the divergence below has not been judged either way", len(doc.Chapters))
|
||||
}
|
||||
chunks := SplitChunks(doc.Chapters, testSeg(), rule, testAbbrevs())
|
||||
// Guarded rather than indexed straight into: this test is BUILT to fail one day, and whoever reads that
|
||||
// failure has to get the message below, not an index panic.
|
||||
if len(chunks) < 3 {
|
||||
t.Fatalf("the cut produced %d chunks for 3 chapters — %s", len(chunks), closed)
|
||||
}
|
||||
// (2) The chunker does not recognise the prologue's header, so the chapter opens untitled …
|
||||
if chunks[0].Heading != "" {
|
||||
t.Fatalf("the prologue now carries the title %q — %s", chunks[0].Heading, closed)
|
||||
}
|
||||
// (3) … and its raw CJK marker travels to the model inside the text it translates.
|
||||
if !strings.HasPrefix(chunks[0].Text, "第零章") {
|
||||
t.Fatalf("the prologue's source marker no longer reaches the model (%q) — %s", firstRunes(chunks[0].Text, 16), closed)
|
||||
}
|
||||
// (4) The half a reader sees, and it is NOT a divergence pin: a chapter's ORDINAL and the number in its
|
||||
// title are two different quantities, and the prologue makes them differ from chapter 2 on — the cut
|
||||
// numbers 第一章 as chapter 2 while the template renders its own «Глава 1». Correct on both sides, and it
|
||||
// survives the fix in (2): recognising 第零章 gives it a title, it does not renumber anything. Asserted
|
||||
// here because this is the file where the two numbers are measurably apart, and the re-cut design has to
|
||||
// say which of them addresses a chapter (chunk_status is keyed on the ordinal; a reader sees the title).
|
||||
var second *Chunk
|
||||
for i := range chunks {
|
||||
if chunks[i].Chapter == 2 && chunks[i].ChunkIdx == 0 {
|
||||
second = &chunks[i]
|
||||
}
|
||||
}
|
||||
if second == nil {
|
||||
t.Fatalf("the cut has no chapter 2 — %s", closed)
|
||||
}
|
||||
if second.Heading != "Глава 1" {
|
||||
t.Fatalf("chapter 2 (第一章) is titled %q, want «Глава 1»: the ordinal and the title's number came apart differently than measured — this is the addressing question, NOT the row-346 divergence, so read it before assuming a fix landed", second.Heading)
|
||||
}
|
||||
|
||||
// The length half, with its own CONTROL. ⚠ The header must carry a separator after its unit rune
|
||||
// («第四章：…»): without one BOTH sides refuse it for an unrelated reason and the divergence does not
|
||||
// reproduce at any length at all.
|
||||
unit := detectChapterUnit(strings.Split(src, "\n"), st)
|
||||
atCeiling := "第四章：" + strings.Repeat("标", chapterHeaderMaxRunes-4)
|
||||
pastCeiling := "第四章：" + strings.Repeat("标", chapterHeaderMaxRunes-3)
|
||||
if len([]rune(atCeiling)) != chapterHeaderMaxRunes || len([]rune(pastCeiling)) != chapterHeaderMaxRunes+1 {
|
||||
t.Fatalf("the test built the wrong lengths: %d and %d runes around a ceiling of %d",
|
||||
len([]rune(atCeiling)), len([]rune(pastCeiling)), chapterHeaderMaxRunes)
|
||||
}
|
||||
// CONTROL: at exactly the ceiling both halves still say "header". It is what makes the disagreement
|
||||
// below a disagreement about LENGTH rather than about the shape of this fixture.
|
||||
_, _, chunkerAt := matchHeaderLine(atCeiling, rule)
|
||||
if ingestAt := isChapterHeader(atCeiling, unit, st); !ingestAt || !chunkerAt {
|
||||
t.Fatalf("control broken: at exactly %d runes the ingest says %v and the chunker says %v — they must still agree there",
|
||||
chapterHeaderMaxRunes, ingestAt, chunkerAt)
|
||||
}
|
||||
n, _, chunkerPast := matchHeaderLine(pastCeiling, rule)
|
||||
if ingestPast := isChapterHeader(pastCeiling, unit, st); ingestPast || !chunkerPast || n != 4 {
|
||||
t.Fatalf("at %d runes the ingest says header=%v and the chunker says header=%v number=%d; today the ingest refuses it and the chunker reads chapter 4. %s",
|
||||
chapterHeaderMaxRunes+1, ingestPast, chunkerPast, n, closed)
|
||||
}
|
||||
}
|
||||
|
||||
// firstRunes keeps a failure message readable when the text it quotes is a whole chapter.
|
||||
func firstRunes(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
192
backend/internal/config/armparity_test.go
Normal file
192
backend/internal/config/armparity_test.go
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// armparity_test.go: an editor swap-arm exists to isolate ONE variable, and it can only do that if every
|
||||
// other key is the production config's. Three arm headers claimed exactly that and were false by two dozen
|
||||
// keys — the whole bank contour among them (b8154cd gave pipeline-c1.yaml sixty-four lines of it and the
|
||||
// arms a single prompt_version), while the test that called itself the guard of «only the editor differs»
|
||||
// compared three fields. The cost is money, not tidiness, and it runs the OTHER way: with no
|
||||
// mining.contrast_path the arm's own book-level price is zero (priceprojection.go, bookOnceUSD returns 0
|
||||
// when the terminology gate is off or the contrast path is empty), so a deploy flipped to an arm quietly
|
||||
// bought a cheaper book than the one it reported — the bank contour was configured nowhere and priced
|
||||
// nowhere, and nothing said so.
|
||||
//
|
||||
// ⛔ THE FORM IS THE POINT. This compares the WHOLE loaded configs and exempts the editor stage, instead of
|
||||
// listing the keys that must match. A list is what drifted: it can only check what somebody remembered to
|
||||
// put in it, and a key ADDED to c1 is invisible to it by construction.
|
||||
|
||||
func TestAnArmIsTheProductionConfigWithADifferentEditor(t *testing.T) {
|
||||
m, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
|
||||
if err != nil {
|
||||
t.Fatalf("load the shipping models.yaml: %v", err)
|
||||
}
|
||||
// ⛔ THE ARMS ARE DISCOVERED, NOT LISTED. A list here would be the very thing the header refuses one
|
||||
// paragraph up: a new pipeline-arm-*.yaml would join the repository outside every gate, and the file
|
||||
// carrying the money defect (an arm with no mining.contrast_path) would be the one nobody loaded.
|
||||
arms, err := filepath.Glob(filepath.Join("..", "..", "configs", "pipeline-arm-*.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(arms) == 0 {
|
||||
t.Fatal("no configs/pipeline-arm-*.yaml in the repository — this gate would be enforcing nothing")
|
||||
}
|
||||
load := func(t *testing.T, file string) *Pipeline {
|
||||
t.Helper()
|
||||
p, lerr := LoadPipeline(file, m, "zh-ru", nil)
|
||||
if lerr != nil {
|
||||
// ⚠ t is a PARAMETER, not the enclosing test's: a Fatalf on the outer t from inside a subtest
|
||||
// kills the parent, and every arm after this one goes unchecked.
|
||||
t.Fatalf("load %s: %v", file, lerr)
|
||||
}
|
||||
return p
|
||||
}
|
||||
base := load(t, filepath.Join("..", "..", "configs", "pipeline-c1.yaml"))
|
||||
baseEditor := editorOf(t, base)
|
||||
t.Logf("arms discovered: %d %v; production editor: %s", len(arms), armNames(arms), baseEditor.Model)
|
||||
for _, file := range arms {
|
||||
t.Run(filepath.Base(file), func(t *testing.T) {
|
||||
arm := load(t, file)
|
||||
editor := editorOf(t, arm).Model
|
||||
// An arm whose editor is the production editor isolates nothing, and a file that isolates
|
||||
// nothing is not an arm — it is a second copy of the production config that has to be kept in
|
||||
// step forever. Said as a gate, because it is a state the interim editor can drift INTO
|
||||
// (D39.22 calls the current one interim): the day c1 adopts an arm's editor, that arm has to be
|
||||
// retired or re-pointed, deliberately.
|
||||
if baseEditor.Model == editor {
|
||||
t.Fatalf("the production editor is already %q, so this file isolates no variable: retire it or point it at a different editor", editor)
|
||||
}
|
||||
// The exemptions are NAMED in the log, so a green run is never read as "every key was compared".
|
||||
// reasoning_max_tokens is exempt only where the loader forces it — today that is one model of the
|
||||
// nine in models.yaml (grok-4.3, the only `additive`), and no arm runs it.
|
||||
exempt := "model, resolved model, escalate_to, few_shot"
|
||||
if m.providerReasoning(editorOf(t, arm).Model) == "additive" {
|
||||
exempt += ", reasoning_max_tokens (additive billing forces it; nothing compares its VALUE)"
|
||||
}
|
||||
t.Logf("%s: editor %q; exempt from the comparison: %s", filepath.Base(file), editor, exempt)
|
||||
for _, d := range pipelineFieldDiffs(withoutEditorModel(t, m, base), withoutEditorModel(t, m, arm)) {
|
||||
t.Errorf("%s is not pipeline-c1.yaml with a different editor — %s", filepath.Base(file), d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// armNames renders the discovered files for a log line.
|
||||
func armNames(paths []string) []string {
|
||||
out := make([]string, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
out = append(out, filepath.Base(p))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// editorOf returns the pipeline's editor stage, failing loudly when there is none.
|
||||
func editorOf(t *testing.T, p *Pipeline) *Stage {
|
||||
t.Helper()
|
||||
for i := range p.Stages {
|
||||
if p.Stages[i].Role == "editor" {
|
||||
return &p.Stages[i]
|
||||
}
|
||||
}
|
||||
t.Fatal("the config has no editor stage")
|
||||
return nil
|
||||
}
|
||||
|
||||
// withoutEditorModel is the config with the editor's MODEL blanked — the model, what it resolves to, and the
|
||||
// knobs the model choice FORCES: the few-shot policy (a reasoning editor drops the hand examples, a
|
||||
// non-reasoning one keeps them; TestSwapArmConfigs pins which is which), and reasoning_max_tokens on an
|
||||
// ADDITIVE-billing provider, where LoadPipeline refuses the stage without a buffer. Everything else —
|
||||
// temperature, the reasoning knob, the prompt and its label — is COMPARED: chosen, not forced, and each is a
|
||||
// second variable the arm would isolate without saying so.
|
||||
//
|
||||
// ⛔ THE CONDITION IS THE POINT, and blanking unconditionally traded one hole for another. On a subset-billed
|
||||
// editor reasoning_max_tokens is optional and unchecked, yet it sizes the spend estimate — so an arm could
|
||||
// carry a buffer production does not have and stay green. Exempt exactly what the other rule forces.
|
||||
//
|
||||
// ⛔ Blanking the WHOLE stage was this gate's own first hole: an arm could then differ from production on
|
||||
// temperature and reasoning too and stay green, isolating three variables while its header claimed one.
|
||||
//
|
||||
// The stage SLOT stays in place, so a stage list of a different length or order still differs. The Stages
|
||||
// slice is copied because a Pipeline value shares its backing array with the original.
|
||||
func withoutEditorModel(t *testing.T, m *Models, p *Pipeline) Pipeline {
|
||||
t.Helper()
|
||||
out := *p
|
||||
out.Stages = append([]Stage(nil), p.Stages...)
|
||||
found := false
|
||||
for i := range out.Stages {
|
||||
if out.Stages[i].Role != "editor" {
|
||||
continue
|
||||
}
|
||||
// Read the model from p: out's copy is blanked two lines below.
|
||||
if m.providerReasoning(p.Stages[i].Model) == "additive" {
|
||||
out.Stages[i].ReasoningMaxTokens = 0
|
||||
}
|
||||
out.Stages[i].Model, out.Stages[i].ResolvedModel = "", ""
|
||||
out.Stages[i].EscalateTo, out.Stages[i].ResolvedHop = "", ""
|
||||
out.Stages[i].FewShot = nil
|
||||
found = true
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("no editor stage: this comparison would exempt nothing, and the two files would not be an arm and its base")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// pipelineFieldDiffs names the top-level fields of two loaded pipelines that differ, with both values.
|
||||
// Top-level is deliberate: it points a reader at the BLOCK (Gates, Mining, Stages), and the two YAML files
|
||||
// are then a three-line diff apart — where a recursive differ would be a second, hand-written model of a
|
||||
// struct that already knows how to compare itself.
|
||||
// renderField prints a field for the failure message with POINTERS DEREFERENCED. Stage.FewShot is a *bool
|
||||
// — a plain %+v prints its address, which is the one thing an operator cannot compare between two configs,
|
||||
// and few-shot is exactly the key an arm is allowed to differ on for a documented reason.
|
||||
func renderField(v reflect.Value) string {
|
||||
switch v.Kind() {
|
||||
case reflect.Pointer:
|
||||
if v.IsNil() {
|
||||
return "<nil>"
|
||||
}
|
||||
return "&" + renderField(v.Elem())
|
||||
case reflect.Slice, reflect.Array:
|
||||
parts := make([]string, 0, v.Len())
|
||||
for i := 0; i < v.Len(); i++ {
|
||||
parts = append(parts, renderField(v.Index(i)))
|
||||
}
|
||||
return "[" + strings.Join(parts, " ") + "]"
|
||||
case reflect.Struct:
|
||||
parts := make([]string, 0, v.NumField())
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
if !v.Field(i).CanInterface() {
|
||||
continue
|
||||
}
|
||||
parts = append(parts, v.Type().Field(i).Name+":"+renderField(v.Field(i)))
|
||||
}
|
||||
return "{" + strings.Join(parts, " ") + "}"
|
||||
default:
|
||||
return fmt.Sprintf("%+v", v.Interface())
|
||||
}
|
||||
}
|
||||
|
||||
func pipelineFieldDiffs(a, b Pipeline) []string {
|
||||
va, vb := reflect.ValueOf(a), reflect.ValueOf(b)
|
||||
var out []string
|
||||
for i := 0; i < va.NumField(); i++ {
|
||||
// CanInterface, because the first unexported field added to Pipeline would otherwise turn this gate
|
||||
// into a panic instead of a message — and an unexported field is invisible to this comparison in
|
||||
// any case, which is worth saying out loud rather than crashing over.
|
||||
if !va.Field(i).CanInterface() {
|
||||
continue
|
||||
}
|
||||
if reflect.DeepEqual(va.Field(i).Interface(), vb.Field(i).Interface()) {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s differs:\n c1 = %s\n arm = %s",
|
||||
va.Type().Field(i).Name, renderField(va.Field(i)), renderField(vb.Field(i))))
|
||||
}
|
||||
return out
|
||||
}
|
||||
128
backend/internal/pipeline/classifiervocab_test.go
Normal file
128
backend/internal/pipeline/classifiervocab_test.go
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/terminology"
|
||||
)
|
||||
|
||||
// classifiervocab_test.go: the classifier's answer vocabulary is a CLOSED set of engine identifiers, and
|
||||
// the only place a model is ever told what they are is the pair's own prompt. Nothing checked that the
|
||||
// prompt says them. Measured: a prompts/ja-ru/classifier.md whose classes read 人名 · 地名 · 称号 · 用語
|
||||
// passes the entire battery — every off-vocabulary reply is counted and dropped with a warning, never
|
||||
// refused — so the pair works, silently mistyped, and the project's default review question («will a pair
|
||||
// that is not in the repository work without editing Go?») gets the answer «yes, and wrongly».
|
||||
//
|
||||
// ⛔ IT LINTS THE CANONICAL FORM, comments stripped, and that is not a detail. Every one of these
|
||||
// identifiers also appears in the zh-ru prompt's HTML header, so a lint that grepped the raw file would
|
||||
// pass a pack whose instructions to the model are in a different vocabulary entirely. The negative fixture
|
||||
// below is built with exactly that shape, so a regression to reading raw bytes fails this test.
|
||||
//
|
||||
// ⚠ WHAT THIS LINT CANNOT DO, said here rather than left for a reader to discover: it proves the
|
||||
// identifiers are ON THE WIRE, not that the model is told to ANSWER in them. A pack that glosses its own
|
||||
// classes bilingually — «人名 (name)» — and then instructs the model to reply in its own language passes
|
||||
// this and is exactly as broken as the pack the row measured. Closing THAT needs the engine to refuse an
|
||||
// off-vocabulary reply instead of counting and dropping it (terminologist.go, Log.WarnContext), which is a
|
||||
// change of paid-run behaviour and is not this pack's to make. What is closed here is the measured case:
|
||||
// a pack ported by translating everything, in which the identifiers appear nowhere the model can see.
|
||||
|
||||
// promptPlaceholder matches a `{{name}}` slot. The engine substitutes these before the call (render.go), so
|
||||
// their NAMES are not text the model is instructed with.
|
||||
var promptPlaceholder = regexp.MustCompile(`\{\{[^}]*\}\}`)
|
||||
|
||||
// classifierVocabulary is what a classifier prompt has to name: the class identifiers the parser keeps and
|
||||
// the gender words it keeps. Read off the engine's own sets, so a class added in Go joins the lint by
|
||||
// being added rather than by anyone remembering this file.
|
||||
func classifierVocabulary() []string {
|
||||
return append(terminology.TypeNames(terminology.Types), terminology.TypeNames(terminology.Genders)...)
|
||||
}
|
||||
|
||||
// classifierVocabularyGaps names the identifiers a prompt never says WHERE THE CLASSIFIER PUTS IT ON THE
|
||||
// WIRE: the canonical form (comments stripped, which is also what the snapshot folds), System and User
|
||||
// parts only.
|
||||
//
|
||||
// ⛔ THE FEW-SHOT BLOCK IS DELIBERATELY EXCLUDED, and reading it was this lint's own first bug. The
|
||||
// classifier renders through MessagesWithInjection (render.go), which takes tpl.System — NOT
|
||||
// SystemFor(fewShotOn) — so a ---FEWSHOT--- block never reaches this role's wire at all. Linting it made
|
||||
// a pack whose entire English vocabulary sat in the few-shot block read as healthy while the model was
|
||||
// shown Japanese.
|
||||
func classifierVocabularyGaps(t *testing.T, path string) []string {
|
||||
t.Helper()
|
||||
tpl, err := LoadPromptTemplate(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load %s in its canonical form: %v", path, err)
|
||||
}
|
||||
// ⛔ PLACEHOLDERS ARE REMOVED BEFORE THE SEARCH. `{{title}}` is the BOOK's title, substituted before the
|
||||
// call, and it satisfied a word-boundary search for the class `title` — so a pack that localised the
|
||||
// class everywhere the model reads it still counted as naming it. What must be searched is the prompt's
|
||||
// own words, not the names of the values poured into it.
|
||||
shown := promptPlaceholder.ReplaceAllString(strings.Join([]string{tpl.System, tpl.User}, "\n"), " ")
|
||||
var missing []string
|
||||
for _, w := range classifierVocabulary() {
|
||||
// Word boundaries, not substrings: `term` sits inside `terminology`, and an en-ru pack that only
|
||||
// discussed terminology would otherwise read as though it had named the class.
|
||||
if !regexp.MustCompile(`\b` + regexp.QuoteMeta(w) + `\b`).MatchString(shown) {
|
||||
missing = append(missing, w)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
func TestEveryClassifierPromptSpeaksTheEnginesVocabulary(t *testing.T) {
|
||||
packs, err := filepath.Glob(filepath.Join("..", "..", "prompts", "*", "classifier.md"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(packs) == 0 {
|
||||
t.Fatal("the repository has no prompts/<pair>/classifier.md at all — this lint would be enforcing nothing")
|
||||
}
|
||||
// ⛔ AND THE OTHER HALF OF THE SAME CONTROL. This lint pins ONE direction — every word the engine keeps
|
||||
// is named in the prompt — so it shrinks with its own subject: drop a class from terminology.Types and
|
||||
// the assertion about that class disappears with it, silently (measured: removing `title` leaves the
|
||||
// whole battery at 19 ok / 0 FAIL). At the limit an empty vocabulary makes this test AND both of its
|
||||
// negative fixtures vacuous, since they compare len(missing) against len(vocabulary) — 0 != 0.
|
||||
if len(classifierVocabulary()) == 0 {
|
||||
t.Fatal("terminology.Types and terminology.Genders are both empty — the lint, and the two negative cases below, would all be comparing nothing with nothing")
|
||||
}
|
||||
for _, p := range packs {
|
||||
if missing := classifierVocabularyGaps(t, p); len(missing) > 0 {
|
||||
t.Errorf("%s never names %v where the model can see it. The engine KEEPS only these identifiers "+
|
||||
"(terminology.Types and terminology.Genders) and counts-and-drops every other answer, so a pack "+
|
||||
"that answers in its own words buys a paid pass and changes nothing.", p, missing)
|
||||
}
|
||||
}
|
||||
t.Logf("classifier prompts linted: %d; vocabulary asserted: %v", len(packs), classifierVocabulary())
|
||||
|
||||
// THE NEGATIVE CASE, IN THE SAME RUN. A pack authored the way the row describes — the zh-ru file
|
||||
// translated, its classes and genders now in the pair's own language, the English identifiers left
|
||||
// behind in the header comment where they do the model no good. A lint reading raw bytes passes this.
|
||||
localized := filepath.Join(t.TempDir(), "classifier.md")
|
||||
writeFile(t, localized,
|
||||
"<!-- ported from zh-ru: name / place / title / term; male | female | neuter | none -->\n"+
|
||||
"用語を分類してください。クラス: 人名 · 地名 · 称号 · 用語。性別: 男性 · 女性 · 中性 · 不明。\n"+
|
||||
"---USER---\n用語:\n\n{{text}}\n")
|
||||
missing := classifierVocabularyGaps(t, localized)
|
||||
if len(missing) != len(classifierVocabulary()) {
|
||||
t.Fatalf("a fully localized pack was reported to be missing only %v of %v — the lint is reading the "+
|
||||
"prompt's COMMENTS, where the identifiers still are, instead of the form the model is shown",
|
||||
missing, classifierVocabulary())
|
||||
}
|
||||
|
||||
// THE SECOND NEGATIVE CASE, and the more likely one: a pack that keeps the English identifiers but puts
|
||||
// them in the ---FEWSHOT--- block. That block is real prompt text, it is NOT a comment, and it still
|
||||
// never reaches this role's wire — MessagesWithInjection renders tpl.System. A lint that read the whole
|
||||
// template would pass this pack while the model saw only Japanese.
|
||||
fewShotOnly := filepath.Join(t.TempDir(), "classifier.md")
|
||||
writeFile(t, fewShotOnly,
|
||||
"用語を分類してください。クラス: 人名 · 地名 · 称号 · 用語。性別: 男性 · 女性 · 中性 · 不明。\n"+
|
||||
"---FEWSHOT---\nAnswer with one of: name, place, title, term; gender male, female, neuter, none.\n"+
|
||||
"---USER---\n用語:\n\n{{text}}\n")
|
||||
if missing := classifierVocabularyGaps(t, fewShotOnly); len(missing) != len(classifierVocabulary()) {
|
||||
t.Fatalf("a pack whose vocabulary lives only in ---FEWSHOT--- was reported to be missing only %v of %v "+
|
||||
"— that block never reaches the classifier's wire, so linting it passes a pack the model cannot read",
|
||||
missing, classifierVocabulary())
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue