package pipeline import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "os" "sort" "strings" "testing" "textmachine/backend/internal/lang" "textmachine/backend/internal/langscreen" "textmachine/backend/internal/llm" ) // offtarget_corpus_test.go: backlog row 113 — the numbers the whole target-language screen (row 46) is // argued from were produced by a TEMPORARY harness in this package that the session deleted, so «18 of 20 // caught, 0 false positives on 1055, 13× margin» went into a contract with nothing left that could // reproduce it. This is the base any proposal for row 46 has to beat, and it lives in the repository now. // // HOW THE CORPUS WAS TAKEN. Every checkpoint in every stand database under `/books/gu-zhenren` // (162 files), put through the engine's OWN preprocessing — splitBanknote then checks.StripThink, which is // the text the gate actually sees (stagerun cuts the banknote, then classifies) — deduplicated by the // hash of that text, and restricted to the SHIPPING roles: draft/translator and edit/editor. Bank-role // replies are excluded because their answers legitimately consist of source terms and the runner exempts // them (SourceEchoExpected); including them would measure the exemption, not the gate. That leaves 1162 // outputs, of which 30 are off-target and 1132 are healthy. // // HOW A SPECIMEN WAS LABELLED, and why not by the gate. An output is off-target when its share of // CYRILLIC letters is below 0.5 — a property of the TARGET side, independent of the source-script gate // under test, so the label cannot be a restatement of the verdict. The corpus separates completely on it: // every healthy output is above 0.9 and every off-target one is at 0.000, with the band between them // EMPTY. Two of the thirty are fluent English (4690 and 4811 letters — the same two the original // measurement names), one is Japanese, twenty-seven are Chinese echoes. // // ⚠ THE FIXTURE IS A SAMPLE OF THAT CORPUS, NOT THE CORPUS, and the numbers below are established over the // sample. All THIRTY off-target specimens are kept — the class is scarce and every member is a distinct // failure — while the healthy class is drawn deliberately hard rather than at random: the twenty with the // HIGHEST source-script share (nearest the echo gate), the fifteen with the LOWEST target-script share // (nearest the target screen), the three that fall under the evidence floor, and a spread of the rest. So // «0 false positives» here means «0 on the forty-four hardest of the 1132», which is the useful claim; it // is a weaker statement than 0 on all 1132, and saying otherwise would be the over-claim this header exists // to prevent. // // ⚠ WHAT THE FIXTURE CARRIES, AND WHY NOT PROSE. Each row is the specimen's rune ACCOUNTING — // total runes, runes in the source script, letters — plus a sixty-rune head for a human, its provenance, // and its label. It is deliberately not the texts: those are a copyrighted web novel and its machine // translations, which this repository keeps out of git on purpose (the stand lives in a separate // repository; `example/` ships an own-authored fragment for the same reason). The accounting is // sufficient BY CONSTRUCTION for what is under test — sourceScriptShare is a ratio of rune classes and // reads nothing else — and the tests below feed the real function a script-faithful surrogate built from // the counts, so the function stays exercised rather than replaced by arithmetic. // The named LOSS: a future detector that reads words, n-grams or orthography cannot be measured on this // fixture, and re-taking the corpus (the walk above) is what that would need. const offTargetCorpusFile = "testdata/offtarget-corpus.json" type offTargetSpecimen struct { ID string `json:"id"` Label string `json:"label"` // off_target | healthy SourceScriptRunes int `json:"source_script_runes"` TotalRunes int `json:"total_runes"` Letters int `json:"letters"` // TargetScriptLetters is the other half of the accounting: how many of Letters are written in the // TARGET's script. It is what the row-46 screen weighs, and it is recorded beside the source-side // counts so one corpus answers both questions and the two can never drift onto different samples. TargetScriptLetters int `json:"target_script_letters"` Provenance string `json:"provenance"` Head string `json:"head"` } // share is the specimen's recorded source-script share. func (s offTargetSpecimen) share() float64 { return float64(s.SourceScriptRunes) / float64(s.TotalRunes) } // surrogate renders the specimen's rune accounting as a string the real detector can read: exactly // SourceScriptRunes Han runes followed by exactly (TotalRunes-SourceScriptRunes) Cyrillic ones. Any two // runes of the right classes would do — the function counts classes, not characters. func (s offTargetSpecimen) surrogate() string { var b strings.Builder b.Grow(s.TotalRunes * 3) for i := 0; i < s.SourceScriptRunes; i++ { b.WriteRune('源') } for i := s.SourceScriptRunes; i < s.TotalRunes; i++ { b.WriteRune('а') } return b.String() } // targetShare is the specimen's recorded target-script share, over letters. func (s offTargetSpecimen) targetShare() float64 { if s.Letters == 0 { return 0 } return float64(s.TargetScriptLetters) / float64(s.Letters) } // targetSurrogate renders the specimen's LETTER accounting for the target screen the way surrogate() does // for the echo gate: exactly TargetScriptLetters Cyrillic letters and the rest Latin, so the real screen // weighs the specimen's real proportions. func (s offTargetSpecimen) targetSurrogate() string { var b strings.Builder b.Grow(s.Letters * 2) for i := 0; i < s.TargetScriptLetters; i++ { b.WriteRune('а') } for i := s.TargetScriptLetters; i < s.Letters; i++ { b.WriteRune('a') } return b.String() } func loadOffTargetCorpus(t *testing.T) []offTargetSpecimen { t.Helper() raw, err := os.ReadFile(offTargetCorpusFile) if err != nil { t.Fatalf("read the off-target corpus fixture: %v", err) } var out []offTargetSpecimen if err := json.Unmarshal(raw, &out); err != nil { t.Fatalf("the off-target corpus fixture is not readable JSON: %v", err) } if len(out) == 0 { t.Fatal("the fixture is empty — this test would be enforcing nothing") } return out } // TestTheOffTargetCorpusIsWhatItClaims guards the fixture itself before anything is measured on it. A // corpus that quietly lost its off-target class, or whose labels stopped separating, would make every // number below look excellent. func TestTheOffTargetCorpusIsWhatItClaims(t *testing.T) { corpus := loadOffTargetCorpus(t) counts := map[string]int{} seen := map[string]bool{} for _, s := range corpus { counts[s.Label]++ if seen[s.ID] { t.Errorf("duplicate specimen %s — a repeated row would weight one output twice", s.ID) } seen[s.ID] = true if s.TotalRunes <= 0 || s.SourceScriptRunes < 0 || s.SourceScriptRunes > s.TotalRunes { t.Errorf("%s: impossible accounting %+v", s.ID, s) } if s.Letters > s.TotalRunes { t.Errorf("%s: more letters than runes %+v", s.ID, s) } if s.TargetScriptLetters < 0 || s.TargetScriptLetters > s.Letters { t.Errorf("%s: impossible target accounting %+v", s.ID, s) } if s.Provenance == "" { t.Errorf("%s: a specimen with no provenance cannot be re-taken", s.ID) } } if counts["off_target"] < 20 { t.Errorf("the off-target class is the scarce one and the fixture must keep it: %d rows", counts["off_target"]) } if counts["healthy"] < 40 { t.Errorf("the healthy class is the false-positive denominator: %d rows", counts["healthy"]) } if len(counts) != 2 { t.Errorf("labels must be exactly {off_target, healthy}, got %v", counts) } } // TestTheCorpusStillHoldsItsHardestSpecimens is the fixture's LEDGER, and it exists because a floor is not // a measurement. The margin test asserts «at least 3×», which is the STOP row 113 names — and a fixture // quietly softened until the hard cases were gone would sail through it reporting a BETTER number than the // corpus ever had. Found by planting into the DATA: zeroing the source runes of the single worst healthy // specimen left the whole battery green while making the margin larger. // // So the numbers the contract is argued from are RECORDED here, exactly as the prompt-label ledger records // a prompt's bytes, and for the same reason: what must never happen silently is the corpus and the claim // drifting apart. ⚠ RE-TAKING THE CORPUS IS LEGITIMATE, and these numbers are then re-recorded in the same // commit — the point is that it costs an edit here and a sentence in the report, not nothing. func TestTheCorpusStillHoldsItsHardestSpecimens(t *testing.T) { // Measured over 1162 shipping completions of the stand (162 databases), of which the fixture keeps all // 30 off-target and the 44 hardest healthy. See the file header for how the healthy class was drawn. const ( wantOffTargets = 30 wantHealthy = 44 wantWorstHealthySrc = 0.011655 // f3367ee44f56 — the specimen the 12.87× margin rests on wantWorstHealthyTgt = 0.976706 // 905834e2606c — the specimen the 0.50 floor is placed under wantShortestOffChars = 346 // 1612c74012cc — what the 200-letter evidence floor is calibrated against tol = 1e-6 // wantBestOffTargetTgt is the OTHER side of the interval, and its absence was a hole: the screen's // doc comment argues «no near-miss at all on the other side» and the threshold test compares that // claim's constant against another constant. The data itself was unpinned — planted, an English // draft's target share moved 0.000 → 0.490 and every corpus test stayed green, which would have left // the 0.50 floor sitting directly on top of a specimen while the ledger said the band was empty. wantBestOffTargetTgt = 0.0 // corpusDigest is the POPULATION, not its extrema. The ledger above records one number per axis, so // every specimen except the worst one could be softened silently: planted, the second-worst healthy // source share went 42 → 0 runes, and separately a hard row was overwritten with an easy row's // accounting under its own id — both invisible, because the extremum still held. The corpus is a // frozen measurement, so a digest over every row's accounting is the honest pin: re-taking it is a // deliberate act that re-records this line in the same commit. // ⚠ IT COVERS EVERY FIELD, INCLUDING THE ONES NO MEASUREMENT READS. The first version folded only the // six counting fields, and a planting that rewrote a specimen's `head` — and, worse, its // `provenance` — passed with the whole ledger green. `provenance` is the ADDRESS a specimen was // taken from and the address it would be re-taken from; a corpus whose numbers are right and whose // provenance is wrong is unreproducible in the one way that matters, and nothing else in this file // looks at that field beyond `!= ""`. corpusDigest = "ac4c14ef47e0c42807f47c88abc49da672884864392930fb07ecf4fe71f8fa23" ) corpus := loadOffTargetCorpus(t) var offTargets, healthy int worstSrc, worstTgt, shortestOff, bestOffTgt := 0.0, 1.0, 1<<30, 0.0 for _, s := range corpus { switch s.Label { case "off_target": offTargets++ if s.Letters < shortestOff { shortestOff = s.Letters } if sh := s.targetShare(); sh > bestOffTgt { bestOffTgt = sh } default: healthy++ if sh := s.share(); sh > worstSrc { worstSrc = sh } if sh := s.targetShare(); sh < worstTgt { worstTgt = sh } } } for _, c := range []struct { name string gotN, wantN int }{ {"off-target rows", offTargets, wantOffTargets}, {"healthy rows", healthy, wantHealthy}, {"shortest off-target, letters", shortestOff, wantShortestOffChars}, } { if c.gotN != c.wantN { t.Errorf("%s = %d, recorded %d — if the corpus was legitimately re-taken, re-record it here in the same commit and say so in the report", c.name, c.gotN, c.wantN) } } for _, c := range []struct { name string got, want float64 }{ {"worst healthy source-script share", worstSrc, wantWorstHealthySrc}, {"worst healthy target-script share", worstTgt, wantWorstHealthyTgt}, {"best off-target target-script share", bestOffTgt, wantBestOffTargetTgt}, } { if diff := c.got - c.want; diff > tol || diff < -tol { t.Errorf("%s = %.6f, recorded %.6f — the number the contract is argued from moved; re-record it deliberately, or find out what edited the fixture", c.name, c.got, c.want) } } // The whole population, so that a row which is not an extremum cannot be softened, relabelled, swapped // for an easier one under its own id, or duplicated, without saying so. if got := offTargetCorpusDigest(corpus); got != corpusDigest { t.Errorf("the corpus accounting changed:\n is: %s\n recorded: %s\nAll EIGHT fields of every "+ "row feed this — id, label, the four counts, provenance and head. If the corpus was legitimately "+ "re-taken, re-record the digest in the same commit and say so in the report; if it was not, find "+ "out what edited the fixture.", got, corpusDigest) } } // offTargetCorpusDigest folds every specimen's identity and accounting — sorted by id, so the digest is a // property of the SET and not of the file's line order. func offTargetCorpusDigest(corpus []offTargetSpecimen) string { rows := append([]offTargetSpecimen(nil), corpus...) sort.Slice(rows, func(i, j int) bool { return rows[i].ID < rows[j].ID }) h := sha256.New() for _, s := range rows { fmt.Fprintf(h, "%s|%s|%d|%d|%d|%d|%s|%s\n", s.ID, s.Label, s.SourceScriptRunes, s.TotalRunes, s.Letters, s.TargetScriptLetters, s.Provenance, s.Head) } return hex.EncodeToString(h.Sum(nil)) } // TestTheLiveEchoGateKeepsItsMeasuredMargin is row 113 itself: the number the shipping gate actually // achieves, computed by the shipping function, over a corpus that lives in the repository. // // ⛔ ITS THRESHOLDS ARE A STOP, not a target. Row 113 says so in as many words: if the fixture stops // reproducing the recall at zero false positives, or the margin falls under 3×, the premise row 46 is // built on has fallen and building on it is a waste. They are asserted here so that a later edit to the // detector, to the threshold or to the preprocessing has to face them. func TestTheLiveEchoGateKeepsItsMeasuredMargin(t *testing.T) { corpus := loadOffTargetCorpus(t) scripts := lang.LangScripts("zh") if len(scripts) == 0 { t.Fatal("the source language declares no scripts — the detector would measure nothing (D39.64 П2)") } var caught, offTargets, healthy, falsePositives int worstHealthy := 0.0 var missed []string for _, s := range corpus { // The REAL function, over a surrogate carrying the specimen's own rune accounting. got := sourceScriptShare(s.surrogate(), scripts) if want := s.share(); got != want { t.Fatalf("%s: the detector reads %.6f off its own accounting, want %.6f — the surrogate and the fixture disagree", s.ID, got, want) } switch s.Label { case "off_target": offTargets++ if got > cjkEchoThreshold { caught++ } else { missed = append(missed, fmt.Sprintf("%s (%s, %d letters, share %.4f)", s.ID, s.Provenance, s.Letters, got)) } default: healthy++ if got > cjkEchoThreshold { falsePositives++ } if got > worstHealthy { worstHealthy = got } } } // ⛔ STOP 1 — zero false positives. A gate that flags healthy Russian is not a margin question at all. if falsePositives != 0 { t.Fatalf("%d of %d healthy outputs are flagged by the live gate — row 46's premise has fallen", falsePositives, healthy) } // ⛔ STOP 2 — the recall the contract quotes. Stated as a RATIO so the corpus may grow: the original // number was 18 of 20 on a narrower slice of the same stand, and this wider one is 28 of 30. const wantRecall = 18.0 / 20.0 // ⚠ THE DENOMINATOR IS CHECKED FIRST, and not out of tidiness: 0/0 is NaN and every comparison against // NaN is false, so a fixture whose off-target class had been emptied would sail through the recall STOP // below reporting nothing. A guard whose own arithmetic can excuse it is not a guard. if offTargets == 0 { t.Fatal("the corpus carries no off-target specimen — the recall STOP below cannot fail and this test would be enforcing nothing") } if got := float64(caught) / float64(offTargets); got < wantRecall { t.Fatalf("the live gate catches %d of %d off-target outputs (%.3f), below the quoted %.3f — the base row 46 must beat has moved. Missed: %s", caught, offTargets, got, wantRecall, strings.Join(missed, "; ")) } // ⛔ STOP 3 — the REAL margin, which is the number nobody had recorded: the worst HEALTHY output's // source-script share against the threshold that judges it. if worstHealthy <= 0 { t.Fatal("no healthy specimen carries any source-script rune — the margin would be infinite and meaningless; the corpus must keep the hard cases") } margin := cjkEchoThreshold / worstHealthy if margin < 3 { t.Fatalf("the live gate's margin is %.2f× (worst healthy %.6f against threshold %.2f), under the 3× floor row 113 stops at", margin, worstHealthy, cjkEchoThreshold) } t.Logf("live echo gate: caught %d/%d off-target, %d false positives on %d healthy, worst healthy %.6f, margin %.2f×", caught, offTargets, falsePositives, healthy, worstHealthy, margin) } // TestTheGateMissesExactlyTheLatinOffTargetClass names WHAT the residual hole is, because «28 of 30» on // its own invites the reading that two specimens are noise. They are not: the misses are the two fluent // ENGLISH drafts, and they are missed BY CONSTRUCTION — an output with no source-script rune in it scores // zero on a source-script detector however wrong it is. That is precisely the hole row 46 exists to close, // and it is the reason a target-side screen is a different mechanism rather than a better threshold. func TestTheGateMissesExactlyTheLatinOffTargetClass(t *testing.T) { corpus := loadOffTargetCorpus(t) scripts := lang.LangScripts("zh") for _, s := range corpus { if s.Label != "off_target" { continue } if sourceScriptShare(s.surrogate(), scripts) > cjkEchoThreshold { continue } // A missed specimen must carry NO source script at all. A miss with source runes in it would be a // threshold problem — a different defect with a different cure — and must not hide in this class. if s.SourceScriptRunes != 0 { t.Errorf("%s carries %d source-script runes and is still missed: that is a THRESHOLD miss, not the Latin class (%s)", s.ID, s.SourceScriptRunes, s.Provenance) } } } // TestTheTargetScreenChangesExactlyTheTwoLinesItWasAskedTo is EFFORT_HANDLE §5.4 step 5 executed, and it // is the acceptance criterion the plan set before the code existed: run the proposed verdict offline over // the corpus, diff the dispositions, and expect exactly the two English drafts to move. «STOP and escalate // if a single HEALTHY line changes» — so the healthy half is a Fatal here, not an Error. // // It is the whole justification for the screen in one number. The echo gate already catches 28 of 30 at // zero false positives, so the screen is only worth its snapshot bump if it closes that residue and // touches nothing else; a screen that also moved healthy lines would be paying for a re-purchase of every // book with false flags. func TestTheTargetScreenChangesExactlyTheTwoLinesItWasAskedTo(t *testing.T) { corpus := loadOffTargetCorpus(t) srcScripts, tgtScripts := lang.LangScripts("zh"), lang.LangScripts("ru") if len(srcScripts) == 0 || len(tgtScripts) == 0 { t.Fatal("both languages must declare their scripts — the rules are data-driven (D39.64 П2)") } var newlyFlagged, healthyMoved []string for _, s := range corpus { // What the gate said BEFORE the screen existed: the echo rule alone. echoed := sourceScriptShare(s.surrogate(), srcScripts) > cjkEchoThreshold // What it says now, for a specimen the echo rule let through. The surrogate is bound to the // specimen's own accounting the same way the echo half is, so a fixture edited on one side and not // the other cannot make this measurement quietly meaningless. scr := langscreen.Screen(s.targetSurrogate(), tgtScripts) if scr.Verdict != langscreen.Abstain && scr.Share != s.targetShare() { t.Fatalf("%s: the screen reads %.6f off its own accounting, want %.6f", s.ID, scr.Share, s.targetShare()) } screened := !echoed && scr.Verdict == langscreen.OffTarget if !screened { continue } line := fmt.Sprintf("%s (%s, %d letters, target share %.4f)", s.ID, s.Provenance, s.Letters, s.targetShare()) newlyFlagged = append(newlyFlagged, line) if s.Label == "healthy" { healthyMoved = append(healthyMoved, line) } } // ⛔ THE STOP. A healthy completion newly flagged is a paid unit lost and a book re-purchased for // nothing; the plan says escalate rather than proceed. if len(healthyMoved) > 0 { t.Fatalf("the screen flags %d HEALTHY completion(s) — STOP: %s", len(healthyMoved), strings.Join(healthyMoved, "; ")) } if len(newlyFlagged) != 2 { t.Fatalf("the screen must move exactly the two off-target completions the echo rule cannot see, moved %d: %s", len(newlyFlagged), strings.Join(newlyFlagged, "; ")) } t.Logf("the target screen newly flags: %s", strings.Join(newlyFlagged, "; ")) } // TestTheTwoScreensTogetherLeaveNoOffTargetBehind is the pair's joint recall, which is the number that // actually answers row 46: not «the screen works» but «what still ships silently after both rules run». func TestTheTwoScreensTogetherLeaveNoOffTargetBehind(t *testing.T) { corpus := loadOffTargetCorpus(t) srcScripts, tgtScripts := lang.LangScripts("zh"), lang.LangScripts("ru") var offTargets, caught, falsePositives int var escaped []string for _, s := range corpus { flagged := sourceScriptShare(s.surrogate(), srcScripts) > cjkEchoThreshold || langscreen.Screen(s.targetSurrogate(), tgtScripts).Verdict == langscreen.OffTarget switch s.Label { case "off_target": offTargets++ if flagged { caught++ } else { escaped = append(escaped, fmt.Sprintf("%s (%s)", s.ID, s.Provenance)) } default: if flagged { falsePositives++ } } } if falsePositives != 0 { t.Fatalf("%d healthy completions are flagged by the pair of rules", falsePositives) } if caught != offTargets { t.Fatalf("%d of %d off-target completions still ship as ok: %s", offTargets-caught, offTargets, strings.Join(escaped, "; ")) } t.Logf("echo gate + target screen: %d/%d off-target caught, %d false positives", caught, offTargets, falsePositives) } // TestABankRoleReplyIsExemptFromBothRules is the false-positive class that would arrive WITH this pack and // not before it. A term table's letters are engine identifiers and source terms, so it scores near zero on // the target screen and high on the echo rule — every healthy classifier batch would be flagged twice. // The classifier was already flagged by the echo rule and it was invisible because no shipping config ran // it; backlog row 140 runs it. func TestABankRoleReplyIsExemptFromBothRules(t *testing.T) { table := strings.Repeat("元石\tterm\tnone\n方源\tname\tmale\n", 40) in := classifyInput{ Output: table, Finish: llm.FinishStop, TargetLang: "ru", SourceScripts: lang.LangScripts("zh"), TargetScripts: lang.LangScripts("ru"), } if got := classify(in); got.Reason == reasonOK { t.Fatal("premise broken: without the exemption this reply must be flagged, or the test proves nothing") } in.NonProseReply = true if got := classify(in); got.Reason != reasonOK { t.Fatalf("a bank role's table must pass both rules, got %q (%s)", got.Reason, got.Detail) } // And the exemption must be keyed on the role SET, not on one role: the classifier's reply has the // same shape as the terminologist's and was outside it. for _, role := range []string{roleTerminologist, roleClassifier} { if !isBankRole(role) { t.Errorf("%s answers a term table and must be exempt", role) } } if isBankRole(roleTranslator) || isBankRole(roleEditor) { t.Error("a prose role must NOT be exempt — the exemption is what makes the rules meaningless") } } // TestTheScreenConstantsStillSitWhereTheCorpusPutThem binds langscreen's two calibrated numbers to the // corpus they were taken from. They live in that package as bare constants with the measurement written in // prose beside them, and prose does not fail: a fixture edited until the gap closed, or a floor raised past // the healthy minimum, would leave every test in that package green because they compare the constants with // constants. Here they are compared with the data. func TestTheScreenConstantsStillSitWhereTheCorpusPutThem(t *testing.T) { corpus := loadOffTargetCorpus(t) worstHealthy, bestOffTarget, shortestOffTarget := 1.0, 0.0, 1<<30 for _, s := range corpus { switch s.Label { case "off_target": if sh := s.targetShare(); sh > bestOffTarget { bestOffTarget = sh } if s.Letters < shortestOffTarget { shortestOffTarget = s.Letters } default: if sh := s.targetShare(); sh < worstHealthy { worstHealthy = sh } } } if !(bestOffTarget < langscreen.OnTargetFloor && langscreen.OnTargetFloor < worstHealthy) { t.Fatalf("the floor %.4f no longer sits in the corpus's empty interval [%.4f, %.4f]", langscreen.OnTargetFloor, bestOffTarget, worstHealthy) } // ⛔ The evidence floor may not rise past the SHORTEST off-target completion: above it, the screen would // abstain on a real event, which is the one thing an abstention must never hide. if langscreen.MinLetters > shortestOffTarget { t.Fatalf("the evidence floor %d is above the shortest off-target completion (%d letters) — a real event would be excused as too short", langscreen.MinLetters, shortestOffTarget) } }