textmachine/backend/internal/membank/memconsistency_test.go

package membank

// memconsistency_test.go: the book-wide consistency measure (backlog row 406). Every test here exists to
// pin a distinction the post-check next door cannot make, so each one is built so that the OLD question
// would answer differently — a fixture on which both questions agree would pin nothing.

import (
	"slices"
	"testing"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/store"
	"textmachine/backend/internal/text"
)

// ruBank materializes a bank with the real ru stemmer, which is what the shipping path builds
// (bankmaterialize.go) and what every count here depends on.
func ruBank(rows []store.GlossaryEntry) *Bank {
	return MaterializeBank(BankInput{Rows: rows, TargetStemmer: lang.NewTargetStemmer(lang.TargetChecksFor("ru"))}, false)
}

func scanOne(b *Bank, src, dst string) TermShipping {
	for _, t := range b.ScanShipping(src, dst, 1) {
		return t
	}
	return TermShipping{}
}

// TestTheCountAnswersHowOftenNotWhether is the whole reason this file exists: the post-check asks whether an
// accepted rendering is present ANYWHERE in the output and stops at the first hit, so a term rendered the
// bank's way once and some other way four times satisfies it completely (backlog row 407). The counts here
// separate those two facts, and the test asserts BOTH sides — that the old question passes this text and the
// new one does not — because asserting only the new number would not show that anything was gained.
func TestTheCountAnswersHowOftenNotWhether(t *testing.T) {
	rows := []store.GlossaryEntry{gl("蛊师", "гу-мастер", "", "approved")}
	b := ruBank(rows)
	// Five occurrences of the key in the source; the shipped text uses the bank's rendering for THREE of
	// them and a competing one for the other two.
	//
	// ⚠ Three rather than one, and that is not decoration: with a single occurrence of the bank's rendering
	// a counter capped at one is indistinguishable from a correct counter, and a planted cap survived this
	// test until the fixture was rebuilt. The assertion has to be able to tell "counted" from "found".
	src := "蛊师。蛊师！蛊师?蛊师、蛊师。"
	shipped := "Гу-мастер вошёл. Гу-мастер молчал. Заклинатель ждал. Гу-мастер встал. Заклинатель ушёл."

	// PREMISE, and it is the point of the test: the post-check's own predicate is satisfied by this text.
	// If this ever stops holding, the fixture no longer demonstrates a gap and the assertion below is empty.
	e := &b.entries[0]
	nout := text.NormalizeTargetForm(shipped)
	if !dstFormPresent(e, []rune(nout), lang.TokenizeWords(nout), b.stemmer) {
		t.Fatalf("premise broken: the post-check must call this output CLEAN — otherwise the new count pins nothing")
	}

	got := scanOne(b, src, shipped)
	if got.Fired != 5 {
		t.Errorf("fired: got %d, want 5 occurrences of 蛊师 in the source", got.Fired)
	}
	if got.Shipped != 3 {
		t.Errorf("shipped: got %d, want 3 — the bank's rendering appears three times against five firings; a count capped at 1 would also 'find' it", got.Shipped)
	}
	if !got.Partial() {
		t.Errorf("a term rendered the bank's way three times in five firings must read as PARTIAL, got fired=%d shipped=%d", got.Fired, got.Shipped)
	}
	// And the anchored column must agree here: nothing in this text is one inflection away from the
	// rendering, so a difference between the columns would mean the relaxed rule is finding something that
	// is not there.
	if got.ShippedRelaxed != got.Shipped {
		t.Errorf("the relaxed rule found %d where the strict one found %d, on a text with no inflected occurrence to recover", got.ShippedRelaxed, got.Shipped)
	}
}

// TestANeverFiredRowIsNotAnAbsentRendering pins the control the whole measure rests on. "The bank's
// rendering is not in the text" and "this term is not in this book" are the same zero to any instrument that
// does not look at the SOURCE, and the second one is not a finding. The two rows here differ only in whether
// their key occurs in the source, and they must come out in different categories.
func TestANeverFiredRowIsNotAnAbsentRendering(t *testing.T) {
	fired := gl("蛊师", "гу-мастер", "", "approved")
	absent := gl("仙人", "небожитель", "", "approved")
	b := ruBank([]store.GlossaryEntry{fired, absent})

	got := b.ScanShipping("蛊师вошёл。", "Заклинатель вошёл.", 1)
	if len(got) != 1 {
		t.Fatalf("want exactly one record — only the row whose key is in the source is a fact about this span; got %d: %+v", len(got), got)
	}
	if got[0].Src != "蛊师" {
		t.Fatalf("the wrong row was reported: got %q, want 蛊师", got[0].Src)
	}
	if !got[0].Absent() {
		t.Errorf("a row whose key fired and whose rendering is missing must read as ABSENT, got fired=%d shipped=%d", got[0].Fired, got[0].Shipped)
	}
	// And the row that never fired must not be reported at all — not reported as a clean zero, which is how
	// it would join the denominator of terms that were actually judged.
	for _, r := range got {
		if r.Src == "仙人" {
			t.Errorf("a row whose key never occurred in the source was judged anyway: %+v", r)
		}
	}
}

// TestANestedRowEatenByALongerKeyIsCounted pins the silent half of backlog row 407, and the fixture is built
// so the silence is UNAVOIDABLE: 古月 is a bank row of its own with a signed rendering, and it sits inside
// the alias 古月方源 of a longer row. suppressContained drops the shorter match, the longer row answers for
// its own rendering, and nothing ever asks whether «Гуюэ» survived — so an output that spells it «Гу Юэ»
// passes every check the engine has.
//
// ⚠ The run of 11.09 could not have shown this: that book had zero aliases in 69 rows and the suppressor
// fired not once. A fixture is the only place the class is reachable, which is why it is here and not a
// measurement.
func TestANestedRowEatenByALongerKeyIsCounted(t *testing.T) {
	long := gl("方源", "Фан Юань", "", "approved")
	long.Aliases = []store.GlossaryAlias{{Alias: "古月方源", AliasType: "name"}}
	nested := gl("古月", "Гуюэ", "", "approved")
	b := ruBank([]store.GlossaryEntry{long, nested})

	// The output honours the LONGER row and breaks the nested one's signed canon.
	got := b.ScanShipping("古月方源来了。", "Гу Юэ Фан Юань пришёл.", 1)
	var eaten, whole *TermShipping
	for i := range got {
		switch got[i].Src {
		case "古月":
			eaten = &got[i]
		case "方源":
			whole = &got[i]
		}
	}
	if whole == nil || whole.Shipped == 0 {
		t.Fatalf("premise broken: the LONGER row must be satisfied by this output — that is what makes the nested row's breach silent; got %+v", got)
	}
	if eaten == nil {
		t.Fatalf("the nested row was not reported at all: %+v", got)
	}
	if !eaten.Suppressed {
		t.Errorf("古月 must be marked as eaten by the longer key: %+v", eaten)
	}
	if !eaten.SuppressedCanonAbsent {
		t.Errorf("古月's signed rendering «Гуюэ» is NOT in «Гу Юэ Фан Юань», so the row must be flagged: %+v", eaten)
	}
}

// TestANestedRowWhoseCanonSurvivedIsNotFlagged is the other half of the pin above, and without it that one
// would pass on a bank that flagged every suppression. Same rows, same firing, same suppression — the only
// difference is that the output spells the nested canon correctly, and that must be enough.
func TestANestedRowWhoseCanonSurvivedIsNotFlagged(t *testing.T) {
	long := gl("方源", "Фан Юань", "", "approved")
	long.Aliases = []store.GlossaryAlias{{Alias: "古月方源", AliasType: "name"}}
	nested := gl("古月", "Гуюэ", "", "approved")
	b := ruBank([]store.GlossaryEntry{long, nested})

	for _, r := range b.ScanShipping("古月方源来了。", "Гуюэ Фан Юань пришёл.", 1) {
		if r.Src == "古月" && r.SuppressedCanonAbsent {
			t.Errorf("the nested canon IS in the output («Гуюэ»), so the row must not be flagged: %+v", r)
		}
	}
}

// TestTheAnchoredCountRecoversAnInflectionTheStrictOneLoses pins the two columns apart. The rendering here is
// in the shipped text in an oblique case whose stem runs one rune past the nominative's — «корифей»→«кориф»
// against «корифея»→«корифе» — which SameStem rejects. Eleven of the eighteen misses recorded on the paid run
// of 11.09 were exactly this shape, so a single-column report was wrong on a majority of its own rows.
func TestTheAnchoredCountRecoversAnInflectionTheStrictOneLoses(t *testing.T) {
	b := ruBank([]store.GlossaryEntry{gl("魔道巨擘", "корифей пути демонов", "", "draft")})
	got := scanOne(b, "魔道巨擘来了。", "С точки зрения корифея пути демонов, это пустяк.")
	if got.Fired != 1 {
		t.Fatalf("premise broken: the key must fire once, got %d", got.Fired)
	}
	if got.Shipped != 0 {
		t.Fatalf("premise broken: the STRICT count must miss this inflection — otherwise the columns are not being compared, got %d", got.Shipped)
	}
	if got.ShippedRelaxed != 1 {
		t.Errorf("the anchored count must find «корифея пути демонов» for «корифей пути демонов», got %d", got.ShippedRelaxed)
	}
}

// TestTheAnchoredCountStillRefusesAOneWordCollision is the safety half of the pin above. data/target-ru.txt
// refuses the bare soft sign as a decl_suffix because «Синь» and «синий» would collapse together; the
// anchored count admits the same relation, so what keeps it out of that collision is the anchor — and a
// one-word rendering has none. Without this test the tolerance would be indistinguishable from the global
// suffix that note rejects.
func TestTheAnchoredCountStillRefusesAOneWordCollision(t *testing.T) {
	b := ruBank([]store.GlossaryEntry{gl("辛", "Синь", "", "approved")})
	// PREMISE: the relation itself equates them, so the refusal below is the anchor gate and not the stemmer
	// simply failing to see a resemblance.
	if !b.stemmer.NearStem("Синь", "синий") {
		t.Fatalf("premise broken: NearStem must equate «Синь»/«синий» — that pair IS the hazard being gated")
	}
	got := scanOne(b, "辛在这里。", "Небо было синим, совсем синий день.")
	if got.ShippedRelaxed != 0 {
		t.Errorf("a one-word name must not be found inside an unrelated adjective: «Синь» matched %d time(s) in «синий/синим»", got.ShippedRelaxed)
	}
}

// TestTheStrictColumnMatchesAStoredDeclFormLiterally pins the one thing that makes the two printed columns a
// comparison rather than a tautology: the STRICT column is «what the post-check itself would say», and the
// post-check stems the base dst while matching a stored decl form literally (dstFormPresent). If the strict
// count stemmed decl forms too, the report's «under the post-check's own equality» line would be describing
// a matcher the post-check does not have, and the gap it prints would be attributed to the wrong cause.
//
// The fixture is the shape that made this visible on real data: the seed lists SINGULAR forms only and the
// text uses a plural, so every listed form is one inflection away from what shipped. On the labelled corpus
// all six residual false flags were this, and stemming the decl forms — not the near-stem tolerance —
// is what closed them.
func TestTheStrictColumnMatchesAStoredDeclFormLiterally(t *testing.T) {
	e := gl("蛊虫", "гу-червь", "", "draft")
	e.Decl = declJSON(false, "гу-червя", "гу-червю", "гу-червём", "гу-черве")
	b := ruBank([]store.GlossaryEntry{e})
	got := scanOne(b, "蛊虫在这里。", "Он собирал гу-червей всю ночь.")

	if got.Fired != 1 {
		t.Fatalf("premise broken: the key must fire once, got %d", got.Fired)
	}
	// PREMISE: not one seeded form occurs in the output as WORDS — which is the space the matcher works in,
	// and checking it by substring instead would be a different question («гу-черве» IS a substring of
	// «гу-червей» and is not a match). Without this the strict column could be zero for a reason that has
	// nothing to do with the rule being pinned.
	out := lang.TokenizeWords(text.NormalizeTargetForm("Он собирал гу-червей всю ночь."))
	for _, f := range []string{"гу-червь", "гу-червя", "гу-червю", "гу-червём", "гу-черве"} {
		fw := lang.TokenizeWords(text.NormalizeTargetForm(f))
		for i := 0; i+len(fw) <= len(out); i++ {
			if slices.Equal(out[i:i+len(fw)], fw) {
				t.Fatalf("premise broken: %q occurs in the output as words, so the strict rule is not being tested", f)
			}
		}
	}
	if got.ShippedStrict != 0 {
		t.Errorf("the STRICT column must match a stored decl form literally, as the post-check does: got %d, want 0", got.ShippedStrict)
	}
	// The PRIMARY column stems the stored forms — the relaxation that removed all six residual false flags
	// of the labelled corpus and carries no hazard. The verdict rests on this one, which is why the gap
	// between it and the strict column is what the report prints as «what the shipping path is blind to».
	if got.Shipped != 1 {
		t.Errorf("the primary count stems the stored forms, so «гу-червей» must be found for «гу-червя»: got %d, want 1", got.Shipped)
	}
}

// TestARenderingMayNotSpanASentenceBoundary pins the boundary, and it pins it in the direction that matters:
// without it the two halves of a rendering landing on either side of a full stop COUNT as an occurrence, so
// a term that went out two ways reads as one that went out cleanly. A miscount that hides a finding is worse
// than one that invents it, which is why this is a boundary in the code and not a caveat in the report.
//
// The control case is the same words in one sentence: the rule must not be refusing the rendering itself.
func TestARenderingMayNotSpanASentenceBoundary(t *testing.T) {
	b := ruBank([]store.GlossaryEntry{gl("白家寨", "род Бай", "", "approved")})

	for _, c := range []struct {
		name    string
		shipped string
		want    int
	}{
		{"split by a full stop", "Он вышел из рода. Бай его не знал.", 0},
		{"split by a question mark", "Кто из рода? Бай молчал.", 0},
		{"CONTROL — the same words, one sentence", "Он вышел из рода Бай на рассвете.", 1},
	} {
		got := scanOne(b, "白家寨在这里。", c.shipped)
		if got.Fired != 1 {
			t.Fatalf("%s: premise broken — the key must fire once, got %d", c.name, got.Fired)
		}
		if got.ShippedRelaxed != c.want {
			t.Errorf("%s: shipped=%d, want %d — %q", c.name, got.ShippedRelaxed, c.want, c.shipped)
		}
	}
}

// TestTheStemToleranceIsInertWithoutAStemRegistry pins the generality of the relaxed column, and it is the
// project's default review question asked of this file: does a pair that is NOT in this repo behave?
//
// The tolerance is a relation on STEMS. A target with no decl_suffix registry has no stems — Stem is the
// identity — so a one-rune-prefix rule over raw words would say `cart` and `cars` are the same word, and the
// column would silently become fuzzy prefix matching for every language this engine has not been taught.
// The acceptance found exactly that: the counting path called an ungated free function while the comment
// described a gated method.
func TestTheStemToleranceIsInertWithoutAStemRegistry(t *testing.T) {
	var inert lang.TargetStemmer // what lang.NewTargetStemmer gives a target with no decl_suffix rows
	if inert.Enabled() {
		t.Fatalf("premise broken: the zero stemmer must be inert, or this test is not about the ungated path")
	}
	// ⛔ THE PAIRS ARE ONE-RUNE PREFIXES, and that is the whole test. The relation accepts a pair only when
	// one stem is a strict one-rune prefix of the other, so a same-length pair like «cart»/«cars» is refused
	// by the prefix rule whatever the gate does — and a fixture built on those cannot tell the gate from the
	// rule. The full mutation gate caught exactly that: a planted removal of the Enabled() check SURVIVED
	// here, because this test was asking a question the prefix rule already answered.
	for _, p := range [][2]string{{"car", "cart"}, {"gol", "gold"}, {"min", "mind"}} {
		if inert.NearStems(p[0], p[1]) {
			t.Errorf("an inert stemmer equated %q and %q: with no decl_suffix registry Stem is the identity, so these are two different WORDS and the relation has no stems to be one rune apart", p[0], p[1])
		}
		if inert.SameStem(p[0], p[1]) {
			t.Errorf("premise broken: %q/%q must differ under SameStem too", p[0], p[1])
		}
	}
	// PREMISE: with a LIVE stemmer the very same pairs ARE accepted — which is what proves the assertions
	// above are about the gate and not about the prefix rule refusing them anyway.
	ruLive := lang.NewTargetStemmer(lang.TargetChecksFor("ru"))
	if !ruLive.NearStems("car", "cart") {
		t.Fatalf("premise broken: with a registry the relation must accept a one-rune prefix pair, or this fixture cannot distinguish the gate from the rule")
	}
	// CONTROL — with a registry the relation is alive, so the assertions above are about the GATE and not
	// about the relation simply never holding.
	if !ruLive.NearStems("кориф", "корифе") {
		t.Fatalf("control broken: a live stemmer must still equate «кориф»/«корифе»")
	}
	// And the whole counting path must be inert too, not merely the relation.
	b := MaterializeBank(BankInput{Rows: []store.GlossaryEntry{gl("X", "car wheel", "", "approved")}}, false)
	got := scanOne(b, "X here.", "the cart wheel stood there")
	if got.ShippedRelaxed != 0 {
		t.Errorf("with no stem registry the relaxed count must equal the strict one; it found %d occurrences of «car wheel» in «cart wheel»", got.ShippedRelaxed)
	}
}

// TestAMultiWordRenderingIsNotSatisfiedByADifferentWord is the hazard pin, and it is mandatory rather than
// nice-to-have because the verdict now rests on this column.
//
// The anchor protects a ONE-word rendering absolutely: with nothing else in the window there is no anchor,
// so the tolerance can never carry it alone. In a MULTI-word rendering the other words ARE the anchors, so
// the tolerance is live on the remaining word — and «глава рода Гуюэ» satisfied by «глаза рода Гуюэ» is a
// false SILENCE: the report says a rendering arrived when it did not, which is the direction that hides
// findings.
//
// What closes it is the shape of the relation itself: Stem truncates, so two stems of one word differ only
// in how much was stripped and the shorter must be a PREFIX of the longer. «глав»/«глаз» is a substitution,
// not a stripping, and is refused. Measured on this book's 2094 distinct words, that refuses 135 of the 426
// pairs the loose form admitted — «вред»~«время», «ветви»~«ветром», «весны»~«весь» among them.
func TestAMultiWordRenderingIsNotSatisfiedByADifferentWord(t *testing.T) {
	b := ruBank([]store.GlossaryEntry{gl("古月族长", "глава рода Гуюэ", "", "approved")})

	// The hazard itself: two exact anchors («рода», «Гуюэ») and a different word in the head position.
	for _, out := range []string{
		"глаза рода Гуюэ смотрели на него",
		"глазами рода Гуюэ он видел всё",
	} {
		got := scanOne(b, "古月族长来了。", out)
		if got.ShippedRelaxed != 0 {
			t.Errorf("a DIFFERENT word was credited to this rendering: «глава рода Гуюэ» matched %d time(s) in %q — that is the report saying a rendering arrived when it did not", got.ShippedRelaxed, out)
		}
	}
	// CONTROL — the same rendering genuinely inflected must still be found, or the pin above is passing
	// because the tolerance stopped working rather than because it stopped over-reaching.
	for _, out := range []string{
		"глава рода Гуюэ вошёл",
		"главе рода Гуюэ доложили",
	} {
		got := scanOne(b, "古月族长来了。", out)
		if got.ShippedRelaxed != 1 {
			t.Errorf("control broken: a real inflection of «глава рода Гуюэ» was not found in %q (got %d)", out, got.ShippedRelaxed)
		}
	}
}