textmachine/backend/internal/pipeline/bankbasis_m1_probe_test.go

package pipeline

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/miner"
	"textmachine/backend/internal/obs"
	"textmachine/backend/internal/store"
	"textmachine/backend/internal/terminology"
	"textmachine/backend/internal/text"
)

// bankbasis_m1_probe_test.go: measurement M1 — an opt-in INSTRUMENT over BOUGHT material, not a unit test.
// It answers the question the settled basis has to be designed against before a carrier is written: on the
// runs this project actually paid for, how many candidates would the predicate serve from memory, and for
// which reasons would the rest be bought again.
//
// ⛔ $0. Nothing here reaches a provider. The runner is opened on the READ path (openRunner forWrite=false),
// which skips models.CheckKeys entirely, and the only engine calls made are the $0 assembly ones — ingest,
// split, mine, merge, KWIC, score. runTerminologist, runClassifier and TranslateBook are never called.
//
// HOW TO PREPARE THE MATERIAL (the instrument refuses to guess, and every path is named):
//
//	cp -a /home/ubuntu/tm-coldrun-a/stand/books/bk_ROEHZBD46ALFI43E /tmp/m1/A
//	cp -a /home/ubuntu/tm-coldrun-b/stand/books/bk_NH6275ZIRFFTLZHG /tmp/m1/B
//	rm -f /tmp/m1/*/project.db.lock
//	sed -i 's#/home/ubuntu-26/#/home/ubuntu/#g' /tmp/m1/*/book.yaml
//
//	# ⚠ RUN A NEEDS ONE MORE STEP, and without it the instrument FAILS on A while passing on B — which
//	# reads as a defect of the pack and is a gap in the ARCHIVE. A's pipeline resolves its role prompts
//	# relative to its own directory (`../prompts/zh-ru/…`) and that directory was not archived beside the
//	# config mirror; B's was. The prompts themselves are in the freeze and are byte-identical to B's
//	# archived copy (`diff -rq` empty, 11 files against 11), and they touch no number here — the
//	# instrument makes no calls — so assembling them is restoring the archive, not choosing an input:
//	mkdir -p /tmp/m1/A/cfg && cp -a /home/ubuntu/tm-coldrun-a/mirror/cfg/. /tmp/m1/A/cfg/
//	cp -a /home/ubuntu/tm-coldrun-a/freeze/backend/prompts /tmp/m1/A/prompts
//	sed -i 's#^pipeline: .*#pipeline: /tmp/m1/A/cfg/pipeline-c1.yaml#' /tmp/m1/A/book.yaml
//
//	TM_BASIS_M1=/tmp/m1/A,/tmp/m1/B go test ./internal/pipeline/ -run TestProbeBankBasisM1 -v
//
// ⚠ COPIES, ALWAYS. The material is bought and unrecoverable, and run A carries store schema 16 against a
// head of 18: OpenReadOnly REFUSES it, so the instrument migrates the copy with store.Open first (v17/v18
// are additive — CREATE TABLE IF NOT EXISTS and ADD COLUMN, no DROP/RENAME/DELETE). Run this against an
// original and the original is migrated in place.
//
// ⚠ THE book.yaml OF EVERY BOUGHT RUN POINTS AT /home/ubuntu-26, WHICH DOES NOT EXIST HERE. Re-point it on
// the copy, as the recipe does. Leaving it is not a loud failure in the interesting direction: the config
// loader would take the HEAD configs and the instrument would silently measure a different pipeline than
// the one the money was spent under, which is why the run below PRINTS the knobs it resolved.
func TestProbeBankBasisM1(t *testing.T) {
	roots := os.Getenv("TM_BASIS_M1")
	if roots == "" {
		t.Skip("instrument: set TM_BASIS_M1=<copy of a bought run dir>[,<another>] to measure the settled basis (see the file docblock)")
	}
	for _, dir := range strings.Split(roots, ",") {
		dir = strings.TrimSpace(dir)
		if dir == "" {
			continue
		}
		t.Run(filepath.Base(dir), func(t *testing.T) { probeBasisOneRun(t, dir) })
	}
}

func probeBasisOneRun(t *testing.T, dir string) {
	t.Helper()
	m := openBasisMaterial(t, dir)
	defer m.r.Close()

	// ── THE KNOBS THIS MEASUREMENT WAS TAKEN UNDER ──────────────────────────────────────────────────
	// Printed, not assumed: «the config of the bought run» and «the config of HEAD, silently substituted»
	// produce the same numbers-shaped output, and only these lines tell them apart.
	batchRunes, kwicPer, kwicWidth := m.r.terminologyOpts()
	t.Logf("KNOBS book=%s pair=%s contrast=%s batch_runes=%d kwic_per_term=%d kwic_width=%d basis_width=%d model=%s",
		m.r.Book.BookID, m.r.Book.LangPair(), m.r.Pipeline.Mining.ContrastPath,
		batchRunes, kwicPer, kwicWidth, m.r.basisWidth(), m.r.Pipeline.Gates.Terminology.Model)

	// ── FIDELITY OF THE RECONSTRUCTION ─────────────────────────────────────────────────────────────
	// The instrument rebuilds the candidate list from the source; the run that was paid for left its own
	// list behind in the stop table sidecar. If the two disagree, every number below is about a different
	// book than the one the money was spent on — so this is asserted, not hoped for.
	m.assertMatchesBoughtTable(t)

	// ── THE BASELINE: the basis written by this very purchase, read by the next one on unchanged source ─
	// ⚠ NOT A CONTROL. Two identical runs agreeing measures the determinism of the harness, not the force
	// of the predicate; it is reported as the SAVING, with its denominator, and the discriminating
	// controls are below.
	base := m.verdicts(m.basisFromRun(), m.nsource)
	m.report(t, "BASELINE unchanged-source", base)
	m.controlUnitShape(t, base)
	m.measureAutoTail(t, base)
	m.measureRechunkResidue(t)
	m.measureUnitRuleVariants(t)

	// ── DISCRIMINATING CONTROL 1: the source moves around ONE occurrence of ONE candidate ──────────────
	// Exactly that candidate and its unit must be bought again, and nothing else. A predicate that ignored
	// the source would show 0 here; one that hashed too much would show the whole book.
	m.controlOneOccurrenceMoved(t, base)

	// ── DISCRIMINATING CONTROL 2: two passes under ONE basis produce byte-identical batches ────────────
	m.controlBatchesStable(t, base)

	// ── KNOB INVARIANCE: the window SELECTION must not move the settled set ────────────────────────────
	// This is the property the whole fingerprint shape exists for: the day a stratified selection lands,
	// or anyone tunes how much context the model is shown, no book may lose its basis.
	m.controlKnobsDoNotMatter(t, base)

	// ── THE GROWING BOOK, AS A NUMBER RATHER THAN AS A GUARD ───────────────────────────────────────────
	// The premise «the source arrives whole» is an OPEN question to the owner, so the engine must not
	// answer it with a watchdog. What it can do is say what growth costs.
	m.measureGrowth(t)
}

// ── material ────────────────────────────────────────────────────────────────────────────────────────

// basisMaterial is one bought run, reconstructed far enough to ask the predicate a question.
type basisMaterial struct {
	dir       string
	r         *Runner
	doc       *chunk.Document
	chunks    []chunk.Chunk
	nsource   string                  // the book's normalized source — the one definition, r.bankSrc's own
	cands     []terminology.Candidate // as the stop would have built them, before any paid call
	bank      []store.GlossaryEntry   // the bank as the bought run left it
	rows      map[string]store.GlossaryEntry
	ambiguous map[string]bool
	// consider is the candidates the BANK filter kept — the only population production offers the basis.
	consider    map[string]bool
	bankSettled int
	unitID      map[string]int
	width       int
}

func openBasisMaterial(t *testing.T, dir string) *basisMaterial {
	t.Helper()
	bookPath := filepath.Join(dir, "book.yaml")
	if _, err := os.Stat(bookPath); err != nil {
		t.Fatalf("material: %s has no book.yaml — point TM_BASIS_M1 at a COPY of a bought run dir: %v", dir, err)
	}
	// Migrate the copy first: run A is schema 16 and the read door refuses it outright (backlog row 475).
	// Done explicitly rather than by opening for write, so nothing here takes the run lock or a key check.
	if st, err := store.Open(filepath.Join(dir, "project.db")); err != nil {
		t.Fatalf("material: could not open/migrate the COPY at %s: %v", dir, err)
	} else {
		st.Close()
	}
	r, err := openRunner(bookPath, obs.NewLogger(), false)
	if err != nil {
		t.Fatalf("material: %s does not load — did you re-point its /home/ubuntu-26 paths? %v", dir, err)
	}
	if r.pack == nil {
		t.Fatalf("material: %s loaded no langpack for pair %q, so the miner cannot run and every count below would be a zero about the instrument", dir, r.Book.LangPair())
	}
	m := &basisMaterial{dir: dir, r: r, width: r.basisWidth()}
	m.build(t)
	return m
}

// build runs the $0 half of the bank-mining stop — the same sequence runBankMiningStop takes before its
// first paid call, in the same order and through the same functions.
func (m *basisMaterial) build(t *testing.T) {
	t.Helper()
	doc, err := m.r.ingestSource()
	if err != nil {
		t.Fatalf("ingest %s: %v", m.dir, err)
	}
	m.doc = doc
	m.chunks, _, _ = chunk.SplitChunksWithChapters(doc.Chapters, m.r.segBudget(), m.r.chapterRule(), m.r.sentenceAbbrevs())
	if len(m.chunks) == 0 {
		t.Fatalf("premise broken: %s produced no chunks", m.dir)
	}
	m.r.bankSrc = newBankSourceIndex(m.chunks)
	m.nsource = m.r.bankSrc.book

	m.bank, err = m.r.Store.GlossaryForBook(m.r.Book.BookID)
	if err != nil {
		t.Fatalf("read the bank of %s: %v", m.dir, err)
	}
	if len(m.bank) == 0 {
		t.Fatalf("premise broken: %s holds an EMPTY bank, so every «settled» count below would be zero for a reason that has nothing to do with the predicate", m.dir)
	}
	m.rows, m.ambiguous = bankBasisRows(m.bank, text.NormalizeSourceKey)
	m.cands = m.candidatesOver(t, m.chunks)
	m.unitID = bankBasisUnits(m.cands, basisSeriesParams(m.r.Book.SourceLang), m.r.familyParams)
	// ⛔ THE INSTRUMENT ASKS THE PREDICATE EXACTLY WHAT THE ENGINE ASKS IT, AND NOT ONE CANDIDATE MORE.
	// Production offers the basis only the candidates the BANK filter kept: a surface the seed renders and
	// every draft agreed with is a different saving with a different sentence on the sheet, and offering it
	// to both is the double-marking this pack had to fix. An instrument that measured over ALL candidates
	// would report a saving the engine never claims — a number about a question nobody asks.
	kept, dropped := m.r.dropBankSettled(context.Background(), m.cands)
	m.consider = map[string]bool{}
	for _, i := range kept {
		m.consider[m.cands[i].Key] = true
	}
	m.bankSettled = dropped
}

// candidatesOver rebuilds the merged candidate list over a given chunk set — parameterised so the growth
// and re-chunk measurements can ask the same question of a different source.
func (m *basisMaterial) candidatesOver(t *testing.T, chunks []chunk.Chunk) []terminology.Candidate {
	t.Helper()
	f, err := os.Open(m.r.Pipeline.Mining.ContrastPath)
	if err != nil {
		t.Fatalf("open the mining contrast %s: %v", m.r.Pipeline.Mining.ContrastPath, err)
	}
	defer f.Close()
	contrast, err := miner.LoadContrast(f)
	if err != nil {
		t.Fatalf("load the mining contrast: %v", err)
	}
	minerChunks := make([]miner.Chunk, len(chunks))
	tchunks := make([]terminology.Chunk, len(chunks))
	for i, ch := range chunks {
		n := text.NormalizeSourceKey(ch.Text)
		minerChunks[i] = miner.Chunk{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, NSource: n}
		tchunks[i] = terminology.Chunk{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, NSource: n}
	}
	rejects, err := m.r.loadMinedRejects()
	if err != nil {
		t.Fatalf("load the mined rejects: %v", err)
	}
	mined, _ := miner.MineBankStats(minerChunks, contrast, unsignedEngineSurfaces(m.bank), rejects, miner.FrozenConfig(), m.r.pack)
	observed, _, err := m.r.bankObservedForBook()
	if err != nil {
		t.Fatalf("read the banknote proposals: %v", err)
	}
	return m.r.buildBankCandidates(mined, observed, tchunks)
}

// ── fidelity ────────────────────────────────────────────────────────────────────────────────────────

// assertMatchesBoughtTable compares the reconstructed candidate list against the one the PAID run wrote to
// its stop-table sidecar.
//
// ⛔ WHY THIS IS AN ASSERT AND NOT A LOG LINE. Every number this file prints is a statement about the runs
// the project paid for. Reconstructed from a source with the wrong chunker, the wrong contrast or the HEAD
// config, the instrument still prints a full, plausible, well-formed table — about a book nobody bought.
// The sidecar is the bought run's own account of which candidates existed, so agreeing with it is the one
// cheap proof that the subject here is the subject there.
func (m *basisMaterial) assertMatchesBoughtTable(t *testing.T) {
	t.Helper()
	raw, err := os.ReadFile(m.r.bankStopTablePath())
	if err != nil {
		t.Fatalf("fidelity: %s has no bought stop table beside its DB, so the reconstruction cannot be checked against anything: %v", m.dir, err)
	}
	bought := map[string]bool{}
	for _, ln := range strings.Split(string(raw), "\n") {
		// One term opens a block with «src<TAB>dst»; every other line of the block is indented.
		if ln == "" || strings.HasPrefix(ln, " ") || !strings.Contains(ln, "\t") {
			continue
		}
		bought[text.NormalizeSourceKey(strings.SplitN(ln, "\t", 2)[0])] = true
	}
	if len(bought) == 0 {
		t.Fatalf("fidelity: parsed ZERO terms out of the bought stop table %s (%d bytes) — the parser, not the book", m.r.bankStopTablePath(), len(raw))
	}
	got := map[string]bool{}
	for _, c := range m.cands {
		got[c.Key] = true
	}
	missing, extra := diffKeys(bought, got), diffKeys(got, bought)
	t.Logf("FIDELITY bought_table_terms=%d reconstructed_candidates=%d missing=%d extra=%d",
		len(bought), len(got), len(missing), len(extra))
	if len(missing) > 0 || len(extra) > 0 {
		t.Fatalf("fidelity: the reconstruction is NOT the bought candidate set — every number below would be about another book.\n  in the bought table, not reconstructed (%d): %s\n  reconstructed, not in the bought table (%d): %s",
			len(missing), strings.Join(capList(missing, 12), ", "), len(extra), strings.Join(capList(extra, 12), ", "))
	}
}

// ── the predicate ───────────────────────────────────────────────────────────────────────────────────

// basisFromRun is the memory the bought purchase WOULD have written at its boundary: for every candidate
// the bank holds as a decided engine row, the answer it holds and the fingerprint of the evidence as it
// stands now.
//
// ⚠ THE FINGERPRINT IS TAKEN ON THE PRE-ROLE CANDIDATES on purpose, and the production writer must do the
// same: the classifier pass MUTATES Candidate.Type in place (applyTypes), so a fingerprint computed after
// the roles have run would hash the CLASSIFIER's type — and then learning whether the terminologist may be
// skipped would require buying the classifier first, every purchase, which is the cost the basis exists
// to remove.
func (m *basisMaterial) basisFromRun() map[string]bankBasisRecord {
	canon := approvedNeighbours(m.bank)
	out := make(map[string]bankBasisRecord, len(m.cands))
	for _, c := range m.cands {
		row, ok := m.rows[c.Key]
		if !ok || row.Status != "draft" || strings.TrimSpace(row.Dst) == "" {
			continue // the purchase settled nothing about this surface, so it writes no record
		}
		out[c.Key] = bankBasisRecord{
			Key:    c.Key,
			FP:     candidateBasisFingerprint(c, m.nsource, canon, m.width),
			Answer: bankBasisAnswer{Dst: row.Dst, Type: row.Type, Gender: row.Gender},
		}
	}
	return out
}

// verdicts runs the predicate over the current candidates against a given basis and source.
func (m *basisMaterial) verdicts(basis map[string]bankBasisRecord, nsource string) map[string]bankBasisVerdict {
	canon := approvedNeighbours(m.bank)
	fps := make(map[string]bankBasisFP, len(m.cands))
	for _, c := range m.cands {
		fps[c.Key] = candidateBasisFingerprint(c, nsource, canon, m.width)
	}
	return m.scoped(bankBasisSettle(m.cands, fps, basis, m.rows, m.ambiguous, m.unitID, "s"))
}

// scoped drops the candidates the bank filter already took, so every count below is about the population
// the engine actually offers the basis.
func (m *basisMaterial) scoped(v map[string]bankBasisVerdict) map[string]bankBasisVerdict {
	out := make(map[string]bankBasisVerdict, len(v))
	for k, verdict := range v {
		if !m.consider[k] {
			continue
		}
		out[k] = verdict
	}
	return out
}

// report prints the counts with every denominator beside them.
func (m *basisMaterial) report(t *testing.T, label string, v map[string]bankBasisVerdict) {
	t.Helper()
	served, byReason := countBasisReasons(v)
	parts := make([]string, 0, len(byReason))
	for _, r := range byReason {
		parts = append(parts, fmt.Sprintf("%s=%d", r.Reason, r.N))
	}
	signed := 0
	for _, e := range m.bank {
		if e.Status == "approved" {
			signed++
		}
	}
	t.Logf("%s candidates=%d offered_to_the_basis=%d (bank filter took %d) bank_rows=%d signed_rows=%d | SERVED=%d re-asked=%d | %s",
		label, len(m.cands), len(m.consider), m.bankSettled, len(m.bank), signed, served, len(m.consider)-served, strings.Join(parts, " "))
}

// ── controls ────────────────────────────────────────────────────────────────────────────────────────

// controlOneOccurrenceMoved is the first discriminating control: the source moves around ONE occurrence of
// ONE settled candidate, and the set bought again afterwards must be EXACTLY the set the edit can reach.
//
// ⛔ WHAT «EXACTLY» MEANS HERE, and why the obvious formulation is wrong. The first version of this control
// asserted «the victim and its unit, and nobody else», and it failed on real material — correctly. An edit
// is a point in the text, and it lands inside the evidence window of every surface whose occurrence is
// near it, including surfaces the victim CONTAINS (三转蛊师 contains 蛊师) and neighbours a few runes away.
// Re-asking those is not coupling, it is the mechanism working: their evidence really did change. Asserting
// otherwise would have forced the fingerprint to read LESS of the source than it must.
//
// So the assertion is a CROSS-CHECK between two independent routes to the same set: the predicate's verdict
// (a hash over evidence + canon + identity) against a direct positional walk of the windows alone. They
// agree only if nothing except the windows contributed to the change — which is the property that matters,
// and which the naive formulation could not see at all.
func (m *basisMaterial) controlOneOccurrenceMoved(t *testing.T, base map[string]bankBasisVerdict) {
	t.Helper()
	var victim terminology.Candidate
	for _, c := range m.cands {
		if base[c.Key].Reason == basisServed && strings.Contains(m.nsource, c.Key) {
			victim = c
			break
		}
	}
	if victim.Key == "" {
		t.Fatalf("control 1: no served candidate occurs in the source, so this control would pass without measuring anything")
	}
	at := strings.Index(m.nsource, victim.Key)
	moved := m.nsource[:at] + "ЗЗЗ" + m.nsource[at:]

	// ROUTE A — the direct positional truth: whose windows actually moved. Uses no hash, no basis, no unit.
	touched := map[string]bool{}
	for _, c := range m.cands {
		if !equalStrings(terminology.EvidenceWindows(m.nsource, c.Key, m.width),
			terminology.EvidenceWindows(moved, c.Key, m.width)) {
			touched[c.Key] = true
		}
	}
	if !touched[victim.Key] {
		t.Fatalf("control 1: the edit did not even move the windows of its own victim %q — the fixture, not the predicate", victim.Src)
	}
	if len(touched) == len(m.cands) {
		t.Fatalf("control 1: a single edit moved the windows of EVERY candidate (%d) — the control cannot separate the classes on this material", len(m.cands))
	}
	// …expanded by the unit rule, which is the predicate's own promise: a family is decided whole.
	hotUnits := map[int]bool{}
	for k := range touched {
		hotUnits[m.unitID[k]] = true
	}
	expected := map[string]bool{}
	for _, c := range m.cands {
		if hotUnits[m.unitID[c.Key]] && base[c.Key].Reason == basisServed {
			expected[c.Key] = true
		}
	}

	// ROUTE B — the predicate.
	after := m.verdicts(m.basisFromRun(), moved)
	observed := map[string]bool{}
	for _, c := range m.cands {
		if base[c.Key].Reason == basisServed && after[c.Key].Reason != basisServed {
			observed[c.Key] = true
		}
	}
	missing, extra := diffKeys(expected, observed), diffKeys(observed, expected)
	t.Logf("CONTROL-1 one-occurrence-moved victim=%q windows_moved=%d expected_re-asked=%d observed_re-asked=%d missing=%d extra=%d | %s",
		victim.Src, len(touched), len(expected), len(observed), len(missing), len(extra),
		strings.Join(capList(sortedSet(observed), 8), ", "))
	if len(observed) == 0 {
		t.Fatalf("control 1: moving the source around %q re-asked NOTHING — the fingerprint does not read the source, and the whole predicate is blind", victim.Src)
	}
	if len(missing) > 0 || len(extra) > 0 {
		t.Fatalf("control 1: the predicate's re-ask set is NOT the set the edit can reach.\n  windows moved, predicate kept serving (%d): %s\n  predicate re-asked, windows did not move (%d): %s",
			len(missing), strings.Join(capList(missing, 10), ", "), len(extra), strings.Join(capList(extra, 10), ", "))
	}
}

// controlUnitShape is not a pass/fail control but the measurement the unit rule has to be judged by: how
// much of the saving the «a family is decided whole» rule costs, and whether the cost comes from the rule
// or from the unit DETECTION being run over a different population than the batcher's.
//
// ⚠ IT PRINTS BOTH POPULATIONS ON PURPOSE. The predicate detects units over ALL candidates (bankBasisUnits)
// while the batcher detects them over the PAID subset; that difference is a decision of this pack, and a
// decision whose price is not printed is a decision nobody can review.
func (m *basisMaterial) controlUnitShape(t *testing.T, base map[string]bankBasisVerdict) {
	t.Helper()
	sizes := map[int]int{}
	for _, c := range m.cands {
		sizes[m.unitID[c.Key]]++
	}
	biggest, singles := 0, 0
	for _, n := range sizes {
		if n > biggest {
			biggest = n
		}
		if n == 1 {
			singles++
		}
	}
	// What the answer would be with no unit rule at all — the ceiling the rule is spending against.
	own := 0
	for _, c := range m.cands {
		if base[c.Key].Reason == basisServed || base[c.Key].Reason == basisUnit {
			own++
		}
	}
	// Which candidates do the blocking, and why: the operator's question is «who is holding my family open».
	blockers := map[basisReason][]string{}
	blocked := map[int]bool{}
	for _, c := range m.cands {
		if r := base[c.Key].Reason; r != basisServed && r != basisUnit {
			blocked[m.unitID[c.Key]] = true
			blockers[r] = append(blockers[r], c.Src)
		}
	}
	t.Logf("UNIT-SHAPE units=%d singletons=%d largest_unit=%d | settled_on_their_own=%d served_after_unit_rule=%d cost_of_the_rule=%d | units_blocked=%d",
		len(sizes), singles, biggest, own, countServed(base), own-countServed(base), len(blocked))
	for _, r := range basisReasonNames {
		if names := blockers[r]; len(names) > 0 {
			sort.Strings(names)
			t.Logf("UNIT-SHAPE   blocked by %-16s n=%d | %s", r, len(names), strings.Join(capList(names, 10), ", "))
		}
	}
}

// controlBatchesStable is the second discriminating control: with one basis, two passes build byte-identical
// batches. This is what makes a resume replay for $0 — a bank batch is addressed by the hash of its request,
// so a composition that differs between passes finds no checkpoint and buys the pass again.
func (m *basisMaterial) controlBatchesStable(t *testing.T, base map[string]bankBasisVerdict) {
	t.Helper()
	batchRunes, _, _ := m.r.terminologyOpts()
	render := func() string {
		var paid []terminology.Candidate
		for _, c := range m.cands {
			if base[c.Key].Reason != basisServed {
				paid = append(paid, c)
			}
		}
		// The batcher exactly as production runs it: units detected over the PAID subset, not over all
		// candidates. The predicate uses its own unit pass (bankBasisUnits) and must not have moved this.
		seriesID := terminology.DetectSeries(paid, basisSeriesParams(m.r.Book.SourceLang))
		fams := terminology.DetectFamilies(paid, m.r.familyParams)
		unitID, _ := terminology.MergeUnits(seriesID, fams, m.r.familyParams)
		var b strings.Builder
		for i, batch := range terminology.Batch(paid, batchRunes, unitID) {
			fmt.Fprintf(&b, "#%d\n%s\n", i, terminology.RenderBatch(batch))
		}
		return b.String()
	}
	first, second := render(), render()
	t.Logf("CONTROL-2 batches-under-one-basis paid_candidates=%d rendered_bytes=%d identical=%v",
		len(m.cands)-countServed(base), len(first), first == second)
	if first != second {
		t.Fatalf("control 2: two passes under ONE basis rendered DIFFERENT batches — every resume would re-buy the pass")
	}
	if len(first) == 0 {
		t.Fatalf("control 2: the batcher rendered ZERO bytes, so identical-to-itself proves nothing")
	}
}

// controlKnobsDoNotMatter is the property the fingerprint shape exists for: turning the knobs that decide
// WHICH contexts the model is shown must not move the settled set.
//
// ⛔ THE ONE KNOB THAT IS ALLOWED TO MOVE IT is basis_width, and that is stated here rather than discovered:
// it is the pair's statement of what counts as the same evidence, so changing it IS a change of the
// question. Every other window knob — how many contexts, how wide the model's own view — is prompt shape.
func (m *basisMaterial) controlKnobsDoNotMatter(t *testing.T, base map[string]bankBasisVerdict) {
	t.Helper()
	basis := m.basisFromRun()
	for _, knob := range []struct {
		name          string
		kwicPer       int
		kwicWidth     int
		basisWidth    int
		mustNotChange bool
	}{
		{"kwic_per_term 3→1", 1, 0, 0, true},
		{"kwic_per_term 3→40", 40, 0, 0, true},
		{"kwic_width 40→120", 0, 120, 0, true},
		{"basis_width 40→80 (ALLOWED to move it)", 0, 0, 80, false},
	} {
		_, per, wid := m.r.terminologyOpts()
		if knob.kwicPer > 0 {
			per = knob.kwicPer
		}
		if knob.kwicWidth > 0 {
			wid = knob.kwicWidth
		}
		// Rebuild the candidates with the knob turned: KWIC is attached during the merge, so a knob that
		// leaks into the fingerprint can only be caught by rebuilding, never by re-hashing the same slice.
		turned := m.rebuiltWithKWIC(t, per, wid)
		width := m.width
		if knob.basisWidth > 0 {
			width = knob.basisWidth
		}
		canon := approvedNeighbours(m.bank)
		fps := make(map[string]bankBasisFP, len(turned))
		for _, c := range turned {
			fps[c.Key] = candidateBasisFingerprint(c, m.nsource, canon, width)
		}
		after := m.scoped(bankBasisSettle(turned, fps, basis, m.rows, m.ambiguous, m.unitID, "s"))
		lost := 0
		for _, c := range m.cands {
			if base[c.Key].Reason == basisServed && after[c.Key].Reason != basisServed {
				lost++
			}
		}
		t.Logf("CONTROL-3 knob=%-40s served_before=%d lost_from_settled=%d", knob.name, countServed(base), lost)
		if knob.mustNotChange && lost > 0 {
			t.Fatalf("control 3: turning %s lost %d settled candidate(s) — a prompt-shape knob is re-buying banks", knob.name, lost)
		}
		if !knob.mustNotChange && lost == 0 {
			t.Fatalf("control 3: turning %s lost NOTHING, so this row proves the instrument cannot see the class at all", knob.name)
		}
	}
}

// rebuiltWithKWIC re-attaches KWIC at other settings over the same merged candidates.
func (m *basisMaterial) rebuiltWithKWIC(t *testing.T, per, width int) []terminology.Candidate {
	t.Helper()
	tchunks := make([]terminology.Chunk, len(m.chunks))
	for i, ch := range m.chunks {
		tchunks[i] = terminology.Chunk{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, NSource: text.NormalizeSourceKey(ch.Text)}
	}
	out := make([]terminology.Candidate, len(m.cands))
	copy(out, m.cands)
	for i := range out {
		out[i].KWIC = nil
	}
	return terminology.AttachKWIC(out, tchunks, per, width)
}

// measureGrowth answers the growing-book question with a number instead of a watchdog: decide the bank on a
// PREFIX of the book, then let the book grow, and count how many of the decided rows the predicate buys
// again.
//
// ⚠ THIS IS THE COST OF THE DESIGN, NOT A DEFECT OF IT. A fingerprint over every occurrence necessarily
// moves when new chapters add occurrences of an old term — and that is the honest reading: a term the new
// chapters talk about HAS new evidence. The number below is what the owner needs in order to answer whether
// a growing serial wants a different rule, and it is the one thing the engine must not decide for them.
func (m *basisMaterial) measureGrowth(t *testing.T) {
	t.Helper()
	chapters := map[int]bool{}
	for _, ch := range m.chunks {
		chapters[ch.Chapter] = true
	}
	if len(chapters) < 2 {
		t.Logf("GROWTH skipped: the book has %d chapter(s), so a prefix cannot be formed — the class is unmeasured here, not absent", len(chapters))
		return
	}
	last := 0
	for c := range chapters {
		if c > last {
			last = c
		}
	}
	var prefix []chunk.Chunk
	for _, ch := range m.chunks {
		if ch.Chapter < last {
			prefix = append(prefix, ch)
		}
	}
	prefixSrc := newBankSourceIndex(prefix).book
	// ⛔ THE PREFIX HAS TO BE SMALLER, AND IT IS ASSERTED RATHER THAN ASSUMED. A chapter rule that did not
	// fire leaves every chunk in «chapter 0», the prefix then equals the whole book, and the measurement
	// below prints a beautifully plausible «growth costs nothing» — a right-looking number from a method
	// that never grew anything.
	if len(prefix) == 0 || len(prefix) == len(m.chunks) {
		t.Fatalf("growth: the prefix is %d chunk(s) of %d — the chapter split did not separate a prefix, so nothing below would be about growth", len(prefix), len(m.chunks))
	}
	prefixBytes, wholeBytes := len(prefixSrc), len(m.nsource)
	prefixCands := m.candidatesOver(t, prefix)
	canon := approvedNeighbours(m.bank)

	// The basis as the PREFIX purchase would have written it.
	basis := map[string]bankBasisRecord{}
	for _, c := range prefixCands {
		row, ok := m.rows[c.Key]
		if !ok || row.Status != "draft" || strings.TrimSpace(row.Dst) == "" {
			continue
		}
		basis[c.Key] = bankBasisRecord{Key: c.Key,
			FP:     candidateBasisFingerprint(c, prefixSrc, canon, m.width),
			Answer: bankBasisAnswer{Dst: row.Dst, Type: row.Type, Gender: row.Gender}}
	}
	// The next purchase: the whole book, the prefix's basis.
	after := m.verdicts(basis, m.nsource)
	kept, reasked := 0, 0
	for key := range basis {
		if after[key].Reason == basisServed {
			kept++
		} else {
			reasked++
		}
	}
	// The breakdown matters more than the total: a decision re-asked because ITS OWN evidence grew is the
	// mechanism working, while one re-asked because a 44-member unit holds a single grown member is the
	// UNIT RULE's bill, and the two have entirely different cures.
	own, byUnit := 0, 0
	for key := range basis {
		switch after[key].Reason {
		case basisServed:
		case basisUnit:
			byUnit++
		default:
			own++
		}
	}
	t.Logf("GROWTH chapters=%d prefix_chapters=%d prefix_chunks=%d/%d prefix_bytes=%d/%d prefix_candidates=%d decided_in_prefix=%d | still_served=%d re-asked=%d (own_evidence_grew=%d held_by_unit=%d)",
		len(chapters), len(chapters)-1, len(prefix), len(m.chunks), prefixBytes, wholeBytes,
		len(prefixCands), len(basis), kept, reasked, own, byUnit)
	if len(basis) == 0 {
		t.Fatalf("growth: the prefix purchase decided NOTHING, so both numbers above are zeros about the fixture rather than about growth")
	}
}

// ── small helpers ───────────────────────────────────────────────────────────────────────────────────

func countServed(v map[string]bankBasisVerdict) int {
	n := 0
	for _, x := range v {
		if x.Reason == basisServed {
			n++
		}
	}
	return n
}

func diffKeys(a, b map[string]bool) []string {
	var out []string
	for k := range a {
		if !b[k] {
			out = append(out, k)
		}
	}
	sort.Strings(out)
	return out
}

func capList(xs []string, n int) []string {
	if len(xs) <= n {
		return xs
	}
	return append(append([]string{}, xs[:n]...), fmt.Sprintf("…(+%d more)", len(xs)-n))
}

func equalStrings(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

func sortedSet(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

// measureUnitRuleVariants prices the «a family is decided whole» rule against the alternatives, so the rule
// is CHOSEN with its cost printed rather than inherited from a sentence.
//
// ⛔ ONE OF THESE ROWS WAS A LIE IN ITS FIRST VERSION, and the correction is the reason the row survives.
// It was labelled «signed rows count as settled» and implemented by handing the predicate a basis RECORD
// for each signed row — which measures nothing, because the predicate then rejects the row on its bank
// STATUS (`approved`, not `draft`) and the unit stays blocked exactly as before. The row printed the same
// number as the row above it and read like a finding. A variant that does not implement its own label is
// not a weak measurement, it is a false one.
//
// What the row asks now is the question that is actually open: a signed row IS still bought today, on
// purpose (the two-condition filter pays for banked surfaces so the sheet keeps its dispute marks, backlog
// row 447) — so should it also HOLD ITS FAMILY OPEN? Being bought and blocking a family are two different
// consequences, and only the second is free to change without touching product behaviour.
func (m *basisMaterial) measureUnitRuleVariants(t *testing.T) {
	t.Helper()
	basis := m.basisFromRun()
	canon := approvedNeighbours(m.bank)
	fps := make(map[string]bankBasisFP, len(m.cands))
	for _, c := range m.cands {
		fps[c.Key] = candidateBasisFingerprint(c, m.nsource, canon, m.width)
	}
	familyOnly := func() map[string]int {
		fams := terminology.DetectFamilies(m.cands, m.r.familyParams)
		id := map[string]int{}
		for i, f := range fams {
			for _, k := range f.Keys {
				if _, taken := id[k]; !taken {
					id[k] = i + 1
				}
			}
		}
		return totalUnits(m.cands, id)
	}()
	signed := map[string]bool{}
	for _, c := range m.cands {
		if row, ok := m.rows[c.Key]; ok && row.Status == "approved" && strings.TrimSpace(row.Dst) != "" {
			signed[c.Key] = true
		}
	}
	// own-verdicts: the per-candidate half, computed once — every variant differs only in how units block.
	own := make(map[string]bankBasisVerdict, len(m.cands))
	for _, c := range m.cands {
		own[c.Key] = basisVerdictFor(c, fps[c.Key], basis, m.rows, m.ambiguous, "s")
	}
	// ⛔ basisSigned IS EXEMPT IN EVERY ROW, because production exempts it (bankBasisSettle) and a row
	// labelled «the design» that does not implement the design is not a weak measurement, it is a false
	// one — the very thing this function's own docblock refuses. Measured before the fix: the row printed
	// 62 of 66 where the engine gives 64, so a reader concluded the signature exemption was an unbuilt
	// alternative worth +2, when it was already built. Found by adversarial review.
	count := func(units map[string]int, exempt map[string]bool) int {
		blocked := map[int]bool{}
		for _, c := range m.cands {
			if r := own[c.Key].Reason; r != basisServed && r != basisSigned && !exempt[c.Key] {
				blocked[units[c.Key]] = true
			}
		}
		n := 0
		for _, c := range m.cands {
			if own[c.Key].Reason == basisServed && !blocked[units[c.Key]] {
				n++
			}
		}
		return n
	}
	none := map[string]int{}
	for i, c := range m.cands {
		none[c.Key] = i + 1
	}
	for _, v := range []struct {
		name   string
		units  map[string]int
		exempt map[string]bool
	}{
		{"none — no unit rule at all (the ceiling)", none, nil},
		{"family-only", familyOnly, nil},
		{"merged series+family (THE DESIGN, signature exempt)", m.unitID, nil},
		{"merged, a signed row ALSO counted as a hole (pre-17.09 shape)", m.unitID, nil},
	} {
		n := count(v.units, v.exempt)
		t.Logf("UNIT-RULE %-48s SERVED=%2d of %d (%.0f%%)", v.name, n, len(m.cands),
			100*float64(n)/float64(len(m.cands)))
	}
	t.Logf("UNIT-RULE   signed rows on this book: %d (%s) — they are BOUGHT either way (backlog row 447); the rows above only ask whether they hold a family open",
		len(signed), strings.Join(capList(sortedSet(signed), 6), ", "))
}

// totalUnits turns a PARTIAL unit map into a total one, giving every unlisted candidate a unit of its own.
// Same trap as bankBasisUnits guards, same cure: a missing key reads as unit 0, and unit 0 is a unit.
func totalUnits(cands []terminology.Candidate, partial map[string]int) map[string]int {
	out := make(map[string]int, len(cands))
	next := 1 << 30
	for _, c := range cands {
		if id, ok := partial[c.Key]; ok && id != 0 {
			out[c.Key] = id
			continue
		}
		next++
		out[c.Key] = next
	}
	return out
}

// measureAutoTail prints the price the consilium named and nobody had put a number on: a surface the model
// steadily declines to render is never SETTLED, so it goes to the paid role on EVERY purchase, on input
// that has not changed by a byte. The basis cannot help it — «no rendering» is not a decision, and storing
// it would let a later purchase serve emptiness back — so this is a standing cost of the mechanism rather
// than a defect of it, and a cost with no number attached is one nobody can decide about.
func (m *basisMaterial) measureAutoTail(t *testing.T, base map[string]bankBasisVerdict) {
	t.Helper()
	tail, noRow, byStatus := 0, 0, map[string]int{}
	for _, c := range m.cands {
		row, ok := m.rows[c.Key]
		if !ok {
			noRow++
			continue
		}
		byStatus[row.Status]++
		if row.Status != "approved" && (row.Status != "draft" || strings.TrimSpace(row.Dst) == "") {
			tail++
		}
	}
	// ⚠ THE DENOMINATOR AND THE STATUS HISTOGRAM RIDE WITH IT, because a zero here has two readings that
	// look identical: «this book has no undecided tail» and «this book's rows are not where the instrument
	// looked». The histogram is what separates them.
	t.Logf("AUTO-TAIL candidates=%d bank_rows=%d with_no_bank_row=%d | re-asked EVERY purchase on unchanged input=%d | statuses=%v served_now=%d",
		len(m.cands), len(m.bank), noRow, tail, byStatus, countServed(base))
}

// measureRechunkResidue puts a NUMBER on the one residue of the fingerprint's shape its author can name but
// could not, at first, measure.
//
// ⚠ THE RESIDUE. The windows are taken from the book's normalized source as the engine holds it —
// r.bankSrc.book, the chunks' normalized texts joined by a newline. That is ONE definition of «the book's
// source», shared with the banknote attestation check, and keeping one definition is worth more than the
// precision it costs. What it costs is this: an occurrence sitting within basis_width runes of a CHUNK JOIN
// carries that join's newline inside its window, so re-chunking the book at another segmentation budget
// moves that window and re-asks the term. A re-cut of CHAPTERS is a named, priced event; a change of the
// chunk budget is not, and «I named the residue but did not measure it» is a blind spot rather than a
// property. This is the measurement.
func (m *basisMaterial) measureRechunkResidue(t *testing.T) {
	t.Helper()
	base := m.r.segBudget()
	canon := approvedNeighbours(m.bank)
	for _, factor := range []float64{0.5, 2.0} {
		b := base
		b.DraftBudgetOut = base.DraftBudgetOut * factor
		other, _, _ := chunk.SplitChunksWithChapters(m.doc.Chapters, b, m.r.chapterRule(), m.r.sentenceAbbrevs())
		if len(other) == len(m.chunks) {
			t.Logf("RECHUNK budget×%.1f produced the SAME %d chunk(s): this factor measures nothing on this book", factor, len(other))
			continue
		}
		src := newBankSourceIndex(other).book
		moved, attested := 0, 0
		for _, c := range m.cands {
			if len(terminology.EvidenceWindows(m.nsource, c.Key, m.width)) > 0 {
				attested++
			}
			if candidateBasisFingerprint(c, m.nsource, canon, m.width).Whole !=
				candidateBasisFingerprint(c, src, canon, m.width).Whole {
				moved++
			}
		}
		// The denominator and the attested count ride along: a zero over candidates the source does not
		// contain would be a zero about the fixture, not about the residue.
		t.Logf("RECHUNK budget×%.1f chunks %d→%d bytes %d→%d | fingerprints moved=%d of %d candidates (attested in the source: %d)",
			factor, len(m.chunks), len(other), len(m.nsource), len(src), moved, len(m.cands), attested)
	}
}