textmachine/backend/internal/pipeline/bookconsistency.go

package pipeline

import (
	"sort"
	"strings"

	"textmachine/backend/internal/membank"
	"textmachine/backend/internal/store"
)

// bookconsistency.go: the book-wide consistency answer (backlog row 406) — the first measure in this engine
// that asks the owner's priority-#1 question of the text a READER actually gets.
//
// Everything that judged terminology before it judged a CHUNK, in flight, against the injection: the
// post-check per chunk, the mining-stop spread over DRAFTS. Neither can answer "is this term one rendering
// across the whole book", because neither ever holds two chapters at once and the stop-border spread is
// computed before the editor wave rewrites anything. This runs over the SHIPPED text of every unit, in
// manifest order, and aggregates per term — so chapter 12 and chapter 901 are finally compared.
//
// Two invariants, reported apart because they fail for different reasons and cost different things:
//
//   - I1, one form per term. Detected WITHOUT naming the second form: a term whose bank rendering reached
//     the reader in some chapters and not in others went out at least two ways, and so did one that got the
//     bank's rendering for some of its occurrences inside a single unit.
//   - I2, the bank's form is the one shipped (the D39.104 law). A term whose key fired and whose accepted
//     rendering is absent everywhere it fired.
//
// ⛔ IT DOES NOT SPLIT TERMS BY SIGNING STATUS, and that is a correction, not an omission. There is no such
// product unit as a signed TERM: a bank is signed AS A WHOLE or the book travels with an unsigned one
// (D39.144 — per-term signing is named there as explicitly NOT the product model, being hundreds of
// clicks; the engine's default is to auto-continue with an unsigned bank, and `--verify-bank` is a pause
// to look, not a march down the rows). The law on the wire (D39.104) is one law for every row, and a
// single term's status neither switches it on nor off. So the signing state is stated ONCE, about the
// book, and the term counts are not cut by it.
//
// ⚠ THE ENGINE DOES NOT AGREE WITH THAT MODEL, and the disagreement is worth more than this file:
// glossary rows carry a per-term machine `auto|draft|approved` (store/migrate.go:192), dispositionFor
// promotes only `approved` to Confirmed (membank/memory.go:704), and the post-check counts ONLY a
// Confirmed miss (membank/mempostcheck.go:154). That is why the paid run of 11.09 reported
// `n_postcheck_miss = 0` while naming eighteen deviations: the counter was deciding on a unit the product
// does not have. Not this pack's to change — it needs its own order — but it is the real cause, and the
// missing report surface was only its symptom.

// BookConsistency is the whole-book verdict, with the denominators that make its zeroes readable. A zero
// here is worth nothing on its own — "no term went out two ways" and "no term was judged" are the same
// number — so every count ships beside the population it was drawn from.
type BookConsistency struct {
	// UnitsJudged is how many shipped units the scan actually read. A unit that shipped no text (dropped,
	// or withheld by the glossary gate) is not judged: its terms did not reach the reader in ANY form, and
	// counting them as missing renderings would blame the shipping decision on the editor.
	UnitsJudged int
	// BankRows / Judged / NoDst partition the bank. A row with no dst has no target surface to have
	// shipped, so it cannot be judged — it is reported, not silently dropped out of the denominator.
	BankRows int
	Judged   int
	NoDst    int
	// FiredTotal / ShippedTotal are the OCCURRENCE denominators, as opposed to the term ones above: how many
	// times a bank key occurred in the source this scan JUDGED — occurrences in a chapter the spoiler window
	// excluded are not among them, because the bank withheld the row there on purpose — and how many times
	// an accepted rendering came out. Term counts alone cannot say whether a book's terminology is mostly reaching the reader —
	// «5 terms absent» reads very differently against 40 firings and against 4000.
	FiredTotal   int
	ShippedTotal int
	// NeverFired is the control the whole measure rests on: rows whose SOURCE key never occurred in any
	// judged unit. Without it "the rendering is absent" and "the term is not in this book" are one number,
	// and the second one is not a finding.
	NeverFired int
	// BlockedOnly counts rows whose every firing fell in a chapter the spoiler window excluded. The bank
	// deliberately withheld them, so their absence is the window working, not a consistency failure.
	BlockedOnly int
	// Covered / Absent / Split partition the rows that fired, judged under the ANCHORED count.
	//
	// ⛔ THE VERDICT RESTS ON THE ANCHORED COLUMN, and the reason is the corpus, not the rule. The safe rule
	// alone leans on a term's stored decl forms — and no MACHINE-produced bank in this tree carries any:
	// zero decl rows across every mined export measured, against 35–42 of ~50 in every hand-written seed.
	// On the product's real path — a book with no hand seed, its bank mined — the stemmer has nothing to
	// stem, and the one-rune tolerance is the only thing that reaches an inflected rendering at all. A
	// verdict taken on the safe column there would report drift that is not happening.
	Covered int
	Absent  int
	Split   int
	// UnsignedRows is the BOOK's signing state, said once: how many of the bank's rows carry a rendering
	// nobody approved. Non-zero means this book's bank travelled unsigned — which is the engine's default —
	// and the number is here rather than as a cut through the term counts because signing is a property of
	// the bank, not of a term.
	UnsignedRows int
	// The same three under the post-check's OWN equality. They are carried, and printed, because the gap
	// between the two columns is the only way to see what the stemmer costs: on the paid run of 11.09 the
	// strict column called eleven renderings missing that were in the shipped text, inflected. A report that
	// printed one column would have been wrong by that much and said so nowhere.
	CoveredStrict int
	AbsentStrict  int
	SplitStrict   int
	// And the same three WITHOUT the stem tolerance — the safe rule alone. The gap between these and the
	// verdict above is what the one-strip stemmer costs on this book, which is a different question from
	// what the post-check is blind to.
	CoveredSafe int
	AbsentSafe  int
	SplitSafe   int
	// ⛔ MaskCandidates IS THE REPORT'S OWN BIAS, COUNTED. The target side counts a rendering ANYWHERE in
	// the unit, so a common-noun rendering occurring away from its term inflates `shipped` — and an
	// inflated `shipped` can cover a real absence. The bias therefore runs ONE WAY, towards reassurance,
	// and a reader of I2 has to know that. This is the population where it can happen at all: terms whose
	// rendering was found MORE often than their key fired, which is only possible if some of those
	// occurrences do not belong to the term. On the cold run it is small but not zero (`魔道`: fired 1,
	// found 3), and printing it is what keeps I2 from reading as a flat verdict.
	//
	// It is a CANDIDATE count, not a defect count: a legitimate plural or a repeated mention inside one
	// unit produces the same shape. Named as such in the report.
	MaskCandidates int
	// ⛔ SuppressedSilent IS ORTHOGONAL TO THE PARTITION ABOVE, not a slice of it. A term can be rendered
	// perfectly in chapter 1 and be eaten with its canon lost in chapter 3; folding the two into one
	// per-term bucket let a single clean firing erase the eaten case entirely, which is how the acceptance
	// found this surface reporting zero on a book that had one. It is counted per TERM-with-any-such-chapter
	// and the chapters are named, and it does not enter the Judged partition.
	//
	// It counts rows whose key fired but was eaten by a longer key of equal-or-higher trust
	// AND whose own rendering did not reach the shipped text. The bank suppresses the shorter match on
	// purpose and says nothing about it; when the longer rendering then fails to carry the shorter one's
	// canon, nothing in the engine notices. That is the second root of backlog row 407, and until now it had
	// no number at all.
	SuppressedSilent int
	// Terms carries the named failures — every Absent and every Split row, in the order the scan first met
	// them, which is the book's own reading order (units are walked in manifest order). Deterministic, and
	// more useful than the bank's internal order: an editor reads a book front to back.
	// Covered rows are not listed: the count is the fact, and a full listing would bury the six lines an
	// operator has to act on under sixty they do not.
	Terms []TermConsistency
}

// TermConsistency is one term's fate across the book: how often it occurred, how often the bank's rendering
// came out, and — the part that makes the line actionable — WHICH CHAPTERS disagree.
type TermConsistency struct {
	Src     string
	Dst     string
	Status  string
	Verdict string // covered | absent | split — a covered row is listed only when a chapter ate it
	Fired   int
	// Shipped is the anchored count (the verdict's own); ShippedStrict is what the post-check's equality
	// would have counted. Both travel so a reader can see which of the two the verdict rests on.
	Shipped       int
	ShippedStrict int
	// WithForm / WithoutForm are the chapters where the term fired and the bank's rendering did / did not
	// come out; SplitIn are chapters where both happened inside one unit. Sorted, deduped.
	WithForm    []int
	WithoutForm []int
	SplitIn     []int
	// EatenIn are the chapters where a longer key swallowed this row and its own rendering did not come out.
	// Separate from WithoutForm because the row never fired there in its own right — nothing asked about it.
	EatenIn []int
}

// consistencyScan accumulates the per-unit scans into one per-term picture. Kept apart from QualityReport's
// loop body so the aggregation can be tested on its own.
type consistencyScan struct {
	bank  *membank.Bank
	units int
	terms map[string]*termAcc
	order []string // first-seen order, so the output never depends on map iteration
}

type termAcc struct {
	rec       membank.TermShipping
	fired     int
	firedUnbl int // firings in chapters the spoiler window did NOT block
	// Two parallel pictures of the same term: what the post-check's equality sees, and what the anchored
	// count sees. Kept side by side rather than folded so the report can print the difference instead of
	// choosing a side.
	strict  verdictAcc
	primary verdictAcc
	relaxed verdictAcc
	// eatenIn are the chapters where a longer key of equal-or-higher trust swallowed this row AND its own
	// rendering is absent from what shipped; eatenCovered are the chapters where it was swallowed and the
	// rendering came out anyway — the longer match carried it, which is the suppressor working.
	//
	// ⛔ eatenCovered EXISTS BECAUSE ITS ABSENCE WAS A LIE IN THE PRINTED NUMBERS. A swallowed row has no
	// firings of its own (its spans are removed before counting), so without this it fell out of every
	// bucket and subtraction filed it under «key never fired in the source» — the one line this measure
	// calls its control, inflated by every correctly-rendered nested row, i.e. on any book with aliases.
	eatenIn      map[int]bool
	eatenCovered map[int]bool
}

// verdictAcc is one term's per-chapter picture under one counting rule.
type verdictAcc struct {
	shipped     int
	withForm    map[int]bool
	withoutForm map[int]bool
	splitIn     map[int]bool
}

func newVerdictAcc() verdictAcc {
	return verdictAcc{withForm: map[int]bool{}, withoutForm: map[int]bool{}, splitIn: map[int]bool{}}
}

// observe folds one span's counts for one term into the picture.
func (v *verdictAcc) observe(chapter, fired, shipped int) {
	v.shipped += shipped
	switch {
	case shipped == 0:
		v.withoutForm[chapter] = true
	case shipped < fired:
		v.withForm[chapter] = true
		v.splitIn[chapter] = true
	default:
		v.withForm[chapter] = true
	}
}

// verdict classifies the term under this rule. "" means covered — a single rendering, the bank's.
func (v verdictAcc) verdict() string {
	switch {
	case len(v.withForm) == 0:
		return "absent"
	case len(v.withoutForm) > 0 || len(v.splitIn) > 0:
		return "split"
	default:
		return ""
	}
}

func newConsistencyScan(bank *membank.Bank) *consistencyScan {
	return &consistencyScan{bank: bank, terms: map[string]*termAcc{}}
}

// add scans one shipped unit and folds it into the running picture. A nil bank (a book with no glossary at
// all) makes this inert, so the caller needs no branch.
func (c *consistencyScan) add(chapter int, source, shipped string) {
	if c == nil || c.bank == nil {
		return
	}
	c.units++
	for _, t := range c.bank.ScanShipping(source, shipped, chapter) {
		a := c.terms[t.ID]
		if a == nil {
			a = &termAcc{strict: newVerdictAcc(), primary: newVerdictAcc(), relaxed: newVerdictAcc(),
				eatenIn: map[int]bool{}, eatenCovered: map[int]bool{}}
			c.terms[t.ID] = a
			c.order = append(c.order, t.ID)
		}
		a.rec = t
		a.fired += t.Fired
		switch {
		case t.SuppressedCanonAbsent:
			a.eatenIn[chapter] = true
		case t.Suppressed:
			a.eatenCovered[chapter] = true
		}
		if t.Fired == 0 || t.Blocked {
			continue // a rendering surfacing where the term did not fire is not a fact about the term's fate
		}
		a.firedUnbl += t.Fired
		a.strict.observe(chapter, t.Fired, t.ShippedStrict)
		a.primary.observe(chapter, t.Fired, t.Shipped)
		a.relaxed.observe(chapter, t.Fired, t.ShippedRelaxed)
	}
}

// finish turns the accumulated scans into the report, using the bank rows the caller already read as the
// denominator — the SAME rows QualityReport counts unsigned terms from, so one report describes one bank.
func (c *consistencyScan) finish(rows []store.GlossaryEntry) *BookConsistency {
	if c == nil || c.bank == nil {
		return nil
	}
	out := &BookConsistency{UnitsJudged: c.units, BankRows: len(rows)}
	for _, row := range rows {
		// TrimSpace, not =="": the bank treats a whitespace-only dst as no rendering at all
		// (MaterializeBank, ScanShipping), and two emptiness tests for one concept put such a row in the
		// judgeable denominator and then reported it as «key never fired» — the one line this measure calls
		// its control.
		if strings.TrimSpace(row.Dst) == "" {
			out.NoDst++
			continue
		}
		out.Judged++
		if row.Status != "approved" {
			out.UnsignedRows++
		}
	}
	for _, id := range c.order {
		a := c.terms[id]
		// A term eaten by a longer key whose canon SURVIVED did occur in the source and did reach the
		// reader — the suppressor doing its job. It is covered, not «never fired»: the row has no firings of
		// its own only because its spans were removed before counting.
		if a.firedUnbl == 0 && len(a.eatenCovered) > 0 && len(a.eatenIn) == 0 {
			out.Covered++
			out.CoveredStrict++
			continue
		}
		if len(a.eatenIn) > 0 {
			out.SuppressedSilent++ // orthogonal to the partition — see the field doc
		}
		if a.fired == 0 && len(a.eatenIn) == 0 {
			continue // surfaced without firing — left to NeverFired below
		}
		if a.firedUnbl == 0 && len(a.eatenIn) == 0 {
			out.BlockedOnly++
			continue
		}
		out.FiredTotal += a.firedUnbl
		out.ShippedTotal += a.relaxed.shipped
		if a.relaxed.shipped > a.firedUnbl {
			out.MaskCandidates++
		}
		// The two CONTEXT columns, neither of which the verdict rests on: what the post-check's own equality
		// would say, and what the safe rule alone (no stem tolerance) would say.
		switch a.strict.verdict() {
		case "absent":
			out.AbsentStrict++
		case "split":
			out.SplitStrict++
		default:
			out.CoveredStrict++
		}
		switch a.primary.verdict() {
		case "absent":
			out.AbsentSafe++
		case "split":
			out.SplitSafe++
		default:
			out.CoveredSafe++
		}
		v := a.relaxed.verdict()
		switch v {
		case "absent":
			out.Absent++
		case "split":
			out.Split++
		default:
			out.Covered++
		}
		if v == "" && len(a.eatenIn) == 0 {
			continue // clean on every axis — the count is the fact, no line to print
		}
		// ⚠ THE VERDICT STAYS THE CONSISTENCY VERDICT even when a chapter ate this row. The two answer
		// different questions — «did one rendering reach the reader» and «was this row ever swallowed by a
		// longer key» — and overwriting the first with the second made a term whose rendering DID arrive
		// read as a failure. The eaten chapters travel in their own column.
		if v == "" {
			v = "covered"
		}
		out.Terms = append(out.Terms, a.term(v))
	}
	// NeverFired is what is LEFT of the judgeable rows once every row that fired is accounted for. Derived
	// by subtraction rather than counted directly because the scan only reports rows it saw: a row that
	// fired nowhere produces no record at all, and that silence is exactly the number wanted.
	out.NeverFired = out.Judged - out.Covered - out.Absent - out.Split - out.BlockedOnly
	if out.NeverFired < 0 {
		// Only reachable if the scan saw a row the caller's `rows` no longer carries — the two come from
		// separate reads, the scan from the materialized bank and this from a fresh GlossaryForBook. The
		// clamp keeps the printed control from going negative; it does not hide a defect, because the
		// partition is asserted to close by TestEveryZeroCarriesThePopulationItCameFrom on matching inputs.
		out.NeverFired = 0
	}
	return out
}

func (a *termAcc) term(verdict string) TermConsistency {
	return TermConsistency{
		Src: a.rec.Src, Dst: a.rec.Dst, Status: a.rec.Status, Verdict: verdict,
		Fired: a.fired, Shipped: a.relaxed.shipped, ShippedStrict: a.strict.shipped,
		WithForm:    sortedChapters(a.relaxed.withForm),
		WithoutForm: sortedChapters(a.relaxed.withoutForm),
		SplitIn:     sortedChapters(a.relaxed.splitIn),
		EatenIn:     sortedChapters(a.eatenIn),
	}
}

func sortedChapters(m map[int]bool) []int {
	if len(m) == 0 {
		return nil
	}
	out := make([]int, 0, len(m))
	for ch := range m {
		out = append(out, ch)
	}
	sort.Ints(out)
	return out
}