textmachine/backend/internal/membank/memconsistency.go

package membank

import (
	"sort"
	"strings"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/text"
)

// memconsistency.go: the book-wide consistency measure (backlog row 406) — "did one term reach the reader
// as ONE rendering, and was that rendering the bank's".
//
// It is a different QUESTION from the post-check next door, and the difference is the whole point. Postcheck
// asks a BOOLEAN of one chunk: is an accepted dst form present ANYWHERE in this output? That question cannot
// see a term rendered the bank's way once and some other way nine times — the one presence satisfies it
// (row 407). This asks two COUNTS of the same span: how many times did the row's source key fire, and how
// many times did an accepted rendering come out. The pair is what makes partial coverage visible, and
// partial coverage is precisely "the term went out more than one way".
//
// It reuses the bank's own machinery rather than standing a second matcher beside it — the source side runs
// the SAME automaton and the SAME two suppressions Select runs (a private matcher of our own would re-earn
// their bugs, and the polygon's hand-rolled one returned three false zeroes by missing inflected lower-case
// forms), and the target side runs the SAME normalizer and the SAME stemmer the post-check accepts forms by.
// That is also why it lives in this package: the row, the automaton and the accepted-form set are unexported,
// and a caller outside cannot reach them without a second implementation of all three.
//
// WHAT IT CANNOT SEE, named rather than implied: the competing rendering itself. A second form is detected
// by the bank's form being ABSENT where the key fired, never by recognising what stands there instead —
// the drafts' proposals live only in the mining-stop projections (pipeline.BankExportProposal.Variants) and
// are not persisted per row, so no reader of the glossary can name them. The consequence is one blind class:
// a rendering that CONTAINS the bank's form in full ("великая церемония открытия апертуры" over "церемония
// открытия апертуры") counts as the bank's form here, because in the target text it is one. Closing that
// needs the variants channel, not a cleverer matcher, and guessing from neighbouring words would be the
// heuristic this file exists to avoid.

// TermShipping is one bank row's fate over one span of shipped text: how often the row's source key fired in
// that span's SOURCE, and how often an accepted rendering of it came out in the span's SHIPPED text.
//
// The two are counted over the SAME span and are meaningful only together. Fired without Shipped is the
// interesting case (the term occurred and its rendering did not); Shipped without Fired is not an error but
// a category — a common-noun dst recurring away from its term, the recall cost dstFormPresent already
// documents.
type TermShipping struct {
	ID  string // the bank row's stable id — the aggregation key across spans
	Src string // the row's source surface
	Dst string // the row's approved rendering (raw, as the operator reads it)
	// Status is the ENGINE's per-term machine (auto|draft|approved). It is carried because the mechanism
	// acts on it — dispositionFor promotes only `approved`, and the post-check counts only that — NOT
	// because the report splits terms by it: there is no signed TERM in the product model, a bank is signed
	// whole (D39.144). The consistency counts therefore ignore this field, and the book's signing state is
	// said once, as a row count. See bookconsistency.go for the finding that distinction uncovered.
	Status string
	// Via is the LONGEST key that fired for this row in this span, as Select records it: a row can own its
	// src and several aliases, and which one fired decides the trust the injection carried.
	Via string
	// Fired counts non-overlapping occurrences of the row's keys in the source span, after the same
	// word-boundary and longest-match suppressions Select applies. It is the denominator of the question.
	Fired int
	// Shipped is the PRIMARY count — the one every verdict is taken on. It is the post-check's rule plus the
	// one relaxation that carries no hazard: a stored decl form is stemmed rather than matched literally.
	//
	// ⛔ THE PRIMARY COLUMN IS THIS AND NOT THE RELAXED ONE, and the choice is a measurement, not a taste.
	// On the only labelled corpus this engine has, stemming the decl forms removes ALL SIX residual false
	// flags while the one-rune tolerance removes NONE; and the tolerance has a real, measured hazard on
	// multi-word renderings («глава рода Гуюэ» is satisfied by «глаза рода Гуюэ»). A verdict must not rest
	// on the relaxation that buys nothing and risks something.
	Shipped int
	// ShippedStrict is what the post-check's OWN equality counts — the column that says what the engine
	// itself would report — and ShippedRelaxed adds the one-rune stem tolerance on top of the primary rule.
	// Both are context, never a verdict: the first shows what the shipping path is blind to, the second what
	// the stemmer's one-strip limit costs. ⚠ Relaxed is ALMOST always ≥ Shipped; a longer form admitted only
	// by the tolerance consumes more words per match, so a contrived form set can invert them (4 in 40 000;
	// no plausible glossary row produced it).
	ShippedStrict  int
	ShippedRelaxed int
	// Suppressed marks a row whose key fired but was eaten by a LONGER key of equal-or-higher trust
	// (suppressContained). The bank does that silently and on purpose — the longer surface is the better
	// match — but the consequence is that the shorter row's own canon is never checked: 方源 firing through
	// its alias 古月方源 answers for «Фан Юань» and nothing answers for the signed 古月 → «Гуюэ» inside it,
	// so «Гу Юэ Фан Юань» passes (backlog row 407, the second root). This column is that silence made
	// countable; it is observability and changes no disposition.
	Suppressed bool
	// SuppressedCanonAbsent lists the suppressed row's own rendering when it is NOT in the shipped span —
	// i.e. the longer match ate the row AND the row's canon did not survive into the text. That pair is the
	// whole finding: suppression alone is correct behaviour, suppression plus an absent canon is not.
	SuppressedCanonAbsent bool
	// Blocked marks a row the spoiler window excluded for this chapter. It fired but was never injected, so
	// judging its absence would flag the window doing its job. Reported as its own category, not dropped —
	// a silent drop is how a denominator quietly stops meaning what it says.
	Blocked bool
}

// Absent reports the strong failure OVER THIS SPAN: the key fired here and no accepted rendering came out.
//
// ⚠ IT IS NOT THE BOOK'S VERDICT, and the distinction is the reason this measure exists. A term absent from
// one chapter and present in another is not an absent term — it is a term rendered two ways, which is the
// worse finding and is only visible with both chapters in hand. The book-level classification therefore
// lives in ONE place, the aggregation (pipeline.consistencyScan), and this predicate is what one span says.
func (t TermShipping) Absent() bool { return t.Fired > 0 && t.Shipped == 0 }

// Partial reports the split: some firings got the bank's rendering and some did not, inside ONE span. That
// is more than one rendering of one term in one place, and it is the case the presence-shaped post-check
// cannot reach.
//
// ⚠ It is an UPPER bound on renderings, not a proof of two: a target language may legitimately answer
// several source occurrences with one word (a Russian plural, or a pronoun after the first mention). The
// count is reported as a count, never as a verdict, for exactly that reason — and like Absent, it speaks
// for one span, not for the book.
func (t TermShipping) Partial() bool { return t.Fired > 0 && t.Shipped > 0 && t.Shipped < t.Fired }

// ScanShipping measures one span — a source text and the shipped text produced from it — against every bank
// row, and returns a record for each row that either fired or surfaced. Pure and deterministic: rows come
// back in the bank's frozen entry order, never map order.
//
// Rows with no dst are skipped: MaterializeBank gives them no target surface, so there is nothing to have
// shipped, and counting them would put rows that CANNOT be judged into the denominator of rows that were.
// The caller counts them separately from the glossary — see pipeline.QualityReport.
func (b *Bank) ScanShipping(source, shipped string, chapter int) []TermShipping {
	// ZERO-VALUE GUARD, for the same reason Select carries one: Bank is exported, so a caller outside this
	// package can construct one whose automaton is nil, and a nil automaton has nothing to match.
	if b == nil || b.ac == nil {
		return nil
	}
	fired, via, eaten := b.firingCounts(source, chapter)
	tgt := b.tokenizeTarget(shipped)

	out := make([]TermShipping, 0, len(fired))
	for ei := range b.entries {
		e := &b.entries[ei]
		if strings.TrimSpace(e.dst) == "" {
			continue // no target surface: nothing could have shipped, so the row is not judgeable here
		}
		n := fired[ei]
		strict, primary, relaxed := b.countShipped(e, tgt)
		if n == 0 && relaxed == 0 && !eaten[ei] {
			continue // the row is absent from this span on every side — not a fact about the span
		}
		out = append(out, TermShipping{
			ID: e.id, Src: e.src, Dst: e.dst, Status: e.status, Via: via[ei],
			Fired: n, Shipped: primary, ShippedStrict: strict, ShippedRelaxed: relaxed,
			Blocked:    spoilerBlocked(e, chapter),
			Suppressed: eaten[ei], SuppressedCanonAbsent: eaten[ei] && primary == 0,
		})
	}
	return out
}

// targetText is one shipped span tokenized ONCE, with every word's stem taken ONCE.
//
// ⛔ IT EXISTS FOR A MEASURED REASON, not for tidiness. The scan asks every bank row about every output
// word, and stemming inside that loop re-derived the same few thousand stems once per row: at 200 rows over
// one unit of the cold run's size that measured 4.4 SECONDS, which on a 2283-chapter book is hours for a
// report that is supposed to be free. Hoisting the stems out and rejecting a row whose first word is not in
// the span at all bought ~70×. The COMPARISON is unchanged — stems are what SameStem compares, taken with
// the same stemmer — so no count moves.
//
// ⚠ IT IS NOT LINEAR IN THE SPAN, and an earlier version of this comment said it was. The rejection scans
// the span's stem set for a near neighbour when the exact stem is missing, so the cost still carries a
// rows×span term: measured by the acceptance at roughly 40–73 s for 3200 units, not the couple of minutes a
// linear claim implies. Tolerable for a report and named rather than rounded off; a stem-prefix index over
// the span would remove the remaining term if it ever stops being tolerable.
type targetText struct {
	words []string
	stems []string
	// present is the set of stems in this span, for O(1) rejection of a row that cannot be here at all.
	// Most rows of a book's bank do not occur in most of its units, and before this they each cost a full
	// pass over the span to discover that.
	present map[string]bool
	// opensSentence[i] marks a word that begins a new sentence, i.e. one a rendering may not reach back
	// across. Without it a two-word rendering is satisfied by its halves landing on either side of a full
	// stop — «род Бай» by «Он из рода. Бай его не знал» — which inflates the count and can turn a term that
	// went out two ways into one that reads clean. That direction HIDES findings, which is why the boundary
	// is tracked here and not left as the inherited behaviour of declinedFormPresent (where the same lapse
	// only ever made a boolean more generous).
	opensSentence []bool
}

// sentenceBreaks are the runes that end a clause or sentence in the normalized target. Deliberately
// punctuation and not a language model: the question is only «may a rendering span this gap», and a full
// stop is the one answer that holds for every target this engine has. The colon and semicolon are included
// on the same reasoning — a multi-word rendering does not straddle them in well-formed text — and the cost
// of being wrong about one of them is a rendering NOT counted, i.e. a louder report, never a quieter one. A paragraph break does not survive NormalizeTargetForm
// (it folds every space run to one space), so it is caught only when it follows a stop — which is the usual
// case and is named here rather than implied.
const sentenceBreaks = ".!?…:;"

func (b *Bank) tokenizeTarget(shipped string) targetText {
	norm := text.NormalizeTargetForm(shipped)
	out := targetText{}
	// Tokenize each SENTENCE with the shared tokenizer rather than writing a second one: the word list must
	// be identical to lang.TokenizeWords over the whole span, and splitting first is what adds the boundary
	// without changing a single token.
	for _, seg := range strings.FieldsFunc(norm, func(r rune) bool { return strings.ContainsRune(sentenceBreaks, r) }) {
		w := lang.TokenizeWords(seg)
		for i, word := range w {
			out.words = append(out.words, word)
			out.opensSentence = append(out.opensSentence, i == 0)
		}
	}
	out.stems = make([]string, len(out.words))
	out.present = make(map[string]bool, len(out.words))
	for i, w := range out.words {
		st := b.stemmer.Stem(w)
		out.stems[i] = st
		out.present[st] = true
	}
	return out
}

// firingCounts runs the source span

// firingCounts runs the source span through the SAME pipeline Select's match half runs — normalize, the
// shared automaton, the phonetic word-boundary suppression, the trust-aware longest-match suppression — and
// returns per-entry occurrence COUNTS plus the longest key each entry fired on.
//
// The only difference from Select is arity: Select needs to know THAT a row fired, so it keeps one record
// per entry; this needs to know HOW OFTEN, because "the rendering is present" and "the rendering is present
// as often as the term occurs" are the two different questions row 407 is about. Overlapping keys of the
// SAME entry (its src and an alias covering the same characters) are folded to one occurrence, so an entry
// with more aliases does not out-count an entry with fewer on identical text.
func (b *Bank) firingCounts(source string, chapter int) (counts map[int]int, via map[int]string, eaten map[int]bool) {
	ntext := []rune(text.NormalizeSourceKey(source))
	occ := b.ac.matches(ntext)
	occ = b.suppressUnboundedPhonetic(occ, ntext)
	// Which entries the longest-match suppression removed, taken as the DIFFERENCE between the two sets
	// rather than re-derived by a containment rule of our own — the rule has three conditions (strictly
	// longer span, spoiler-valid at this chapter, trust at least as high) and a second copy of it would
	// drift from the one that actually fires.
	before := b.owners(occ)
	occ, _ = b.suppressContained(occ, chapter)
	after := b.owners(occ)
	eaten = map[int]bool{}
	for ei := range before {
		if !after[ei] {
			eaten[ei] = true
		}
	}

	// Group the surviving spans by the entry that owns the key, then fold spans that overlap within one
	// entry. Sorted by (start, -length) so the longest span at a position wins and the shorter nested one
	// is dropped — the same preference suppressContained applies across entries.
	spans := map[int][]acMatch{}
	for _, m := range occ {
		k := b.ac.keys[m.keyIdx]
		for _, ei := range b.keyOwners[k] {
			spans[ei] = append(spans[ei], m)
		}
	}
	counts, via = make(map[int]int, len(spans)), make(map[int]string, len(spans))
	for ei, ms := range spans {
		sort.Slice(ms, func(i, j int) bool {
			if ms[i].start != ms[j].start {
				return ms[i].start < ms[j].start
			}
			return ms[i].end > ms[j].end
		})
		last := -1
		for _, m := range ms {
			if m.start < last {
				continue // nested in or overlapping the span already counted for this entry
			}
			last = m.end
			counts[ei]++
			if k := b.ac.keys[m.keyIdx]; len([]rune(k)) > len([]rune(via[ei])) {
				via[ei] = k
			}
		}
	}
	return counts, via, eaten
}

// owners is the set of entries any of these key occurrences belongs to.
func (b *Bank) owners(ms []acMatch) map[int]bool {
	out := map[int]bool{}
	for _, m := range ms {
		for _, ei := range b.keyOwners[b.ac.keys[m.keyIdx]] {
			out[ei] = true
		}
	}
	return out
}

// countShipped counts non-overlapping occurrences of the entry's accepted renderings in the tokenized
// shipped text, and returns TWO counts rather than one, because the difference between them is a number
// the operator has to see.
//
// STRICT is the post-check's own equality: the SAME union of accepted forms (base dst plus stored decl
// forms, D24.4), the base stemmed and a stored decl form matched literally, exactly as dstFormPresent does.
//
// ⚠ «EXACTLY» HAS TWO MEASURED EXCEPTIONS, and they are named because an unqualified claim here would
// mis-attribute the gap the report prints beside this column. Both live in WORD space versus RUNE space:
// dstFormPresent matches a stored decl form with containsWholeWord over runes, this counts over tokens.
//   - Punctuation INSIDE a form is invisible here: a decl form «гу-мастера» is found in «гу мастера», where
//     the post-check would not find it. Measured at 23 disagreements in 30 000 random triples, all in this
//     direction, so the strict column can be marginally LOOSER than the post-check on a book whose decl
//     forms carry punctuation the text does not.
//   - A form with NO letters at all (digits only) is dropped by acceptedFormWords and would be found by
//     containsWholeWord — the one direction in which strict is tighter.
//
// Neither is reachable on a book with no decl forms, which is every book measured so far (run A: 0 of 69).
//
// ANCHORED relaxes that in the two ways the ladder below names, and they were measured SEPARATELY because
// together they credit the wrong one: a stored decl form is stemmed rather than matched literally, and a
// word's stem may run one rune past the stem it is compared with (lang.NearStems). The first closed all six
// residual false flags of the labelled corpus (the seed lists singular forms, the text uses a plural) and
// carries no hazard. The second closed ELEVEN of the eighteen misses recorded on the paid run of 11.09, and
// it DOES carry one — so it is gated exactly as the note in data/target-ru.txt requires: the window must
// contain a word that matched outright, which a single-word rendering can never have, so the «Синь»/«синий»
// collapse that decl_suffix refuses the soft sign over cannot come back in through here.
//
// ⚠ The two are reported side by side and neither is called the truth. Strict under-counts by the
// stemmer's limit; anchored can over-count where two different words of one phrase merely start alike.
// Publishing one number would have hidden whichever error it carried.
//
// Counting runs in word space rather than rune space because that is where the decl branch already lives.
// On an inert stemmer (a target with no decl_suffix registry) SameStem degrades to exact equality and
// NearStem to the same, so both counts collapse to exact matching and the measure stays meaningful with
// no per-language code.
//
// Longest form first at every position, and each walk skips past a match, so a rendering whose decl form
// contains its base ("гу-мастер третьего оборота" over "гу-мастер") is counted ONCE, not twice.
func (b *Bank) countShipped(e *entry, out targetText) (strict, primary, relaxed int) {
	forms := b.acceptedFormWords(e)
	if len(forms) == 0 {
		return 0, 0, 0
	}
	// A row whose first word's stem is nowhere in this span cannot match at any position. Checking that
	// once is what keeps the scan linear in the span rather than in rows×span.
	if !anyFormReachable(forms, out, b.stemmer) {
		return 0, 0, 0
	}
	return countForms(forms, out, b.stemmer, matchStrict),
		countForms(forms, out, b.stemmer, matchStemAll),
		countForms(forms, out, b.stemmer, matchRelaxed)
}

// anyFormReachable reports whether any accepted form's first word could match somewhere in the span. It is
// deliberately keyed on the STEM and deliberately loose: the relaxed rule accepts a stem one rune off, so a
// stem miss is not proof of absence and the check also passes a form whose first-word stem is a one-rune
// neighbour of one present. Cheap and never the thing that decides a count.
func anyFormReachable(forms []acceptedForm, out targetText, stemmer lang.TargetStemmer) bool {
	for _, f := range forms {
		if out.present[f.stems[0]] {
			return true
		}
		for st := range out.present {
			if stemmer.NearStems(st, f.stems[0]) {
				return true
			}
		}
	}
	return false
}

// The three matching rules the counters can run under. They are an ordered ladder — each admits everything
// the one before it does — and they are SEPARATE rather than one boolean because the two relaxations have
// different causes and different risks, and a measurement that bundled them would credit the wrong one.
const (
	// matchStrict is exactly what dstFormPresent accepts: the base dst stem-aware, a stored decl form
	// literally.
	matchStrict = iota
	// matchStemAll additionally stems the STORED decl forms. It costs nothing in safety — a decl form is
	// an author-supplied rendering of this very term — and on the labelled corpus it is what removes the
	// singular-only seed gap («гу-червь» seeded, «гу-червей» shipped).
	matchStemAll
	// matchRelaxed additionally accepts a one-rune-off near stem, anchored by another word of the same
	// rendering that matched outright. This is the one with a hazard (data/target-ru.txt's «Синь»/«синий»),
	// and the anchor is what contains it.
	matchRelaxed
)

// acceptedForm is one rendering the entry may legitimately appear as, tokenized, plus the ONE bit that
// decides how strictly it may be matched. dstFormPresent treats the two sources differently and the
// difference is load-bearing: the base dst is matched stem-aware (declinedFormPresent), a STORED decl form
// only literally. Carrying the distinction is what lets the strict column here mean exactly "what the
// post-check would have said" — without it the column would quietly be a third rule, and the gap printed
// beside it would be attributed to the wrong cause.
type acceptedForm struct {
	words []string
	stems []string // taken once, with the bank's stemmer — the same values SameStem would compare
	base  bool     // the entry's own dst, the only form the post-check stems
}

// dstFormPresentAnchored is the post-check's presence question asked under the anchored tolerance — the
// same predicate dstFormPresent answers, with the one-rune stem slack countShipped uses. It exists so the
// labelled-corpus harness can measure what the tolerance does to PRECISION on the same rows the ratified
// baseline was measured on, rather than on an argument. Nothing on the paid path calls it.
func dstFormPresentAnchored(b *Bank, e *entry, shipped string, rule int) bool {
	return countForms(b.acceptedFormWords(e), b.tokenizeTarget(shipped), b.stemmer, rule) > 0
}

// countForms is one walk over the output. relax picks the anchored rule; see countShipped.
func countForms(forms []acceptedForm, out targetText, stemmer lang.TargetStemmer, rule int) int {
	n := 0
	for i := 0; i < len(out.words); {
		matched := 0
		for _, f := range forms {
			if len(f.words) == 0 || i+len(f.words) > len(out.words) {
				continue
			}
			if windowMatches(out, i, f, stemmer, rule) {
				matched = len(f.words)
				break // forms are longest-first, so the first hit is the longest at this position
			}
		}
		if matched == 0 {
			i++
			continue
		}
		n++
		i += matched
	}
	return n
}

// windowMatches reports whether a window of output words renders the form. Under the strict rule every word
// must share a stem. Under the relaxed rule a word may instead be a one-rune-off near stem — but the window
// must still contain at least one word that matched OUTRIGHT, and that requirement is the whole safety of
// the relaxation.
//
// ⛔ THE ANCHOR IS WHAT KEEPS «Синь» OUT OF «синий», and it is the ONLY thing that does. A one-word
// rendering has no other word to anchor it, so `anchor` never becomes true and the relaxation can never
// carry it alone — which is exactly the collapse data/target-ru.txt refuses the bare soft sign over. That
// half is verified exhaustively rather than argued: over the shipped book's 2097 distinct words, every
// one-word form against every word, the relaxation admitted nothing the un-relaxed rule did not, against
// 850 opportunities where the relation held.
//
// ⚠ AND IT GUARDS ONLY THAT HALF. In a MULTI-word rendering the anchors are the other words, so one word
// may still be credited to a different word that merely starts alike: «глава рода Гуюэ» is satisfied by
// «глаза рода Гуюэ», anchored by «рода» and «Гуюэ». On the shipped book of run A, NearStems equates 425
// real word pairs SameStem does not — «глава»~«глаз», «воле»~«волна», «вред»~«время» — so this is a
// measured exposure, not a hypothetical one. It does not fire on run A (the text says «глава рода Гуюэ»
// three times and «глаза рода Гуюэ» never), and it is the reason the relaxed column is printed BESIDE the
// strict one and neither is called the truth.
func windowMatches(out targetText, at int, form acceptedForm, stemmer lang.TargetStemmer, rule int) bool {
	// A rendering may not reach back across a sentence boundary: its words standing on either side of a
	// full stop are two sentences, not one rendering.
	for k := 1; k < len(form.words); k++ {
		if out.opensSentence[at+k] {
			return false
		}
	}
	anchor := false
	for k, fw := range form.words {
		switch {
		case out.words[at+k] == fw:
			anchor = true
		// A stored decl form is matched literally under the strict rule, because that is all
		// dstFormPresent does with it; only the base dst gets the stemmer there.
		case !form.base && rule == matchStrict:
			return false
		case out.stems[at+k] == form.stems[k]:
			anchor = true // the stems are pre-taken with the same stemmer SameStem compares by
		case rule == matchRelaxed && stemmer.NearStems(out.stems[at+k], form.stems[k]):
			// accepted only if some other word anchors this window — checked after the loop, because the
			// anchor may be a word we have not reached yet.
		default:
			return false
		}
	}
	return anchor
}

// acceptedFormWords is the entry's accepted rendering set, tokenized and ordered longest-first. Deduped by
// the tokenization rather than by the raw string, because two raw forms differing only in punctuation or
// case ("Гу-мастер", "гу мастер") are ONE form once tokenized, and counting them twice would let the same
// output words satisfy two forms at one position.
func (b *Bank) acceptedFormWords(e *entry) []acceptedForm {
	seen := map[string]bool{}
	var out []acceptedForm
	add := func(s string, base bool) {
		w := lang.TokenizeWords(text.NormalizeTargetForm(s))
		if len(w) == 0 {
			return
		}
		k := strings.Join(w, "\x1f")
		if seen[k] {
			return
		}
		seen[k] = true
		st := make([]string, len(w))
		for i, word := range w {
			st[i] = b.stemmer.Stem(word)
		}
		out = append(out, acceptedForm{words: w, stems: st, base: base})
	}
	add(e.dst, true)
	for _, f := range e.declForms {
		add(f, false)
	}
	sort.SliceStable(out, func(i, j int) bool { return len(out[i].words) > len(out[j].words) })
	return out
}