textmachine/backend/internal/pipeline/bankmaterialize.go

package pipeline

import (
	"fmt"
	"sort"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/membank"
	"textmachine/backend/internal/store"
)

// bankmaterialize.go: the book's bank fold, split so that the SAME fold can be performed with or without the
// store round-trip that sits in the middle of it (row 231 / D39.165 §3, errata 28.08-к).
//
// THE PROBLEM IT EXISTS FOR. `seedGlossary` folds the bank from FILES (the curated seed, the ruby
// readings, the owner's mined-delta, the engine's auto-bank), writes the result with ReplaceBank, reads
// it straight back with GlossaryForBook and materializes THAT. Every read-only surface, however, had no
// way to reach the same answer: it cannot write, so `projectStoredMemory` materialized the glossary the
// LAST run stored — i.e. the previous fold. Right after a `bank-apply` the decision files hold rows the
// stored glossary does not, so a read-only projection answered "nothing moved" while the very next
// `translate` would re-seed, see the rows and bill for them. The projection and the run disagreed about
// money, which is the one thing they may never do.
//
// WHY THE ROUND-TRIP IS NOT A FORMALITY, and what makes skipping it PROVABLE rather than plausible.
// The store contributes exactly one thing to the fold: an ORDER. `GlossaryForBook` returns
// `ORDER BY src, sense, since_ch, until_ch, status, dst`, and that order is load-bearing twice over —
// `MaterializeBank` iterates the rows as given (it never re-sorts), so the order decides both the bank's
// content hash (ComputeVersionScopedIn streams a SHA-256 over the rows in iteration order) and, through
// Select's stable budget sort, the literal LINE ORDER of the rendered glossary block, whose bytes are
// hashed into chunk_status.content_hash. Reproduce the set but not the order and the free estimate names
// one number while the paid run charges another.
//
// The reproduction is exact, and the reason is a constraint rather than an observation: `glossary`
// carries UNIQUE (book_id, src, sense, since_ch, until_ch) (store/migrate.go), so the ORDER BY's first
// four columns CANNOT tie within one book — the order is total and fully determined by values, with no
// dependence on rowid or insertion order (`glossary.id` is deliberately excluded from the fold, see
// membank.ComputeVersion). The schema declares no COLLATE anywhere, so SQLite compares TEXT with its
// default BINARY collation — byte-wise, which is exactly what Go's `<` on a string does. Aliases come
// back `ORDER BY term_id, alias`, i.e. sorted by alias within an entry. Voice profiles and address pairs
// each have their own ORDER BY matching their own UNIQUE key, so the same argument covers them.
//
// storeOrder below therefore reproduces the round-trip's ONE contribution, and bankmaterialize_identity_test.go
// proves the equality by running both paths over the same inputs rather than by asserting this comment.

// bankInputs is everything the deterministic $0 prelude gathers about a book's bank BEFORE anything is
// written: the glossary rows in the order seedGlossary builds them (which is the order ReplaceBank
// inserts them in), the two pack-19 record types, and the non-fatal remarks the gather produced.
//
// It is deliberately the PRE-store form. Turning it into what a run actually materializes is storeOrder's
// job, and it is one function so a caller cannot accidentally materialize the unsorted build order.
type bankInputs struct {
	entries []store.GlossaryEntry
	voices  []store.VoiceProfile
	pairs   []store.AddressPair
	// remarks are the gather's non-fatal findings, in the order they were made. seedGlossary logs them;
	// a read-only projection drops them, because a $0 read must not narrate a run it is not making.
	remarks []bankRemark
}

// bankRemark is one non-fatal finding of the gather, carried rather than logged so the two callers can
// decide for themselves whether their surface is one that speaks.
type bankRemark struct {
	msg  string
	args []any
}

// gatherBankInputs performs the pure, $0, WRITE-FREE half of the bank fold: it reads the book's
// deterministic inputs and assembles the exact row set seedGlossary hands to ReplaceBank.
//
// Everything fatal here is fatal for the same reason it was fatal inside seedGlossary — a collision that
// would crash ReplaceBank mid-run, or a voice row naming a character the bank does not have, is a broken
// book and not a projection detail. What the two callers differ in is what they DO with that error:
// `translate` fails the run, a read-only projection falls back and says so.
func (r *Runner) gatherBankInputs() (bankInputs, error) {
	var in bankInputs
	if r.Book.GlossarySeed != "" {
		bank, err := membank.LoadBankSeed(r.Book.GlossarySeed)
		if err != nil {
			return in, err
		}
		in.entries = append(in.entries, bank.Terms...)
		in.voices, in.pairs = bank.Voices, bank.Pairs
	}
	ruby, err := r.Store.RubyReadingsForBook(r.Book.BookID)
	if err != nil {
		return in, fmt.Errorf("pipeline: read ruby readings for %s: %w", r.Book.BookID, err)
	}
	// D16.4: attach a manual term's kana ruby-reading as an alias (kana spelling matchable)
	// BEFORE appending the auto-candidates, so it only touches the curated manual entries. A
	// reading that would collide with a different seeded term (homophone) is skipped+logged,
	// not attached (which would fail the book loud on an alias the operator cannot edit out).
	if skipped := membank.AttachRubyAliasesToManual(in.entries, ruby); len(skipped) > 0 {
		in.remark("ruby kana-alias skipped as a homophone collision (kana form left unmatchable; disambiguate in the seed if needed)",
			"skipped", joinSemi(skipped))
	}
	// Mined-write path (R1, plan §1(в)/F2): the owner-curated mined-delta file joins the seed as
	// Source:"mined" (loadMinedDelta re-stamps the seed loader's Source), so its terms fold into the
	// ENRICHED bank version but NOT the base — a re-run that adds signed mined terms moves ONLY snapshot_W2.
	minedDelta, err := r.loadMinedDelta()
	if err != nil {
		return in, err
	}
	// D39.20 deviation-#1 fix: a mined-delta term whose UNIQUE key (src, sense, since_ch, until_ch —
	// store/migrate.go glossary UNIQUE) already exists in the SIGNED seed makes the flat INSERT in
	// ReplaceGlossary crash on that constraint and abort the whole paid run. membank.ApprovedSharedKeyCollisions
	// below does NOT catch it (it skips a SAME-dst duplicate, and keys on the firing surface, not the
	// UNIQUE tuple). Fail LOUD here with the duplicate list + a fix hint (edit the seed OR drop it from
	// the delta) — NEVER a silent merge over the signed seed. Checked BEFORE the append so the delta rows
	// are still separable. Deterministic (seed order).
	if dups := membank.MinedDeltaSeedCollisions(in.entries, minedDelta); len(dups) > 0 {
		return in, fmt.Errorf("pipeline: mined-delta %s duplicates term(s) already in the signed seed (would crash ReplaceGlossary on the glossary UNIQUE(book_id,src,sense,since_ch,until_ch)):\n  - %s\n  fix: correct the term in the seed, or remove it from the mined-delta file — never both (no silent merge over the signed seed)",
			r.Book.MinedDelta, joinLines(dups))
	}
	in.entries = append(in.entries, minedDelta...)
	// AUTO-BANK (pack-20 / D39.42 п.3): the engine's own unsigned rows — what the terminologist consolidated
	// on the last run's bank-mining boundary. They load exactly like the owner-curated delta (Source:"mined",
	// so base-excluded and only the edit-wave snapshot moves) but they are NOT signed: their status is
	// auto/draft, which is what routes them to the editor's separately-headed unverified section instead of
	// the canon list.
	//
	// Loading them HERE rather than injecting them mid-run is load-bearing for money. The gather runs before
	// checkRebillConsent, so the re-payment projection sees the same bank the edit wave will use; if the rows
	// only appeared after the projection, the next run would compare stored edit rows against a bank that has
	// not been rebuilt yet and project a re-payment of the whole edit wave that is not real.
	autoBank, dropped, err := r.loadAutoBank(in.entries)
	if err != nil {
		return in, err
	}
	if len(dropped) > 0 {
		// A collision with a SIGNED row cannot abort a paid run over an engine-written proposal — the signed
		// row simply wins and the proposal is dropped, loudly.
		in.remark("auto-bank rows dropped: their key is already held by a signed term (the signed term wins)",
			"book", r.Book.BookID, "dropped", joinSemi(dropped))
	}
	in.entries = append(in.entries, autoBank...)
	for i := range in.entries {
		in.entries[i].BookID = r.Book.BookID
	}
	// Task-6 re-audit: fail loud on a firing key shared by two DIFFERENT approved terms with
	// different dst + overlapping windows (the alias generalization of the D16.1 polysemy
	// livelock) — checked over the FULL entry set (incl. ruby-attached aliases) before persisting.
	if cols := membank.ApprovedSharedKeyCollisions(in.entries); len(cols) > 0 {
		return in, fmt.Errorf("pipeline: glossary shared-key collisions (A2 / D16.1 livelock class):\n  - %s", joinLines(cols))
	}
	// A voice/address row naming a character the bank does not have is SILENTLY inert — nothing can ever
	// attribute a reply to it — which is the A-class hole this bank exists to close, so it stops the run.
	// Checked over the FULL entry set (seed + ruby + mined + auto), because a character may legitimately be
	// signed in a delta rather than in the base seed.
	if unknown := membank.UnknownVoiceCharacters(in.entries, in.voices, in.pairs); len(unknown) > 0 {
		return in, fmt.Errorf("pipeline: voice/address rows name characters absent from the bank (a profile for a term that does not exist can never fire):\n  - %s", joinLines(unknown))
	}
	return in, nil
}

func (in *bankInputs) remark(msg string, args ...any) {
	in.remarks = append(in.remarks, bankRemark{msg: msg, args: args})
}

// storeOrder reproduces, WITHOUT a store, exactly what a ReplaceBank+read-back round-trip returns: the
// same rows in the store's ORDER BY, each entry's aliases in the store's alias order, and the same for
// voice profiles and address pairs. The inputs are left untouched — a caller may still hand the ORIGINAL
// slice to ReplaceBank and get the insert order it has always had.
//
// It REFUSES the states the store itself would refuse rather than papering over them, because a
// projection of a book whose fold would crash the next run is not a projection of anything: a duplicate
// UNIQUE key would abort ReplaceBank's flat INSERT mid-transaction, and reporting a tidy number for it
// would promise a run that cannot start. The three refusals correspond one-to-one to the three UNIQUE
// constraints the round-trip passes through.
func storeOrder(in bankInputs) (rows []store.GlossaryEntry, voices []store.VoiceProfile, pairs []store.AddressPair, err error) {
	type gkey struct {
		src, sense   string
		since, until int
	}
	rows = append(rows, in.entries...)
	seen := make(map[gkey]bool, len(rows))
	for i := range rows {
		k := gkey{rows[i].Src, rows[i].Sense, rows[i].SinceCh, rows[i].UntilCh}
		if seen[k] {
			return nil, nil, nil, fmt.Errorf("pipeline: the folded bank holds two rows for (src=%q, sense=%q, since_ch=%d, until_ch=%d) — the glossary UNIQUE key; ReplaceBank would abort on it", k.src, k.sense, k.since, k.until)
		}
		seen[k] = true
		// The store returns a term's aliases ORDER BY alias (within its term_id), and refuses a repeated
		// one on UNIQUE (book_id, term_id, alias). The aliases are copied before sorting so an input slice
		// shared with the write path keeps the order ReplaceBank inserts in.
		if len(rows[i].Aliases) > 1 {
			al := append([]store.GlossaryAlias(nil), rows[i].Aliases...)
			sort.SliceStable(al, func(a, b int) bool { return al[a].Alias < al[b].Alias })
			for j := 1; j < len(al); j++ {
				if al[j].Alias == al[j-1].Alias {
					return nil, nil, nil, fmt.Errorf("pipeline: the folded bank gives term %q the alias %q twice — the glossary_aliases UNIQUE key; ReplaceBank would abort on it", rows[i].Src, al[j].Alias)
				}
			}
			rows[i].Aliases = al
		}
	}
	// GlossaryForBook: ORDER BY src, sense, since_ch, until_ch, status, dst. The first four columns are the
	// table's UNIQUE key, so status/dst can never actually decide anything — they are kept so this reads as
	// the SQL it reproduces and would still hold if that constraint were ever widened.
	sort.SliceStable(rows, func(a, b int) bool { return glossaryLess(rows[a], rows[b]) })

	voices = append(voices, in.voices...)
	vseen := make(map[gkey]bool, len(voices))
	for _, v := range voices {
		k := gkey{v.Src, v.Sense, v.SinceCh, v.UntilCh}
		if vseen[k] {
			return nil, nil, nil, fmt.Errorf("pipeline: the folded bank holds two voice profiles for (src=%q, sense=%q, since_ch=%d, until_ch=%d) — the voice_profiles UNIQUE key; ReplaceBank would abort on it", k.src, k.sense, k.since, k.until)
		}
		vseen[k] = true
	}
	// VoiceProfilesForBook: ORDER BY src, sense, since_ch, until_ch.
	sort.SliceStable(voices, func(a, b int) bool {
		x, y := voices[a], voices[b]
		return lessKey4(x.Src, y.Src, x.Sense, y.Sense, x.SinceCh, y.SinceCh, x.UntilCh, y.UntilCh)
	})

	type pkey struct {
		spSrc, spSense, adSrc, adSense string
		since, until                   int
	}
	pairs = append(pairs, in.pairs...)
	pseen := make(map[pkey]bool, len(pairs))
	for _, p := range pairs {
		k := pkey{p.SpeakerSrc, p.SpeakerSense, p.AddresseeSrc, p.AddresseeSense, p.SinceCh, p.UntilCh}
		if pseen[k] {
			return nil, nil, nil, fmt.Errorf("pipeline: the folded bank holds two address pairs for speaker %q → addressee %q over the same window — the address_pairs UNIQUE key; ReplaceBank would abort on it", k.spSrc, k.adSrc)
		}
		pseen[k] = true
	}
	// AddressPairsForBook: ORDER BY speaker_src, speaker_sense, addressee_src, addressee_sense, since_ch, until_ch.
	sort.SliceStable(pairs, func(a, b int) bool {
		x, y := pairs[a], pairs[b]
		if x.SpeakerSrc != y.SpeakerSrc {
			return x.SpeakerSrc < y.SpeakerSrc
		}
		if x.SpeakerSense != y.SpeakerSense {
			return x.SpeakerSense < y.SpeakerSense
		}
		if x.AddresseeSrc != y.AddresseeSrc {
			return x.AddresseeSrc < y.AddresseeSrc
		}
		if x.AddresseeSense != y.AddresseeSense {
			return x.AddresseeSense < y.AddresseeSense
		}
		if x.SinceCh != y.SinceCh {
			return x.SinceCh < y.SinceCh
		}
		return x.UntilCh < y.UntilCh
	})
	return rows, voices, pairs, nil
}

// glossaryLess is GlossaryForBook's ORDER BY as a Go comparison. Both sides are byte-wise on TEXT (the
// schema declares no COLLATE, so SQLite uses BINARY) and numeric on the two INTEGER columns.
func glossaryLess(x, y store.GlossaryEntry) bool {
	if x.Src != y.Src {
		return x.Src < y.Src
	}
	if x.Sense != y.Sense {
		return x.Sense < y.Sense
	}
	if x.SinceCh != y.SinceCh {
		return x.SinceCh < y.SinceCh
	}
	if x.UntilCh != y.UntilCh {
		return x.UntilCh < y.UntilCh
	}
	if x.Status != y.Status {
		return x.Status < y.Status
	}
	return x.Dst < y.Dst
}

func lessKey4(s1, s2, e1, e2 string, a1, a2, b1, b2 int) bool {
	if s1 != s2 {
		return s1 < s2
	}
	if e1 != e2 {
		return e1 < e2
	}
	if a1 != a2 {
		return a1 < a2
	}
	return b1 < b2
}

// materializeBanks builds the two banks a run works against — the ENRICHED one the editor sees and the
// BASE one (Source:"mined" excluded) the draft wave selects over — from bank content already in the
// store's order. It is the shared tail of both folds, so the write path and the read-only projection can
// never materialize the same rows two different ways.
func (r *Runner) materializeBanks(rows []store.GlossaryEntry, voices []store.VoiceProfile, pairs []store.AddressPair) {
	// InjectVoice is FALSE and has no config knob: pack-19 builds the schema and the flagger, and D21 п.2
	// holds the injection conditional until the polygon experiment. It is the fold's condition, so while
	// it is false a book with voice rows hashes exactly as it did without them and nobody re-pays for
	// authoring a profile. Wiring the injection means setting it and accepting a full --resnapshot.
	// The §3 decl stemmer is target data, constant per book; baseIn copies bankIn below, so both banks share
	// it. A target with no decl_suffix registry yields an inert stemmer → exact-match post-check as before.
	// The rows are retained because a caller that has just folded the bank must be able to say things
	// ABOUT it — how many of its terms are unsigned, for one — without going back to the store and
	// getting a DIFFERENT bank than the one it just materialized. A status document that folds the
	// decision files for its money figures and counts unsigned terms off the stored glossary is one
	// document describing two banks.
	r.bankRows = rows
	// The wire-hash memo belongs to the bank that produced it: a new materialization invalidates every
	// hash rendered against the old one (cachedRenderedContentHashes).
	r.contentHashes = nil
	bankIn := membank.BankInput{Rows: rows, Voices: voices, Pairs: pairs,
		TargetStemmer: lang.NewTargetStemmer(lang.TargetChecksFor(r.Book.TargetLang))}
	r.memory = membank.MaterializeBank(bankIn, r.Pipeline.Gates.Glossary.PostcheckGate)
	// The DRAFT wave selects over a BASE-scoped bank (Source:mined excluded) so its injection is
	// byte-identical across a bank-mining enrichment — matching the draft-wave snapshot (baseMemoryVersion),
	// which keeps «one re-payment» honest at the WIRE level, not only the version-hash level. Only the
	// editor sees mined terms (the enriched `memory`). When there are no mined rows (every $0 test / the
	// golden) the base bank IS the enriched one — share the object, no double materialization, no drift.
	baseRows := rows[:0:0]
	hasMined := false
	for _, row := range rows {
		if row.Source == "mined" {
			hasMined = true
			continue
		}
		baseRows = append(baseRows, row)
	}
	if hasMined {
		baseIn := bankIn
		baseIn.Rows = baseRows
		r.baseMemory = membank.MaterializeBank(baseIn, r.Pipeline.Gates.Glossary.PostcheckGate)
	} else {
		r.baseMemory = r.memory
	}
}

// projectFoldedMemory materializes BOTH banks from the book's deterministic inputs WITHOUT writing
// anything — the read-only twin of seedGlossary, and the answer to backlog row 231.
//
// What it fixes is a disagreement about money. `translate` re-seeds from the FILES and does it BEFORE
// checkRebillConsent, so its consent gate sees a `bank-apply` edit the moment the files change; the
// read-only surfaces materialized the STORED glossary instead — last run's fold — and therefore reported
// "nothing moved" for exactly the edit the next run would bill for. Reading the files rather than the
// store keeps every read-only guarantee intact: this writes nothing at all, so a store opened
// query_only(1) is untouched, and `status` remains the purely-reading verb it is ratified as.
//
// It also sets r.baseMemory, which projectStoredMemory never did — see the note on projectStoredMemory.
func (r *Runner) projectFoldedMemory() error {
	in, err := r.gatherBankInputs()
	if err != nil {
		return err
	}
	rows, voices, pairs, err := storeOrder(in)
	if err != nil {
		return err
	}
	r.materializeBanks(rows, voices, pairs)
	return nil
}

// The values of StatusReport.RebillBasis — see its doc comment for what each one promises a reader.
// Exported because they are a WIRE vocabulary: the CLI renders them and a consumer branches on them, and
// a second hand-typed copy of an enum in the renderer is how the two drift apart.
const (
	RebillBasisPending = "pending"
	RebillBasisStored  = "stored"
	RebillBasisNone    = "none"
	RebillBasisFailed  = "failed"
)

// foldMemoryForRead materializes the bank a READ-ONLY surface must judge against, and reports WHICH bank
// it managed to get. It is the one place the fallback lives, so status and export can never end up
// judging drift against two different banks.
//
// The preferred answer is the FOLD of the book's current inputs, because that is the bank the next
// `translate` will build (it re-seeds before its consent gate) — projecting anything else is projecting a
// run nobody is going to make. The fallback exists because the fold can legitimately refuse: a seed with a
// shared-key collision, a mined-delta duplicating a signed term, a voice row naming an absent character.
// Those are broken books, and `translate` fails loudly on them — but a READ command must not, or the book
// that most needs inspecting becomes the one that cannot be inspected (the D20.4 property rebill.go leans
// on: "a book that needs consent stays fully inspectable"). So the read falls back to the glossary the
// last run stored, and says so through the basis rather than passing the older number off as the answer.
//
// The returned error is the fold's refusal, for the caller to LOG in its own idiom; it is never fatal.
func (r *Runner) foldMemoryForRead() (basis string, warn error) {
	foldErr := r.projectFoldedMemory()
	if foldErr == nil {
		return RebillBasisPending, nil
	}
	if storedErr := r.projectStoredMemory(); storedErr != nil {
		return RebillBasisFailed, fmt.Errorf("the bank fold refused (%w) and the stored glossary could not be materialized either: %w", foldErr, storedErr)
	}
	return RebillBasisStored, foldErr
}

func joinSemi(v []string) string  { return joinWith(v, "; ") }
func joinLines(v []string) string { return joinWith(v, "\n  - ") }

func joinWith(v []string, sep string) string {
	out := ""
	for i, s := range v {
		if i > 0 {
			out += sep
		}
		out += s
	}
	return out
}