textmachine/backend/internal/pipeline/seeding.go

package pipeline

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/membank"
	"textmachine/backend/internal/store"
)

// seeding.go: the job's deterministic $0 preludes — REPLACE-seed the glossary from the
// manual file + ruby readings (D16.4) and persist the ruby aggregates. Run BEFORE
// snapshotID: the frozen approved rows enter memoryVersion (F1).

// seedGlossary REPLACES the book's glossary from its deterministic inputs — the manual
// seed file (curated approved/draft terms) and the captured ruby readings (classified
// into auto candidates) — then MATERIALIZES the frozen bank for this job (r.memory). Run
// once before snapshotID: the frozen APPROVED rows are hashed into memoryVersion (F1), so
// editing the seed is a loud --resnapshot. Idempotent (full replace), $0, no LLM. B2
// approved dst-collisions are logged (not fatal — some collisions are legitimate).
func (r *Runner) seedGlossary(ctx context.Context) error {
	var entries []store.GlossaryEntry
	var voices []store.VoiceProfile
	var pairs []store.AddressPair
	if r.Book.GlossarySeed != "" {
		bank, err := membank.LoadBankSeed(r.Book.GlossarySeed)
		if err != nil {
			return err
		}
		entries = append(entries, bank.Terms...)
		voices, pairs = bank.Voices, bank.Pairs
	}
	manualSrcs := map[string]bool{}
	for _, e := range entries {
		manualSrcs[e.Src] = true
	}
	ruby, err := r.Store.RubyReadingsForBook(r.Book.BookID)
	if err != nil {
		return fmt.Errorf("pipeline: read ruby readings for %s: %w", r.Book.BookID, err)
	}
	// D16.4: attach a manual term's kana ruby-reading as an alias (kana spelling matchable)
	// BEFORE appending the auto-candidates, so it only touches the curated manual entries. A
	// reading that would collide with a different seeded term (homophone) is skipped+logged,
	// not attached (which would fail the book loud on an alias the operator cannot edit out).
	if skipped := membank.AttachRubyAliasesToManual(entries, ruby); len(skipped) > 0 {
		r.Log.WarnContext(ctx, "ruby kana-alias skipped as a homophone collision (kana form left unmatchable; disambiguate in the seed if needed)",
			"skipped", strings.Join(skipped, "; "))
	}
	// Mined-write path (R1, plan §1(в)/F2): the owner-curated mined-delta file joins the seed as
	// Source:"mined" (loadMinedDelta re-stamps the seed loader's Source), so its terms fold into the
	// ENRICHED bank version but NOT the base — a re-run that adds signed mined terms moves ONLY snapshot_W2.
	// Loaded AFTER the seed/ruby so manualSrcs already reflects the curated seed. A `mined` term that
	// duplicates a seed src is caught by membank.ApprovedSharedKeyCollisions below like any other collision.
	minedDelta, err := r.loadMinedDelta()
	if err != nil {
		return err
	}
	// D39.20 deviation-#1 fix: a mined-delta term whose UNIQUE key (src, sense, since_ch, until_ch —
	// store/migrate.go glossary UNIQUE) already exists in the SIGNED seed makes the flat INSERT in
	// ReplaceGlossary crash on that constraint and abort the whole paid run. membank.ApprovedSharedKeyCollisions
	// below does NOT catch it (it skips a SAME-dst duplicate, and keys on the firing surface, not the
	// UNIQUE tuple). Fail LOUD here with the duplicate list + a fix hint (edit the seed OR drop it from
	// the delta) — NEVER a silent merge over the signed seed. Checked BEFORE the append so the delta rows
	// are still separable. Deterministic (seed order).
	if dups := membank.MinedDeltaSeedCollisions(entries, minedDelta); len(dups) > 0 {
		return fmt.Errorf("pipeline: mined-delta %s duplicates term(s) already in the signed seed (would crash ReplaceGlossary on the glossary UNIQUE(book_id,src,sense,since_ch,until_ch)):\n  - %s\n  fix: correct the term in the seed, or remove it from the mined-delta file — never both (no silent merge over the signed seed)",
			r.Book.MinedDelta, strings.Join(dups, "\n  - "))
	}
	entries = append(entries, minedDelta...)
	// AUTO-BANK (pack-20 / D39.42 п.3): the engine's own unsigned rows — what the terminologist consolidated
	// on the last run's bank-mining boundary. They load exactly like the owner-curated delta (Source:"mined",
	// so base-excluded and only the edit-wave snapshot moves) but they are NOT signed: their status is
	// auto/draft, which is what routes them to the editor's separately-headed unverified section instead of
	// the canon list.
	//
	// Loading them HERE rather than injecting them mid-run is load-bearing for money. seedGlossary runs before
	// checkRebillConsent, so the re-payment projection sees the same bank the edit wave will use; if the rows
	// only appeared after the projection, the next run would compare stored edit rows against a bank that has
	// not been rebuilt yet and project a re-payment of the whole edit wave that is not real.
	autoBank, dropped, err := r.loadAutoBank(entries)
	if err != nil {
		return err
	}
	if len(dropped) > 0 {
		// A collision with a SIGNED row cannot abort a paid run over an engine-written proposal — the signed
		// row simply wins and the proposal is dropped, loudly.
		r.Log.WarnContext(ctx, "auto-bank rows dropped: their key is already held by a signed term (the signed term wins)",
			"book", r.Book.BookID, "dropped", strings.Join(dropped, "; "))
	}
	entries = append(entries, autoBank...)
	for i := range entries {
		entries[i].BookID = r.Book.BookID
	}
	// Task-6 re-audit: fail loud on a firing key shared by two DIFFERENT approved terms with
	// different dst + overlapping windows (the alias generalization of the D16.1 polysemy
	// livelock) — checked over the FULL entry set (incl. ruby-attached aliases) before persisting.
	if cols := membank.ApprovedSharedKeyCollisions(entries); len(cols) > 0 {
		return fmt.Errorf("pipeline: glossary shared-key collisions (A2 / D16.1 livelock class):\n  - %s", strings.Join(cols, "\n  - "))
	}
	// A voice/address row naming a character the bank does not have is SILENTLY inert — nothing can ever
	// attribute a reply to it — which is the A-class hole this bank exists to close, so it stops the run.
	// Checked over the FULL entry set (seed + ruby + mined + auto), because a character may legitimately be
	// signed in a delta rather than in the base seed.
	if unknown := membank.UnknownVoiceCharacters(entries, voices, pairs); len(unknown) > 0 {
		return fmt.Errorf("pipeline: voice/address rows name characters absent from the bank (a profile for a term that does not exist can never fire):\n  - %s", strings.Join(unknown, "\n  - "))
	}
	if err := r.Store.ReplaceBank(r.Book.BookID, entries, voices, pairs); err != nil {
		return fmt.Errorf("pipeline: replace bank for %s (%d terms, %d voices, %d pairs): %w", r.Book.BookID, len(entries), len(voices), len(pairs), err)
	}
	rows, err := r.Store.GlossaryForBook(r.Book.BookID)
	if err != nil {
		return fmt.Errorf("pipeline: read glossary for %s: %w", r.Book.BookID, err)
	}
	storedVoices, err := r.Store.VoiceProfilesForBook(r.Book.BookID)
	if err != nil {
		return fmt.Errorf("pipeline: read voice profiles for %s: %w", r.Book.BookID, err)
	}
	storedPairs, err := r.Store.AddressPairsForBook(r.Book.BookID)
	if err != nil {
		return fmt.Errorf("pipeline: read address pairs for %s: %w", r.Book.BookID, err)
	}
	// A chapter range no profile covers is legitimate but is far more often a typo, and it is invisible in
	// a seed file — so it is logged, never fatal (the optional lint of D39.55).
	if gaps := membank.VoiceWindowGaps(storedVoices); len(gaps) > 0 {
		r.Log.WarnContext(ctx, "voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)",
			"book", r.Book.BookID, "gaps", strings.Join(gaps, "; "))
	}
	// InjectVoice is FALSE and has no config knob: pack-19 builds the schema and the flagger, and D21 п.2
	// holds the injection conditional until the polygon experiment. It is the fold's condition, so while
	// it is false a book with voice rows hashes exactly as it did without them and nobody re-pays for
	// authoring a profile. Wiring the injection means setting it and accepting a full --resnapshot.
	// The §3 decl stemmer is target data, constant per book; baseIn copies bankIn below, so both banks share
	// it. A target with no decl_suffix registry yields an inert stemmer → exact-match post-check as before.
	bankIn := membank.BankInput{Rows: rows, Voices: storedVoices, Pairs: storedPairs,
		TargetStemmer: lang.NewTargetStemmer(lang.TargetChecksFor(r.Book.TargetLang))}
	r.memory = membank.MaterializeBank(bankIn, r.Pipeline.Gates.Glossary.PostcheckGate)
	// The DRAFT wave selects over a BASE-scoped bank (Source:mined excluded) so its injection is
	// byte-identical across a bank-mining enrichment — matching the draft-wave snapshot (baseMemoryVersion),
	// which keeps «one re-payment» honest at the WIRE level, not only the version-hash level. Only the
	// editor sees mined terms (the enriched `memory`). When there are no mined rows (every $0 test / the
	// golden) the base bank IS the enriched one — share the object, no double materialization, no drift.
	baseRows := rows[:0:0]
	hasMined := false
	for _, row := range rows {
		if row.Source == "mined" {
			hasMined = true
			continue
		}
		baseRows = append(baseRows, row)
	}
	if hasMined {
		baseIn := bankIn
		baseIn.Rows = baseRows
		r.baseMemory = membank.MaterializeBank(baseIn, r.Pipeline.Gates.Glossary.PostcheckGate)
	} else {
		r.baseMemory = r.memory
	}
	if cols := membank.InjectivityCollisions(rows); len(cols) > 0 {
		r.Log.WarnContext(ctx, "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)",
			"collisions", strings.Join(cols, "; "))
	}
	// S14: the diagnostic the approved-only checks structurally cannot make — an approved term and an
	// unsigned one contradicting each other on the SAME firing surface. In the auto mode both are injected
	// (canon + working version), so the model is handed two answers to one question; the trust gate keeps
	// the approved one authoritative, but the collision is exactly the silent-degradation source the auto
	// mode needs visible.
	if cols := membank.UnverifiedKeyConflicts(rows); len(cols) > 0 {
		r.Log.WarnContext(ctx, "bank: an UNSIGNED row contradicts an approved term on the same firing surface (the approved rendering keeps precedence; reconcile the seed or decline the proposal)",
			"book", r.Book.BookID, "conflicts", strings.Join(cols, "; "))
	}
	r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID,
		"entries", len(rows), "voices", len(storedVoices), "address_pairs", len(storedPairs),
		"memory_version", r.memory.Version()[:12])
	return nil
}

// persistRuby aggregates the ingested ruby occurrences into one row per
// (base, reading) — first_chapter = MIN, occurrences = full-book count — and
// REPLACES the book's whole ruby set (store.ReplaceRubyReadings). Idempotent: the
// aggregation is recomputed identically on every ingest, so a resume re-writes the
// same rows; a source edit converges every column — a pair removed by the edit
// disappears instead of lingering as a phantom (external-review #6). Called
// unconditionally (even for zero readings — a txt book or a source that dropped its
// furigana) so the full-replace clears any stale rows. The write order is sorted for
// deterministic, test-stable behavior. Memory v2 (step 4) consumes ruby_readings into
// a glossary name-lock (D9); nothing here injects into a prompt (§7d).
func (r *Runner) persistRuby(readings []chunk.RubyReading) error {
	type agg struct {
		first int
		count int
	}
	seen := map[[2]string]*agg{}
	order := make([][2]string, 0, len(readings))
	for _, rr := range readings {
		key := [2]string{rr.Base, rr.Reading}
		a, ok := seen[key]
		if !ok {
			a = &agg{first: rr.Chapter}
			seen[key] = a
			order = append(order, key)
		}
		if rr.Chapter < a.first {
			a.first = rr.Chapter
		}
		a.count++
	}
	sort.Slice(order, func(i, j int) bool {
		if order[i][0] != order[j][0] {
			return order[i][0] < order[j][0]
		}
		return order[i][1] < order[j][1]
	})
	rows := make([]store.RubyReading, 0, len(order))
	for _, key := range order {
		a := seen[key]
		rows = append(rows, store.RubyReading{
			BookID: r.Book.BookID, Base: key[0], Reading: key[1],
			FirstChapter: a.first, Occurrences: a.count,
		})
	}
	return r.Store.ReplaceRubyReadings(r.Book.BookID, rows)
}