textmachine/backend/internal/pipeline/seeding.go

package pipeline

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/membank"
	"textmachine/backend/internal/store"
)

// seeding.go: the job's deterministic $0 preludes — REPLACE-seed the glossary from the
// manual file + ruby readings (D16.4) and persist the ruby aggregates. Run BEFORE
// snapshotID: the frozen approved rows enter memoryVersion (F1).

// seedGlossary REPLACES the book's glossary from its deterministic inputs — the manual
// seed file (curated approved/draft terms) and the captured ruby readings (classified
// into auto candidates) — then MATERIALIZES the frozen bank for this job (r.memory). Run
// once before snapshotID: the frozen APPROVED rows are hashed into memoryVersion (F1), so
// editing the seed is a loud --resnapshot. Idempotent (full replace), $0, no LLM. B2
// approved dst-collisions are logged (not fatal — some collisions are legitimate).
//
// The gather and the materialization live in bankmaterialize.go, because a read-only surface has to reach the
// SAME fold without the write in the middle (row 231): what it cannot do is write, and the write is the
// only step it has to skip. This function is therefore the write path's spelling of one shared fold —
// gather, persist, read back, materialize — and the insert order it hands ReplaceBank is untouched.
func (r *Runner) seedGlossary(ctx context.Context) error {
	in, err := r.gatherBankInputs()
	// The remarks go out BEFORE the error is acted on, and the order is load-bearing rather than tidy.
	// Before the gather was extracted these warnings were emitted inline as they were made, so a book that
	// died on a later collision check still told the operator about the ruby alias it had skipped or the
	// auto-bank rows it had dropped. Logging them after the error return would lose exactly the diagnostics
	// of the run that most needs them — the one that failed. gatherBankInputs returns its partial inputs
	// with the error for this reason.
	for _, rem := range in.remarks {
		r.Log.WarnContext(ctx, rem.msg, rem.args...)
	}
	if err != nil {
		return err
	}
	if err := r.Store.ReplaceBank(r.Book.BookID, in.entries, in.voices, in.pairs); err != nil {
		return fmt.Errorf("pipeline: replace bank for %s (%d terms, %d voices, %d pairs): %w", r.Book.BookID, len(in.entries), len(in.voices), len(in.pairs), err)
	}
	rows, err := r.Store.GlossaryForBook(r.Book.BookID)
	if err != nil {
		return fmt.Errorf("pipeline: read glossary for %s: %w", r.Book.BookID, err)
	}
	storedVoices, err := r.Store.VoiceProfilesForBook(r.Book.BookID)
	if err != nil {
		return fmt.Errorf("pipeline: read voice profiles for %s: %w", r.Book.BookID, err)
	}
	storedPairs, err := r.Store.AddressPairsForBook(r.Book.BookID)
	if err != nil {
		return fmt.Errorf("pipeline: read address pairs for %s: %w", r.Book.BookID, err)
	}
	// A chapter range no profile covers is legitimate but is far more often a typo, and it is invisible in
	// a seed file — so it is logged, never fatal (the optional lint of D39.55).
	if gaps := membank.VoiceWindowGaps(storedVoices); len(gaps) > 0 {
		r.Log.WarnContext(ctx, "voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)",
			"book", r.Book.BookID, "gaps", strings.Join(gaps, "; "))
	}
	r.materializeBanks(rows, storedVoices, storedPairs)
	if cols := membank.InjectivityCollisions(rows); len(cols) > 0 {
		r.Log.WarnContext(ctx, "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)",
			"collisions", strings.Join(cols, "; "))
	}
	// S14: the diagnostic the approved-only checks structurally cannot make — an approved term and an
	// unsigned one contradicting each other on the SAME firing surface. In the auto mode both are injected
	// (canon + working version), so the model is handed two answers to one question; the trust gate keeps
	// the approved one authoritative, but the collision is exactly the silent-degradation source the auto
	// mode needs visible.
	if cols := membank.UnverifiedKeyConflicts(rows); len(cols) > 0 {
		r.Log.WarnContext(ctx, "bank: an UNSIGNED row contradicts an approved term on the same firing surface (the approved rendering keeps precedence; reconcile the seed or decline the proposal)",
			"book", r.Book.BookID, "conflicts", strings.Join(cols, "; "))
	}
	r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID,
		"entries", len(rows), "voices", len(storedVoices), "address_pairs", len(storedPairs),
		"memory_version", r.memory.Version()[:12])
	return nil
}

// persistRuby aggregates the ingested ruby occurrences into one row per
// (base, reading) — first_chapter = MIN, occurrences = full-book count — and
// REPLACES the book's whole ruby set (store.ReplaceRubyReadings). Idempotent: the
// aggregation is recomputed identically on every ingest, so a resume re-writes the
// same rows; a source edit converges every column — a pair removed by the edit
// disappears instead of lingering as a phantom (external-review #6). Called
// unconditionally (even for zero readings — a txt book or a source that dropped its
// furigana) so the full-replace clears any stale rows. The write order is sorted for
// deterministic, test-stable behavior. Memory v2 (step 4) consumes ruby_readings into
// a glossary name-lock (D9); nothing here injects into a prompt (§7d).
func (r *Runner) persistRuby(readings []chunk.RubyReading) error {
	type agg struct {
		first int
		count int
	}
	seen := map[[2]string]*agg{}
	order := make([][2]string, 0, len(readings))
	for _, rr := range readings {
		key := [2]string{rr.Base, rr.Reading}
		a, ok := seen[key]
		if !ok {
			a = &agg{first: rr.Chapter}
			seen[key] = a
			order = append(order, key)
		}
		if rr.Chapter < a.first {
			a.first = rr.Chapter
		}
		a.count++
	}
	sort.Slice(order, func(i, j int) bool {
		if order[i][0] != order[j][0] {
			return order[i][0] < order[j][0]
		}
		return order[i][1] < order[j][1]
	})
	rows := make([]store.RubyReading, 0, len(order))
	for _, key := range order {
		a := seen[key]
		rows = append(rows, store.RubyReading{
			BookID: r.Book.BookID, Base: key[0], Reading: key[1],
			FirstChapter: a.first, Occurrences: a.count,
		})
	}
	return r.Store.ReplaceRubyReadings(r.Book.BookID, rows)
}