136 lines
7 KiB
Go
136 lines
7 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"sort"
|
||
"strings"
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/membank"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// seeding.go: the job's deterministic $0 preludes — REPLACE-seed the glossary from the
|
||
// manual file + ruby readings (D16.4) and persist the ruby aggregates. Run BEFORE
|
||
// snapshotID: the frozen approved rows enter memoryVersion (F1).
|
||
|
||
// seedGlossary REPLACES the book's glossary from its deterministic inputs — the manual
|
||
// seed file (curated approved/draft terms) and the captured ruby readings (classified
|
||
// into auto candidates) — then MATERIALIZES the frozen bank for this job (r.memory). Run
|
||
// once before snapshotID: the frozen APPROVED rows are hashed into memoryVersion (F1), so
|
||
// editing the seed is a loud --resnapshot. Idempotent (full replace), $0, no LLM. B2
|
||
// approved dst-collisions are logged (not fatal — some collisions are legitimate).
|
||
//
|
||
// The gather and the materialization live in bankmaterialize.go, because a read-only surface has to reach the
|
||
// SAME fold without the write in the middle (row 231): what it cannot do is write, and the write is the
|
||
// only step it has to skip. This function is therefore the write path's spelling of one shared fold —
|
||
// gather, persist, read back, materialize — and the insert order it hands ReplaceBank is untouched.
|
||
func (r *Runner) seedGlossary(ctx context.Context) error {
|
||
in, err := r.gatherBankInputs()
|
||
// The remarks go out BEFORE the error is acted on, and the order is load-bearing rather than tidy.
|
||
// Before the gather was extracted these warnings were emitted inline as they were made, so a book that
|
||
// died on a later collision check still told the operator about the ruby alias it had skipped or the
|
||
// auto-bank rows it had dropped. Logging them after the error return would lose exactly the diagnostics
|
||
// of the run that most needs them — the one that failed. gatherBankInputs returns its partial inputs
|
||
// with the error for this reason.
|
||
for _, rem := range in.remarks {
|
||
r.Log.WarnContext(ctx, rem.msg, rem.args...)
|
||
}
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err := r.Store.ReplaceBank(r.Book.BookID, in.entries, in.voices, in.pairs); err != nil {
|
||
return fmt.Errorf("pipeline: replace bank for %s (%d terms, %d voices, %d pairs): %w", r.Book.BookID, len(in.entries), len(in.voices), len(in.pairs), err)
|
||
}
|
||
rows, err := r.Store.GlossaryForBook(r.Book.BookID)
|
||
if err != nil {
|
||
return fmt.Errorf("pipeline: read glossary for %s: %w", r.Book.BookID, err)
|
||
}
|
||
storedVoices, err := r.Store.VoiceProfilesForBook(r.Book.BookID)
|
||
if err != nil {
|
||
return fmt.Errorf("pipeline: read voice profiles for %s: %w", r.Book.BookID, err)
|
||
}
|
||
storedPairs, err := r.Store.AddressPairsForBook(r.Book.BookID)
|
||
if err != nil {
|
||
return fmt.Errorf("pipeline: read address pairs for %s: %w", r.Book.BookID, err)
|
||
}
|
||
// A chapter range no profile covers is legitimate but is far more often a typo, and it is invisible in
|
||
// a seed file — so it is logged, never fatal (the optional lint of D39.55).
|
||
if gaps := membank.VoiceWindowGaps(storedVoices); len(gaps) > 0 {
|
||
r.Log.WarnContext(ctx, "voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)",
|
||
"book", r.Book.BookID, "gaps", strings.Join(gaps, "; "))
|
||
}
|
||
// The same lint on the TERM side, and it had no guard at all until now (backlog row 490). The loader
|
||
// refuses an OVERLAP of windows; a HOLE between them — «until chapter 3» followed by «since chapter 5» —
|
||
// is not a schema violation and passes everything, leaving chapter 4 with no law for that term inside a
|
||
// book whose whole point is one word per term. Logged and never fatal, exactly like its voice sibling:
|
||
// a deliberate gap is a real editorial shape, and refusing the load would turn one into a broken book.
|
||
if gaps := membank.TermWindowGaps(rows); len(gaps) > 0 {
|
||
r.Log.WarnContext(ctx, "bank term windows leave chapters uncovered — those chapters have NO law for the term and their drafts will render it however they like (deliberate is fine; a typo is not)",
|
||
"book", r.Book.BookID, "gaps", strings.Join(gaps, "; "))
|
||
}
|
||
r.materializeBanks(rows, storedVoices, storedPairs)
|
||
if cols := membank.InjectivityCollisions(rows); len(cols) > 0 {
|
||
r.Log.WarnContext(ctx, "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)",
|
||
"collisions", strings.Join(cols, "; "))
|
||
}
|
||
// S14: the diagnostic the approved-only checks cannot make — two rows contradicting each other on the
|
||
// SAME firing surface, at least one of them unsigned. Both are injected, and which is signed changes
|
||
// nothing about that (D39.104 п.2: no fork by status). Where a signature exists the trust gate still
|
||
// decides precedence; an unsigned×unsigned pair has none, so it is simply two renderings of one term.
|
||
if cols := membank.UnverifiedKeyConflicts(rows); len(cols) > 0 {
|
||
r.Log.WarnContext(ctx, "bank: two rows render ONE firing surface differently and at least one is unsigned (the deterministic matcher cannot pick between them; reconcile the seed, the delta or the proposal)",
|
||
"book", r.Book.BookID, "conflicts", strings.Join(cols, "; "))
|
||
}
|
||
r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID,
|
||
"entries", len(rows), "voices", len(storedVoices), "address_pairs", len(storedPairs),
|
||
"memory_version", r.memory.Version()[:12])
|
||
return nil
|
||
}
|
||
|
||
// persistRuby aggregates the ingested ruby occurrences into one row per
|
||
// (base, reading) — first_chapter = MIN, occurrences = full-book count — and
|
||
// REPLACES the book's whole ruby set (store.ReplaceRubyReadings). Idempotent: the
|
||
// aggregation is recomputed identically on every ingest, so a resume re-writes the
|
||
// same rows; a source edit converges every column — a pair removed by the edit
|
||
// disappears instead of lingering as a phantom (external-review #6). Called
|
||
// unconditionally (even for zero readings — a txt book or a source that dropped its
|
||
// furigana) so the full-replace clears any stale rows. The write order is sorted for
|
||
// deterministic, test-stable behavior. Memory v2 (step 4) consumes ruby_readings into
|
||
// a glossary name-lock (D9); nothing here injects into a prompt (§7d).
|
||
func (r *Runner) persistRuby(readings []chunk.RubyReading) error {
|
||
type agg struct {
|
||
first int
|
||
count int
|
||
}
|
||
seen := map[[2]string]*agg{}
|
||
order := make([][2]string, 0, len(readings))
|
||
for _, rr := range readings {
|
||
key := [2]string{rr.Base, rr.Reading}
|
||
a, ok := seen[key]
|
||
if !ok {
|
||
a = &agg{first: rr.Chapter}
|
||
seen[key] = a
|
||
order = append(order, key)
|
||
}
|
||
if rr.Chapter < a.first {
|
||
a.first = rr.Chapter
|
||
}
|
||
a.count++
|
||
}
|
||
sort.Slice(order, func(i, j int) bool {
|
||
if order[i][0] != order[j][0] {
|
||
return order[i][0] < order[j][0]
|
||
}
|
||
return order[i][1] < order[j][1]
|
||
})
|
||
rows := make([]store.RubyReading, 0, len(order))
|
||
for _, key := range order {
|
||
a := seen[key]
|
||
rows = append(rows, store.RubyReading{
|
||
BookID: r.Book.BookID, Base: key[0], Reading: key[1],
|
||
FirstChapter: a.first, Occurrences: a.count,
|
||
})
|
||
}
|
||
return r.Store.ReplaceRubyReadings(r.Book.BookID, rows)
|
||
}
|