package pipeline import ( "context" "fmt" "sort" "strings" "textmachine/backend/internal/chunk" "textmachine/backend/internal/lang" "textmachine/backend/internal/membank" "textmachine/backend/internal/store" ) // seeding.go: the job's deterministic $0 preludes — REPLACE-seed the glossary from the // manual file + ruby readings (D16.4) and persist the ruby aggregates. Run BEFORE // snapshotID: the frozen approved rows enter memoryVersion (F1). // seedGlossary REPLACES the book's glossary from its deterministic inputs — the manual // seed file (curated approved/draft terms) and the captured ruby readings (classified // into auto candidates) — then MATERIALIZES the frozen bank for this job (r.memory). Run // once before snapshotID: the frozen APPROVED rows are hashed into memoryVersion (F1), so // editing the seed is a loud --resnapshot. Idempotent (full replace), $0, no LLM. B2 // approved dst-collisions are logged (not fatal — some collisions are legitimate). func (r *Runner) seedGlossary(ctx context.Context) error { var entries []store.GlossaryEntry var voices []store.VoiceProfile var pairs []store.AddressPair if r.Book.GlossarySeed != "" { bank, err := membank.LoadBankSeed(r.Book.GlossarySeed) if err != nil { return err } entries = append(entries, bank.Terms...) voices, pairs = bank.Voices, bank.Pairs } manualSrcs := map[string]bool{} for _, e := range entries { manualSrcs[e.Src] = true } ruby, err := r.Store.RubyReadingsForBook(r.Book.BookID) if err != nil { return fmt.Errorf("pipeline: read ruby readings for %s: %w", r.Book.BookID, err) } // D16.4: attach a manual term's kana ruby-reading as an alias (kana spelling matchable) // BEFORE appending the auto-candidates, so it only touches the curated manual entries. A // reading that would collide with a different seeded term (homophone) is skipped+logged, // not attached (which would fail the book loud on an alias the operator cannot edit out). if skipped := membank.AttachRubyAliasesToManual(entries, ruby); len(skipped) > 0 { r.Log.WarnContext(ctx, "ruby kana-alias skipped as a homophone collision (kana form left unmatchable; disambiguate in the seed if needed)", "skipped", strings.Join(skipped, "; ")) } // Mined-write path (R1, plan §1(в)/F2): the owner-curated mined-delta file joins the seed as // Source:"mined" (loadMinedDelta re-stamps the seed loader's Source), so its terms fold into the // ENRICHED bank version but NOT the base — a re-run that adds signed mined terms moves ONLY snapshot_W2. // Loaded AFTER the seed/ruby so manualSrcs already reflects the curated seed. A `mined` term that // duplicates a seed src is caught by membank.ApprovedSharedKeyCollisions below like any other collision. minedDelta, err := r.loadMinedDelta() if err != nil { return err } // D39.20 deviation-#1 fix: a mined-delta term whose UNIQUE key (src, sense, since_ch, until_ch — // store/migrate.go glossary UNIQUE) already exists in the SIGNED seed makes the flat INSERT in // ReplaceGlossary crash on that constraint and abort the whole paid run. membank.ApprovedSharedKeyCollisions // below does NOT catch it (it skips a SAME-dst duplicate, and keys on the firing surface, not the // UNIQUE tuple). Fail LOUD here with the duplicate list + a fix hint (edit the seed OR drop it from // the delta) — NEVER a silent merge over the signed seed. Checked BEFORE the append so the delta rows // are still separable. Deterministic (seed order). if dups := membank.MinedDeltaSeedCollisions(entries, minedDelta); len(dups) > 0 { return fmt.Errorf("pipeline: mined-delta %s duplicates term(s) already in the signed seed (would crash ReplaceGlossary on the glossary UNIQUE(book_id,src,sense,since_ch,until_ch)):\n - %s\n fix: correct the term in the seed, or remove it from the mined-delta file — never both (no silent merge over the signed seed)", r.Book.MinedDelta, strings.Join(dups, "\n - ")) } entries = append(entries, minedDelta...) // AUTO-BANK (pack-20 / D39.42 п.3): the engine's own unsigned rows — what the terminologist consolidated // on the last run's bank-mining boundary. They load exactly like the owner-curated delta (Source:"mined", // so base-excluded and only the edit-wave snapshot moves) but they are NOT signed: their status is // auto/draft, which is what routes them to the editor's separately-headed unverified section instead of // the canon list. // // Loading them HERE rather than injecting them mid-run is load-bearing for money. seedGlossary runs before // checkRebillConsent, so the re-payment projection sees the same bank the edit wave will use; if the rows // only appeared after the projection, the next run would compare stored edit rows against a bank that has // not been rebuilt yet and project a re-payment of the whole edit wave that is not real. autoBank, dropped, err := r.loadAutoBank(entries) if err != nil { return err } if len(dropped) > 0 { // A collision with a SIGNED row cannot abort a paid run over an engine-written proposal — the signed // row simply wins and the proposal is dropped, loudly. r.Log.WarnContext(ctx, "auto-bank rows dropped: their key is already held by a signed term (the signed term wins)", "book", r.Book.BookID, "dropped", strings.Join(dropped, "; ")) } entries = append(entries, autoBank...) for i := range entries { entries[i].BookID = r.Book.BookID } // Task-6 re-audit: fail loud on a firing key shared by two DIFFERENT approved terms with // different dst + overlapping windows (the alias generalization of the D16.1 polysemy // livelock) — checked over the FULL entry set (incl. ruby-attached aliases) before persisting. if cols := membank.ApprovedSharedKeyCollisions(entries); len(cols) > 0 { return fmt.Errorf("pipeline: glossary shared-key collisions (A2 / D16.1 livelock class):\n - %s", strings.Join(cols, "\n - ")) } // A voice/address row naming a character the bank does not have is SILENTLY inert — nothing can ever // attribute a reply to it — which is the A-class hole this bank exists to close, so it stops the run. // Checked over the FULL entry set (seed + ruby + mined + auto), because a character may legitimately be // signed in a delta rather than in the base seed. if unknown := membank.UnknownVoiceCharacters(entries, voices, pairs); len(unknown) > 0 { return fmt.Errorf("pipeline: voice/address rows name characters absent from the bank (a profile for a term that does not exist can never fire):\n - %s", strings.Join(unknown, "\n - ")) } if err := r.Store.ReplaceBank(r.Book.BookID, entries, voices, pairs); err != nil { return fmt.Errorf("pipeline: replace bank for %s (%d terms, %d voices, %d pairs): %w", r.Book.BookID, len(entries), len(voices), len(pairs), err) } rows, err := r.Store.GlossaryForBook(r.Book.BookID) if err != nil { return fmt.Errorf("pipeline: read glossary for %s: %w", r.Book.BookID, err) } storedVoices, err := r.Store.VoiceProfilesForBook(r.Book.BookID) if err != nil { return fmt.Errorf("pipeline: read voice profiles for %s: %w", r.Book.BookID, err) } storedPairs, err := r.Store.AddressPairsForBook(r.Book.BookID) if err != nil { return fmt.Errorf("pipeline: read address pairs for %s: %w", r.Book.BookID, err) } // A chapter range no profile covers is legitimate but is far more often a typo, and it is invisible in // a seed file — so it is logged, never fatal (the optional lint of D39.55). if gaps := membank.VoiceWindowGaps(storedVoices); len(gaps) > 0 { r.Log.WarnContext(ctx, "voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)", "book", r.Book.BookID, "gaps", strings.Join(gaps, "; ")) } // InjectVoice is FALSE and has no config knob: pack-19 builds the schema and the flagger, and D21 п.2 // holds the injection conditional until the polygon experiment. It is the fold's condition, so while // it is false a book with voice rows hashes exactly as it did without them and nobody re-pays for // authoring a profile. Wiring the injection means setting it and accepting a full --resnapshot. // The §3 decl stemmer is target data, constant per book; baseIn copies bankIn below, so both banks share // it. A target with no decl_suffix registry yields an inert stemmer → exact-match post-check as before. bankIn := membank.BankInput{Rows: rows, Voices: storedVoices, Pairs: storedPairs, TargetStemmer: lang.NewTargetStemmer(lang.TargetChecksFor(r.Book.TargetLang))} r.memory = membank.MaterializeBank(bankIn, r.Pipeline.Gates.Glossary.PostcheckGate) // The DRAFT wave selects over a BASE-scoped bank (Source:mined excluded) so its injection is // byte-identical across a bank-mining enrichment — matching the draft-wave snapshot (baseMemoryVersion), // which keeps «one re-payment» honest at the WIRE level, not only the version-hash level. Only the // editor sees mined terms (the enriched `memory`). When there are no mined rows (every $0 test / the // golden) the base bank IS the enriched one — share the object, no double materialization, no drift. baseRows := rows[:0:0] hasMined := false for _, row := range rows { if row.Source == "mined" { hasMined = true continue } baseRows = append(baseRows, row) } if hasMined { baseIn := bankIn baseIn.Rows = baseRows r.baseMemory = membank.MaterializeBank(baseIn, r.Pipeline.Gates.Glossary.PostcheckGate) } else { r.baseMemory = r.memory } if cols := membank.InjectivityCollisions(rows); len(cols) > 0 { r.Log.WarnContext(ctx, "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)", "collisions", strings.Join(cols, "; ")) } // S14: the diagnostic the approved-only checks structurally cannot make — an approved term and an // unsigned one contradicting each other on the SAME firing surface. In the auto mode both are injected // (canon + working version), so the model is handed two answers to one question; the trust gate keeps // the approved one authoritative, but the collision is exactly the silent-degradation source the auto // mode needs visible. if cols := membank.UnverifiedKeyConflicts(rows); len(cols) > 0 { r.Log.WarnContext(ctx, "bank: an UNSIGNED row contradicts an approved term on the same firing surface (the approved rendering keeps precedence; reconcile the seed or decline the proposal)", "book", r.Book.BookID, "conflicts", strings.Join(cols, "; ")) } r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID, "entries", len(rows), "voices", len(storedVoices), "address_pairs", len(storedPairs), "memory_version", r.memory.Version()[:12]) return nil } // persistRuby aggregates the ingested ruby occurrences into one row per // (base, reading) — first_chapter = MIN, occurrences = full-book count — and // REPLACES the book's whole ruby set (store.ReplaceRubyReadings). Idempotent: the // aggregation is recomputed identically on every ingest, so a resume re-writes the // same rows; a source edit converges every column — a pair removed by the edit // disappears instead of lingering as a phantom (external-review #6). Called // unconditionally (even for zero readings — a txt book or a source that dropped its // furigana) so the full-replace clears any stale rows. The write order is sorted for // deterministic, test-stable behavior. Memory v2 (step 4) consumes ruby_readings into // a glossary name-lock (D9); nothing here injects into a prompt (§7d). func (r *Runner) persistRuby(readings []chunk.RubyReading) error { type agg struct { first int count int } seen := map[[2]string]*agg{} order := make([][2]string, 0, len(readings)) for _, rr := range readings { key := [2]string{rr.Base, rr.Reading} a, ok := seen[key] if !ok { a = &agg{first: rr.Chapter} seen[key] = a order = append(order, key) } if rr.Chapter < a.first { a.first = rr.Chapter } a.count++ } sort.Slice(order, func(i, j int) bool { if order[i][0] != order[j][0] { return order[i][0] < order[j][0] } return order[i][1] < order[j][1] }) rows := make([]store.RubyReading, 0, len(order)) for _, key := range order { a := seen[key] rows = append(rows, store.RubyReading{ BookID: r.Book.BookID, Base: key[0], Reading: key[1], FirstChapter: a.first, Occurrences: a.count, }) } return r.Store.ReplaceRubyReadings(r.Book.BookID, rows) }