package pipeline import ( "encoding/json" "fmt" "os" "sort" "strings" "unicode" "gopkg.in/yaml.v3" "textmachine/backend/internal/store" ) // memseed.go: the DETERMINISTIC seeding of the glossary from its two milestone inputs — // a manual seed file (curated approved/draft terms) and the captured ruby readings // (classified into auto candidates). No LLM (autopopulation/adjudication are a later // milestone). The output is a []store.GlossaryEntry that seedGlossary REPLACES into the // book (idempotent full-replace: same inputs → same rows, an edit converges every column). // --- manual seed file (YAML) ---------------------------------------------------- type seedFile struct { Terms []seedTerm `yaml:"terms"` } type seedTerm struct { Src string `yaml:"src"` Dst string `yaml:"dst"` Type string `yaml:"type"` Sense string `yaml:"sense"` Gender string `yaml:"gender"` Speech string `yaml:"speech"` Decl *seedDecl `yaml:"decl"` TranslitPolicy string `yaml:"translit_policy"` FirstPerson string `yaml:"first_person"` NicknameTranslation string `yaml:"nickname_translation"` SinceCh int `yaml:"since_ch"` UntilCh int `yaml:"until_ch"` Status string `yaml:"status"` AllowShort bool `yaml:"allow_short"` Note string `yaml:"note"` Aliases []seedAlias `yaml:"aliases"` } type seedDecl struct { Invariant bool `yaml:"invariant"` Forms []string `yaml:"forms"` } type seedAlias struct { Alias string `yaml:"alias"` Type string `yaml:"type"` } // loadGlossarySeed parses a glossary seed YAML into store entries. A manual seed is // curated, so an empty status defaults to "approved" (the author downgrades explicitly // with status: draft|auto). Validation is fail-loud: a term without src, or with an // unknown status, is a config error (a silently-dropped term is exactly the A-class // hole this bank exists to close). func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) { raw, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("pipeline: read glossary seed %s: %w", path, err) } var sf seedFile if err := yaml.Unmarshal(raw, &sf); err != nil { return nil, fmt.Errorf("pipeline: parse glossary seed %s: %w", path, err) } var out []store.GlossaryEntry var problems []string termKeys := map[[4]string]bool{} // (src,sense,since,until) → detect a duplicate term key for i, t := range sf.Terms { if strings.TrimSpace(t.Src) == "" { problems = append(problems, fmt.Sprintf("term %d: src is required", i)) continue } // A duplicate (src, sense, window) would crash ReplaceGlossary on the store's // UNIQUE constraint mid-run (aborting the whole book) — catch it here as a clear // config error instead (self-review #5, sibling of the alias case below). tk := [4]string{t.Src, t.Sense, fmt.Sprint(t.SinceCh), fmt.Sprint(t.UntilCh)} if termKeys[tk] { problems = append(problems, fmt.Sprintf("term %q: duplicate (src, sense=%q, since_ch=%d, until_ch=%d) — a term may appear once per spoiler window", t.Src, t.Sense, t.SinceCh, t.UntilCh)) continue } termKeys[tk] = true status := t.Status if status == "" { status = "approved" } switch status { case "auto", "draft", "approved": default: problems = append(problems, fmt.Sprintf("term %q: status must be auto|draft|approved, got %q", t.Src, status)) continue } // An approved OR draft term with no dst is silently inert (not matchable — it renders // nothing) yet reads as an intended rendering: fail loud (external-review minor #5). // Only status=auto (a raw candidate — e.g. a ruby reading awaiting a dst) may be empty. if (status == "approved" || status == "draft") && strings.TrimSpace(t.Dst) == "" { problems = append(problems, fmt.Sprintf("term %q: a %s term must have a non-empty dst (an empty one is silently inert; only status=auto may lack a dst)", t.Src, status)) continue } decl := "" if t.Decl != nil { b, mErr := json.Marshal(declInfo{Invariant: t.Decl.Invariant, Forms: t.Decl.Forms}) if mErr != nil { return nil, mErr } decl = string(b) } e := store.GlossaryEntry{ BookID: "", Src: t.Src, Dst: t.Dst, Type: t.Type, Sense: t.Sense, Gender: t.Gender, Speech: t.Speech, Decl: decl, TranslitPolicy: t.TranslitPolicy, FirstPerson: t.FirstPerson, NicknameTranslation: t.NicknameTranslation, SinceCh: t.SinceCh, UntilCh: t.UntilCh, Status: status, AllowShort: t.AllowShort, Source: "seed", Note: t.Note, } // Dedup aliases within a term by surface (keep the first type). A repeated alias // surface — a copy-paste, or the same name annotated with two types — would // otherwise crash ReplaceGlossary on UNIQUE(book_id,term_id,alias) and abort the // whole book run (self-review #5). Deduping is the forgiving fix: the surface is // what matters for matching, and a redundant alias is a harmless authoring slip. seenAlias := map[string]bool{} for _, a := range t.Aliases { if strings.TrimSpace(a.Alias) == "" { problems = append(problems, fmt.Sprintf("term %q: an alias is empty", t.Src)) continue } if seenAlias[a.Alias] { continue } seenAlias[a.Alias] = true e.Aliases = append(e.Aliases, store.GlossaryAlias{Alias: a.Alias, AliasType: a.Type}) } out = append(out, e) } // Overlapping spoiler windows for the SAME (src,sense) with DIFFERENT dst inject two // contradictory renderings at a chapter in the overlap — silently (external-review // minor #4). Fail loud. UNIQUE(src,sense,window) already blocks identical windows, so // this catches the partial-overlap case. Iterate `out` in order (deterministic message). for i := range out { for j := 0; j < i; j++ { a, b := out[i], out[j] if a.Src == b.Src && a.Sense == b.Sense && a.Dst != b.Dst && windowsOverlap(a.SinceCh, a.UntilCh, b.SinceCh, b.UntilCh) { problems = append(problems, fmt.Sprintf("term %q (sense %q): spoiler windows [%d,%d] and [%d,%d] overlap with different dst (%q vs %q) — a term has ONE rendering per chapter", a.Src, a.Sense, b.SinceCh, b.UntilCh, a.SinceCh, a.UntilCh, b.Dst, a.Dst)) } } } if len(problems) > 0 { return nil, fmt.Errorf("glossary seed %s:\n - %s", path, strings.Join(problems, "\n - ")) } return out, nil } // windowsOverlap reports whether two spoiler windows share any chapter. since_ch 0 means // "from chapter 1"; until_ch 0 means "no end". func windowsOverlap(since1, until1, since2, until2 int) bool { const inf = 1 << 62 s1, s2 := since1, since2 if s1 == 0 { s1 = 1 } if s2 == 0 { s2 = 1 } u1, u2 := until1, until2 if u1 == 0 { u1 = inf } if u2 == 0 { u2 = inf } lo, hi := s1, u1 if s2 > lo { lo = s2 } if u2 < hi { hi = u2 } return lo <= hi } // --- ruby → auto candidates ----------------------------------------------------- // rubyToCandidates classifies the captured ruby readings into auto glossary candidates // (registry §Ruby-seed / §8). One candidate per BASE (the dominant reading by occurrence, // ties broken lexically) so variant readings never collide on UNIQUE(src,sense,window) // — the (base,reading) source of truth stays in ruby_readings. A base already covered by // the manual seed is skipped (the curated entry wins). Every candidate is status=auto // with NO dst: the reading is the Polivanov bridge (B6, Phase 2), not a dst — so nothing // is auto-injected, and a human/batch-judge promotes it (never a blind name-lock, which // would be a mistranslation). The ruby_class is a deterministic HINT for that promotion. func rubyToCandidates(readings []store.RubyReading, manualSrcs map[string]bool) []store.GlossaryEntry { // Group by base → the dominant (base,reading). type best struct { reading string occ int firstCh int } byBase := map[string]best{} order := []string{} for _, rr := range readings { if manualSrcs[rr.Base] { continue // the curated seed already owns this src } cur, ok := byBase[rr.Base] if !ok { byBase[rr.Base] = best{rr.Reading, rr.Occurrences, rr.FirstChapter} order = append(order, rr.Base) continue } // Dominant reading: higher occurrences wins; tie → lexically smaller reading. if rr.Occurrences > cur.occ || (rr.Occurrences == cur.occ && rr.Reading < cur.reading) { cur.reading, cur.occ = rr.Reading, rr.Occurrences } if rr.FirstChapter < cur.firstCh { // earliest appearance across readings cur.firstCh = rr.FirstChapter } byBase[rr.Base] = cur } sort.Strings(order) // deterministic output regardless of input order var out []store.GlossaryEntry for _, base := range order { b := byBase[base] class := classifyRubyReading(base, b.reading) typ := "" if class == rubyClassName { typ = "name" } out = append(out, store.GlossaryEntry{ Src: base, Dst: "", Type: typ, Status: "auto", Source: "ruby", RubyReading: b.reading, RubyClass: class, SinceCh: b.firstCh, Confidence: b.occ, Note: "ruby auto-candidate (" + class + "); reading is the Polivanov bridge, not a dst — promote before use", }) } return out } // ruby classifier classes (deterministic HINTS for human promotion). const ( rubyClassName = "name_candidate" // all-Han base + all-kana reading: the furigana-name SHAPE rubyClassGloss = "gloss_candidate" // reading carries Han: likely an author double-reading → footnote, NOT a lock rubyClassAmbiguous = "ambiguous" // anything else → flag to a human ) // classifyRubyReading is the deterministic clear-case rule (§8). It NEVER auto-locks: // the all-Han+all-kana shape is a name-lock CANDIDATE (it could still be a double-reading // like 強敵→とも "friend", indistinguishable without a yomi dictionary — Phase 2), so it is // flagged for human confirmation, not locked. A reading that carries kanji is a clear // author's double-reading/gloss → a footnote candidate. Over-flagging is safe; a blind // lock is a mistranslation. Pure and deterministic. func classifyRubyReading(base, reading string) string { if runesAnyHan(reading) { return rubyClassGloss // the "reading" is itself a word (semantic gloss / double-reading) } if runesAllHan(base) && runesAllKana(reading) { return rubyClassName } return rubyClassAmbiguous } func runesAllHan(s string) bool { if s == "" { return false } for _, r := range s { if !unicode.Is(unicode.Han, r) { return false } } return true } func runesAnyHan(s string) bool { for _, r := range s { if unicode.Is(unicode.Han, r) { return true } } return false } func runesAllKana(s string) bool { if s == "" { return false } for _, r := range s { if r == 'ー' || r == '・' { // prolonged-sound mark, middle dot — allowed within kana readings continue } if !unicode.Is(unicode.Hiragana, r) && !unicode.Is(unicode.Katakana, r) { return false } } return true }