752 lines
34 KiB
Go
752 lines
34 KiB
Go
package membank
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"maps"
|
|
"os"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"unicode"
|
|
|
|
"textmachine/backend/internal/seed"
|
|
"textmachine/backend/internal/store"
|
|
"textmachine/backend/internal/text"
|
|
)
|
|
|
|
// memseed.go: the DETERMINISTIC seeding of the glossary from its two milestone inputs —
|
|
// a manual seed file (curated approved/draft terms) and the captured ruby readings
|
|
// (classified into auto candidates). No LLM (autopopulation/adjudication are a later
|
|
// milestone). The output is a []store.GlossaryEntry that seedGlossary REPLACES into the
|
|
// book (idempotent full-replace: same inputs → same rows, an edit converges every column).
|
|
|
|
// --- manual seed file (YAML) ----------------------------------------------------
|
|
|
|
// LoadBankSeed parses a seed YAML into the bank's three record types. A manual seed is
|
|
// curated, so an empty status defaults to "approved" (the author downgrades explicitly
|
|
// with status: draft|auto). Validation is fail-loud: a term without src, or with an
|
|
// unknown status, is a config error (a silently-dropped term is exactly the A-class
|
|
// hole this bank exists to close). The voices:/addresses: sections follow the same
|
|
// contract — see loadVoiceSections.
|
|
func LoadBankSeed(path string) (BankSeed, error) {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return BankSeed{}, fmt.Errorf("membank: read glossary seed %s: %w", path, err)
|
|
}
|
|
return ParseBankSeed(path, raw)
|
|
}
|
|
|
|
// LoadEngineBankSeed is LoadBankSeed for a document THIS ENGINE WROTE — see ParseEngineBankSeed for the
|
|
// one rule that differs and why.
|
|
func LoadEngineBankSeed(path string) (BankSeed, error) {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return BankSeed{}, fmt.Errorf("membank: read engine-written bank document %s: %w", path, err)
|
|
}
|
|
return ParseEngineBankSeed(path, raw)
|
|
}
|
|
|
|
// ParseBankSeed is LoadBankSeed over bytes already in hand. It exists because `bank-apply` has to know
|
|
// whether the document it is ABOUT to write would load — asking that question by writing the file first
|
|
// and reading it back is the one answer a door that refuses on all-or-nothing cannot use. `name` is what
|
|
// error messages call the document; for a file it is its path.
|
|
func ParseBankSeed(name string, raw []byte) (BankSeed, error) {
|
|
return parseBankSeed(name, raw, false)
|
|
}
|
|
|
|
// ParseEngineBankSeed is ParseBankSeed for a document THIS ENGINE WROTE — the mined delta and the
|
|
// auto-bank, both of which are built from MODEL OUTPUT rather than typed by a person.
|
|
//
|
|
// ⛔ THE DIFFERENCE IS ONE RULE AND IT IS ABOUT WHOSE ARTIFACT IT IS. The wire fence refuses a value that
|
|
// could write into a system message, and for the operator's own seed refusing the DOCUMENT is right: it
|
|
// is theirs to fix, and dropping the row would leave a term they believe is in force silently absent
|
|
// from every request. For a document the engine wrote from a model's answer the same refusal is wrong in
|
|
// both directions — it aborts a PAID run over the engine's own output (mining.go's loadMinedDelta and
|
|
// the auto-bank both hand their error straight up, and bankmaterialize turns it into a dead run), and it
|
|
// asks a person to hand-edit a file no person authored. So an unfit row is DROPPED here: it never enters
|
|
// the bank, therefore never the wire, and the run continues.
|
|
//
|
|
// Found by acceptance (F8), which named the mined delta; the auto-bank is the same class and is included
|
|
// because it is written from model output on EVERY run, which makes it the likelier of the two.
|
|
func ParseEngineBankSeed(name string, raw []byte) (BankSeed, error) {
|
|
return parseBankSeed(name, raw, true)
|
|
}
|
|
|
|
// parseBankSeed is the shared core. `engineWritten` decides only what an unfit row costs — see
|
|
// ParseEngineBankSeed.
|
|
func parseBankSeed(name string, raw []byte, engineWritten bool) (BankSeed, error) {
|
|
var bs BankSeed
|
|
path := name
|
|
// STRICT (seed.DecodeFile): yaml.v3 drops an unknown key silently, so `gendr:` for `gender:` used to
|
|
// be thrown away BEFORE any validation ran — and seed-lint could not see it either, because it
|
|
// validates VALUES and never saw the key at all (backlog row 212). The files this loader reads are
|
|
// the ones the owner edits by hand most often.
|
|
sf, err := seed.DecodeFile(raw)
|
|
if err != nil {
|
|
return bs, fmt.Errorf("membank: parse glossary seed %s: %w", path, err)
|
|
}
|
|
var out []store.GlossaryEntry
|
|
var problems problemList
|
|
termKeys := map[[4]string]bool{} // (src,sense,since,until) → detect a duplicate term key
|
|
for i, t := range sf.Terms {
|
|
// ⚠ A TERM THAT FAILS THIS LOADER IS STILL BUILT AND STILL RETURNED (see the bottom of the
|
|
// function), because the checks that live BELOW this loader — collisions, gender vocabulary,
|
|
// voice coverage — are asked about the same document by the decisions door, and a term the loader
|
|
// DROPPED was a term they could not see. That blindness is what made a repair of one term's
|
|
// loadability read as introducing every fault the drop had been hiding. Nothing downstream of a
|
|
// FAILED parse builds a bank, so a faulty row in the returned set can only inform a verdict.
|
|
//
|
|
// The one exception is a term with no `src`: it has no identity at all, so it can be neither the
|
|
// subject of a fault nor an input to a check, and carrying it forward would only make a SECOND
|
|
// src-less term look like a duplicate of the first.
|
|
// Trim surrounding whitespace on the KEYING surfaces before anything reads them.
|
|
// text.NormalizeSourceKey does not trim, so a raw " 強敵" would key as " 強敵" and never
|
|
// match "強敵" — a silently-inert approved term (the A-class hole this bank closes).
|
|
// A trailing space on sense also makes two identical senses look distinct, slipping
|
|
// past BOTH the duplicate-key and the same-sense contradiction guards below.
|
|
// Surrounding whitespace is never intentional; trim it (a forgiving fix, like the
|
|
// alias dedup). Internal spacing of a multi-word surface is preserved.
|
|
t.Src = strings.TrimSpace(t.Src)
|
|
t.Sense = strings.TrimSpace(t.Sense)
|
|
t.Dst = strings.TrimSpace(t.Dst)
|
|
if t.Src == "" {
|
|
// Keyed WITHOUT the position: it is the one message whose text moves when an EARLIER term is
|
|
// removed, and the decisions door compares these verdicts before and after a call.
|
|
problems.addKeyed("term: src is required", fmt.Sprintf("term %d: src is required", i))
|
|
continue
|
|
}
|
|
// A duplicate (src, sense, window) would crash ReplaceGlossary on the store's
|
|
// UNIQUE constraint mid-run (aborting the whole book) — catch it here as a clear
|
|
// config error instead (self-review #5, sibling of the alias case below).
|
|
tk := [4]string{t.Src, t.Sense, fmt.Sprint(t.SinceCh), fmt.Sprint(t.UntilCh)}
|
|
if termKeys[tk] {
|
|
problems.addKeyed(subjectOf("term-duplicate", t.Src, t.Sense, t.SinceCh, t.UntilCh),
|
|
fmt.Sprintf("term %q: duplicate (src, sense=%q, since_ch=%d, until_ch=%d) — a term may appear once per spoiler window", t.Src, t.Sense, t.SinceCh, t.UntilCh))
|
|
}
|
|
termKeys[tk] = true
|
|
status := t.Status
|
|
if status == "" {
|
|
status = "approved"
|
|
}
|
|
switch status {
|
|
case "auto", "draft", "approved":
|
|
default:
|
|
problems.addKeyed(subjectOf("term-status", t.Src, t.Sense, t.SinceCh, t.UntilCh),
|
|
fmt.Sprintf("term %q: status must be auto|draft|approved, got %q", t.Src, status))
|
|
}
|
|
// An approved OR draft term with no dst is silently inert (not matchable — it renders
|
|
// nothing) yet reads as an intended rendering: fail loud (external-review minor #5).
|
|
// Only status=auto (a raw candidate — e.g. a ruby reading awaiting a dst) may be empty.
|
|
if (status == "approved" || status == "draft") && t.Dst == "" {
|
|
problems.addKeyed(subjectOf("term-dst", t.Src, t.Sense, t.SinceCh, t.UntilCh),
|
|
fmt.Sprintf("term %q: a %s term must have a non-empty dst (an empty one is silently inert; only status=auto may lack a dst)", t.Src, status))
|
|
}
|
|
// THE WIRE FENCE (wirefence.go, backlog row 271): src and dst are concatenated into a system
|
|
// message on every paid call, so a value that can end its own line writes into that block. Refused
|
|
// at LOAD because a seed file is the operator's own artifact and theirs to fix — the alternative,
|
|
// dropping the row on the way to the wire, would leave a term the operator believes is in force
|
|
// silently absent from every request. `%q` is load-bearing in the message: it Go-quotes the very
|
|
// runes being refused, so the fault can be printed without reproducing it.
|
|
if reasons := WireUnfitReasons(t.Src, t.Dst); len(reasons) > 0 {
|
|
if engineWritten {
|
|
// A document the ENGINE wrote from a model's answer: drop the row and carry on. Refusing
|
|
// would abort a paid run over our own output — see ParseEngineBankSeed. ⚠ The drop is
|
|
// RECORDED, not silent: BankSeed.Dropped carries it out to a caller that has a log.
|
|
bs.Dropped = append(bs.Dropped, t.Src)
|
|
continue
|
|
}
|
|
for _, why := range reasons {
|
|
problems.addKeyed(subjectOf("term-wire", t.Src, t.Sense, t.SinceCh, t.UntilCh),
|
|
fmt.Sprintf("term %q: %s", t.Src, why))
|
|
}
|
|
}
|
|
decl := ""
|
|
if t.Decl != nil {
|
|
b, mErr := json.Marshal(declInfo{Invariant: t.Decl.Invariant, Forms: t.Decl.Forms})
|
|
if mErr != nil {
|
|
return bs, mErr
|
|
}
|
|
decl = string(b)
|
|
}
|
|
e := store.GlossaryEntry{
|
|
BookID: "", Src: t.Src, Dst: t.Dst, Type: t.Type, Sense: t.Sense,
|
|
Gender: t.Gender, Speech: t.Speech, Decl: decl,
|
|
TranslitPolicy: t.TranslitPolicy, FirstPerson: t.FirstPerson,
|
|
NicknameTranslation: t.NicknameTranslation,
|
|
SinceCh: t.SinceCh, UntilCh: t.UntilCh, Status: status,
|
|
AllowShort: t.AllowShort, Source: "seed", Note: t.Note,
|
|
}
|
|
// Dedup aliases within a term by surface (keep the first type). A repeated alias
|
|
// surface — a copy-paste, or the same name annotated with two types — would
|
|
// otherwise crash ReplaceGlossary on UNIQUE(book_id,term_id,alias) and abort the
|
|
// whole book run (self-review #5). Deduping is the forgiving fix: the surface is
|
|
// what matters for matching, and a redundant alias is a harmless authoring slip.
|
|
seenAlias := map[string]bool{}
|
|
for _, a := range t.Aliases {
|
|
a.Alias = strings.TrimSpace(a.Alias) // same silent-inert risk as src (an alias is a match key)
|
|
if a.Alias == "" {
|
|
problems.addKeyed(subjectOf("term-alias", t.Src, t.Sense, t.SinceCh, t.UntilCh),
|
|
fmt.Sprintf("term %q: an alias is empty", t.Src))
|
|
continue
|
|
}
|
|
if seenAlias[a.Alias] {
|
|
continue
|
|
}
|
|
seenAlias[a.Alias] = true
|
|
e.Aliases = append(e.Aliases, store.GlossaryAlias{Alias: a.Alias, AliasType: a.Type})
|
|
}
|
|
out = append(out, e)
|
|
}
|
|
// Overlapping spoiler windows for the SAME src inject CONTRADICTORY renderings at a
|
|
// chapter in the overlap — silently (external-review minor #4; D16.1). The deterministic
|
|
// matcher keys ONLY on src (it cannot disambiguate sense within a chunk), so BOTH rows
|
|
// fire together (keyOwners maps the shared key to both indices) and the post-check is
|
|
// guaranteed to miss one — with the gate ON, a deterministic per-chunk livelock. Fail
|
|
// loud. UNIQUE(src,sense,window) already blocks identical (src,sense,window), so this
|
|
// catches partial overlap and the different-SENSE case the old same-sense-only guard let
|
|
// through. Iterate `out` in order (deterministic message).
|
|
for i := range out {
|
|
for j := 0; j < i; j++ {
|
|
a, b := out[i], out[j]
|
|
if a.Src != b.Src || a.Dst == b.Dst || !windowsOverlap(a.SinceCh, a.UntilCh, b.SinceCh, b.UntilCh) {
|
|
continue
|
|
}
|
|
if a.Sense == b.Sense {
|
|
problems.addKeyed(subjectOf("term-window-overlap/"+b.Sense, a.Src, a.Sense, a.SinceCh, a.UntilCh),
|
|
fmt.Sprintf("term %q (sense %q): spoiler windows [%d,%d] and [%d,%d] overlap with different dst (%q vs %q) — a term has ONE rendering per chapter",
|
|
a.Src, a.Sense, b.SinceCh, b.UntilCh, a.SinceCh, a.UntilCh, b.Dst, a.Dst))
|
|
continue
|
|
}
|
|
// Different sense, overlapping windows, different dst (D16.1 polysemy livelock).
|
|
// Scope to a MATCHABLE src: an A3-banned single-char src (道 — text.SignificantLen below
|
|
// the min-key floor, never in the automaton) is inert and cannot livelock, so its
|
|
// polysemy stays allowed. Both rows must be matchable for the contradiction to fire.
|
|
if srcKeyFires(a.Src, a.AllowShort) && srcKeyFires(b.Src, b.AllowShort) {
|
|
problems.addKeyed(subjectOf("term-polysemy/"+b.Sense, a.Src, a.Sense, a.SinceCh, a.UntilCh),
|
|
fmt.Sprintf("term %q: senses %q→%q and %q→%q have OVERLAPPING spoiler windows [%d,%d] and [%d,%d] — the deterministic matcher keys only on src and cannot pick a sense, so both inject as authoritative and the post-check is guaranteed to miss one (give them non-overlapping windows, merge the sense, or mark one status: auto)",
|
|
a.Src, b.Sense, b.Dst, a.Sense, a.Dst, b.SinceCh, b.UntilCh, a.SinceCh, a.UntilCh))
|
|
}
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
// The entries that WERE built travel with the refusal. A caller that must not load a broken
|
|
// document ignores them (every one does — they check err first); the decisions door reads them,
|
|
// because "what else is wrong with this file" is a question it has to answer about a document
|
|
// that does not load.
|
|
bs.Terms = out
|
|
return bs, SeedProblems{Path: path, Problems: problems}
|
|
}
|
|
voices, pairs, verr := loadVoiceSections(path, &sf)
|
|
if verr != nil {
|
|
return BankSeed{}, verr
|
|
}
|
|
bs.Terms, bs.Voices, bs.Pairs = out, voices, pairs
|
|
return bs, nil
|
|
}
|
|
|
|
// ApprovedSharedKeyCollisions detects the alias-generalization of the D16.1 polysemy livelock
|
|
// (Task-6 re-audit, confirmed by execution): a firing KEY (normalized src OR alias) shared by
|
|
// two approved terms with DIFFERENT dst and OVERLAPPING spoiler windows. The matcher keys only
|
|
// on the surface (keyOwners maps the shared key to BOTH terms), so both inject as authoritative
|
|
// and the post-check is guaranteed to miss one — a deterministic chunk livelock with the gate
|
|
// ON, exactly like same-src polysemy but reached via a shared nickname (老赵 for two 赵). It does
|
|
// NOT skip same-src pairs: LoadGlossarySeed's D16.1 check scopes its fail-loud to a MATCHABLE
|
|
// src (srcKeyFires), so a same-src pair whose src is single-key-BANNED (道) but that shares an
|
|
// ELIGIBLE alias (大道) slips D16.1 entirely — this catches it via the shared alias key (self-
|
|
// review finding #1). In the common case (an eligible shared src) LoadGlossarySeed fails loud
|
|
// FIRST and seedGlossary never reaches here, so there is no double report. Returns human
|
|
// messages (deterministic order); seedGlossary fails loud on any. A shared key with the SAME
|
|
// dst (a legitimate merge) or NON-overlapping windows (a spoiler handoff) is not a contradiction.
|
|
func ApprovedSharedKeyCollisions(entries []store.GlossaryEntry) []string {
|
|
return problemTexts(approvedSharedKeyCollisions(entries))
|
|
}
|
|
|
|
// approvedSharedKeyCollisions is the same check with the SUBJECT of each collision beside its message —
|
|
// the shared key and the two terms' identities, never their renderings (see subjectOf).
|
|
func approvedSharedKeyCollisions(entries []store.GlossaryEntry) []Problem {
|
|
owners := map[string][]int{} // normalized firing key → indices of approved terms that own it
|
|
for i, e := range entries {
|
|
if e.Status != "approved" || strings.TrimSpace(e.Dst) == "" {
|
|
continue
|
|
}
|
|
for _, k := range entryFiringKeys(e) {
|
|
owners[k] = append(owners[k], i)
|
|
}
|
|
}
|
|
keys := slices.Sorted(maps.Keys(owners))
|
|
var out []Problem
|
|
seenPair := map[[2]int]bool{}
|
|
for _, k := range keys {
|
|
idxs := owners[k]
|
|
for a := 0; a < len(idxs); a++ {
|
|
for b := a + 1; b < len(idxs); b++ {
|
|
i, j := idxs[a], idxs[b]
|
|
ei, ej := entries[i], entries[j]
|
|
if ei.Dst == ej.Dst {
|
|
continue // same dst → both render identically, no contradiction (incl. an alias of one entry)
|
|
}
|
|
if !windowsOverlap(ei.SinceCh, ei.UntilCh, ej.SinceCh, ej.UntilCh) {
|
|
continue
|
|
}
|
|
pair := [2]int{i, j}
|
|
if seenPair[pair] {
|
|
continue
|
|
}
|
|
seenPair[pair] = true
|
|
out = append(out, Problem{
|
|
Subject: subjectOf("shared-key/"+k+"/"+ej.Src, ei.Src, ei.Sense, ei.SinceCh, ei.UntilCh),
|
|
Text: fmt.Sprintf("firing key %q is shared by different terms %q→%q and %q→%q with overlapping spoiler windows — both inject as authoritative and the deterministic matcher cannot pick one (livelock; give non-overlapping windows, a more specific alias, or drop the shared surface)",
|
|
k, ei.Src, ei.Dst, ej.Src, ej.Dst)})
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// entryFiringKeys returns the normalized keys of a term ELIGIBLE to enter the automaton (the
|
|
// same eligibility Materialize applies: src + aliases, each passing the per-language
|
|
// single-key floor or allow_short). Mirrors the matcher so the collision check sees exactly the
|
|
// keys that would fire.
|
|
func entryFiringKeys(e store.GlossaryEntry) []string {
|
|
surfaces := []string{e.Src}
|
|
for _, a := range e.Aliases {
|
|
surfaces = append(surfaces, a.Alias)
|
|
}
|
|
var keys []string
|
|
seen := map[string]bool{}
|
|
for _, s := range surfaces {
|
|
nk := text.NormalizeSourceKey(s)
|
|
if nk == "" || seen[nk] {
|
|
continue
|
|
}
|
|
if text.SignificantLen(nk) >= minKeyLenFor(nk) || e.AllowShort {
|
|
seen[nk] = true
|
|
keys = append(keys, nk)
|
|
}
|
|
}
|
|
return keys
|
|
}
|
|
|
|
// srcKeyFires reports whether src normalizes to a key eligible to ENTER the matcher's
|
|
// automaton — it passes the per-language single-key floor (minKeyLenFor), or allow_short
|
|
// overrides it. Mirrors Materialize's eligibility test. An ineligible src (a single-
|
|
// char Han like 道, banned by A3) never fires on the hot path, so a polysemy contradiction on
|
|
// it is inert and must not fail the seed load (D16.1 scoping).
|
|
func srcKeyFires(src string, allowShort bool) bool {
|
|
nk := text.NormalizeSourceKey(src)
|
|
if nk == "" {
|
|
return false
|
|
}
|
|
return allowShort || text.SignificantLen(nk) >= minKeyLenFor(nk)
|
|
}
|
|
|
|
// windowsOverlap reports whether two spoiler windows share any chapter. since_ch 0 means
|
|
// "from chapter 1"; until_ch 0 means "no end".
|
|
func windowsOverlap(since1, until1, since2, until2 int) bool {
|
|
const inf = 1 << 62
|
|
s1, s2 := since1, since2
|
|
if s1 == 0 {
|
|
s1 = 1
|
|
}
|
|
if s2 == 0 {
|
|
s2 = 1
|
|
}
|
|
u1, u2 := until1, until2
|
|
if u1 == 0 {
|
|
u1 = inf
|
|
}
|
|
if u2 == 0 {
|
|
u2 = inf
|
|
}
|
|
lo, hi := s1, u1
|
|
if s2 > lo {
|
|
lo = s2
|
|
}
|
|
if u2 < hi {
|
|
hi = u2
|
|
}
|
|
return lo <= hi
|
|
}
|
|
|
|
// --- ruby → auto candidates -----------------------------------------------------
|
|
|
|
// rubyToCandidates classifies the captured ruby readings into auto glossary candidates
|
|
// (registry §Ruby-seed / §8). One candidate per BASE (the dominant reading by occurrence,
|
|
// ties broken lexically) so variant readings never collide on UNIQUE(src,sense,window)
|
|
// — the (base,reading) source of truth stays in ruby_readings. A base already covered by
|
|
// the manual seed is skipped (the curated entry wins). Every candidate is status=auto
|
|
// with NO dst: the reading is the Polivanov bridge (B6, Phase 2), not a dst — so nothing
|
|
// is auto-injected, and a human/batch-judge promotes it (never a blind name-lock, which
|
|
// would be a mistranslation). The ruby_class is a deterministic HINT for that promotion.
|
|
func rubyToCandidates(readings []store.RubyReading, manualSrcs map[string]bool) []store.GlossaryEntry {
|
|
// Group by base → the dominant (base,reading).
|
|
type best struct {
|
|
reading string
|
|
occ int
|
|
firstCh int
|
|
}
|
|
byBase := map[string]best{}
|
|
order := []string{}
|
|
for _, rr := range readings {
|
|
if manualSrcs[rr.Base] {
|
|
continue // the curated seed already owns this src
|
|
}
|
|
cur, ok := byBase[rr.Base]
|
|
if !ok {
|
|
byBase[rr.Base] = best{rr.Reading, rr.Occurrences, rr.FirstChapter}
|
|
order = append(order, rr.Base)
|
|
continue
|
|
}
|
|
// Dominant reading: higher occurrences wins; tie → lexically smaller reading.
|
|
if rr.Occurrences > cur.occ || (rr.Occurrences == cur.occ && rr.Reading < cur.reading) {
|
|
cur.reading, cur.occ = rr.Reading, rr.Occurrences
|
|
}
|
|
if rr.FirstChapter < cur.firstCh { // earliest appearance across readings
|
|
cur.firstCh = rr.FirstChapter
|
|
}
|
|
byBase[rr.Base] = cur
|
|
}
|
|
sort.Strings(order) // deterministic output regardless of input order
|
|
var out []store.GlossaryEntry
|
|
for _, base := range order {
|
|
b := byBase[base]
|
|
class := classifyRubyReading(base, b.reading)
|
|
typ := ""
|
|
if class == rubyClassName {
|
|
typ = "name"
|
|
}
|
|
out = append(out, store.GlossaryEntry{
|
|
Src: base, Dst: "", Type: typ, Status: "auto", Source: "ruby",
|
|
RubyReading: b.reading, RubyClass: class, SinceCh: b.firstCh,
|
|
Confidence: b.occ,
|
|
Note: "ruby auto-candidate (" + class + "); reading is the Polivanov bridge, not a dst — promote before use",
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// AttachRubyAliasesToManual makes the kana READING of a manually-seeded name matchable
|
|
// (D16.4): a manual term 鈴木 that appears in a kana-written chunk as すずき would otherwise
|
|
// miss — rubyToCandidates SKIPS manual bases, so the reading is discarded, a "silently empty"
|
|
// recall hole on the ja acceptance book. For each name-shape reading (all-Han base +
|
|
// all-kana reading — the furigana-name form classifyRubyReading calls rubyClassName) whose
|
|
// base is a manual src, it appends the reading as an ALIAS of that manual entry IN PLACE,
|
|
// deduped. The reading then fires with the ordinary alias disposition — AMBIGUOUS when
|
|
// short/collision-prone (post-checked), so a possible double-reading (強敵→とも "friend") is
|
|
// never blindly trusted; full offline disambiguation is B6. Auto-candidates with no dst stay
|
|
// unmatchable (deliberate). Deterministic: readings are processed sorted, and GlossaryForBook
|
|
// re-sorts aliases on read, so the materialized bank is stable regardless of input order.
|
|
//
|
|
// A ruby reading is auto-derived (not curated), so a reading that would give a FIRING key to
|
|
// two seeded terms with different dst (homophone names 高橋/高梁 both read たかはし) must NOT be
|
|
// attached to either — the reading is genuinely ambiguous between them. External-review MAJOR
|
|
// (homophone first-wins): the previous pass attached-then-blocked, so the alphabetically-first
|
|
// base won the reading as an ALIAS while the other was skipped; at ≥4 kana that alias is not
|
|
// collision-prone → it fired CONFIRMED → every kana occurrence injected as an authoritative
|
|
// (and wrong) rendering for whichever base sorted first (class A2). The key must therefore be
|
|
// decided BEFORE any attach, for ALL its owners at once: a contested firing key (≥2 would-fire
|
|
// owners with different dst over overlapping spoiler windows) skips EVERY proposed attach and
|
|
// logs it (not silent) — the kana form stays unmatchable until the seed disambiguates it with a
|
|
// curated alias (whose own collisions DO fail loud, as an authoring error). Attaching would also
|
|
// trip ApprovedSharedKeyCollisions loud on an alias the operator cannot edit out of the seed.
|
|
//
|
|
// Window/dst-eligibility awareness (review minor): only a dst-bearing owner fires (Materialize
|
|
// builds no surface for a no-dst row), and two owners with the SAME dst (a merge) or NON-overlapping
|
|
// windows (a spoiler handoff) do not contradict — so those are no longer falsely skipped.
|
|
func AttachRubyAliasesToManual(entries []store.GlossaryEntry, readings []store.RubyReading) (skipped []string) {
|
|
idx := map[string][]int{}
|
|
for i := range entries {
|
|
idx[entries[i].Src] = append(idx[entries[i].Src], i)
|
|
}
|
|
dstBearing := func(e *store.GlossaryEntry) bool { return strings.TrimSpace(e.Dst) != "" }
|
|
|
|
// wouldOwn[rk] = set of term indices that would own firing key rk AFTER the proposed
|
|
// attaches: the pre-existing dst-bearing owners (src + eligible aliases already on the seed)
|
|
// plus every deferred (fires) proposal. Only dst-bearing entries fire, so only they can
|
|
// collide. addOwner keeps a set (an index is never paired with itself in contested()).
|
|
wouldOwn := map[string]map[int]bool{}
|
|
addOwner := func(rk string, ti int) {
|
|
if wouldOwn[rk] == nil {
|
|
wouldOwn[rk] = map[int]bool{}
|
|
}
|
|
wouldOwn[rk][ti] = true
|
|
}
|
|
for i := range entries {
|
|
if !dstBearing(&entries[i]) {
|
|
continue
|
|
}
|
|
for _, k := range entryFiringKeys(entries[i]) {
|
|
addOwner(k, i)
|
|
}
|
|
}
|
|
|
|
sorted := append([]store.RubyReading(nil), readings...)
|
|
sort.Slice(sorted, func(i, j int) bool {
|
|
if sorted[i].Base != sorted[j].Base {
|
|
return sorted[i].Base < sorted[j].Base
|
|
}
|
|
return sorted[i].Reading < sorted[j].Reading
|
|
})
|
|
|
|
// Pass 1 — gather proposals. A proposal that would actually FIRE (dst-bearing target + a key
|
|
// past the per-language floor / allow_short) is DEFERRED to the contested-key decision so a
|
|
// homophone key is resolved for all its owners together, never first-wins. An inert attach (no
|
|
// dst, or a sub-floor key) can never fire → attach it immediately; it round-trips the reading
|
|
// but cannot collide.
|
|
type prop struct {
|
|
rk, reading, base string
|
|
ti int
|
|
}
|
|
var deferred []prop
|
|
for _, rr := range sorted {
|
|
targets, ok := idx[rr.Base]
|
|
if !ok {
|
|
continue // the reading's base is not a manual term
|
|
}
|
|
if classifyRubyReading(rr.Base, rr.Reading) != rubyClassName {
|
|
continue // only the all-kana furigana-NAME shape is a match surface; a kanji-bearing gloss/double-reading is a footnote, not an alias
|
|
}
|
|
rk := text.NormalizeSourceKey(rr.Reading)
|
|
for _, ti := range targets {
|
|
e := &entries[ti]
|
|
if rr.Reading == e.Src || hasAliasSurface(e.Aliases, rr.Reading) {
|
|
continue
|
|
}
|
|
fires := dstBearing(e) && rk != "" && (text.SignificantLen(rk) >= minKeyLenFor(rk) || e.AllowShort)
|
|
if !fires {
|
|
e.Aliases = append(e.Aliases, store.GlossaryAlias{Alias: rr.Reading, AliasType: "reading"})
|
|
continue
|
|
}
|
|
deferred = append(deferred, prop{rk: rk, reading: rr.Reading, base: rr.Base, ti: ti})
|
|
addOwner(rk, ti)
|
|
}
|
|
}
|
|
|
|
// contested(rk): among ALL would-fire owners of rk, two render DIFFERENT dst over OVERLAPPING
|
|
// spoiler windows — attaching for any of them would inject a wrong authoritative rendering (and
|
|
// trip the loud shared-key check). Deterministic (owner set → sorted index pairs).
|
|
contested := func(rk string) bool {
|
|
is := make([]int, 0, len(wouldOwn[rk]))
|
|
for i := range wouldOwn[rk] {
|
|
is = append(is, i)
|
|
}
|
|
sort.Ints(is)
|
|
for a := 0; a < len(is); a++ {
|
|
for b := a + 1; b < len(is); b++ {
|
|
ei, ej := entries[is[a]], entries[is[b]]
|
|
if ei.Dst != ej.Dst && windowsOverlap(ei.SinceCh, ei.UntilCh, ej.SinceCh, ej.UntilCh) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Pass 2 — attach every deferred reading whose key is uncontested; skip+log the contested ones
|
|
// (for ALL owners symmetrically, closing the first-wins hole).
|
|
for _, p := range deferred {
|
|
if contested(p.rk) {
|
|
skipped = append(skipped, fmt.Sprintf("%q reading %q (homophone of another seeded term — kana left unmatchable; disambiguate in the seed if needed)", p.base, p.reading))
|
|
continue
|
|
}
|
|
e := &entries[p.ti]
|
|
if hasAliasSurface(e.Aliases, p.reading) { // a same-key sibling proposal already attached this exact surface
|
|
continue
|
|
}
|
|
e.Aliases = append(e.Aliases, store.GlossaryAlias{Alias: p.reading, AliasType: "reading"})
|
|
}
|
|
return skipped
|
|
}
|
|
|
|
// hasAliasSurface reports whether an alias with the given raw surface already exists on a term.
|
|
func hasAliasSurface(aliases []store.GlossaryAlias, surface string) bool {
|
|
for _, a := range aliases {
|
|
if a.Alias == surface {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// SeedLint dry-runs the REAL LoadGlossarySeed fail-louds + the approved shared-key collision check over
|
|
// a glossary seed YAML (a manual seed OR the emitted mined delta), returning nil on a clean seed or a
|
|
// loud error listing every problem (WS3 (д): "a dry run of the real LoadGlossarySeed fail-louds over
|
|
// the delta"). $0, no store, no LLM — it runs the same loader translate would, so a delta that lints clean
|
|
// here is guaranteed loadable, and a shared-key livelock / duplicate / missing-dst is caught BEFORE the
|
|
// bank-mining reseed instead of aborting a paid run.
|
|
func SeedLint(path string) error {
|
|
entries, err := LoadGlossarySeed(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if problems := ApprovedSharedKeyCollisions(entries); len(problems) > 0 {
|
|
return fmt.Errorf("glossary seed %s shared-key collisions:\n - %s", path, strings.Join(problems, "\n - "))
|
|
}
|
|
if bad := GenderVocabViolations(entries); len(bad) > 0 {
|
|
return fmt.Errorf("glossary seed %s unknown gender values (each would SILENTLY inject no gender directive — row 84):\n - %s", path, strings.Join(bad, "\n - "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// knownGenders is the gender vocabulary genderConstraintNote understands. Matched EXACTLY (no trim/case
|
|
// fold), mirroring that switch: a value it does not recognise injects no directive at all, so " male" or
|
|
// "Male" is as silent a no-op as a typo and is flagged the same way.
|
|
var knownGenders = map[string]bool{
|
|
"": true, "male": true, "m": true, "female": true, "f": true, "neuter": true, "n": true, "hidden": true,
|
|
}
|
|
|
|
// GenderVocabViolations lists seed rows whose gender is outside the known vocabulary — the row-84 class where
|
|
// gender:neuter (now valid) was an unnoticed no-op. Deterministic, seed order preserved.
|
|
func GenderVocabViolations(entries []store.GlossaryEntry) []string {
|
|
return problemTexts(genderVocabViolations(entries))
|
|
}
|
|
|
|
// genderVocabViolations is the same check with a subject. ⚠ The message names the RENDERING and the
|
|
// subject must not: this door has no `gender` channel at all, so a fault it cannot fix must never look
|
|
// new — see subjectOf for the refusal that produced.
|
|
func genderVocabViolations(entries []store.GlossaryEntry) []Problem {
|
|
var out []Problem
|
|
for _, e := range entries {
|
|
if !knownGenders[e.Gender] {
|
|
out = append(out, Problem{
|
|
Subject: subjectOf("gender", e.Src, e.Sense, e.SinceCh, e.UntilCh),
|
|
Text: fmt.Sprintf("%s→%s: gender %q is not one of male|female|neuter|hidden", e.Src, e.Dst, e.Gender)})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ruby classifier classes (deterministic HINTS for human promotion).
|
|
const (
|
|
rubyClassName = "name_candidate" // all-Han base + all-kana reading: the furigana-name SHAPE
|
|
rubyClassGloss = "gloss_candidate" // reading carries Han: likely an author double-reading → footnote, NOT a lock
|
|
rubyClassAmbiguous = "ambiguous" // anything else → flag to a human
|
|
)
|
|
|
|
// classifyRubyReading is the deterministic clear-case rule (§8). It NEVER auto-locks:
|
|
// the all-Han+all-kana shape is a name-lock CANDIDATE (it could still be a double-reading
|
|
// like 強敵→とも "friend", indistinguishable without a yomi dictionary — Phase 2), so it is
|
|
// flagged for human confirmation, not locked. A reading that carries kanji is a clear
|
|
// author's double-reading/gloss → a footnote candidate. Over-flagging is safe; a blind
|
|
// lock is a mistranslation. Pure and deterministic.
|
|
func classifyRubyReading(base, reading string) string {
|
|
if runesAnyHan(reading) {
|
|
return rubyClassGloss // the "reading" is itself a word (semantic gloss / double-reading)
|
|
}
|
|
if runesAllHan(base) && runesAllKana(reading) {
|
|
return rubyClassName
|
|
}
|
|
return rubyClassAmbiguous
|
|
}
|
|
|
|
func runesAllHan(s string) bool {
|
|
if s == "" {
|
|
return false
|
|
}
|
|
for _, r := range s {
|
|
if !unicode.Is(unicode.Han, r) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func runesAnyHan(s string) bool {
|
|
for _, r := range s {
|
|
if unicode.Is(unicode.Han, r) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func runesAllKana(s string) bool {
|
|
if s == "" {
|
|
return false
|
|
}
|
|
for _, r := range s {
|
|
if r == 'ー' || r == '・' { // prolonged-sound mark, middle dot — allowed within kana readings
|
|
continue
|
|
}
|
|
if !unicode.Is(unicode.Hiragana, r) && !unicode.Is(unicode.Katakana, r) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// UnverifiedKeyConflicts reports the pairs S14 named: an APPROVED term and an UNSIGNED one that share a
|
|
// firing surface with DIFFERENT renderings and overlapping windows. Both are injected — the approved one
|
|
// as canon, the unsigned one as a working version — so the model receives two contradictory lines about
|
|
// the same surface. That is the likeliest source of a silent degradation in the auto mode, and until
|
|
// pack-20 no diagnostic could see it: ApprovedSharedKeyCollisions and InjectivityCollisions are both
|
|
// approved-only, by construction.
|
|
//
|
|
// It is a WARNING, not a fail-loud, and deliberately so: the unsigned row is engine-produced, and
|
|
// aborting a paid run over the engine's own proposal would be a self-inflicted outage. The trust gate
|
|
// already protects the reader (the approved rendering keeps its precedence); this makes the collision
|
|
// VISIBLE so the owner can reconcile it. Deterministic (sorted keys, sorted pairs).
|
|
func UnverifiedKeyConflicts(entries []store.GlossaryEntry) []string {
|
|
approved := map[string][]int{}
|
|
unsigned := map[string][]int{}
|
|
for i, e := range entries {
|
|
if strings.TrimSpace(e.Dst) == "" {
|
|
continue
|
|
}
|
|
target := unsigned
|
|
if e.Status == "approved" {
|
|
target = approved
|
|
}
|
|
for _, k := range entryFiringKeys(e) {
|
|
target[k] = append(target[k], i)
|
|
}
|
|
}
|
|
var out []string
|
|
for _, k := range slices.Sorted(maps.Keys(approved)) {
|
|
for _, ai := range approved[k] {
|
|
for _, ui := range unsigned[k] {
|
|
ae, ue := entries[ai], entries[ui]
|
|
if ae.Dst == ue.Dst || !windowsOverlap(ae.SinceCh, ae.UntilCh, ue.SinceCh, ue.UntilCh) {
|
|
continue
|
|
}
|
|
out = append(out, fmt.Sprintf("firing key %q: approved %q→%q vs unsigned %s %q→%q (the model is shown two renderings of one surface)",
|
|
k, ae.Src, ae.Dst, ue.Status, ue.Src, ue.Dst))
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// MinedDeltaSeedCollisions returns a human message for every mined-delta entry whose store UNIQUE key
|
|
// (src, sense, since_ch, until_ch) already exists among the seed/ruby entries — the D39.20 deviation-#1
|
|
// crash class. It keys on the FULL uniqueness tuple (not the firing surface, unlike
|
|
// membank.ApprovedSharedKeyCollisions, and regardless of dst — a SAME-dst duplicate crashes the INSERT just the
|
|
// same, yet membank.ApprovedSharedKeyCollisions deliberately skips it). Pure and deterministic (seed order); an
|
|
// empty result means the delta is disjoint from the seed on the uniqueness axis and can be appended safely.
|
|
func MinedDeltaSeedCollisions(seedEntries, mined []store.GlossaryEntry) []string {
|
|
return problemTexts(minedDeltaSeedCollisions(seedEntries, mined))
|
|
}
|
|
|
|
// minedDeltaSeedCollisions is the same check with a subject — the colliding uniqueness tuple, which is
|
|
// what the collision IS, and not the two renderings the message quotes.
|
|
func minedDeltaSeedCollisions(seedEntries, mined []store.GlossaryEntry) []Problem {
|
|
type key struct {
|
|
src, sense string
|
|
since, until int
|
|
}
|
|
seen := map[key]store.GlossaryEntry{}
|
|
for _, e := range seedEntries {
|
|
seen[key{e.Src, e.Sense, e.SinceCh, e.UntilCh}] = e
|
|
}
|
|
var out []Problem
|
|
for _, m := range mined {
|
|
if prior, ok := seen[key{m.Src, m.Sense, m.SinceCh, m.UntilCh}]; ok {
|
|
out = append(out, Problem{
|
|
Subject: subjectOf("seed-delta-collision", m.Src, m.Sense, m.SinceCh, m.UntilCh),
|
|
Text: fmt.Sprintf("src %q (sense %q, window [%d,%d]) is in both the seed (%q→%q) and the mined-delta (%q→%q)",
|
|
m.Src, m.Sense, m.SinceCh, m.UntilCh, prior.Src, prior.Dst, m.Src, m.Dst)})
|
|
}
|
|
}
|
|
return out
|
|
}
|