package pipeline import ( "fmt" "sort" "gopkg.in/yaml.v3" "textmachine/backend/internal/lang" "textmachine/backend/internal/store" ) // miner_emit.go: the WS3 seed-delta emission (§C2-7) — the miner's owner-facing output. It runs the // default-B detector, applies the subsumption + fragment + type + freq FILTERS (never a raw 13618-dump, // which carries §B2 fragment noise), clusters aliases (tier-1), and emits one MinedTerm per NEW entity. // // Discipline: the miner NEVER writes `approved` — every emitted term is a `draft`/`auto` PROPOSAL for the // owner to sign (§C2). In default B the WHAT (dst) is delivered by the banknote (WS4) at the W1.5 sign // boundary, so the miner emits WHICH-only candidates (status:auto, no dst — inert until a dst + owner // promotion). A candidate that clusters with an existing seed surface is an alias-of-existing and is left // for the owner to attach (not emitted as a new entity), so the delta is genuinely new terms. // emitMinFreq is the emission frequency floor (emit_owner_sheets.build_precision30: freq ≥ 5). const emitMinFreq = 5 // emitRankCap is the top-N cap applied to the ranked candidate list BEFORE the emission filters (FL-3), // mirroring the reference's a3[:200] slice (emit_owner_sheets / alias.py:165). It bounds the owner-signed // signature-map volume to the reference; it does NOT touch the miner's WHICH-invariant SET/recall/screen // (those are properties of mr.ranked, computed before emission). const emitRankCap = 200 // MinedTerm is one WHICH candidate the miner proposes (a seed-delta entry). The miner never writes a dst // (default B) or `approved`; the caller persists it via the mined-write path (Source:"mined", status // auto) and the owner signs it at W1.5. type MinedTerm struct { Src string Type string // name|place|title|term (the candidate's first pattern type; "" → term) SinceCh int // first chapter of appearance across the term + its aliases (auto) Freq int Aliases []string // identity-cluster co-surfaces (normalized), sorted; the entity's other surfaces Evidence []string // up to 3 pattern-evidence tags (for the owner sidecar) } // MineBank runs the full default-B miner over the normalized chunks and emits the alias-clustered, // filtered mined delta (WHICH candidates for owner sign). seed is the current glossary (its surfaces // scope the non-seed filter, the fragment guard, and the alias entity/metadata). Deterministic and $0. func MineBank(chunks []MinerChunk, contrast *Contrast, seed []store.GlossaryEntry, cfg MinerConfig, pack *lang.Pack) []MinedTerm { mr := mineDetect(chunks, contrast, cfg, pack) // Seed surfaces (normalized) + their metadata for the alias rules and the non-seed / fragment guards. seedSurfaces := map[string]bool{} seedMeta := map[string]aliasSurface{} for _, e := range seed { for _, surf := range append([]string{e.Src}, aliasStrings(e.Aliases)...) { nk := normalizeSourceKey(surf) if nk == "" { continue } seedSurfaces[nk] = true // The primary src carries the dst/gender/type; an alias inherits them (same entity). if _, ok := seedMeta[nk]; !ok { meta := aliasSurface{src: nk, typ: e.Type} if e.Status == "approved" { meta.approvedDst = e.Dst } meta.gender = e.Gender seedMeta[nk] = meta } } } // Qualifying candidate pool (build_precision30 filters), in ranked order (the ranked order is the // representative-selection order below — the top-ranked cluster member owns the aliases). The ranked // list is capped to the top emitRankCap BEFORE the eligibility filters (FL-3), mirroring the // reference's a3[:200] slice (emit_owner_sheets.build_precision30 / alias.py:165): the cap bounds the // owner-signed signature map to the reference's volume, so an eligible candidate below the cap is not // surfaced. WHICH-invariants (the 13618-member SET, recall, the catastrophe screen) are unaffected — // they are properties of mr.ranked itself, not of the capped emission slice. ranked := mr.ranked if len(ranked) > emitRankCap { ranked = ranked[:emitRankCap] } var pool []ScoredCand for _, c := range ranked { if !emissionEligible(c, mr.subsumed, seedSurfaces, pack) { continue } pool = append(pool, c) } // Alias universe = qualifying candidate surfaces ∪ seed surfaces; run tier-1 clustering. surfaces := map[string]aliasSurface{} for _, c := range pool { typ := "" if len(c.Types) > 0 { typ = c.Types[0] } surfaces[c.Src] = aliasSurface{src: c.Src, typ: typ} } for nk, meta := range seedMeta { surfaces[nk] = meta // seed metadata wins (gender/approved_dst) } universe := make([]string, 0, len(surfaces)) for s := range surfaces { universe = append(universe, s) } sort.Strings(universe) ident, _, _ := proposeAliasEdges(surfaces, chunks, seedSurfaces, pack) clusters := clusterAlias(ident, universe) // clusterOf maps a surface to its identity cluster (members); a surface not in a multi-cluster maps // to a singleton. clusterOf := map[string][]string{} for _, cl := range clusters { for _, s := range cl { clusterOf[s] = cl } } var out []MinedTerm emitted := map[string]bool{} for _, c := range pool { // ranked order → the top-ranked cluster member is the representative if seedSurfaces[c.Src] || emitted[c.Src] { continue } cl := clusterOf[c.Src] // A cluster touching a seed surface is an alias-of-existing entity → the owner attaches it; skip. if clusterHasSeed(cl, seedSurfaces) { markEmitted(cl, emitted) continue } var aliases []string if len(cl) > 1 { for _, m := range cl { if m != c.Src && !seedSurfaces[m] { aliases = append(aliases, m) } } sort.Strings(aliases) markEmitted(cl, emitted) // the whole cluster is represented by this one term } else { emitted[c.Src] = true } typ := "term" if len(c.Types) > 0 { typ = c.Types[0] } out = append(out, MinedTerm{ Src: c.Src, Type: typ, Freq: c.Freq, Aliases: aliases, Evidence: c.Evidence, SinceCh: entitySinceCh(append([]string{c.Src}, aliases...), chunks), }) } // Deterministic output order: by src (the caller may re-sort, but pin a stable delta). sort.Slice(out, func(i, j int) bool { return out[i].Src < out[j].Src }) return out } // MinedDeltaYAML serializes the mined delta into the seedTerm YAML schema (§C2-7) — the owner-sign // artifact and the `tmctl seed-lint` input. Every term is status:auto with NO dst (WHICH-only, default // B; the WHAT is delivered by the banknote at W1.5). It reuses the EXISTING seedTerm schema (no new // fields — §3(б)); evidence / zones live in the sidecar sign-map, not the seed. The output is guaranteed // loadable by loadGlossarySeed (status:auto may lack a dst); seed-lint proves it against the real loader. func MinedDeltaYAML(mined []MinedTerm) (string, error) { sf := seedFile{} for _, m := range mined { st := seedTerm{Src: m.Src, Type: m.Type, Status: "auto", SinceCh: m.SinceCh} for _, a := range m.Aliases { st.Aliases = append(st.Aliases, seedAlias{Alias: a, Type: "mined"}) } if len(m.Evidence) > 0 { st.Note = "mined WHICH candidate; evidence: " + fmt.Sprint(m.Evidence) } sf.Terms = append(sf.Terms, st) } b, err := yaml.Marshal(sf) if err != nil { return "", fmt.Errorf("pipeline: marshal mined delta: %w", err) } return string(b), nil } // emissionEligible applies the build_precision30 filters: type ∈ {name,place,title}, freq ≥ 5, src not // subsumed, len ≥ 2 rune, not a boundary fragment. func emissionEligible(c ScoredCand, subsumed, seedSurfaces map[string]bool, pack *lang.Pack) bool { if !hasAnyType(c.Types, "name", "place", "title") { return false } if c.Freq < emitMinFreq || subsumed[c.Src] || runeLen(c.Src) < 2 { return false } if isFragment(c.Src, seedSurfaces, pack) { return false } return true } func hasAnyType(types []string, want ...string) bool { for _, t := range types { for _, w := range want { if t == w { return true } } } return false } func clusterHasSeed(cl []string, seedSurfaces map[string]bool) bool { for _, s := range cl { if seedSurfaces[s] { return true } } return false } func markEmitted(cl []string, emitted map[string]bool) { for _, s := range cl { emitted[s] = true } } // entitySinceCh is the earliest chapter any of the entity's surfaces appears in (canon.since_ch = first // chapter of appearance). func entitySinceCh(surfaces []string, chunks []MinerChunk) int { min := 0 for _, s := range surfaces { for ch := range candidateChapters(s, chunks) { if min == 0 || ch < min { min = ch } } } return min } func aliasStrings(as []store.GlossaryAlias) []string { out := make([]string, len(as)) for i, a := range as { out[i] = a.Alias } return out }