package miner import ( "fmt" "sort" "strings" "gopkg.in/yaml.v3" "textmachine/backend/internal/lang" "textmachine/backend/internal/seed" "textmachine/backend/internal/store" "textmachine/backend/internal/text" ) // miner_emit.go: the WS3 seed-delta emission (§C2-7) — the miner's owner-facing output. It runs the // default-B detector, applies the subsumption + fragment + type + freq FILTERS (never a raw 13618-dump, // which carries §B2 fragment noise), clusters aliases (tier-1), and emits one Term per NEW entity. // // Discipline: the miner NEVER writes `approved` — every emitted term is a `draft`/`auto` PROPOSAL for the // owner to sign (§C2). In default B the WHAT (dst) is delivered by the banknote (WS4) at the bank-mining sign // boundary, so the miner emits WHICH-only candidates (status:auto, no dst — inert until a dst + owner // promotion). A candidate that clusters with an existing seed surface is an alias-of-existing and is left // for the owner to attach (not emitted as a new entity), so the delta is genuinely new terms. // emitMinFreq is the emission frequency floor (emit_owner_sheets.build_precision30: freq ≥ 5). const emitMinFreq = 5 // emitRankCap is the top-N cap applied to the ranked candidate list BEFORE the emission filters (FL-3), // mirroring the reference's a3[:200] slice (emit_owner_sheets / alias.py:165). It bounds the owner-signed // signature-map volume to the reference; it does NOT touch the miner's WHICH-invariant SET/recall/screen // (those are properties of mr.ranked, computed before emission). const emitRankCap = 200 // EmitRankCap exposes that bound to the other producer of signature-map rows — the bank stop's reverse // section (banknote-only surfaces the miner structurally cannot see). One definition, so the volume the // owner is asked to sign cannot be capped at 200 through one door and uncapped through the other. func EmitRankCap() int { return emitRankCap } // Term is one WHICH candidate the miner proposes (a seed-delta entry). The miner never writes a dst // (default B) or `approved`; the caller persists it via the mined-write path (Source:"mined", status // auto) and the owner signs it at the bank-mining stop. type Term struct { Src string Type string // name|place|title|term (the candidate's first pattern type; "" → term) SinceCh int // first chapter of appearance across the term + its aliases (auto) Freq int Aliases []string // identity-cluster co-surfaces (normalized), sorted; the entity's other surfaces Evidence []string // up to 3 pattern-evidence tags (for the owner sidecar) // Dst is the CONSOLIDATED rendering the terminologist produced over the whole book's contexts (pack-20, // D39.42 п.1). The MINER never fills it — it is attached by the caller after the terminologist runs, and // it is what selects the emission MODE in DeltaYAML (§C2-7). Empty when the role is off or declined the // term: the term then emits status:auto, inert until someone signs it. Dst string } // MineBank runs the full default-B miner over the normalized chunks and emits the alias-clustered, // filtered mined delta (WHICH candidates for owner sign). seed is the current glossary (its surfaces // scope the non-seed filter, the fragment guard, and the alias entity/metadata). rejects is the owner's // declined-term set (normalized src, R1-FL-B): a candidate whose entity touches a rejected surface is // dropped from the delta so a declined term never re-fires the stop (a nil/empty rejects is a no-op — the // emission is byte-identical to before, preserving the frozen parity). Deterministic and $0. func MineBank(chunks []Chunk, contrast *Contrast, seed []store.GlossaryEntry, rejects map[string]bool, cfg Config, pack *lang.Pack) []Term { out, _ := MineBankStats(chunks, contrast, seed, rejects, cfg, pack) return out } // EmissionStats is the funnel BEHIND the emitted delta — how many candidates the detector's alphabet held // and where they were lost. It exists because an empty delta is otherwise unreadable (G10, polygon package // seven): `emitted=0` looks exactly the same whether the book has no new terms or whether a thousand // candidates were ranked and every one of them was cut by the top-N slice, the type filter or the seed. A // stop that cannot tell those apart tells the owner "your book is clean" when it means "I looked through a // keyhole". Pure counters, no behaviour: every number here is read off the same pass that already runs. type EmissionStats struct { Ranked int // the detector's whole ranked candidate set — the "alphabet" AfterCap int // after the top-emitRankCap slice Eligible int // after the emission eligibility filters (type, length, subsumption) SeedSkipped int // clusters dropped as aliases-of-existing (a seed surface is in the cluster) Rejected int // clusters dropped by the owner's reject list Emitted int // what the delta actually carries } // MineBankStats is MineBank plus the funnel counters. Same emission, byte for byte — the counters are // read-only. func MineBankStats(chunks []Chunk, contrast *Contrast, seed []store.GlossaryEntry, rejects map[string]bool, cfg Config, pack *lang.Pack) ([]Term, EmissionStats) { var stats EmissionStats mr := mineDetect(chunks, contrast, cfg, pack) stats.Ranked = len(mr.ranked) // Seed surfaces (normalized) + their metadata for the alias rules and the non-seed / fragment guards. seedSurfaces := map[string]bool{} seedMeta := map[string]aliasSurface{} for _, e := range seed { for _, surf := range append([]string{e.Src}, aliasStrings(e.Aliases)...) { nk := text.NormalizeSourceKey(surf) if nk == "" { continue } seedSurfaces[nk] = true // The primary src carries the dst/gender/type; an alias inherits them (same entity). if _, ok := seedMeta[nk]; !ok { meta := aliasSurface{src: nk, typ: e.Type} if e.Status == "approved" { meta.approvedDst = e.Dst } meta.gender = e.Gender seedMeta[nk] = meta } } } // Qualifying candidate pool (build_precision30 filters), in ranked order (the ranked order is the // representative-selection order below — the top-ranked cluster member owns the aliases). The ranked // list is capped to the top emitRankCap BEFORE the eligibility filters (FL-3), mirroring the // reference's a3[:200] slice (emit_owner_sheets.build_precision30 / alias.py:165): the cap bounds the // owner-signed signature map to the reference's volume, so an eligible candidate below the cap is not // surfaced. WHICH-invariants (the 13618-member SET, recall, the catastrophe screen) are unaffected — // they are properties of mr.ranked itself, not of the capped emission slice. ranked := mr.ranked if len(ranked) > emitRankCap { ranked = ranked[:emitRankCap] } stats.AfterCap = len(ranked) var pool []scoredCand for _, c := range ranked { if !emissionEligible(c, mr.subsumed, seedSurfaces, pack) { continue } pool = append(pool, c) } stats.Eligible = len(pool) // Alias universe = qualifying candidate surfaces ∪ seed surfaces; run tier-1 clustering. surfaces := map[string]aliasSurface{} for _, c := range pool { typ := "" if len(c.Types) > 0 { typ = c.Types[0] } surfaces[c.Src] = aliasSurface{src: c.Src, typ: typ} } for nk, meta := range seedMeta { surfaces[nk] = meta // seed metadata wins (gender/approved_dst) } universe := make([]string, 0, len(surfaces)) for s := range surfaces { universe = append(universe, s) } sort.Strings(universe) ident, _, _ := proposeAliasEdges(surfaces, chunks, seedSurfaces, pack) clusters := clusterAlias(ident, universe) // clusterOf maps a surface to its identity cluster (members); a surface not in a multi-cluster maps // to a singleton. clusterOf := map[string][]string{} for _, cl := range clusters { for _, s := range cl { clusterOf[s] = cl } } var out []Term emitted := map[string]bool{} for _, c := range pool { // ranked order → the top-ranked cluster member is the representative if seedSurfaces[c.Src] || emitted[c.Src] { continue } cl := clusterOf[c.Src] // A cluster touching a seed surface is an alias-of-existing entity → the owner attaches it; skip. if clusterTouches(cl, seedSurfaces) { markEmitted(cl, emitted) stats.SeedSkipped++ continue } // A declined entity (R1-FL-B) is dropped WITH its whole cluster, so it never re-fires the stop: // marking the cluster emitted stops a lower-ranked member from re-emitting the same entity next run // (which would keep the delta non-empty forever). rejects[c.Src] covers a SINGLETON representative // (cl is nil for a candidate in no multi-cluster); clusterTouches covers a rejected alias of a // multi-member cluster. markEmitted(nil,...) is a harmless no-op for the singleton case. if rejects[c.Src] || clusterTouches(cl, rejects) { markEmitted(cl, emitted) stats.Rejected++ continue } var aliases []string if len(cl) > 1 { for _, m := range cl { if m != c.Src && !seedSurfaces[m] { aliases = append(aliases, m) } } sort.Strings(aliases) markEmitted(cl, emitted) // the whole cluster is represented by this one term } else { emitted[c.Src] = true } typ := "term" if len(c.Types) > 0 { typ = c.Types[0] } out = append(out, Term{ Src: c.Src, Type: typ, Freq: c.Freq, Aliases: aliases, Evidence: c.Evidence, SinceCh: entitySinceCh(append([]string{c.Src}, aliases...), chunks), }) } // Deterministic output order: by src (the caller may re-sort, but pin a stable delta). sort.Slice(out, func(i, j int) bool { return out[i].Src < out[j].Src }) stats.Emitted = len(out) return out, stats } // DeltaYAML serializes the mined delta into the seed.Term YAML schema (§C2-7) — the owner-sign artifact // and the `tmctl seed-lint` input. The miner supplies the WHICH; `proposals` (keyed by normalized src) // supplies the WHAT the banknote channel collected during the draft wave, so the owner signs a term that // already carries the translator's proposed rendering instead of a bare source surface. // // Every term stays status:auto — that is the load-bearing line. Attaching a dst delivers EVIDENCE, not // trust: an auto term is outside the CONFIRMED-only injection (D39.2 trust gate), so nothing the model // proposed reaches the book until the owner signs it. Before this join the parsed dst was dropped into // `_` (D39.36) and the owner met the stop holding bare terms, having to invent the Russian for a word // the model had already rendered. // // nil/empty proposals reproduce the previous output BYTE FOR BYTE (WHICH-only, no dst), so a book // running with the channel off is unaffected. Alternative renderings are listed in the note with their // chunk counts — the disagreement IS the reason to sign (one term came back three ways in the 25.07 // mini-run). It reuses the EXISTING seed.Term schema (no new fields — §3(б)); evidence / zones live in // the sidecar sign-map. The output is guaranteed loadable by loadGlossarySeed (status:auto may lack a // dst); seed-lint proves it against the real loader. func DeltaYAML(mined []Term, proposals map[string][]DstProposal) (string, error) { sf := seed.File{} for _, m := range mined { st := seed.Term{Src: m.Src, Type: m.Type, Status: "auto", SinceCh: m.SinceCh} for _, a := range m.Aliases { st.Aliases = append(st.Aliases, seed.Alias{Alias: a, Type: "mined"}) } note := "" if len(m.Evidence) > 0 { note = "mined WHICH candidate; evidence: " + fmt.Sprint(m.Evidence) } // The join key is the miner's own normalized surface, so a proposal written with different // orthography still lands on its term (the same normalization the seed-surface exclusion uses). if props := proposals[text.NormalizeSourceKey(m.Src)]; len(props) > 0 { st.Dst = props[0].Dst note = appendProposalNote(note, props) } // TWO-MODE EMISSION (§C2-7, ratified and until pack-20 unbuilt): a term that arrives with a // CONSOLIDATED canon proposal emits `draft` — the mode whose injection carries ⟨проверить⟩ — while a // term with no dst stays `auto`, inert. The mode turns on the CONSOLIDATED dst only, never on a raw // per-chunk banknote guess: a proposal one chunk's translator improvised is evidence for the owner, // and lifting it to draft would let the model's first thought become the book's working canon. Both // modes are still PROPOSALS — neither is `approved`, and the miner has no path to write that word. if m.Dst != "" { st.Dst, st.Status = m.Dst, "draft" note = appendConsolidatedNote(note) } st.Note = note sf.Terms = append(sf.Terms, st) } b, err := yaml.Marshal(sf) if err != nil { return "", fmt.Errorf("miner: marshal mined delta: %w", err) } return string(b), nil } // appendProposalNote documents WHERE the attached dst came from and what else was proposed. The // provenance line is not decoration: a dst in a sign map that does not say "a model proposed this, // nobody approved it" invites the owner to read it as already-canon. func appendProposalNote(note string, props []DstProposal) string { var b strings.Builder if note != "" { b.WriteString(note) b.WriteString("; ") } fmt.Fprintf(&b, "dst PROPOSED by the translator (banknote, %d chunk(s)) — NOT approved, sign or replace it", props[0].Chunks) if len(props) > 1 { b.WriteString("; other proposals: ") for i, p := range props[1:] { if i > 0 { b.WriteString(", ") } fmt.Fprintf(&b, "%q ×%d", p.Dst, p.Chunks) } } return b.String() } // appendConsolidatedNote records that the dst on this row is the TERMINOLOGIST's consolidation, not a // per-chunk guess and not a signature. The distinction is the whole trust story of the auto mode: a // `draft` row reaches the wire with an unverified marker, so the sign map must say, in the artifact // itself, who produced the rendering and that nobody has approved it. func appendConsolidatedNote(note string) string { const s = "dst CONSOLIDATED by the terminologist over the whole book's contexts — a PROPOSAL (status:draft), not approved" if note == "" { return s } return note + "; " + s } // emissionEligible applies the build_precision30 filters: type ∈ {name,place,title}, freq ≥ 5, src not // subsumed, len ≥ 2 rune, not a boundary fragment. func emissionEligible(c scoredCand, subsumed, seedSurfaces map[string]bool, pack *lang.Pack) bool { if !hasAnyType(c.Types, "name", "place", "title") { return false } if c.Freq < emitMinFreq || subsumed[c.Src] || runeLen(c.Src) < 2 { return false } if isFragment(c.Src, seedSurfaces, pack) { return false } return true } func hasAnyType(types []string, want ...string) bool { for _, t := range types { for _, w := range want { if t == w { return true } } } return false } // clusterTouches reports whether any surface of the identity cluster is in `set` — used both for the seed // surfaces (an alias-of-existing entity) and the reject surfaces (a declined entity), which are treated the // same way at emission: the whole cluster is suppressed. func clusterTouches(cl []string, set map[string]bool) bool { for _, s := range cl { if set[s] { return true } } return false } func markEmitted(cl []string, emitted map[string]bool) { for _, s := range cl { emitted[s] = true } } // entitySinceCh is the earliest chapter any of the entity's surfaces appears in (canon.since_ch = first // chapter of appearance). func entitySinceCh(surfaces []string, chunks []Chunk) int { min := 0 for _, s := range surfaces { for ch := range candidateChapters(s, chunks) { if min == 0 || ch < min { min = ch } } } return min } func aliasStrings(as []store.GlossaryAlias) []string { out := make([]string, len(as)) for i, a := range as { out[i] = a.Alias } return out } // DstProposal is one PROPOSED rendering of a mined term, delivered by the banknote channel (WS4) and // attached to the signature map at sign time. It is evidence for the owner, never a canon: the emitted // term keeps status:auto, so a proposal enters the bank only through a signature. // // Chunks is how many chunks proposed this exact rendering — the disagreement signal. One term coming // back with three renderings is the drift a canon closes, and hiding that behind a single "winner" // would throw away the reason to sign it at all. type DstProposal struct { Dst string Type string Chunks int }