textmachine/backend/internal/miner/miner_emit.go

package miner

import (
	"fmt"
	"sort"
	"strings"

	"gopkg.in/yaml.v3"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/seed"
	"textmachine/backend/internal/store"
	"textmachine/backend/internal/text"
)

// miner_emit.go: the WS3 seed-delta emission (§C2-7) — the miner's owner-facing output. It runs the
// default-B detector, applies the subsumption + fragment + type + freq FILTERS (never a raw 13618-dump,
// which carries §B2 fragment noise), clusters aliases (tier-1), and emits one Term per NEW entity.
//
// Discipline: the miner NEVER writes `approved` — every emitted term is a `draft`/`auto` PROPOSAL for the
// owner to sign (§C2). In default B the WHAT (dst) is delivered by the banknote (WS4) at the bank-mining sign
// boundary, so the miner emits WHICH-only candidates (status:auto, no dst — inert until a dst + owner
// promotion). A candidate that clusters with an existing seed surface is an alias-of-existing and is left
// for the owner to attach (not emitted as a new entity), so the delta is genuinely new terms.

// emitMinFreq is the emission frequency floor (emit_owner_sheets.build_precision30: freq ≥ 5).
const emitMinFreq = 5

// emitRankCap is the top-N cap applied to the ranked candidate list BEFORE the emission filters (FL-3),
// mirroring the reference's a3[:200] slice (emit_owner_sheets / alias.py:165). It bounds the owner-signed
// signature-map volume to the reference; it does NOT touch the miner's WHICH-invariant SET/recall/screen
// (those are properties of mr.ranked, computed before emission).
const emitRankCap = 200

// EmitRankCap exposes that bound to the other producer of signature-map rows — the bank stop's reverse
// section (banknote-only surfaces the miner structurally cannot see). One definition, so the volume the
// owner is asked to sign cannot be capped at 200 through one door and uncapped through the other.
func EmitRankCap() int { return emitRankCap }

// Term is one WHICH candidate the miner proposes (a seed-delta entry). The miner never writes a dst
// (default B) or `approved`; the caller persists it via the mined-write path (Source:"mined", status
// auto) and the owner signs it at the bank-mining stop.
type Term struct {
	Src      string
	Type     string // name|place|title|term (the candidate's first pattern type; "" → term)
	SinceCh  int    // first chapter of appearance across the term + its aliases (auto)
	Freq     int
	Aliases  []string // identity-cluster co-surfaces (normalized), sorted; the entity's other surfaces
	Evidence []string // up to 3 pattern-evidence tags (for the owner sidecar)
	// Dst is the CONSOLIDATED rendering the terminologist produced over the whole book's contexts (pack-20,
	// D39.42 п.1). The MINER never fills it — it is attached by the caller after the terminologist runs, and
	// it is what selects the emission MODE in DeltaYAML (§C2-7). Empty when the role is off or declined the
	// term: the term then emits status:auto, inert until someone signs it.
	Dst string
	// Gender is the entity's grammatical gender (male|female|neuter), the datum the injection turns into a
	// DC3 directive. Like Dst the MINER never fills it — it is stamped by the caller from the §2
	// classifier's answer (backlog row 210, the first automatic producer this field has ever had). Empty
	// for every term the classifier answered `none` for, did not answer, or never saw, which is the state
	// the whole field was in before: a gender reached the bank only from a hand-written seed.
	Gender string
}

// MineBank runs the full default-B miner over the normalized chunks and emits the alias-clustered,
// filtered mined delta (WHICH candidates for owner sign). seed is the current glossary (its surfaces
// scope the non-seed filter, the fragment guard, and the alias entity/metadata). rejects is the owner's
// declined-term set (normalized src, R1-FL-B): a candidate whose entity touches a rejected surface is
// dropped from the delta, so a declined term never re-enters a map or the auto-bank (a nil/empty rejects
// is a no-op — the emission is byte-identical to before, preserving the frozen parity). Deterministic
// and $0.
func MineBank(chunks []Chunk, contrast *Contrast, seed []store.GlossaryEntry, rejects map[string]bool, cfg Config, pack *lang.Pack) []Term {
	out, _ := MineBankStats(chunks, contrast, seed, rejects, cfg, pack)
	return out
}

// EmissionStats is the funnel BEHIND the emitted delta — how many candidates the detector's alphabet held
// and where they were lost. It exists because an empty delta is otherwise unreadable (G10, polygon package
// seven): `emitted=0` looks exactly the same whether the book has no new terms or whether a thousand
// candidates were ranked and every one of them was cut by the top-N slice, the type filter or the seed. A
// stop that cannot tell those apart tells the owner "your book is clean" when it means "I looked through a
// keyhole". Pure counters, no behaviour: every number here is read off the same pass that already runs.
type EmissionStats struct {
	Ranked      int // the detector's whole ranked candidate set — the "alphabet"
	AfterCap    int // after the top-emitRankCap slice
	Eligible    int // after the emission eligibility filters (type, length, subsumption)
	SeedSkipped int // clusters dropped as aliases-of-existing (a seed surface is in the cluster)
	Rejected    int // clusters dropped by the owner's reject list
	Emitted     int // what the delta actually carries
}

// MineBankStats is MineBank plus the funnel counters. Same emission, byte for byte — the counters are
// read-only.
func MineBankStats(chunks []Chunk, contrast *Contrast, seed []store.GlossaryEntry, rejects map[string]bool, cfg Config, pack *lang.Pack) ([]Term, EmissionStats) {
	var stats EmissionStats
	mr := mineDetect(chunks, contrast, cfg, pack)
	stats.Ranked = len(mr.ranked)

	// Seed surfaces (normalized) + their metadata for the alias rules and the non-seed / fragment guards.
	seedSurfaces := map[string]bool{}
	seedMeta := map[string]aliasSurface{}
	for _, e := range seed {
		for _, surf := range append([]string{e.Src}, aliasStrings(e.Aliases)...) {
			nk := text.NormalizeSourceKey(surf)
			if nk == "" {
				continue
			}
			seedSurfaces[nk] = true
			// The primary src carries the dst/gender/type; an alias inherits them (same entity).
			if _, ok := seedMeta[nk]; !ok {
				meta := aliasSurface{src: nk, typ: e.Type}
				if e.Status == "approved" {
					meta.approvedDst = e.Dst
				}
				meta.gender = e.Gender
				seedMeta[nk] = meta
			}
		}
	}

	// Qualifying candidate pool (build_precision30 filters), in ranked order (the ranked order is the
	// representative-selection order below — the top-ranked cluster member owns the aliases). The ranked
	// list is capped to the top emitRankCap BEFORE the eligibility filters (FL-3), mirroring the
	// reference's a3[:200] slice (emit_owner_sheets.build_precision30 / alias.py:165): the cap bounds the
	// owner-signed signature map to the reference's volume, so an eligible candidate below the cap is not
	// surfaced. WHICH-invariants (the 13618-member SET, recall, the catastrophe screen) are unaffected —
	// they are properties of mr.ranked itself, not of the capped emission slice.
	ranked := mr.ranked
	if len(ranked) > emitRankCap {
		ranked = ranked[:emitRankCap]
	}
	stats.AfterCap = len(ranked)
	var pool []scoredCand
	for _, c := range ranked {
		if !emissionEligible(c, mr.subsumed, seedSurfaces, pack) {
			continue
		}
		pool = append(pool, c)
	}
	stats.Eligible = len(pool)

	// Alias universe = qualifying candidate surfaces ∪ seed surfaces; run tier-1 clustering.
	surfaces := map[string]aliasSurface{}
	for _, c := range pool {
		typ := ""
		if len(c.Types) > 0 {
			typ = c.Types[0]
		}
		surfaces[c.Src] = aliasSurface{src: c.Src, typ: typ}
	}
	for nk, meta := range seedMeta {
		surfaces[nk] = meta // seed metadata wins (gender/approved_dst)
	}
	universe := make([]string, 0, len(surfaces))
	for s := range surfaces {
		universe = append(universe, s)
	}
	sort.Strings(universe)
	ident, _, _ := proposeAliasEdges(surfaces, chunks, seedSurfaces, pack)
	clusters := clusterAlias(ident, universe)

	// clusterOf maps a surface to its identity cluster (members); a surface not in a multi-cluster maps
	// to a singleton.
	clusterOf := map[string][]string{}
	for _, cl := range clusters {
		for _, s := range cl {
			clusterOf[s] = cl
		}
	}

	var out []Term
	emitted := map[string]bool{}
	for _, c := range pool { // ranked order → the top-ranked cluster member is the representative
		if seedSurfaces[c.Src] || emitted[c.Src] {
			continue
		}
		cl := clusterOf[c.Src]
		// A cluster touching a seed surface is an alias-of-existing entity → the owner attaches it; skip.
		if clusterTouches(cl, seedSurfaces) {
			markEmitted(cl, emitted)
			stats.SeedSkipped++
			continue
		}
		// A declined entity (R1-FL-B) is dropped WITH its whole cluster, so no spelling of it re-enters a
		// map: marking the cluster emitted stops a lower-ranked member from re-emitting the same entity
		// next run — the owner declined the ENTITY, not one surface. rejects[c.Src] covers a SINGLETON representative
		// (cl is nil for a candidate in no multi-cluster); clusterTouches covers a rejected alias of a
		// multi-member cluster. markEmitted(nil,...) is a harmless no-op for the singleton case.
		if rejects[c.Src] || clusterTouches(cl, rejects) {
			markEmitted(cl, emitted)
			stats.Rejected++
			continue
		}
		var aliases []string
		if len(cl) > 1 {
			for _, m := range cl {
				if m != c.Src && !seedSurfaces[m] {
					aliases = append(aliases, m)
				}
			}
			sort.Strings(aliases)
			markEmitted(cl, emitted) // the whole cluster is represented by this one term
		} else {
			emitted[c.Src] = true
		}
		typ := "term"
		if len(c.Types) > 0 {
			typ = c.Types[0]
		}
		out = append(out, Term{
			Src: c.Src, Type: typ, Freq: c.Freq, Aliases: aliases, Evidence: c.Evidence,
			SinceCh: entitySinceCh(append([]string{c.Src}, aliases...), chunks),
		})
	}
	// Deterministic output order: by src (the caller may re-sort, but pin a stable delta).
	sort.Slice(out, func(i, j int) bool { return out[i].Src < out[j].Src })
	stats.Emitted = len(out)
	return out, stats
}

// DeltaYAML serializes the mined delta into the seed.Term YAML schema (§C2-7) — the owner-sign artifact
// and the `tmctl seed-lint` input. The miner supplies the WHICH; `proposals` (keyed by normalized src)
// supplies the WHAT the banknote channel collected during the draft wave, so the owner signs a term that
// already carries the translator's proposed rendering instead of a bare source surface.
//
// Every term stays status:auto — that is the load-bearing line. Attaching a dst delivers EVIDENCE, not
// trust: an auto term is outside the CONFIRMED-only injection (D39.2 trust gate), so nothing the model
// proposed reaches the book until the owner signs it. Before this join the parsed dst was dropped into
// `_` (D39.36) and the owner met the stop holding bare terms, having to invent the Russian for a word
// the model had already rendered.
//
// nil/empty proposals reproduce the previous output BYTE FOR BYTE (WHICH-only, no dst), so a book
// running with the channel off is unaffected. Alternative renderings are listed in the note with their
// chunk counts — the disagreement IS the reason to sign (one term came back three ways in the 25.07
// mini-run). It reuses the EXISTING seed.Term schema (no new fields — §3(б)); evidence / zones live in
// the sidecar sign-map. The output is guaranteed loadable by loadGlossarySeed (status:auto may lack a
// dst); seed-lint proves it against the real loader.
//
// Two renderings share this one construction: DeltaYAML (the bare document — the auto-bank file) and
// the signature map, which wraps DeltaFile's result in the seam envelope (membank.RenderSignatureMap).
func DeltaYAML(mined []Term, proposals map[string][]DstProposal) (string, error) {
	b, err := yaml.Marshal(DeltaFile(mined, proposals))
	if err != nil {
		return "", fmt.Errorf("miner: marshal mined delta: %w", err)
	}
	return string(b), nil
}

// DeltaFile builds the mined delta as the seed document itself — see DeltaYAML for the contract.
func DeltaFile(mined []Term, proposals map[string][]DstProposal) seed.File {
	sf := seed.File{}
	for _, m := range mined {
		st := seed.Term{Src: m.Src, Type: m.Type, Gender: m.Gender, Status: "auto", SinceCh: m.SinceCh}
		for _, a := range m.Aliases {
			st.Aliases = append(st.Aliases, seed.Alias{Alias: a, Type: "mined"})
		}
		note := ""
		if len(m.Evidence) > 0 {
			note = "mined WHICH candidate; evidence: " + fmt.Sprint(m.Evidence)
		}
		// The join key is the miner's own normalized surface, so a proposal written with different
		// orthography still lands on its term (the same normalization the seed-surface exclusion uses).
		if props := proposals[text.NormalizeSourceKey(m.Src)]; len(props) > 0 {
			// The dst comes from a proposal made ON THIS SURFACE. A rendering that reached the term through an
			// ALIAS of its cluster is evidence — it is listed in the note with the surface that proposed it —
			// but the miner's clustering is itself unverified (measured: a surname cluster holding a clan
			// title), and promoting an alias's word to the term's own rendering is the clustering DECISION,
			// which belongs to the owner's signature and to nothing else.
			if props[0].Via == "" {
				st.Dst = props[0].Dst
			}
			note = appendProposalNote(note, props)
		}
		// TWO-MODE EMISSION (§C2-7, ratified and until pack-20 unbuilt): a term that arrives with a
		// CONSOLIDATED canon proposal emits `draft` — the mode whose injection carries a rendering — while a
		// term with no dst stays `auto`, inert. The mode turns on the CONSOLIDATED dst only, never on a raw
		// per-chunk banknote guess: a proposal one chunk's translator improvised is evidence for the owner,
		// and lifting it to draft would let the model's first thought become the book's working canon. Both
		// modes are still PROPOSALS — neither is `approved`, and the miner has no path to write that word.
		if m.Dst != "" {
			st.Dst, st.Status = m.Dst, "draft"
			note = appendConsolidatedNote(note)
		}
		st.Note = note
		sf.Terms = append(sf.Terms, st)
	}
	return sf
}

// appendProposalNote documents WHERE the attached dst came from and what else was proposed. The
// provenance line is not decoration: a dst in a sign map that does not say "a model proposed this,
// nobody approved it" invites the owner to read it as already-canon.
func appendProposalNote(note string, props []DstProposal) string {
	var b strings.Builder
	if note != "" {
		b.WriteString(note)
		b.WriteString("; ")
	}
	if props[0].Via != "" {
		// Nothing was proposed on this surface itself: every rendering came through an alias, so the row
		// carries no dst and the note says who proposed what for which surface.
		b.WriteString("dst PROPOSED only through cluster alias(es) — NOT attached, sign or replace it: ")
		for i, p := range props {
			if i > 0 {
				b.WriteString(", ")
			}
			fmt.Fprintf(&b, "%q ×%d%s", p.Dst, p.Chunks, viaNote(p.Via))
		}
		return b.String()
	}
	fmt.Fprintf(&b, "dst PROPOSED by the translator (banknote, %d chunk(s)) — NOT approved, sign or replace it",
		props[0].Chunks)
	if len(props) > 1 {
		b.WriteString("; other proposals: ")
		for i, p := range props[1:] {
			if i > 0 {
				b.WriteString(", ")
			}
			fmt.Fprintf(&b, "%q ×%d%s", p.Dst, p.Chunks, viaNote(p.Via))
		}
	}
	return b.String()
}

// viaNote renders the alias provenance of a proposal, or "" for one made on the term's own surface.
func viaNote(via string) string {
	if via == "" {
		return ""
	}
	return " [proposed for " + via + "]"
}

// appendConsolidatedNote records that the dst on this row is the TERMINOLOGIST's consolidation, not a
// per-chunk guess and not a signature. The distinction is the whole trust story of the auto mode: a
// `draft` row reaches the wire with an unverified marker, so the sign map must say, in the artifact
// itself, who produced the rendering and that nobody has approved it.
func appendConsolidatedNote(note string) string {
	const s = "dst CONSOLIDATED by the terminologist over the whole book's contexts — a PROPOSAL (status:draft), not approved"
	if note == "" {
		return s
	}
	return note + "; " + s
}

// emissionEligible applies the build_precision30 filters: type ∈ {name,place,title}, freq ≥ 5, src not
// subsumed, len ≥ 2 rune, not a boundary fragment.
func emissionEligible(c scoredCand, subsumed, seedSurfaces map[string]bool, pack *lang.Pack) bool {
	if !hasAnyType(c.Types, "name", "place", "title") {
		return false
	}
	if c.Freq < emitMinFreq || subsumed[c.Src] || runeLen(c.Src) < 2 {
		return false
	}
	if isFragment(c.Src, seedSurfaces, pack) {
		return false
	}
	return true
}

func hasAnyType(types []string, want ...string) bool {
	for _, t := range types {
		for _, w := range want {
			if t == w {
				return true
			}
		}
	}
	return false
}

// clusterTouches reports whether any surface of the identity cluster is in `set` — used both for the seed
// surfaces (an alias-of-existing entity) and the reject surfaces (a declined entity), which are treated the
// same way at emission: the whole cluster is suppressed.
func clusterTouches(cl []string, set map[string]bool) bool {
	for _, s := range cl {
		if set[s] {
			return true
		}
	}
	return false
}

func markEmitted(cl []string, emitted map[string]bool) {
	for _, s := range cl {
		emitted[s] = true
	}
}

// entitySinceCh is the earliest chapter any of the entity's surfaces appears in (canon.since_ch = first
// chapter of appearance).
func entitySinceCh(surfaces []string, chunks []Chunk) int {
	min := 0
	for _, s := range surfaces {
		for ch := range candidateChapters(s, chunks) {
			if min == 0 || ch < min {
				min = ch
			}
		}
	}
	return min
}

func aliasStrings(as []store.GlossaryAlias) []string {
	out := make([]string, len(as))
	for i, a := range as {
		out[i] = a.Alias
	}
	return out
}

// DstProposal is one PROPOSED rendering of a mined term, delivered by the banknote channel (WS4) and
// attached to the signature map at sign time. It is evidence for the owner, never a canon: the emitted
// term keeps status:auto, so a proposal enters the bank only through a signature.
//
// Chunks is how many chunks proposed this exact rendering — the disagreement signal. One term coming
// back with three renderings is the drift a canon closes, and hiding that behind a single "winner"
// would throw away the reason to sign it at all.
type DstProposal struct {
	Dst    string
	Type   string
	Chunks int
	// Via names the surface that actually proposed this rendering when it is not the term's own — an ALIAS
	// the miner clustered onto it. It travels into the note because the alternative is a sign map that
	// presents a rendering proposed FOR ANOTHER SURFACE as a rendering of this one, and the clustering that
	// joined them is itself unverified (the polygon's surname cluster holding a clan title).
	Via string
}