textmachine/backend/internal/miner/miner_emit.go

package miner

import (
	"fmt"
	"sort"

	"gopkg.in/yaml.v3"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/seed"
	"textmachine/backend/internal/store"
	"textmachine/backend/internal/text"
)

// miner_emit.go: the WS3 seed-delta emission (§C2-7) — the miner's owner-facing output. It runs the
// default-B detector, applies the subsumption + fragment + type + freq FILTERS (never a raw 13618-dump,
// which carries §B2 fragment noise), clusters aliases (tier-1), and emits one Term per NEW entity.
//
// Discipline: the miner NEVER writes `approved` — every emitted term is a `draft`/`auto` PROPOSAL for the
// owner to sign (§C2). In default B the WHAT (dst) is delivered by the banknote (WS4) at the bank-mining sign
// boundary, so the miner emits WHICH-only candidates (status:auto, no dst — inert until a dst + owner
// promotion). A candidate that clusters with an existing seed surface is an alias-of-existing and is left
// for the owner to attach (not emitted as a new entity), so the delta is genuinely new terms.

// emitMinFreq is the emission frequency floor (emit_owner_sheets.build_precision30: freq ≥ 5).
const emitMinFreq = 5

// emitRankCap is the top-N cap applied to the ranked candidate list BEFORE the emission filters (FL-3),
// mirroring the reference's a3[:200] slice (emit_owner_sheets / alias.py:165). It bounds the owner-signed
// signature-map volume to the reference; it does NOT touch the miner's WHICH-invariant SET/recall/screen
// (those are properties of mr.ranked, computed before emission).
const emitRankCap = 200

// Term is one WHICH candidate the miner proposes (a seed-delta entry). The miner never writes a dst
// (default B) or `approved`; the caller persists it via the mined-write path (Source:"mined", status
// auto) and the owner signs it at the bank-mining stop.
type Term struct {
	Src      string
	Type     string // name|place|title|term (the candidate's first pattern type; "" → term)
	SinceCh  int    // first chapter of appearance across the term + its aliases (auto)
	Freq     int
	Aliases  []string // identity-cluster co-surfaces (normalized), sorted; the entity's other surfaces
	Evidence []string // up to 3 pattern-evidence tags (for the owner sidecar)
}

// MineBank runs the full default-B miner over the normalized chunks and emits the alias-clustered,
// filtered mined delta (WHICH candidates for owner sign). seed is the current glossary (its surfaces
// scope the non-seed filter, the fragment guard, and the alias entity/metadata). rejects is the owner's
// declined-term set (normalized src, R1-FL-B): a candidate whose entity touches a rejected surface is
// dropped from the delta so a declined term never re-fires the stop (a nil/empty rejects is a no-op — the
// emission is byte-identical to before, preserving the frozen parity). Deterministic and $0.
func MineBank(chunks []Chunk, contrast *Contrast, seed []store.GlossaryEntry, rejects map[string]bool, cfg Config, pack *lang.Pack) []Term {
	mr := mineDetect(chunks, contrast, cfg, pack)

	// Seed surfaces (normalized) + their metadata for the alias rules and the non-seed / fragment guards.
	seedSurfaces := map[string]bool{}
	seedMeta := map[string]aliasSurface{}
	for _, e := range seed {
		for _, surf := range append([]string{e.Src}, aliasStrings(e.Aliases)...) {
			nk := text.NormalizeSourceKey(surf)
			if nk == "" {
				continue
			}
			seedSurfaces[nk] = true
			// The primary src carries the dst/gender/type; an alias inherits them (same entity).
			if _, ok := seedMeta[nk]; !ok {
				meta := aliasSurface{src: nk, typ: e.Type}
				if e.Status == "approved" {
					meta.approvedDst = e.Dst
				}
				meta.gender = e.Gender
				seedMeta[nk] = meta
			}
		}
	}

	// Qualifying candidate pool (build_precision30 filters), in ranked order (the ranked order is the
	// representative-selection order below — the top-ranked cluster member owns the aliases). The ranked
	// list is capped to the top emitRankCap BEFORE the eligibility filters (FL-3), mirroring the
	// reference's a3[:200] slice (emit_owner_sheets.build_precision30 / alias.py:165): the cap bounds the
	// owner-signed signature map to the reference's volume, so an eligible candidate below the cap is not
	// surfaced. WHICH-invariants (the 13618-member SET, recall, the catastrophe screen) are unaffected —
	// they are properties of mr.ranked itself, not of the capped emission slice.
	ranked := mr.ranked
	if len(ranked) > emitRankCap {
		ranked = ranked[:emitRankCap]
	}
	var pool []scoredCand
	for _, c := range ranked {
		if !emissionEligible(c, mr.subsumed, seedSurfaces, pack) {
			continue
		}
		pool = append(pool, c)
	}

	// Alias universe = qualifying candidate surfaces ∪ seed surfaces; run tier-1 clustering.
	surfaces := map[string]aliasSurface{}
	for _, c := range pool {
		typ := ""
		if len(c.Types) > 0 {
			typ = c.Types[0]
		}
		surfaces[c.Src] = aliasSurface{src: c.Src, typ: typ}
	}
	for nk, meta := range seedMeta {
		surfaces[nk] = meta // seed metadata wins (gender/approved_dst)
	}
	universe := make([]string, 0, len(surfaces))
	for s := range surfaces {
		universe = append(universe, s)
	}
	sort.Strings(universe)
	ident, _, _ := proposeAliasEdges(surfaces, chunks, seedSurfaces, pack)
	clusters := clusterAlias(ident, universe)

	// clusterOf maps a surface to its identity cluster (members); a surface not in a multi-cluster maps
	// to a singleton.
	clusterOf := map[string][]string{}
	for _, cl := range clusters {
		for _, s := range cl {
			clusterOf[s] = cl
		}
	}

	var out []Term
	emitted := map[string]bool{}
	for _, c := range pool { // ranked order → the top-ranked cluster member is the representative
		if seedSurfaces[c.Src] || emitted[c.Src] {
			continue
		}
		cl := clusterOf[c.Src]
		// A cluster touching a seed surface is an alias-of-existing entity → the owner attaches it; skip.
		if clusterTouches(cl, seedSurfaces) {
			markEmitted(cl, emitted)
			continue
		}
		// A declined entity (R1-FL-B) is dropped WITH its whole cluster, so it never re-fires the stop:
		// marking the cluster emitted stops a lower-ranked member from re-emitting the same entity next run
		// (which would keep the delta non-empty forever). rejects[c.Src] covers a SINGLETON representative
		// (cl is nil for a candidate in no multi-cluster); clusterTouches covers a rejected alias of a
		// multi-member cluster. markEmitted(nil,...) is a harmless no-op for the singleton case.
		if rejects[c.Src] || clusterTouches(cl, rejects) {
			markEmitted(cl, emitted)
			continue
		}
		var aliases []string
		if len(cl) > 1 {
			for _, m := range cl {
				if m != c.Src && !seedSurfaces[m] {
					aliases = append(aliases, m)
				}
			}
			sort.Strings(aliases)
			markEmitted(cl, emitted) // the whole cluster is represented by this one term
		} else {
			emitted[c.Src] = true
		}
		typ := "term"
		if len(c.Types) > 0 {
			typ = c.Types[0]
		}
		out = append(out, Term{
			Src: c.Src, Type: typ, Freq: c.Freq, Aliases: aliases, Evidence: c.Evidence,
			SinceCh: entitySinceCh(append([]string{c.Src}, aliases...), chunks),
		})
	}
	// Deterministic output order: by src (the caller may re-sort, but pin a stable delta).
	sort.Slice(out, func(i, j int) bool { return out[i].Src < out[j].Src })
	return out
}

// DeltaYAML serializes the mined delta into the seed.Term YAML schema (§C2-7) — the owner-sign
// artifact and the `tmctl seed-lint` input. Every term is status:auto with NO dst (WHICH-only, default
// B; the WHAT is delivered by the banknote at the bank-mining stop). It reuses the EXISTING seed.Term schema (no new
// fields — §3(б)); evidence / zones live in the sidecar sign-map, not the seed. The output is guaranteed
// loadable by loadGlossarySeed (status:auto may lack a dst); seed-lint proves it against the real loader.
func DeltaYAML(mined []Term) (string, error) {
	sf := seed.File{}
	for _, m := range mined {
		st := seed.Term{Src: m.Src, Type: m.Type, Status: "auto", SinceCh: m.SinceCh}
		for _, a := range m.Aliases {
			st.Aliases = append(st.Aliases, seed.Alias{Alias: a, Type: "mined"})
		}
		if len(m.Evidence) > 0 {
			st.Note = "mined WHICH candidate; evidence: " + fmt.Sprint(m.Evidence)
		}
		sf.Terms = append(sf.Terms, st)
	}
	b, err := yaml.Marshal(sf)
	if err != nil {
		return "", fmt.Errorf("miner: marshal mined delta: %w", err)
	}
	return string(b), nil
}

// emissionEligible applies the build_precision30 filters: type ∈ {name,place,title}, freq ≥ 5, src not
// subsumed, len ≥ 2 rune, not a boundary fragment.
func emissionEligible(c scoredCand, subsumed, seedSurfaces map[string]bool, pack *lang.Pack) bool {
	if !hasAnyType(c.Types, "name", "place", "title") {
		return false
	}
	if c.Freq < emitMinFreq || subsumed[c.Src] || runeLen(c.Src) < 2 {
		return false
	}
	if isFragment(c.Src, seedSurfaces, pack) {
		return false
	}
	return true
}

func hasAnyType(types []string, want ...string) bool {
	for _, t := range types {
		for _, w := range want {
			if t == w {
				return true
			}
		}
	}
	return false
}

// clusterTouches reports whether any surface of the identity cluster is in `set` — used both for the seed
// surfaces (an alias-of-existing entity) and the reject surfaces (a declined entity), which are treated the
// same way at emission: the whole cluster is suppressed.
func clusterTouches(cl []string, set map[string]bool) bool {
	for _, s := range cl {
		if set[s] {
			return true
		}
	}
	return false
}

func markEmitted(cl []string, emitted map[string]bool) {
	for _, s := range cl {
		emitted[s] = true
	}
}

// entitySinceCh is the earliest chapter any of the entity's surfaces appears in (canon.since_ch = first
// chapter of appearance).
func entitySinceCh(surfaces []string, chunks []Chunk) int {
	min := 0
	for _, s := range surfaces {
		for ch := range candidateChapters(s, chunks) {
			if min == 0 || ch < min {
				min = ch
			}
		}
	}
	return min
}

func aliasStrings(as []store.GlossaryAlias) []string {
	out := make([]string, len(as))
	for i, a := range as {
		out[i] = a.Alias
	}
	return out
}