textmachine/backend/internal/pipeline/miner_emit.go

package pipeline

import (
	"fmt"
	"sort"

	"gopkg.in/yaml.v3"

	"textmachine/backend/internal/store"
)

// miner_emit.go: the WS3 seed-delta emission (§C2-7) — the miner's owner-facing output. It runs the
// default-B detector, applies the subsumption + fragment + type + freq FILTERS (never a raw 13618-dump,
// which carries §B2 fragment noise), clusters aliases (tier-1), and emits one MinedTerm per NEW entity.
//
// Discipline: the miner NEVER writes `approved` — every emitted term is a `draft`/`auto` PROPOSAL for the
// owner to sign (§C2). In default B the WHAT (dst) is delivered by the banknote (WS4) at the W1.5 sign
// boundary, so the miner emits WHICH-only candidates (status:auto, no dst — inert until a dst + owner
// promotion). A candidate that clusters with an existing seed surface is an alias-of-existing and is left
// for the owner to attach (not emitted as a new entity), so the delta is genuinely new terms.

// emitMinFreq is the emission frequency floor (emit_owner_sheets.build_precision30: freq ≥ 5).
const emitMinFreq = 5

// MinedTerm is one WHICH candidate the miner proposes (a seed-delta entry). The miner never writes a dst
// (default B) or `approved`; the caller persists it via the mined-write path (Source:"mined", status
// auto) and the owner signs it at W1.5.
type MinedTerm struct {
	Src      string
	Type     string // name|place|title|term (the candidate's first pattern type; "" → term)
	SinceCh  int    // first chapter of appearance across the term + its aliases (auto)
	Freq     int
	Aliases  []string // identity-cluster co-surfaces (normalized), sorted; the entity's other surfaces
	Evidence []string // up to 3 pattern-evidence tags (for the owner sidecar)
}

// MineBank runs the full default-B miner over the normalized chunks and emits the alias-clustered,
// filtered mined delta (WHICH candidates for owner sign). seed is the current glossary (its surfaces
// scope the non-seed filter, the fragment guard, and the alias entity/metadata). Deterministic and $0.
func MineBank(chunks []MinerChunk, contrast *Contrast, seed []store.GlossaryEntry, cfg MinerConfig) []MinedTerm {
	mr := mineDetect(chunks, contrast, cfg)

	// Seed surfaces (normalized) + their metadata for the alias rules and the non-seed / fragment guards.
	seedSurfaces := map[string]bool{}
	seedMeta := map[string]aliasSurface{}
	for _, e := range seed {
		for _, surf := range append([]string{e.Src}, aliasStrings(e.Aliases)...) {
			nk := normalizeSourceKey(surf)
			if nk == "" {
				continue
			}
			seedSurfaces[nk] = true
			// The primary src carries the dst/gender/type; an alias inherits them (same entity).
			if _, ok := seedMeta[nk]; !ok {
				meta := aliasSurface{src: nk, typ: e.Type}
				if e.Status == "approved" {
					meta.approvedDst = e.Dst
				}
				meta.gender = e.Gender
				seedMeta[nk] = meta
			}
		}
	}

	// Qualifying candidate pool (build_precision30 filters), in ranked order (the ranked order is the
	// representative-selection order below — the top-ranked cluster member owns the aliases).
	var pool []ScoredCand
	for _, c := range mr.ranked {
		if !emissionEligible(c, mr.subsumed, seedSurfaces) {
			continue
		}
		pool = append(pool, c)
	}

	// Alias universe = qualifying candidate surfaces ∪ seed surfaces; run tier-1 clustering.
	surfaces := map[string]aliasSurface{}
	for _, c := range pool {
		typ := ""
		if len(c.Types) > 0 {
			typ = c.Types[0]
		}
		surfaces[c.Src] = aliasSurface{src: c.Src, typ: typ}
	}
	for nk, meta := range seedMeta {
		surfaces[nk] = meta // seed metadata wins (gender/approved_dst)
	}
	universe := make([]string, 0, len(surfaces))
	for s := range surfaces {
		universe = append(universe, s)
	}
	sort.Strings(universe)
	ident, _, _ := proposeAliasEdges(surfaces, chunks, seedSurfaces)
	clusters := clusterAlias(ident, universe)

	// clusterOf maps a surface to its identity cluster (members); a surface not in a multi-cluster maps
	// to a singleton.
	clusterOf := map[string][]string{}
	for _, cl := range clusters {
		for _, s := range cl {
			clusterOf[s] = cl
		}
	}

	var out []MinedTerm
	emitted := map[string]bool{}
	for _, c := range pool { // ranked order → the top-ranked cluster member is the representative
		if seedSurfaces[c.Src] || emitted[c.Src] {
			continue
		}
		cl := clusterOf[c.Src]
		// A cluster touching a seed surface is an alias-of-existing entity → the owner attaches it; skip.
		if clusterHasSeed(cl, seedSurfaces) {
			markEmitted(cl, emitted)
			continue
		}
		var aliases []string
		if len(cl) > 1 {
			for _, m := range cl {
				if m != c.Src && !seedSurfaces[m] {
					aliases = append(aliases, m)
				}
			}
			sort.Strings(aliases)
			markEmitted(cl, emitted) // the whole cluster is represented by this one term
		} else {
			emitted[c.Src] = true
		}
		typ := "term"
		if len(c.Types) > 0 {
			typ = c.Types[0]
		}
		out = append(out, MinedTerm{
			Src: c.Src, Type: typ, Freq: c.Freq, Aliases: aliases, Evidence: c.Evidence,
			SinceCh: entitySinceCh(append([]string{c.Src}, aliases...), chunks),
		})
	}
	// Deterministic output order: by src (the caller may re-sort, but pin a stable delta).
	sort.Slice(out, func(i, j int) bool { return out[i].Src < out[j].Src })
	return out
}

// MinedDeltaYAML serializes the mined delta into the seedTerm YAML schema (§C2-7) — the owner-sign
// artifact and the `tmctl seed-lint` input. Every term is status:auto with NO dst (WHICH-only, default
// B; the WHAT is delivered by the banknote at W1.5). It reuses the EXISTING seedTerm schema (no new
// fields — §3(б)); evidence / zones live in the sidecar sign-map, not the seed. The output is guaranteed
// loadable by loadGlossarySeed (status:auto may lack a dst); seed-lint proves it against the real loader.
func MinedDeltaYAML(mined []MinedTerm) (string, error) {
	sf := seedFile{}
	for _, m := range mined {
		st := seedTerm{Src: m.Src, Type: m.Type, Status: "auto", SinceCh: m.SinceCh}
		for _, a := range m.Aliases {
			st.Aliases = append(st.Aliases, seedAlias{Alias: a, Type: "mined"})
		}
		if len(m.Evidence) > 0 {
			st.Note = "mined WHICH candidate; evidence: " + fmt.Sprint(m.Evidence)
		}
		sf.Terms = append(sf.Terms, st)
	}
	b, err := yaml.Marshal(sf)
	if err != nil {
		return "", fmt.Errorf("pipeline: marshal mined delta: %w", err)
	}
	return string(b), nil
}

// emissionEligible applies the build_precision30 filters: type ∈ {name,place,title}, freq ≥ 5, src not
// subsumed, len ≥ 2 rune, not a boundary fragment.
func emissionEligible(c ScoredCand, subsumed, seedSurfaces map[string]bool) bool {
	if !hasAnyType(c.Types, "name", "place", "title") {
		return false
	}
	if c.Freq < emitMinFreq || subsumed[c.Src] || runeLen(c.Src) < 2 {
		return false
	}
	if isFragment(c.Src, seedSurfaces) {
		return false
	}
	return true
}

func hasAnyType(types []string, want ...string) bool {
	for _, t := range types {
		for _, w := range want {
			if t == w {
				return true
			}
		}
	}
	return false
}

func clusterHasSeed(cl []string, seedSurfaces map[string]bool) bool {
	for _, s := range cl {
		if seedSurfaces[s] {
			return true
		}
	}
	return false
}

func markEmitted(cl []string, emitted map[string]bool) {
	for _, s := range cl {
		emitted[s] = true
	}
}

// entitySinceCh is the earliest chapter any of the entity's surfaces appears in (canon.since_ch = first
// chapter of appearance).
func entitySinceCh(surfaces []string, chunks []MinerChunk) int {
	min := 0
	for _, s := range surfaces {
		for ch := range candidateChapters(s, chunks) {
			if min == 0 || ch < min {
				min = ch
			}
		}
	}
	return min
}

func aliasStrings(as []store.GlossaryAlias) []string {
	out := make([]string, len(as))
	for i, a := range as {
		out[i] = a.Alias
	}
	return out
}