textmachine/backend/internal/pipeline/miner_patterns.go

package pipeline

import "textmachine/backend/internal/lang"

// miner_patterns.go: the V-C pattern channels (WS3 — a Go port of exp16 patterns.py). A language×genre
// pattern pack for zh (§B5 plugin P3, universal set only — owner decision 18.07: genre packs are NOT
// built). Each channel proposes TYPED candidates (name/title/place/term) with evidence, closing V-A's
// frequency-blind classes (rare surname-anchored names, rank/grade titles, one-off realia). Versioned
// data (surnames / title affixes / topo suffixes) + a book-adaptive productive-morphology detector
// (channel 4 — auto-detects the domain formant, NOT hardcoded 蛊). Pure and deterministic.

// minerPackVersion is the pattern-inventory (channel) version (patterns.PACK_VERSION) — the ALGORITHM
// schema, distinct from the DATA. The surname / title-topo-rank inventories / particle / Palladius tables
// this file's channels consult now live in a langpack (internal/lang, configs/langpacks/), content-hashed
// by Pack.Version(); this const versions the pattern CHANNELS, not the tables (D39.15: data as files).
const minerPackVersion = "zh-universal-v1"

// isSurnameStart returns the surname prefix (compound preferred) if s starts with one, else "" (patterns.
// is_surname_start). s is a []rune (a Han run suffix); the surname sets come from the langpack.
func isSurnameStart(s []rune, p *lang.Pack) string {
	if len(s) >= 2 {
		if pre := string(s[:2]); p.SurnamesCompound[pre] {
			return pre
		}
	}
	if len(s) >= 1 && p.SurnamesSingle[s[0]] {
		return string(s[0])
	}
	return ""
}

// patternInfo is one candidate's channel-proposed types + evidence, in first-seen (deterministic) order.
type patternInfo struct {
	types    []string
	evidence []string
}

// formantInfo describes a detected productive-morphology formant (patterns.detect_formants entry).
type formantInfo struct {
	role    string // suffix|prefix|both
	overRep float64
	nSuffix int
	nPrefix int
}

// runeHasPrefixAt reports whether run[i:] starts with p (patterns' run.startswith(suf, i)).
func runeHasPrefixAt(run []rune, i int, p []rune) bool {
	if i+len(p) > len(run) {
		return false
	}
	for k := range p {
		if run[i+k] != p[k] {
			return false
		}
	}
	return true
}

// formantType maps a formant char to a candidate type (patterns.formant_type): topo→place, 等/转/阶→
// title, else term.
func formantType(c rune, p *lang.Pack) string {
	if p.TopoSuffix[c] {
		return "place"
	}
	if c == '等' || c == '转' || c == '阶' {
		return "title"
	}
	return "term"
}

// bookCharFreq counts Han chars over the normalized sources (patterns.book_char_freq).
func bookCharFreq(chunks []MinerChunk) map[rune]int {
	bc := map[rune]int{}
	for _, c := range chunks {
		for _, r := range c.NSource {
			if isMinerHan(r) {
				bc[r]++
			}
		}
	}
	return bc
}

// detectFormants auto-detects the productive DOMAIN formants (patterns.detect_formants): a Han char that
// binds (suffix OR prefix) with ≥ minPartners DISTINCT content morphemes among candidates AND is
// over-represented in-book vs general zh (over_rep ≥ minOverRep). Over-representation — NOT char-rarity
// — isolates 蛊/窍/虫 while rejecting common chars 师/花/等. Deterministic (per-char, order-independent).
func detectFormants(chunks []MinerChunk, candFreq map[string]int, contrast *Contrast, minPartners int, minOverRep float64) map[rune]formantInfo {
	bc := bookCharFreq(chunks)
	totBook := 0
	for _, v := range bc {
		totBook += v
	}
	if totBook == 0 {
		totBook = 1
	}
	suf := map[rune]map[string]bool{}
	pre := map[rune]map[string]bool{}
	addPartner := func(m map[rune]map[string]bool, key rune, part string) {
		if m[key] == nil {
			m[key] = map[string]bool{}
		}
		m[key][part] = true
	}
	for a := range candFreq {
		ar := []rune(a)
		if len(ar) < 2 {
			continue
		}
		addPartner(suf, ar[len(ar)-1], string(ar[:len(ar)-1]))
		addPartner(pre, ar[0], string(ar[1:]))
	}
	chars := map[rune]bool{}
	for c := range suf {
		chars[c] = true
	}
	for c := range pre {
		chars[c] = true
	}
	formants := map[rune]formantInfo{}
	for c := range chars {
		pBook := float64(bc[c]) / float64(totBook)
		overRep := contrast.charOverRep(c, pBook)
		if overRep < minOverRep {
			continue
		}
		ns, npr := len(suf[c]), len(pre[c])
		role := ""
		if ns >= minPartners {
			role = "suffix"
		}
		if npr >= minPartners {
			if role != "" {
				role = "both"
			} else {
				role = "prefix"
			}
		}
		if role != "" {
			formants[c] = formantInfo{role: role, overRep: overRep, nSuffix: ns, nPrefix: npr}
		}
	}
	return formants
}

// patternCandidates produces the typed pattern candidates from the source (patterns.pattern_candidates).
// It operates over the ALREADY-normalized source and may propose candidates BELOW the V-A freq floor
// (that is the point — patterns close the rare-term blind spot). Returns the candidate→info map and the
// detected formants. Deterministic (fixed chunk/run/position iteration; add() preserves first-seen order).
func patternCandidates(chunks []MinerChunk, candFreq map[string]int, contrast *Contrast, minPartners int, minOverRep float64, p *lang.Pack) (map[string]*patternInfo, map[rune]formantInfo) {
	out := map[string]*patternInfo{}
	add := func(cand []rune, typ, ev string) {
		cn := normalizeSourceKey(string(cand))
		if cn == "" || !minerHanOnly(cn) {
			return
		}
		info := out[cn]
		if info == nil {
			info = &patternInfo{}
			out[cn] = info
		}
		if !containsStr(info.types, typ) {
			info.types = append(info.types, typ)
		}
		if !containsStr(info.evidence, ev) {
			info.evidence = append(info.evidence, ev)
		}
	}

	titleSufR := toRuneSlices(p.TitleSuffix)
	ordinalR := toRuneSlices(p.OrdinalTitle)
	rankR := toRuneSlices(p.RankWord)

	for _, c := range chunks {
		for _, run := range hanRuns(c.NSource) {
			L := len(run)
			for i := 0; i < L; i++ {
				// (1) surname anchor: surname + 1–2 Han given-name window → name
				if sn := isSurnameStart(run[i:], p); sn != "" {
					base := i + len([]rune(sn))
					for _, gl := range [2]int{1, 2} {
						if base+gl <= L {
							full := run[i : base+gl]
							if len(full) >= 2 && len(full) <= 4 {
								add(full, "name", "surname:"+sn)
							}
						}
					}
				}
				// (2) title suffix: content + suffix → title
				for si, suf := range titleSufR {
					if runeHasPrefixAt(run, i, suf) {
						for _, left := range [4]int{3, 2, 1, 0} {
							if i-left >= 0 {
								cand := run[i-left : i+len(suf)]
								if len(cand) >= 2 && len(cand) <= 6 {
									add(cand, "title", "title_suffix:"+p.TitleSuffix[si])
								}
							}
						}
						add([]rune(p.TitleSuffix[si]), "title", "title_bare:"+p.TitleSuffix[si])
					}
				}
				// (2b) ordinal + title (四代族长-class) → title
				for oi, od := range ordinalR {
					if runeHasPrefixAt(run, i, od) {
						for si, suf := range titleSufR {
							end := i + len(od)
							if runeHasPrefixAt(run, end, suf) {
								add(run[i:end+len(suf)], "title", "ordinal_title:"+p.OrdinalTitle[oi]+"+"+p.TitleSuffix[si])
							}
						}
					}
				}
				// (2c) rank/grade compositional: (numeral|grade-prefix) + rank-word → title
				for ri, rw := range rankR {
					if runeHasPrefixAt(run, i, rw) && i >= 1 {
						left := run[i-1]
						if p.Numeral[left] || p.GradePrefix[left] {
							add(run[i-1:i+len(rw)], "title", "rank_grade:"+string(left)+"+"+p.RankWord[ri])
						}
					}
				}
				// (3) topo suffix: 1–3 Han base + topo char → place
				if p.TopoSuffix[run[i]] && i >= 1 {
					for _, left := range [3]int{3, 2, 1} {
						if i-left >= 0 {
							cand := run[i-left : i+1]
							if len(cand) >= 2 && len(cand) <= 4 {
								add(cand, "place", "topo_suffix:"+string(run[i]))
							}
						}
					}
				}
			}
		}
	}

	// (4) productive morphology: for each detected formant, propose all binding n-grams.
	formants := detectFormants(chunks, candFreq, contrast, minPartners, minOverRep)
	for _, c := range chunks {
		for _, run := range hanRuns(c.NSource) {
			L := len(run)
			for i := 0; i < L; i++ {
				info, ok := formants[run[i]]
				if !ok {
					continue
				}
				typ := formantType(run[i], p)
				if info.role == "suffix" || info.role == "both" {
					for _, left := range [3]int{3, 2, 1} {
						if i-left >= 0 {
							cand := run[i-left : i+1]
							if len(cand) >= 2 && len(cand) <= 4 {
								add(cand, typ, "formant_suffix:"+string(run[i]))
							}
						}
					}
				}
				if info.role == "prefix" || info.role == "both" {
					for _, r := range [2]int{2, 3} {
						if i+r <= L {
							cand := run[i : i+r]
							if len(cand) >= 2 && len(cand) <= 4 {
								add(cand, typ, "formant_prefix:"+string(run[i]))
							}
						}
					}
				}
			}
		}
	}
	return out, formants
}

// minerHanOnly reports whether s is non-empty and all Han (exp16_common.han_only).
func minerHanOnly(s string) bool {
	if s == "" {
		return false
	}
	for _, r := range s {
		if !isMinerHan(r) {
			return false
		}
	}
	return true
}

// toRuneSlices converts a []string inventory to []([]rune) once (avoids re-decoding per scan position).
func toRuneSlices(ss []string) [][]rune {
	out := make([][]rune, len(ss))
	for i, s := range ss {
		out[i] = []rune(s)
	}
	return out
}

// containsStr reports whether ss contains s.
func containsStr(ss []string, s string) bool {
	for _, x := range ss {
		if x == s {
			return true
		}
	}
	return false
}