package pipeline import "textmachine/backend/internal/lang" // miner_patterns.go: the V-C pattern channels (WS3 — a Go port of exp16 patterns.py). A language×genre // pattern pack for zh (§B5 plugin P3, universal set only — owner decision 18.07: genre packs are NOT // built). Each channel proposes TYPED candidates (name/title/place/term) with evidence, closing V-A's // frequency-blind classes (rare surname-anchored names, rank/grade titles, one-off realia). Versioned // data (surnames / title affixes / topo suffixes) + a book-adaptive productive-morphology detector // (channel 4 — auto-detects the domain formant, NOT hardcoded 蛊). Pure and deterministic. // minerPackVersion is the pattern-inventory (channel) version (patterns.PACK_VERSION) — the ALGORITHM // schema, distinct from the DATA. The surname / title-topo-rank inventories / particle / Palladius tables // this file's channels consult now live in a langpack (internal/lang, configs/langpacks/), content-hashed // by Pack.Version(); this const versions the pattern CHANNELS, not the tables (D39.15: data as files). const minerPackVersion = "zh-universal-v1" // isSurnameStart returns the surname prefix (compound preferred) if s starts with one, else "" (patterns. // is_surname_start). s is a []rune (a Han run suffix); the surname sets come from the langpack. func isSurnameStart(s []rune, p *lang.Pack) string { if len(s) >= 2 { if pre := string(s[:2]); p.SurnamesCompound[pre] { return pre } } if len(s) >= 1 && p.SurnamesSingle[s[0]] { return string(s[0]) } return "" } // patternInfo is one candidate's channel-proposed types + evidence, in first-seen (deterministic) order. type patternInfo struct { types []string evidence []string } // formantInfo describes a detected productive-morphology formant (patterns.detect_formants entry). type formantInfo struct { role string // suffix|prefix|both overRep float64 nSuffix int nPrefix int } // runeHasPrefixAt reports whether run[i:] starts with p (patterns' run.startswith(suf, i)). func runeHasPrefixAt(run []rune, i int, p []rune) bool { if i+len(p) > len(run) { return false } for k := range p { if run[i+k] != p[k] { return false } } return true } // formantType maps a formant char to a candidate type (patterns.formant_type): topo→place, 等/转/阶→ // title, else term. func formantType(c rune, p *lang.Pack) string { if p.TopoSuffix[c] { return "place" } if c == '等' || c == '转' || c == '阶' { return "title" } return "term" } // bookCharFreq counts Han chars over the normalized sources (patterns.book_char_freq). func bookCharFreq(chunks []MinerChunk) map[rune]int { bc := map[rune]int{} for _, c := range chunks { for _, r := range c.NSource { if isMinerHan(r) { bc[r]++ } } } return bc } // detectFormants auto-detects the productive DOMAIN formants (patterns.detect_formants): a Han char that // binds (suffix OR prefix) with ≥ minPartners DISTINCT content morphemes among candidates AND is // over-represented in-book vs general zh (over_rep ≥ minOverRep). Over-representation — NOT char-rarity // — isolates 蛊/窍/虫 while rejecting common chars 师/花/等. Deterministic (per-char, order-independent). func detectFormants(chunks []MinerChunk, candFreq map[string]int, contrast *Contrast, minPartners int, minOverRep float64) map[rune]formantInfo { bc := bookCharFreq(chunks) totBook := 0 for _, v := range bc { totBook += v } if totBook == 0 { totBook = 1 } suf := map[rune]map[string]bool{} pre := map[rune]map[string]bool{} addPartner := func(m map[rune]map[string]bool, key rune, part string) { if m[key] == nil { m[key] = map[string]bool{} } m[key][part] = true } for a := range candFreq { ar := []rune(a) if len(ar) < 2 { continue } addPartner(suf, ar[len(ar)-1], string(ar[:len(ar)-1])) addPartner(pre, ar[0], string(ar[1:])) } chars := map[rune]bool{} for c := range suf { chars[c] = true } for c := range pre { chars[c] = true } formants := map[rune]formantInfo{} for c := range chars { pBook := float64(bc[c]) / float64(totBook) overRep := contrast.charOverRep(c, pBook) if overRep < minOverRep { continue } ns, npr := len(suf[c]), len(pre[c]) role := "" if ns >= minPartners { role = "suffix" } if npr >= minPartners { if role != "" { role = "both" } else { role = "prefix" } } if role != "" { formants[c] = formantInfo{role: role, overRep: overRep, nSuffix: ns, nPrefix: npr} } } return formants } // patternCandidates produces the typed pattern candidates from the source (patterns.pattern_candidates). // It operates over the ALREADY-normalized source and may propose candidates BELOW the V-A freq floor // (that is the point — patterns close the rare-term blind spot). Returns the candidate→info map and the // detected formants. Deterministic (fixed chunk/run/position iteration; add() preserves first-seen order). func patternCandidates(chunks []MinerChunk, candFreq map[string]int, contrast *Contrast, minPartners int, minOverRep float64, p *lang.Pack) (map[string]*patternInfo, map[rune]formantInfo) { out := map[string]*patternInfo{} add := func(cand []rune, typ, ev string) { cn := normalizeSourceKey(string(cand)) if cn == "" || !minerHanOnly(cn) { return } info := out[cn] if info == nil { info = &patternInfo{} out[cn] = info } if !containsStr(info.types, typ) { info.types = append(info.types, typ) } if !containsStr(info.evidence, ev) { info.evidence = append(info.evidence, ev) } } titleSufR := toRuneSlices(p.TitleSuffix) ordinalR := toRuneSlices(p.OrdinalTitle) rankR := toRuneSlices(p.RankWord) for _, c := range chunks { for _, run := range hanRuns(c.NSource) { L := len(run) for i := 0; i < L; i++ { // (1) surname anchor: surname + 1–2 Han given-name window → name if sn := isSurnameStart(run[i:], p); sn != "" { base := i + len([]rune(sn)) for _, gl := range [2]int{1, 2} { if base+gl <= L { full := run[i : base+gl] if len(full) >= 2 && len(full) <= 4 { add(full, "name", "surname:"+sn) } } } } // (2) title suffix: content + suffix → title for si, suf := range titleSufR { if runeHasPrefixAt(run, i, suf) { for _, left := range [4]int{3, 2, 1, 0} { if i-left >= 0 { cand := run[i-left : i+len(suf)] if len(cand) >= 2 && len(cand) <= 6 { add(cand, "title", "title_suffix:"+p.TitleSuffix[si]) } } } add([]rune(p.TitleSuffix[si]), "title", "title_bare:"+p.TitleSuffix[si]) } } // (2b) ordinal + title (四代族长-class) → title for oi, od := range ordinalR { if runeHasPrefixAt(run, i, od) { for si, suf := range titleSufR { end := i + len(od) if runeHasPrefixAt(run, end, suf) { add(run[i:end+len(suf)], "title", "ordinal_title:"+p.OrdinalTitle[oi]+"+"+p.TitleSuffix[si]) } } } } // (2c) rank/grade compositional: (numeral|grade-prefix) + rank-word → title for ri, rw := range rankR { if runeHasPrefixAt(run, i, rw) && i >= 1 { left := run[i-1] if p.Numeral[left] || p.GradePrefix[left] { add(run[i-1:i+len(rw)], "title", "rank_grade:"+string(left)+"+"+p.RankWord[ri]) } } } // (3) topo suffix: 1–3 Han base + topo char → place if p.TopoSuffix[run[i]] && i >= 1 { for _, left := range [3]int{3, 2, 1} { if i-left >= 0 { cand := run[i-left : i+1] if len(cand) >= 2 && len(cand) <= 4 { add(cand, "place", "topo_suffix:"+string(run[i])) } } } } } } } // (4) productive morphology: for each detected formant, propose all binding n-grams. formants := detectFormants(chunks, candFreq, contrast, minPartners, minOverRep) for _, c := range chunks { for _, run := range hanRuns(c.NSource) { L := len(run) for i := 0; i < L; i++ { info, ok := formants[run[i]] if !ok { continue } typ := formantType(run[i], p) if info.role == "suffix" || info.role == "both" { for _, left := range [3]int{3, 2, 1} { if i-left >= 0 { cand := run[i-left : i+1] if len(cand) >= 2 && len(cand) <= 4 { add(cand, typ, "formant_suffix:"+string(run[i])) } } } } if info.role == "prefix" || info.role == "both" { for _, r := range [2]int{2, 3} { if i+r <= L { cand := run[i : i+r] if len(cand) >= 2 && len(cand) <= 4 { add(cand, typ, "formant_prefix:"+string(run[i])) } } } } } } } return out, formants } // minerHanOnly reports whether s is non-empty and all Han (exp16_common.han_only). func minerHanOnly(s string) bool { if s == "" { return false } for _, r := range s { if !isMinerHan(r) { return false } } return true } // toRuneSlices converts a []string inventory to []([]rune) once (avoids re-decoding per scan position). func toRuneSlices(ss []string) [][]rune { out := make([][]rune, len(ss)) for i, s := range ss { out[i] = []rune(s) } return out } // containsStr reports whether ss contains s. func containsStr(ss []string, s string) bool { for _, x := range ss { if x == s { return true } } return false }