192 lines
6.8 KiB
Go
192 lines
6.8 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"sort"
|
||
"strings"
|
||
|
||
"textmachine/backend/internal/lang"
|
||
)
|
||
|
||
// miner_alias.go: alias tier-1 clustering (WS3 — a Go port of exp16 alias.py, DEFAULT-B). Precision-safe
|
||
// PROP-layer rules over a set of src surfaces:
|
||
//
|
||
// R1 surface containment X ⊂ Y, |X|≥2 → 'extension' edge (方源 ⊂ 古月方源), COMPOSITIONAL guard
|
||
// (古月+族长 = a phrase, not an alias) blocks the clan↔title chaining
|
||
// R2 surname anchor+compose shared surname → FAMILY supercluster (NOT identity)
|
||
// R4 NEGATIVE hard-blocks (checked FIRST): (ii) different confirmed gender; (iii/v) different approved
|
||
// dst → different entities EVEN under containment (族长⊂四代族长, 古月⊂古月方源); (i) same surname +
|
||
// different given names → family, not identity; (iv) co-presence in one source sentence → not identity
|
||
//
|
||
// The R3 "shared ru rendering → identity" edge is OMITTED in default B (it needs the pymorphy3-backed
|
||
// dst-variant lemmas, which the ratified default drops). Precision is measured entity-level (B³, §A5).
|
||
|
||
// aliasSurface is one src surface with the seed metadata the rules read (alias.Surface, minus the
|
||
// default-B-dropped dst_lemmas).
|
||
type aliasSurface struct {
|
||
src string // normalized src surface
|
||
gender string
|
||
approvedDst string
|
||
typ string
|
||
}
|
||
|
||
// aliasEdge is one proposed edge between two surfaces.
|
||
type aliasEdge struct {
|
||
a, b string
|
||
rule string
|
||
kind string
|
||
}
|
||
|
||
// isFragment reports whether src is a known surface + a trailing particle (alias.frag): a boundary
|
||
// artifact, not an independent entity. seedSurfaces is the normalized seed src/alias set.
|
||
func isFragment(src string, seedSurfaces map[string]bool, p *lang.Pack) bool {
|
||
r := []rune(src)
|
||
if len(r) < 2 {
|
||
return false
|
||
}
|
||
return seedSurfaces[string(r[:len(r)-1])] && p.AliasParticle[r[len(r)-1]]
|
||
}
|
||
|
||
// cooccurSameSentence reports whether a and b appear in one source sentence anywhere (alias.
|
||
// cooccur_same_sentence: split on 。！？\n).
|
||
func cooccurSameSentence(chunks []MinerChunk, aNorm, bNorm string) bool {
|
||
for _, c := range chunks {
|
||
for _, sent := range splitMinerSentences(c.NSource) {
|
||
if strings.Contains(sent, aNorm) && strings.Contains(sent, bNorm) {
|
||
return true
|
||
}
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func splitMinerSentences(s string) []string {
|
||
return strings.FieldsFunc(s, func(r rune) bool {
|
||
return r == '。' || r == '！' || r == '？' || r == '\n'
|
||
})
|
||
}
|
||
|
||
// proposeAliasEdges applies the tier-1 rules over the surfaces, returning identity, family and weak
|
||
// edges (alias.propose_edges, default B). seedSurfaces is the normalized seed src/alias set (the R1
|
||
// entity-vs-fragment guard). Deterministic: surfaces are iterated in a fixed sorted order.
|
||
func proposeAliasEdges(surfaces map[string]aliasSurface, chunks []MinerChunk, seedSurfaces map[string]bool, p *lang.Pack) (ident, family, weak []aliasEdge) {
|
||
keys := make([]string, 0, len(surfaces))
|
||
for k := range surfaces {
|
||
keys = append(keys, k)
|
||
}
|
||
sort.Strings(keys)
|
||
for i := 0; i < len(keys); i++ {
|
||
for j := i + 1; j < len(keys); j++ {
|
||
a, b := surfaces[keys[i]], surfaces[keys[j]]
|
||
// R4-ii different confirmed gender (hidden never blocks).
|
||
if a.gender != "" && b.gender != "" && a.gender != b.gender && a.gender != "hidden" && b.gender != "hidden" {
|
||
continue
|
||
}
|
||
// R4-iii/v different approved dst → different entities, even under containment.
|
||
if a.approvedDst != "" && b.approvedDst != "" && normalizeSourceKey(a.approvedDst) != normalizeSourceKey(b.approvedDst) {
|
||
continue
|
||
}
|
||
surA, surB := isSurnameStart([]rune(a.src), p), isSurnameStart([]rune(b.src), p)
|
||
// R1 containment → extension edge (same entity, fuller vs shorter surface).
|
||
if runeLen(a.src) >= 2 && runeLen(b.src) >= 2 && a.src != b.src && (strings.Contains(b.src, a.src) || strings.Contains(a.src, b.src)) {
|
||
short, long := a, b
|
||
if runeLen(b.src) < runeLen(a.src) {
|
||
short, long = b, a
|
||
}
|
||
shortIsEntity := short.typ != "" || seedSurfaces[short.src]
|
||
rest := containmentRest(short.src, long.src)
|
||
restEnt, restKnown := surfaces[rest]
|
||
compositional := rest != "" && seedSurfaces[rest] &&
|
||
(!restKnown || restEnt.typ == "title" || restEnt.typ == "place" || restEnt.typ == "term")
|
||
switch {
|
||
case compositional:
|
||
weak = append(weak, aliasEdge{a.src, b.src, "R1-compositional", "phrase_not_alias"})
|
||
case shortIsEntity:
|
||
ident = append(ident, aliasEdge{a.src, b.src, "R1", "extension"})
|
||
default:
|
||
weak = append(weak, aliasEdge{a.src, b.src, "R1-fragment?", "containment_weak"})
|
||
}
|
||
continue
|
||
}
|
||
// R2 surname anchor → family (NOT identity).
|
||
if surA != "" && surB != "" && surA == surB && a.src != b.src {
|
||
givenA := trimRunePrefix(a.src, surA)
|
||
givenB := trimRunePrefix(b.src, surB)
|
||
if givenA != givenB && givenA != "" && givenB != "" { // R4-i different given names → family only
|
||
if cooccurSameSentence(chunks, a.src, b.src) {
|
||
family = append(family, aliasEdge{a.src, b.src, "R2+R4iv", "family_copresent"})
|
||
} else {
|
||
family = append(family, aliasEdge{a.src, b.src, "R2", "family"})
|
||
}
|
||
continue
|
||
}
|
||
}
|
||
// R3 (shared ru rendering → identity) is OMITTED in default B.
|
||
}
|
||
}
|
||
return ident, family, weak
|
||
}
|
||
|
||
// containmentRest is alias.py's rest = long[len(short):] if long.startswith(short) else long[:-len(short)]
|
||
// (by character). short is contained in long.
|
||
func containmentRest(short, long string) string {
|
||
sr, lr := []rune(short), []rune(long)
|
||
if len(sr) > len(lr) {
|
||
return ""
|
||
}
|
||
if runesEqual(lr[:len(sr)], sr) {
|
||
return string(lr[len(sr):])
|
||
}
|
||
return string(lr[:len(lr)-len(sr)])
|
||
}
|
||
|
||
// trimRunePrefix removes the leading prefix (a surname) from src, by character.
|
||
func trimRunePrefix(src, prefix string) string {
|
||
sr, pr := []rune(src), []rune(prefix)
|
||
if len(pr) <= len(sr) && runesEqual(sr[:len(pr)], pr) {
|
||
return string(sr[len(pr):])
|
||
}
|
||
return src
|
||
}
|
||
|
||
// clusterAlias runs union-find over the identity edges and returns clusters of size > 1, each sorted
|
||
// (alias.cluster). allSurfaces bounds the universe.
|
||
func clusterAlias(ident []aliasEdge, allSurfaces []string) [][]string {
|
||
parent := make(map[string]string, len(allSurfaces))
|
||
for _, s := range allSurfaces {
|
||
parent[s] = s
|
||
}
|
||
var find func(string) string
|
||
find = func(x string) string {
|
||
for parent[x] != x {
|
||
parent[x] = parent[parent[x]]
|
||
x = parent[x]
|
||
}
|
||
return x
|
||
}
|
||
for _, e := range ident {
|
||
if _, ok := parent[e.a]; ok {
|
||
if _, ok := parent[e.b]; ok {
|
||
parent[find(e.a)] = find(e.b)
|
||
}
|
||
}
|
||
}
|
||
groups := map[string][]string{}
|
||
for _, s := range allSurfaces {
|
||
r := find(s)
|
||
groups[r] = append(groups[r], s)
|
||
}
|
||
var out [][]string
|
||
roots := make([]string, 0, len(groups))
|
||
for r := range groups {
|
||
roots = append(roots, r)
|
||
}
|
||
sort.Strings(roots)
|
||
for _, r := range roots {
|
||
if len(groups[r]) > 1 {
|
||
g := append([]string(nil), groups[r]...)
|
||
sort.Strings(g)
|
||
out = append(out, g)
|
||
}
|
||
}
|
||
return out
|
||
}
|