textmachine/backend/internal/pipeline/miner_detect.go

package pipeline

import (
	"math"
	"sort"
	"strings"
	"unicode/utf8"
)

// miner_detect.go: the deterministic V-A detector (WS3 — a Go port of exp16 detectors.py). Frequency ×
// contrast(weirdness) × c-value(nested discount). Output: ranked src candidates. The thresholds (freq
// floor 3, subsume α 0.80) are frozen miner-v1 (tuned on chapters 1–15, §D1). Pure and deterministic —
// map iteration only ever precedes a total sort by (-score, -freq, src).

// minerSubsumeAlpha: drop candidate a if a longer container b has f(b) >= α·f(a) — a is a boundary
// fragment, not an independent term (detectors.SUBSUME_ALPHA). It only cleans the candidate RANKING;
// the occurrence axis (countOccurrences) is suppress-free (D24.2).
const minerSubsumeAlpha = 0.80

// runeLen is the character length of a candidate (Han n-grams are ≤6 runes).
func runeLen(s string) int { return utf8.RuneCountInString(s) }

// minerCandidate is one V-A-scored candidate (detectors.Candidate; the V-B/V-C enrichments live on the
// arm-level ScoredCand, not here).
type minerCandidate struct {
	src       string
	freq      int
	cvalue    float64
	weirdness float64
	termhood  float64
	score     float64
	length    int
}

// computeCValue is the c-value with nested discount (detectors.compute_cvalue). For a candidate a nested
// in longer candidates T_a: cval(a) = g(|a|)·(f(a) − (1/|T_a|)·Σ_{b∈T_a} f(b)); non-nested: g(|a|)·f(a).
// g(L)=log2(L+1). Deterministic (accumulates into maps, order-independent).
func computeCValue(candFreq map[string]int) map[string]float64 {
	tcount := make(map[string]int, len(candFreq))
	tfreqsum := make(map[string]int, len(candFreq))
	for b, fb := range candFreq {
		br := []rune(b)
		Lb := len(br)
		if Lb < 2 {
			continue
		}
		// Distinct proper substrings a of b that are themselves candidates.
		subs := map[string]bool{}
		for n := 1; n < Lb; n++ {
			for i := 0; i+n <= Lb; i++ {
				a := string(br[i : i+n])
				if a == b {
					continue
				}
				if _, ok := candFreq[a]; ok {
					subs[a] = true
				}
			}
		}
		for a := range subs {
			tcount[a]++
			tfreqsum[a] += fb
		}
	}
	cval := make(map[string]float64, len(candFreq))
	for a, fa := range candFreq {
		g := lengthMult(runeLen(a))
		if tcount[a] > 0 {
			cval[a] = g * (float64(fa) - float64(tfreqsum[a])/float64(tcount[a]))
		} else {
			cval[a] = g * float64(fa)
		}
	}
	return cval
}

// subsumedCandidates returns the boundary-fragment candidates to drop from the RANKING (detectors.
// subsumed_candidates): a whose every occurrence is (almost) inside a strictly longer candidate b with
// f(b) ≥ α·f(a). 方源(559) survives 方源的(85) [ratio 0.15]; 花酒行(65) is dropped by 花酒行者(65) [1.0].
func subsumedCandidates(candFreq map[string]int, alpha float64) map[string]bool {
	byLen := map[int][]string{}
	for c := range candFreq {
		L := runeLen(c)
		byLen[L] = append(byLen[L], c)
	}
	drop := map[string]bool{}
	for a, fa := range candFreq {
		La := runeLen(a)
		found := false
		for Lb := La + 1; Lb <= minerNgramMax && !found; Lb++ {
			for _, b := range byLen[Lb] {
				if strings.Contains(b, a) && float64(candFreq[b]) >= alpha*float64(fa) {
					found = true
					break
				}
			}
		}
		if found {
			drop[a] = true
		}
	}
	return drop
}

// vaResult carries the V-A build outputs the arms/patterns layers need.
type vaResult struct {
	candFreq map[string]int     // freq ≥ floor
	nBook    int                // total n-gram token mass (weirdness denominator scale)
	subsumed map[string]bool    // ranking-dropped boundary fragments
	cval     map[string]float64 // per-candidate c-value
	ranked   []minerCandidate   // V-A ranking (subsumed excluded), sorted (-score,-freq,src)
}

// runVA builds and scores V-A over the chunks (detectors.VA.build + score_all). candFreq is the
// freq≥floor multiset; nBook is the FULL (pre-floor) token mass. score = max(cval,0)·termhood, where
// termhood=log1p(weirdness), weirdness = P_book / max(word_rel, char_indep_rel, 1e-9).
func runVA(chunks []MinerChunk, contrast *Contrast, freqFloor int, alpha float64) vaResult {
	allFreq, nBook := enumerateCandidates(chunks)
	candFreq := make(map[string]int, len(allFreq))
	for k, v := range allFreq {
		if v >= freqFloor {
			candFreq[k] = v
		}
	}
	subsumed := subsumedCandidates(candFreq, alpha)
	cval := computeCValue(candFreq)

	const weirdFloor = 1e-9
	var ranked []minerCandidate
	for a, f := range candFreq {
		if subsumed[a] {
			continue
		}
		pBook := float64(f) / float64(nBook)
		gen := math.Max(math.Max(contrast.wordRel(a), contrast.charIndepRel(a)), weirdFloor)
		w := pBook / gen
		termhood := math.Log1p(w)
		cv := cval[a]
		score := math.Max(cv, 0.0) * termhood
		ranked = append(ranked, minerCandidate{
			src: a, freq: f, cvalue: cv, weirdness: w, termhood: termhood, score: score, length: runeLen(a),
		})
	}
	sortCandidates(ranked)
	return vaResult{candFreq: candFreq, nBook: nBook, subsumed: subsumed, cval: cval, ranked: ranked}
}

// sortCandidates orders by (-score, -freq, src) — the frozen miner-v1 ranking key (stable across Go/
// Python via the same tie-break).
func sortCandidates(cs []minerCandidate) {
	sort.Slice(cs, func(i, j int) bool {
		if cs[i].score != cs[j].score {
			return cs[i].score > cs[j].score
		}
		if cs[i].freq != cs[j].freq {
			return cs[i].freq > cs[j].freq
		}
		return cs[i].src < cs[j].src
	})
}