package pipeline import ( "math" "sort" "strings" "unicode/utf8" ) // miner_detect.go: the deterministic V-A detector (WS3 — a Go port of exp16 detectors.py). Frequency × // contrast(weirdness) × c-value(nested discount). Output: ranked src candidates. The thresholds (freq // floor 3, subsume α 0.80) are frozen miner-v1 (tuned on chapters 1–15, §D1). Pure and deterministic — // map iteration only ever precedes a total sort by (-score, -freq, src). // minerSubsumeAlpha: drop candidate a if a longer container b has f(b) >= α·f(a) — a is a boundary // fragment, not an independent term (detectors.SUBSUME_ALPHA). It only cleans the candidate RANKING; // the occurrence axis (countOccurrences) is suppress-free (D24.2). const minerSubsumeAlpha = 0.80 // runeLen is the character length of a candidate (Han n-grams are ≤6 runes). func runeLen(s string) int { return utf8.RuneCountInString(s) } // minerCandidate is one V-A-scored candidate (detectors.Candidate; the V-B/V-C enrichments live on the // arm-level ScoredCand, not here). type minerCandidate struct { src string freq int cvalue float64 weirdness float64 termhood float64 score float64 length int } // computeCValue is the c-value with nested discount (detectors.compute_cvalue). For a candidate a nested // in longer candidates T_a: cval(a) = g(|a|)·(f(a) − (1/|T_a|)·Σ_{b∈T_a} f(b)); non-nested: g(|a|)·f(a). // g(L)=log2(L+1). Deterministic (accumulates into maps, order-independent). func computeCValue(candFreq map[string]int) map[string]float64 { tcount := make(map[string]int, len(candFreq)) tfreqsum := make(map[string]int, len(candFreq)) for b, fb := range candFreq { br := []rune(b) Lb := len(br) if Lb < 2 { continue } // Distinct proper substrings a of b that are themselves candidates. subs := map[string]bool{} for n := 1; n < Lb; n++ { for i := 0; i+n <= Lb; i++ { a := string(br[i : i+n]) if a == b { continue } if _, ok := candFreq[a]; ok { subs[a] = true } } } for a := range subs { tcount[a]++ tfreqsum[a] += fb } } cval := make(map[string]float64, len(candFreq)) for a, fa := range candFreq { g := lengthMult(runeLen(a)) if tcount[a] > 0 { cval[a] = g * (float64(fa) - float64(tfreqsum[a])/float64(tcount[a])) } else { cval[a] = g * float64(fa) } } return cval } // subsumedCandidates returns the boundary-fragment candidates to drop from the RANKING (detectors. // subsumed_candidates): a whose every occurrence is (almost) inside a strictly longer candidate b with // f(b) ≥ α·f(a). 方源(559) survives 方源的(85) [ratio 0.15]; 花酒行(65) is dropped by 花酒行者(65) [1.0]. func subsumedCandidates(candFreq map[string]int, alpha float64) map[string]bool { byLen := map[int][]string{} for c := range candFreq { L := runeLen(c) byLen[L] = append(byLen[L], c) } drop := map[string]bool{} for a, fa := range candFreq { La := runeLen(a) found := false for Lb := La + 1; Lb <= minerNgramMax && !found; Lb++ { for _, b := range byLen[Lb] { if strings.Contains(b, a) && float64(candFreq[b]) >= alpha*float64(fa) { found = true break } } } if found { drop[a] = true } } return drop } // vaResult carries the V-A build outputs the arms/patterns layers need. type vaResult struct { candFreq map[string]int // freq ≥ floor nBook int // total n-gram token mass (weirdness denominator scale) subsumed map[string]bool // ranking-dropped boundary fragments cval map[string]float64 // per-candidate c-value ranked []minerCandidate // V-A ranking (subsumed excluded), sorted (-score,-freq,src) } // runVA builds and scores V-A over the chunks (detectors.VA.build + score_all). candFreq is the // freq≥floor multiset; nBook is the FULL (pre-floor) token mass. score = max(cval,0)·termhood, where // termhood=log1p(weirdness), weirdness = P_book / max(word_rel, char_indep_rel, 1e-9). func runVA(chunks []MinerChunk, contrast *Contrast, freqFloor int, alpha float64) vaResult { allFreq, nBook := enumerateCandidates(chunks) candFreq := make(map[string]int, len(allFreq)) for k, v := range allFreq { if v >= freqFloor { candFreq[k] = v } } subsumed := subsumedCandidates(candFreq, alpha) cval := computeCValue(candFreq) const weirdFloor = 1e-9 var ranked []minerCandidate for a, f := range candFreq { if subsumed[a] { continue } pBook := float64(f) / float64(nBook) gen := math.Max(math.Max(contrast.wordRel(a), contrast.charIndepRel(a)), weirdFloor) w := pBook / gen termhood := math.Log1p(w) cv := cval[a] score := math.Max(cv, 0.0) * termhood ranked = append(ranked, minerCandidate{ src: a, freq: f, cvalue: cv, weirdness: w, termhood: termhood, score: score, length: runeLen(a), }) } sortCandidates(ranked) return vaResult{candFreq: candFreq, nBook: nBook, subsumed: subsumed, cval: cval, ranked: ranked} } // sortCandidates orders by (-score, -freq, src) — the frozen miner-v1 ranking key (stable across Go/ // Python via the same tie-break). func sortCandidates(cs []minerCandidate) { sort.Slice(cs, func(i, j int) bool { if cs[i].score != cs[j].score { return cs[i].score > cs[j].score } if cs[i].freq != cs[j].freq { return cs[i].freq > cs[j].freq } return cs[i].src < cs[j].src }) }