224 lines
7.6 KiB
Go
224 lines
7.6 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"bufio"
|
||
"io"
|
||
"math"
|
||
"strconv"
|
||
"strings"
|
||
"unicode"
|
||
)
|
||
|
||
// miner_substrate.go: the deterministic bank-mining substrate (WS3, слой 4 — a Go port of the FROZEN
|
||
// exp16 miner-v1: exp16_common.py). $0, no LLM, no embeddings. It provides the candidate generator (Han
|
||
// char n-grams 1–6 over the memnorm-normalized source, NO word segmentation — §B1/Qiu&Zhang), the
|
||
// general-zh contrast corpus (a versioned jieba word-freq artifact), overlapping occurrence counting
|
||
// (Aho-Corasick-equivalent WITHOUT the hot-path suppressContained — D24.2), and the frequency strata.
|
||
//
|
||
// It reuses the VALIDATED normalizeSourceKey (memnorm.go, byte-faithful to the eval mem_select port), so
|
||
// a candidate key and a seed/GT key live in the SAME normalized space (trad→simp, NFKC, kana-fold, lower)
|
||
// — comparable to the jieba dict (simplified zh). The whole detector (this + miner_detect/patterns/arms)
|
||
// is the WHICH channel; the dst (WHAT) is delivered by the banknote (WS4), never by co-occurrence
|
||
// (canon-recovery 0.20–0.68 < 70% — research/20). Determinism is load-bearing: no time/rand, fixed
|
||
// iteration; the mined delta is a pure function of (chunks, contrast, config).
|
||
|
||
// Miner n-gram bounds and the V-A frequency floor (exp16_common: NGRAM_MIN/MAX, FREQ_FLOOR).
|
||
const (
|
||
minerNgramMin = 1
|
||
minerNgramMax = 6
|
||
minerFreqFloor = 3 // §A1/§B1: below this V-A does not score (a conscious blind spot patterns close)
|
||
)
|
||
|
||
// MinerChunk is one source chunk for mining: its chapter and the memnorm-normalized zh source. The
|
||
// caller normalizes once (normalizeSourceKey) so the candidate space matches the glossary/GT space.
|
||
type MinerChunk struct {
|
||
Chapter int
|
||
ChunkIdx int
|
||
NSource string // normalizeSourceKey(raw source)
|
||
}
|
||
|
||
// isMinerHan reports whether r is a Han ideograph — the candidate alphabet (§B1). Mirrors the Python
|
||
// reference's \p{Han}: candidate n-grams are runs of Han characters only.
|
||
func isMinerHan(r rune) bool { return unicode.Is(unicode.Han, r) }
|
||
|
||
// hanRuns returns the maximal runs of Han characters in normalized text (each run a []rune slice so
|
||
// n-gram slicing is by CHARACTER, exactly like the Python str slicing).
|
||
func hanRuns(ntext string) [][]rune {
|
||
var runs [][]rune
|
||
var cur []rune
|
||
for _, r := range ntext {
|
||
if isMinerHan(r) {
|
||
cur = append(cur, r)
|
||
} else if len(cur) > 0 {
|
||
runs = append(runs, cur)
|
||
cur = nil
|
||
}
|
||
}
|
||
if len(cur) > 0 {
|
||
runs = append(runs, cur)
|
||
}
|
||
return runs
|
||
}
|
||
|
||
// enumerateCandidates counts every Han n-gram (length minerNgramMin..minerNgramMax) with OVERLAPPING
|
||
// counts across the chunk sources (exp16_common.enumerate_candidates). The annotation chunk is included
|
||
// (blurb terms live there; the GT/seed filter handles the blurb rule separately). Returns the full
|
||
// multiset (before the freq floor) — its total token mass is the weirdness denominator (N_book).
|
||
func enumerateCandidates(chunks []MinerChunk) (freq map[string]int, nBook int) {
|
||
freq = make(map[string]int, 1<<16)
|
||
for _, c := range chunks {
|
||
for _, run := range hanRuns(c.NSource) {
|
||
L := len(run)
|
||
for n := minerNgramMin; n <= minerNgramMax; n++ {
|
||
for i := 0; i+n <= L; i++ {
|
||
freq[string(run[i:i+n])]++
|
||
nBook++
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return freq, nBook
|
||
}
|
||
|
||
// countOccurrences totals the OVERLAPPING occurrences of a normalized needle across the chunk sources
|
||
// (exp16_common.count_occurrences: pos advances by 1, so 重重 counts twice in 重重重). Suppress-free
|
||
// (D24.2) — this is the occurrence axis, never the hot-path longest-match.
|
||
func countOccurrences(needleNorm string, chunks []MinerChunk) int {
|
||
if needleNorm == "" {
|
||
return 0
|
||
}
|
||
total := 0
|
||
for _, c := range chunks {
|
||
hay := c.NSource
|
||
for pos := 0; pos <= len(hay); {
|
||
rel := strings.Index(hay[pos:], needleNorm)
|
||
if rel < 0 {
|
||
break
|
||
}
|
||
total++
|
||
pos = pos + rel + 1 // overlap advance by ONE byte-position (mirrors Python find(needle, pos+1))
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
// candidateChapters returns the set of chapters whose normalized source contains the candidate (for
|
||
// since_ch = the first chapter of appearance, exp16_common.candidate_chapters). Deterministic.
|
||
func candidateChapters(candNorm string, chunks []MinerChunk) map[int]bool {
|
||
chs := map[int]bool{}
|
||
if candNorm == "" {
|
||
return chs
|
||
}
|
||
for _, c := range chunks {
|
||
if strings.Contains(c.NSource, candNorm) {
|
||
chs[c.Chapter] = true
|
||
}
|
||
}
|
||
return chs
|
||
}
|
||
|
||
// freqStratum buckets an occurrence count (exp16_common.freq_stratum).
|
||
func freqStratum(f int) string {
|
||
switch {
|
||
case f >= 10:
|
||
return "f>=10"
|
||
case f >= minerFreqFloor:
|
||
return "f3-9"
|
||
default:
|
||
return "f<3"
|
||
}
|
||
}
|
||
|
||
// --- general-zh contrast corpus (jieba dict.txt: "word freq POS" per line) ------------------------
|
||
|
||
// Contrast is the general-domain zh reference: normalized word frequencies + derived char frequencies,
|
||
// used for the weirdness (domain-specificity) signal. A versioned artifact (the jieba 0.42.1 dict.txt,
|
||
// SHA-pinned, kept OUT of git — reproducible from jieba). Load it once (miner-build time, $0).
|
||
type Contrast struct {
|
||
wordFreq map[string]int // normalized word → max freq (trad/simp collisions fold to the max)
|
||
charFreq map[rune]int // char → summed freq across normalized words
|
||
totalWord int64
|
||
totalChar int64
|
||
nDistinct int // len(charFreq), the Laplace |distinct| term
|
||
}
|
||
|
||
// LoadContrast reads a jieba-style "word freq POS" corpus from r and builds the contrast model. Each
|
||
// word is normalized into the candidate space (normalizeSourceKey) and, on a normalization collision,
|
||
// the MAX frequency is kept (mirrors exp16_common.Contrast). Malformed / non-integer-freq lines are
|
||
// skipped, exactly like the reference.
|
||
func LoadContrast(r io.Reader) (*Contrast, error) {
|
||
c := &Contrast{wordFreq: map[string]int{}, charFreq: map[rune]int{}}
|
||
sc := bufio.NewScanner(r)
|
||
sc.Buffer(make([]byte, 0, 64*1024), 1<<20)
|
||
for sc.Scan() {
|
||
fields := strings.Fields(sc.Text())
|
||
if len(fields) < 2 {
|
||
continue
|
||
}
|
||
w := fields[0]
|
||
fr, err := strconv.Atoi(fields[1])
|
||
if err != nil {
|
||
continue
|
||
}
|
||
wn := normalizeSourceKey(w)
|
||
if wn == "" {
|
||
continue
|
||
}
|
||
if prev, ok := c.wordFreq[wn]; !ok || fr > prev {
|
||
c.wordFreq[wn] = fr
|
||
}
|
||
c.totalWord += int64(fr)
|
||
for _, ch := range wn {
|
||
c.charFreq[ch] += fr
|
||
}
|
||
}
|
||
if err := sc.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
for _, v := range c.charFreq {
|
||
c.totalChar += int64(v)
|
||
}
|
||
c.nDistinct = len(c.charFreq)
|
||
return c, nil
|
||
}
|
||
|
||
// wordRel is P_general(ngram) treated as a whole word (0 for OOV).
|
||
func (c *Contrast) wordRel(ngramNorm string) float64 {
|
||
if c.totalWord == 0 {
|
||
return 0
|
||
}
|
||
return float64(c.wordFreq[ngramNorm]) / float64(c.totalWord)
|
||
}
|
||
|
||
// charIndepRel is the expected P_general(ngram) under char-independence with Laplace smoothing
|
||
// (exp16_common.char_indep_rel): pc = (cf+1)/(total_char+|distinct|), product over chars.
|
||
func (c *Contrast) charIndepRel(ngramNorm string) float64 {
|
||
denom := float64(c.totalChar) + float64(c.nDistinct)
|
||
if denom == 0 {
|
||
return 0
|
||
}
|
||
p := 1.0
|
||
for _, ch := range ngramNorm {
|
||
pc := float64(c.charFreq[ch]+1) / denom
|
||
p *= pc
|
||
}
|
||
return p
|
||
}
|
||
|
||
// charOverRep is p_book(char)/p_general(char) — the domain over-representation used by the formant
|
||
// detector (patterns). p_general uses the same Laplace denominator as charIndepRel.
|
||
func (c *Contrast) charOverRep(ch rune, pBook float64) float64 {
|
||
denom := float64(c.totalChar) + float64(c.nDistinct)
|
||
if denom == 0 {
|
||
return 0
|
||
}
|
||
pGen := float64(c.charFreq[ch]+1) / denom
|
||
if pGen == 0 {
|
||
return 0
|
||
}
|
||
return pBook / pGen
|
||
}
|
||
|
||
// lengthMult is the c-value g(L)=log2(L+1) length multiplier (miner_detect uses it; kept here beside
|
||
// the substrate math). §B1: NOT log2(L) — that zeros |a|=1 (蛊/转) and breaks the catastrophe screen.
|
||
func lengthMult(L int) float64 { return math.Log2(float64(L) + 1) }
|