package pipeline

import (
	"bufio"
	"io"
	"math"
	"strconv"
	"strings"
	"unicode"
)

// miner_substrate.go: the deterministic bank-mining substrate (WS3, слой 4 — a Go port of the FROZEN
// exp16 miner-v1: exp16_common.py). $0, no LLM, no embeddings. It provides the candidate generator (Han
// char n-grams 1–6 over the memnorm-normalized source, NO word segmentation — §B1/Qiu&Zhang), the
// general-zh contrast corpus (a versioned jieba word-freq artifact), overlapping occurrence counting
// (Aho-Corasick-equivalent WITHOUT the hot-path suppressContained — D24.2), and the frequency strata.
//
// It reuses the VALIDATED normalizeSourceKey (memnorm.go, byte-faithful to the eval mem_select port), so
// a candidate key and a seed/GT key live in the SAME normalized space (trad→simp, NFKC, kana-fold, lower)
// — comparable to the jieba dict (simplified zh). The whole detector (this + miner_detect/patterns/arms)
// is the WHICH channel; the dst (WHAT) is delivered by the banknote (WS4), never by co-occurrence
// (canon-recovery 0.20–0.68 < 70% — research/20). Determinism is load-bearing: no time/rand, fixed
// iteration; the mined delta is a pure function of (chunks, contrast, config).

// Miner n-gram bounds and the V-A frequency floor (exp16_common: NGRAM_MIN/MAX, FREQ_FLOOR).
const (
	minerNgramMin  = 1
	minerNgramMax  = 6
	minerFreqFloor = 3 // §A1/§B1: below this V-A does not score (a conscious blind spot patterns close)
)

// MinerChunk is one source chunk for mining: its chapter and the memnorm-normalized zh source. The
// caller normalizes once (normalizeSourceKey) so the candidate space matches the glossary/GT space.
type MinerChunk struct {
	Chapter  int
	ChunkIdx int
	NSource  string // normalizeSourceKey(raw source)
}

// isMinerHan reports whether r is a Han ideograph — the candidate alphabet (§B1). Mirrors the Python
// reference's \p{Han}: candidate n-grams are runs of Han characters only.
func isMinerHan(r rune) bool { return unicode.Is(unicode.Han, r) }

// hanRuns returns the maximal runs of Han characters in normalized text (each run a []rune slice so
// n-gram slicing is by CHARACTER, exactly like the Python str slicing).
func hanRuns(ntext string) [][]rune {
	var runs [][]rune
	var cur []rune
	for _, r := range ntext {
		if isMinerHan(r) {
			cur = append(cur, r)
		} else if len(cur) > 0 {
			runs = append(runs, cur)
			cur = nil
		}
	}
	if len(cur) > 0 {
		runs = append(runs, cur)
	}
	return runs
}

// enumerateCandidates counts every Han n-gram (length minerNgramMin..minerNgramMax) with OVERLAPPING
// counts across the chunk sources (exp16_common.enumerate_candidates). The annotation chunk is included
// (blurb terms live there; the GT/seed filter handles the blurb rule separately). Returns the full
// multiset (before the freq floor) — its total token mass is the weirdness denominator (N_book).
func enumerateCandidates(chunks []MinerChunk) (freq map[string]int, nBook int) {
	freq = make(map[string]int, 1<<16)
	for _, c := range chunks {
		for _, run := range hanRuns(c.NSource) {
			L := len(run)
			for n := minerNgramMin; n <= minerNgramMax; n++ {
				for i := 0; i+n <= L; i++ {
					freq[string(run[i:i+n])]++
					nBook++
				}
			}
		}
	}
	return freq, nBook
}

// countOccurrences totals the OVERLAPPING occurrences of a normalized needle across the chunk sources
// (exp16_common.count_occurrences: pos advances by 1, so 重重 counts twice in 重重重). Suppress-free
// (D24.2) — this is the occurrence axis, never the hot-path longest-match.
func countOccurrences(needleNorm string, chunks []MinerChunk) int {
	if needleNorm == "" {
		return 0
	}
	total := 0
	for _, c := range chunks {
		hay := c.NSource
		for pos := 0; pos <= len(hay); {
			rel := strings.Index(hay[pos:], needleNorm)
			if rel < 0 {
				break
			}
			total++
			pos = pos + rel + 1 // overlap advance by ONE byte-position (mirrors Python find(needle, pos+1))
		}
	}
	return total
}

// candidateChapters returns the set of chapters whose normalized source contains the candidate (for
// since_ch = the first chapter of appearance, exp16_common.candidate_chapters). Deterministic.
func candidateChapters(candNorm string, chunks []MinerChunk) map[int]bool {
	chs := map[int]bool{}
	if candNorm == "" {
		return chs
	}
	for _, c := range chunks {
		if strings.Contains(c.NSource, candNorm) {
			chs[c.Chapter] = true
		}
	}
	return chs
}

// freqStratum buckets an occurrence count (exp16_common.freq_stratum).
func freqStratum(f int) string {
	switch {
	case f >= 10:
		return "f>=10"
	case f >= minerFreqFloor:
		return "f3-9"
	default:
		return "f<3"
	}
}

// --- general-zh contrast corpus (jieba dict.txt: "word freq POS" per line) ------------------------

// Contrast is the general-domain zh reference: normalized word frequencies + derived char frequencies,
// used for the weirdness (domain-specificity) signal. A versioned artifact (the jieba 0.42.1 dict.txt,
// SHA-pinned, kept OUT of git — reproducible from jieba). Load it once (miner-build time, $0).
type Contrast struct {
	wordFreq  map[string]int // normalized word → max freq (trad/simp collisions fold to the max)
	charFreq  map[rune]int   // char → summed freq across normalized words
	totalWord int64
	totalChar int64
	nDistinct int // len(charFreq), the Laplace |distinct| term
}

// LoadContrast reads a jieba-style "word freq POS" corpus from r and builds the contrast model. Each
// word is normalized into the candidate space (normalizeSourceKey) and, on a normalization collision,
// the MAX frequency is kept (mirrors exp16_common.Contrast). Malformed / non-integer-freq lines are
// skipped, exactly like the reference.
func LoadContrast(r io.Reader) (*Contrast, error) {
	c := &Contrast{wordFreq: map[string]int{}, charFreq: map[rune]int{}}
	sc := bufio.NewScanner(r)
	sc.Buffer(make([]byte, 0, 64*1024), 1<<20)
	for sc.Scan() {
		fields := strings.Fields(sc.Text())
		if len(fields) < 2 {
			continue
		}
		w := fields[0]
		fr, err := strconv.Atoi(fields[1])
		if err != nil {
			continue
		}
		wn := normalizeSourceKey(w)
		if wn == "" {
			continue
		}
		if prev, ok := c.wordFreq[wn]; !ok || fr > prev {
			c.wordFreq[wn] = fr
		}
		c.totalWord += int64(fr)
		for _, ch := range wn {
			c.charFreq[ch] += fr
		}
	}
	if err := sc.Err(); err != nil {
		return nil, err
	}
	for _, v := range c.charFreq {
		c.totalChar += int64(v)
	}
	c.nDistinct = len(c.charFreq)
	return c, nil
}

// wordRel is P_general(ngram) treated as a whole word (0 for OOV).
func (c *Contrast) wordRel(ngramNorm string) float64 {
	if c.totalWord == 0 {
		return 0
	}
	return float64(c.wordFreq[ngramNorm]) / float64(c.totalWord)
}

// charIndepRel is the expected P_general(ngram) under char-independence with Laplace smoothing
// (exp16_common.char_indep_rel): pc = (cf+1)/(total_char+|distinct|), product over chars.
func (c *Contrast) charIndepRel(ngramNorm string) float64 {
	denom := float64(c.totalChar) + float64(c.nDistinct)
	if denom == 0 {
		return 0
	}
	p := 1.0
	for _, ch := range ngramNorm {
		pc := float64(c.charFreq[ch]+1) / denom
		p *= pc
	}
	return p
}

// charOverRep is p_book(char)/p_general(char) — the domain over-representation used by the formant
// detector (patterns). p_general uses the same Laplace denominator as charIndepRel.
func (c *Contrast) charOverRep(ch rune, pBook float64) float64 {
	denom := float64(c.totalChar) + float64(c.nDistinct)
	if denom == 0 {
		return 0
	}
	pGen := float64(c.charFreq[ch]+1) / denom
	if pGen == 0 {
		return 0
	}
	return pBook / pGen
}

// lengthMult is the c-value g(L)=log2(L+1) length multiplier (miner_detect uses it; kept here beside
// the substrate math). §B1: NOT log2(L) — that zeros |a|=1 (蛊/转) and breaks the catastrophe screen.
func lengthMult(L int) float64 { return math.Log2(float64(L) + 1) }