package pipeline import ( "fmt" "sort" "unicode" ) // regressionguard.go: the post-reflow regression guard (D38 infra-pack, research/18 §C#5) — two // deterministic OBSERVABILITY flaggers over the DRAFT→FINAL transform, opt-in like a QA gate // (Gates.RegressionGuard) but NEVER a disposition change. The reflow editor legitimately // restructures and merges sentences, so a HARD skip would false-flag a correct edit; instead a hit // is recorded in the cheap-gate observability channel (retrieval-state n_style_flags) and surfaced // in the report for a human. Both flaggers are tuned PRECISION over recall. // // (a) length collapse — the final is drastically SHORTER than the draft (>regressionMaxShrinkPct // of non-space chars), catching a reflow that dropped whole sentences/paragraphs (draft // 5000 → final 600). Reflow SUMMARISES structure into paragraphs; it must not delete content. // (b) number drift — a MULTI-DIGIT arabic number in the draft is ABSENT from the final, or a // new one APPEARS: 四成四=44%→«четыре десятых» (the 44 vanished), or a hallucinated figure. // Single digits are EXCLUDED (they are routinely spelled out — «5»→«пять» — a legit reflow), // and grouped triples are stitched (10 000 ≡ 10000), so the signal is high precision. // // HONEST recall gap: a number rendered as WORDS on BOTH sides (черновик «сорок четыре» → финал // «четыре десятых») is invisible here — offline word-number alignment is a Ф2 concern. And a // legit reflow that converts a multi-digit figure to words («44%»→«сорок четыре процента») will // show as a disappeared number: a FALSE observability flag, tolerable ONLY because this never // drops the chunk (a human glance dismisses it). Precision over recall, observability over gating. const ( // regressionMaxShrinkPct: a final shorter than the draft by MORE than this percent of non-space // characters is a collapse signal. 40% — a reflow that keeps content while merging lines shrinks // only slightly (Russian is ~fertility-neutral draft→final); losing ~half the characters is a // strong omission signal, not a merge. Tuned precision-over-recall; a code const (a change is a // loud cheapGateVersion bump), not config, so the gate carries only an on/off switch. regressionMaxShrinkPct = 40 // regressionMinChars: below this the draft is too short for a percentage ratio to be meaningful // (a 3-word draft legitimately halving is noise), so the length flagger stays silent. regressionMinChars = 200 ) // regressionGuardResult is the guard's per-chunk outcome (folded into the cheap-gate observability // result). LengthCollapse/NumberDrift are 0/1 counts; Detail carries the human-readable lines. type regressionGuardResult struct { LengthCollapse int NumberDrift int Detail []string } // runRegressionGuard compares the DRAFT (first-stage translator output) with the FINAL (last-stage // reflow output). Pure and deterministic (sorted detail), so a resume re-derives identical counts. func runRegressionGuard(draft, final string) regressionGuardResult { var r regressionGuardResult // (a) length collapse — non-space chars, mirroring the coverage gate's length metric. dn, fn := nonSpaceRuneCount(draft), nonSpaceRuneCount(final) if dn >= regressionMinChars && fn*100 < dn*(100-regressionMaxShrinkPct) { r.LengthCollapse = 1 r.Detail = append(r.Detail, fmt.Sprintf( "reflow-регрессия: коллапс длины черновик→финал %d→%d непробельных символов (−%d%%, порог −%d%%) — возможен пропуск", dn, fn, 100-fn*100/dn, regressionMaxShrinkPct)) } // (b) number drift — multi-digit arabic tokens present on one side only. dNums, fNums := arabicNumberTokens(draft), arabicNumberTokens(final) disappeared := numberSetDiff(dNums, fNums) appeared := numberSetDiff(fNums, dNums) if len(disappeared) > 0 || len(appeared) > 0 { r.NumberDrift = len(disappeared) + len(appeared) if len(disappeared) > 0 { r.Detail = append(r.Detail, "reflow-регрессия: числа черновика отсутствуют в финале (возможен пропуск/дрейф разряда): "+joinPreview(disappeared)) } if len(appeared) > 0 { r.Detail = append(r.Detail, "reflow-регрессия: в финале появились числа, которых не было в черновике (возможна добавка): "+joinPreview(appeared)) } } return r } // nonSpaceRuneCount counts non-whitespace runes — the length metric for the collapse flagger. func nonSpaceRuneCount(s string) int { n := 0 for _, r := range s { if !unicode.IsSpace(r) { n++ } } return n } // arabicNumberTokens returns the MULTI-DIGIT (≥2) arabic integers in text, stitching grouping // separators (space / NBSP / comma) between runs of exactly three digits so "10 000" reads as one // "10000", not "10" and "000". Single-digit numbers are dropped (routinely spelled out in prose). func arabicNumberTokens(text string) []string { rs := []rune(text) var out []string for i := 0; i < len(rs); { if !isASCIIDigit(rs[i]) { i++ continue } j := i for j < len(rs) && isASCIIDigit(rs[j]) { j++ } digits := string(rs[i:j]) // Stitch " ddd" / " ddd" / ",ddd" grouped triples. for j < len(rs) { if rs[j] == ' ' || rs[j] == ' ' || rs[j] == ',' { k := j + 1 g := 0 for k < len(rs) && isASCIIDigit(rs[k]) { k++ g++ } if g == 3 { digits += string(rs[j+1 : k]) j = k continue } } break } if len(digits) >= 2 { out = append(out, digits) } i = j } return out } // numberSetDiff returns the distinct members of a that do not appear in b, sorted. A SET diff (not // a multiset): a number present on both sides is covered even if its multiplicity differs, keeping // the precision-over-recall bias (a repeated number is not a drift signal on its own). func numberSetDiff(a, b []string) []string { inB := make(map[string]bool, len(b)) for _, x := range b { inB[x] = true } seen := map[string]bool{} var out []string for _, x := range a { if !inB[x] && !seen[x] { seen[x] = true out = append(out, x) } } sort.Strings(out) return out } // joinPreview renders a sorted number list compactly for the detail line (bounded). func joinPreview(nums []string) string { const max = 8 if len(nums) > max { return preview(fmt.Sprintf("%v …(+%d)", nums[:max], len(nums)-max)) } return fmt.Sprintf("%v", nums) }