221 lines
8.9 KiB
Go
221 lines
8.9 KiB
Go
package checks
|
||
|
||
// labelcandidates_test.go: measures CANDIDATE checker-rule changes against the labelled corpus BEFORE
|
||
// committing them to production — the mandate's "measure the fix on the labelled data, then decide". Each
|
||
// candidate is a local re-implementation of a proposed rule; the test prints its precision/recall delta so
|
||
// a build/close decision is data-grounded, not speculative. Env-gated identically (TM_CHECKER_LABELS=1).
|
||
import (
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"unicode"
|
||
)
|
||
|
||
func TestCheckerLabelsCandidates(t *testing.T) {
|
||
if _, err := os.Stat(filepath.Join(labelsDir, "raw", "corpus.jsonl")); err != nil {
|
||
if os.Getenv("TM_CHECKER_LABELS") == "1" {
|
||
t.Fatalf("TM_CHECKER_LABELS=1 but corpus is missing: %v", err)
|
||
}
|
||
t.Skipf("labelled corpus absent (%v)", err)
|
||
}
|
||
c := buildCheckers(t)
|
||
units := loadCorpus(t)
|
||
|
||
// --- #3: the SHIPPED rule extends the chevron attribution join from the strict comma «X», — to ALSO accept
|
||
// a sentence-final «X!»/«X?»/«X…» — reply, while STILL rejecting a flat «X» — citation at the SHAPE level.
|
||
// (An earlier "just make the comma optional" variant was REFUTED: isSpokenChevronLine alone does not guard
|
||
// the flat citation — a stray speech verb elsewhere on the line trips it — so the sentence-final signal must
|
||
// live in the shape, not be delegated to the verb gate.) BASE = the pre-#3 comma-only shape frozen locally
|
||
// (like latinBaseline) so the "before" 5/1/41/99 reproduces from the tree; CAND = the shipped production rule.
|
||
k4bBase, k4bCand := metric{}, metric{}
|
||
for _, l := range loadJSONL(t, filepath.Join(labelsDir, "labels", "k4.jsonl")) {
|
||
if l.Bucket != "chevron" {
|
||
continue
|
||
}
|
||
line := strings.TrimLeft(lineForLabel(l, units), " \t ")
|
||
rs := []rune(line)
|
||
base := chevronSpeechShapeCommaOnly(rs) && c.isSpokenChevronLine(line)
|
||
cand := chevronSpeechShape(rs) && c.isSpokenChevronLine(line)
|
||
joinK4(&k4bBase, l.Speech, "spoken", base)
|
||
joinK4(&k4bCand, l.Speech, "spoken", cand)
|
||
}
|
||
t.Logf("#3 chevron attribution — BASE (comma-only, pre-#3): %+v prec=%s recall=%s", k4bBase, k4bBase.precision(), k4bBase.recall())
|
||
t.Logf("#3 chevron attribution — CAND (shipped #3) : %+v prec=%s recall=%s", k4bCand, k4bCand.precision(), k4bCand.recall())
|
||
|
||
// --- #6/#7: Latin residue variants over the labelled tokens.
|
||
k2Base, k2Caps, k2Thr2 := metric{}, metric{}, metric{}
|
||
for _, l := range loadJSONL(t, filepath.Join(labelsDir, "labels", "k2.jsonl")) {
|
||
u, ok := units[unitOf(l.ID)]
|
||
if !ok {
|
||
continue
|
||
}
|
||
text := u.Final
|
||
if l.Field == "draft" {
|
||
text = u.Draft
|
||
}
|
||
joinAdd(&k2Base, l.Label, latinHit(text, l.Token, latinBaseline))
|
||
joinAdd(&k2Caps, l.Label, latinHit(text, l.Token, latinCapsAware))
|
||
joinAdd(&k2Thr2, l.Label, latinHit(text, l.Token, latinThreshold2))
|
||
}
|
||
t.Logf("#6/#7 latin — BASELINE(len3,lower): %+v prec=%s recall=%s", k2Base, k2Base.precision(), k2Base.recall())
|
||
t.Logf("#6 latin — CAPS-AWARE : %+v prec=%s recall=%s", k2Caps, k2Caps.precision(), k2Caps.recall())
|
||
t.Logf("#7 latin — THRESHOLD=2 : %+v prec=%s recall=%s", k2Thr2, k2Thr2.precision(), k2Thr2.recall())
|
||
|
||
// --- k4_inverse (D39.64 §5.5 inner_marker-veto-on-dash): a dash-led line whose ATTRIBUTION carries an
|
||
// inner-speech marker («про себя», «себе под нос») is a THOUGHT typeset as spoken dialogue. BASE = no rule
|
||
// (a dash thought is structurally invisible). CAND = the SHIPPED production rule: the marker must sit in the
|
||
// attribution segment (speechAttribution + lineHasInnerMarker, whole-word) — the same predicate measureK4
|
||
// runs. REJECTED = the once-proposed whole-line-substring variant, refuted by adversarial review (a spoken
|
||
// reply that merely SAYS «про себя» tripped it). REJECTED scores IDENTICALLY to CAND on this corpus (3/0/1/71):
|
||
// the labelled set carries none of the divergence, which is exactly why the FP took an adversarial reader, not
|
||
// a label count, to find — so it is kept, explicitly labelled, never presented as the candidate (mirror of
|
||
// chevronSpeechShapeCommaOnly's discipline: a refuted rule must not read as the prod number).
|
||
invBase, invCand, invRej := metric{}, metric{}, metric{}
|
||
for _, l := range loadJSONL(t, filepath.Join(labelsDir, "labels", "k4.jsonl")) {
|
||
if l.Bucket != "em-dash" {
|
||
continue
|
||
}
|
||
line := strings.TrimLeft(lineForLabel(l, units), " \t ")
|
||
rs := []rune(line)
|
||
isDash := len(rs) > 0 && (rs[0] == '—' || rs[0] == '–' || rs[0] == '-')
|
||
attr := speechAttribution(line)
|
||
shipped := isDash && attr != "" && c.lineHasInnerMarker(attr)
|
||
rejected := isDash && hasInnerMarkerWholeLine(c, line)
|
||
joinK4(&invBase, l.Speech, "thought", false)
|
||
joinK4(&invCand, l.Speech, "thought", shipped)
|
||
joinK4(&invRej, l.Speech, "thought", rejected)
|
||
}
|
||
t.Logf("k4_inverse — BASE (no rule) : %+v recall=%s", invBase, invBase.recall())
|
||
t.Logf("k4_inverse — CAND (shipped: dash + inner-marker in attribution) : %+v prec=%s recall=%s", invCand, invCand.precision(), invCand.recall())
|
||
t.Logf("k4_inverse — REJECTED (whole-line substring; refuted by review) : %+v prec=%s recall=%s", invRej, invRej.precision(), invRej.recall())
|
||
|
||
// --- #6 detail: which labelled tokens are caps-shaped, and their label (precision risk audit).
|
||
var capsDefect, capsOK []string
|
||
for _, l := range loadJSONL(t, filepath.Join(labelsDir, "labels", "k2.jsonl")) {
|
||
tok := []rune(l.Token)
|
||
if len(tok) >= 3 && isAllLatinLetters(tok) && hasUpper(tok) {
|
||
switch l.Label {
|
||
case "defect":
|
||
capsDefect = append(capsDefect, l.Token)
|
||
case "ok":
|
||
capsOK = append(capsOK, l.Token)
|
||
}
|
||
}
|
||
}
|
||
t.Logf("#6 caps-shaped labelled tokens: defect=%v ok=%v", capsDefect, capsOK)
|
||
}
|
||
|
||
// chevronSpeechShapeCommaOnly is the pre-#3 baseline FROZEN locally (like latinBaseline): a closing » must be
|
||
// followed by an EXACT attribution comma, then a dash — «Реплика», —. It reproduces the "before" k4b metric
|
||
// (5/1/41/99) from the tree so the #3 delta is measurable without a git checkout. The shipped production rule
|
||
// (chevronSpeechShape) extends this to sentence-final «X!»/«X?»/«X…» replies; the once-proposed
|
||
// "optional-comma" variant — which also matched a flat «X» — citation and was REJECTED by adversarial review
|
||
// (TestChevronCitationNotFlagged) — is deliberately NOT kept here, so a refuted candidate can't be re-measured.
|
||
func chevronSpeechShapeCommaOnly(rs []rune) bool {
|
||
i := 1
|
||
for i < len(rs) && unicode.IsSpace(rs[i]) {
|
||
i++
|
||
}
|
||
if i >= len(rs) || !unicode.IsLetter(rs[i]) {
|
||
return false
|
||
}
|
||
for ; i < len(rs); i++ {
|
||
if rs[i] != '»' {
|
||
continue
|
||
}
|
||
j := i + 1
|
||
if j >= len(rs) || rs[j] != ',' { // the pre-#3 rule REQUIRED the attribution comma
|
||
continue
|
||
}
|
||
j++
|
||
for j < len(rs) && unicode.IsSpace(rs[j]) {
|
||
j++
|
||
}
|
||
if j < len(rs) && (rs[j] == '—' || rs[j] == '–' || rs[j] == '-') {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
type latinRule int
|
||
|
||
const (
|
||
latinBaseline latinRule = iota
|
||
latinCapsAware
|
||
latinThreshold2
|
||
)
|
||
|
||
// latinHit reports whether the given rule flags token as a Latin residue.
|
||
func latinHit(text, token string, rule latinRule) bool {
|
||
// Re-tokenize the field the way lintLatinResidue does and test the specific token surface.
|
||
rs := []rune(text)
|
||
for i := 0; i < len(rs); {
|
||
if !isLatinLetterOrDigit(rs[i]) {
|
||
i++
|
||
continue
|
||
}
|
||
j := i
|
||
hasDigit, hasUp := false, false
|
||
for j < len(rs) && isLatinLetterOrDigit(rs[j]) {
|
||
if rs[j] >= '0' && rs[j] <= '9' {
|
||
hasDigit = true
|
||
}
|
||
if rs[j] >= 'A' && rs[j] <= 'Z' {
|
||
hasUp = true
|
||
}
|
||
j++
|
||
}
|
||
tok := string(rs[i:j])
|
||
i = j
|
||
if tok != token {
|
||
continue
|
||
}
|
||
n := len([]rune(tok))
|
||
switch rule {
|
||
case latinBaseline:
|
||
return !hasDigit && !hasUp && n >= 3 && !isRomanNumeral(tok)
|
||
case latinThreshold2:
|
||
return !hasDigit && !hasUp && n >= 2 && !isRomanNumeral(strings.ToLower(tok))
|
||
case latinCapsAware:
|
||
// flag a lowercase word (baseline) OR a Title/ALL-CAPS Latin word (a brand/leak marker), still
|
||
// excluding digit-bearing and Roman-numeral tokens; ≥3 letters.
|
||
if hasDigit || n < 3 || isRomanNumeral(strings.ToLower(tok)) {
|
||
return false
|
||
}
|
||
return true // all-Latin, ≥3, no digit — lower or capped
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// hasInnerMarkerWholeLine reports whether the line carries an inner-speech marker ANYWHERE on it, as a bare
|
||
// SUBSTRING — the REFUTED k4_inverse variant, kept only to measure it against the shipped attribution-scoped
|
||
// rule (it must never masquerade as the candidate). Production uses speechAttribution + lineHasInnerMarker.
|
||
func hasInnerMarkerWholeLine(c *Checkers, line string) bool {
|
||
low := strings.ToLower(line)
|
||
for _, m := range c.innerMarker {
|
||
if strings.Contains(low, m) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func isAllLatinLetters(rs []rune) bool {
|
||
for _, r := range rs {
|
||
if !((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')) {
|
||
return false
|
||
}
|
||
}
|
||
return len(rs) > 0
|
||
}
|
||
func hasUpper(rs []rune) bool {
|
||
for _, r := range rs {
|
||
if r >= 'A' && r <= 'Z' {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|