72 lines
2.9 KiB
Go
72 lines
2.9 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"strings"
|
|
)
|
|
|
|
// loopguard.go: a degenerate-loop observability check. When a model gets stuck it emits the SAME output
|
|
// for consecutive units; this flags a run of identical consecutive translated segments. The signature is
|
|
// a whitespace-normalized hash of each segment, so cosmetic spacing differences don't hide a loop. It is
|
|
// pure OBSERVABILITY — like the cheap style gates it is never a disposition and never touches the wire —
|
|
// and it is surfaced in the read-only quality report, so a resume re-derives the identical count. Pure and
|
|
// deterministic (no time/rand).
|
|
|
|
// segmentLoopMinRun is the run length that triggers a flag. Three consecutive units with byte-identical
|
|
// (whitespace-normalized) output is already a strong signal for prose, where every unit's source — and so
|
|
// its translation — differs. Precision over recall: a rare false flag on legitimately repeated boilerplate
|
|
// is better than dropping to a noisy threshold.
|
|
const segmentLoopMinRun = 3
|
|
|
|
// segmentLoopRun is one maximal run of identical consecutive non-empty segments.
|
|
type segmentLoopRun struct {
|
|
Signature string // the normalized SHA-256 prefix of the repeated segment
|
|
Start int // 0-based index of the first segment of the run (in reading/manifest order)
|
|
Count int // number of consecutive identical segments (≥ minRun)
|
|
}
|
|
|
|
// segmentLoopRuns scans segments in reading order for maximal runs of ≥ minRun consecutive segments with
|
|
// the same normalized signature, returning one segmentLoopRun per such run. EMPTY / whitespace-only
|
|
// segments never join a run (a pending/flagged unit exports "", which must not read as a loop) and break
|
|
// any run in progress. Deterministic: the signature is a pure function of the segment bytes.
|
|
func segmentLoopRuns(segments []string, minRun int) []segmentLoopRun {
|
|
if minRun < 2 {
|
|
minRun = 2
|
|
}
|
|
var runs []segmentLoopRun
|
|
runStart, runSig, runLen := -1, "", 0
|
|
flush := func() {
|
|
if runLen >= minRun {
|
|
runs = append(runs, segmentLoopRun{Signature: runSig, Start: runStart, Count: runLen})
|
|
}
|
|
runStart, runSig, runLen = -1, "", 0
|
|
}
|
|
for i, seg := range segments {
|
|
sig, empty := loopSignature(seg)
|
|
if empty {
|
|
flush() // an empty export cannot be part of a loop and breaks the run
|
|
continue
|
|
}
|
|
if runLen > 0 && sig == runSig {
|
|
runLen++
|
|
continue
|
|
}
|
|
flush()
|
|
runStart, runSig, runLen = i, sig, 1
|
|
}
|
|
flush()
|
|
return runs
|
|
}
|
|
|
|
// loopSignature returns a segment's whitespace-normalized SHA-256 prefix and whether it is empty. All
|
|
// runs of whitespace collapse to a single space and the ends are trimmed, so cosmetic whitespace between
|
|
// two otherwise-identical outputs does not hide a loop; an empty / whitespace-only segment reports empty.
|
|
func loopSignature(seg string) (sig string, empty bool) {
|
|
norm := strings.Join(strings.Fields(seg), " ")
|
|
if norm == "" {
|
|
return "", true
|
|
}
|
|
sum := sha256.Sum256([]byte(norm))
|
|
return hex.EncodeToString(sum[:])[:16], false
|
|
}
|