textmachine/backend/internal/pipeline/chunkrun.go

package pipeline

import (
	"context"
	"encoding/json"
	"fmt"
	"maps"
	"slices"
	"sort"
	"strings"

	"textmachine/backend/internal/checks"
	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/membank"
	"textmachine/backend/internal/store"
)

// chunkrun.go: the CHUNK-level disposition loop (D2) — stages in order, the first flag
// stops the chunk (everything after is skipped, no paid editing of garbage), plus the hot
// memory path v2 (Select→per-role injection→post-check E1) and the cheap style flaggers.
// New Phase-2 injection-consumer roles (annotator, voice-layer) are added here.

// classifyInputFor assembles the classifyInput for a LIVE classify() call. It exists because there is more
// than one such call, and the field that matters is the one a second site forgets.
//
// ⚠ THE HISTORY IS THE ARGUMENT. `TargetScripts` was added to classifyOutput when the target-language
// screen landed, and the repair re-gate — a separate call built by hand from the same fields — was left
// without it, so the screen abstained inside the re-gate while judging everywhere else: a repair that came
// back in another language passed a gate whose entire job is «is this still an acceptable completion».
// That was found, fixed by hand at the second site, and then a planting proved the fix was held by nothing:
// dropping the field again at that site left the whole battery green. Two hand-built copies of one input is
// the defect; a shared builder is the fix, and the pin below is on the builder, so BOTH sites inherit it.
func (r *Runner) classifyInputFor(source, output, finish string, nonProse bool) classifyInput {
	return classifyInput{
		Source:        source,
		Output:        output,
		Finish:        finish,
		TargetLang:    r.Book.TargetLang,
		SourceScripts: r.checkers.SourceScripts(),
		TargetScripts: lang.LangScripts(r.Book.TargetLang),
		NonProseReply: nonProse,
	}
}

// classifyOutput resolves a completion's disposition: FIRST the always-on intrinsic
// classifier (echo/empty/refusal/length — Milestone 2), then, only if that is ok AND the
// configurable coverage gate is enabled, the excision gate (step 6 / D12 Q3). Both are
// pure and deterministic over (source, output, finish), so a resumed checkpoint
// reproduces the identical verdict for free — the intrinsic part unconditionally, the
// gate part under the same coverage config (folded into the snapshot, so a gate change
// re-pins loudly). The coverage gate runs ONLY on the TRANSLATOR role's output (vs the
// original source): a monolingual editor legitimately restructures sentences, so gating
// it against the source would false-flag a correct edit (see the role check below).
// classifyOutput returns the disposition AND, for a cosmetic sanitizer strip, the resolved EXPORT
// text (the strip remainder) computed ONCE here (T3.4 / L8-stripcosmetic-recomputed-twice): the
// second return travels back to runStage as the authoritative export, so no site re-derives
// checks.StripCosmetic and the "non-empty & clean" invariant is asserted over the very bytes that ship.
// The string is "" for every disposition except FlagSanitizerStripped.
func (r *Runner) classifyOutput(role, source, output, finish string, isFinal bool) (classification, string) {
	cls := classify(r.classifyInputFor(source, output, finish, isBankRole(role)))
	if !cls.ok() {
		return cls, ""
	}
	// Output-sanitizer gate (D30.3): a deterministic verdict-axis check for "instant unreadability"
	// defects no other gate catches — a leaked service preamble, a trailing note/edit block, a markdown
	// ### header, a Latin-script insertion, a contentless CJK-leak run. Opt-in (Gates.Sanitizer). Runs
	// ONLY on the FINAL stage's output (isFinal) — the text that SHIPS: an intermediate translator draft
	// is legitimately rough and the EDITOR's job is to clean it, so sanitizing the draft would wrongly
	// skip the recovering editor and drop the chunk to a placeholder (adversarial review). It is a
	// READABILITY gate whose patterns bake in Cyrillic/Russian conventions, so it is gated behind a
	// Russian target (D39 layer 7, L8-readability-gates-target-blind) — a no-op for any other pair, which
	// would false-flag (prod zh→ru is unaffected). Runs on the completion text like the intrinsic
	// classifier, so a resumed checkpoint re-derives the identical verdict; its rule version folds into
	// the snapshot only when enabled (sanitizerSnapshot).
	if isFinal && r.Pipeline.Gates.Sanitizer.Enabled && r.checkers.TargetActive() {
		if san := r.checkers.SanitizeOutput(output); san.Total() > 0 {
			// Cosmetic-only leak (leading markdown header and/or contentless CJK-leak) → strip it and
			// export the remainder flagged, not drop the whole chunk (D35.4a). The guard is
			// load-bearing: we tag sanitizer_stripped ONLY when the strip yields a NON-EMPTY output that
			// is now clean, and we RETURN that stripped text so runStage/resume ship the exact bytes the
			// guard validated. Anything else falls through to the substantive skip. Pure over `output`,
			// so a resumed checkpoint re-derives the identical verdict.
			if san.CosmeticOnly() {
				if stripped := checks.StripCosmetic(output); strings.TrimSpace(stripped) != "" && r.checkers.SanitizeOutput(stripped).Total() == 0 {
					return classification{FlagSanitizerStripped, san.Summary()}, stripped
				}
			}
			return classification{FlagSanitizerDefect, san.Summary()}, ""
		}
	}
	if !r.Pipeline.Gates.Coverage.Enabled {
		return cls, ""
	}
	// The coverage gate compares the output against the ORIGINAL source, which is a
	// source→target translation only for the TRANSLATOR role. A monolingual editor
	// legitimately restructures/merges sentences, so gating ITS output against the
	// source with the translation corridor false-flags a correct edit as excision
	// (self-review finding). Editor/other-role fidelity is a Phase-2 concern (a
	// bilingual judge over the draft, §04 mode 2), not this excision gate.
	if role != roleTranslator {
		return cls, ""
	}
	// The gate reports the metric breach; the FLAG is the driver's vocabulary (the checks package
	// owns no disposition constants), so the mapping excision → excision_suspect lives here.
	if cov := checks.CoverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); cov.Excision {
		return classification{FlagExcisionSuspect, cov.Detail}, ""
	}
	return cls, ""
}

// injectionRenderer serializes a chunk's selected memory records into a role's injection message, using the
// target-language wire-text (lang.InjectionTexts, pair-14 §2) — a target with no texts renders nothing.
type injectionRenderer func(injected []membank.PickedEntry, tx lang.InjectionTexts) string

// roleInjectionRenderers maps a stage ROLE to the memory-injection renderer it consumes — a registry
// instead of a hand-edited switch (D39 layer 7, L8-role-injection-hardcoded-switch), so a new Phase-2
// consumer role (annotator, voice-layer — README) is DATA + one renderer, not surgery on a switch.
// A role absent from the map gets no injection. Renderers are pure, so the per-role result is
// deterministic regardless of map iteration order (no map-order in OUTPUT — invariant #6).
var roleInjectionRenderers = map[string]injectionRenderer{
	roleTranslator: membank.RenderGlossaryBlock,         // src→dst glossary block
	roleEditor:     membank.RenderEditorConstraintBlock, // CONFIRMED dst forms (target-consistency constraints)
}

// persistRetrievalState writes the per-chunk observability record from the deterministic
// selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED
// records (what the model saw); spoiler/eviction are the dropped-and-logged totals;
// post-check misses are recorded only when the translator actually produced text.
func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel membank.Selection, misses membank.PostcheckResult, outputChecked bool, cheap checks.CheapGateResult, bank bankFlags, bankProps string, voice checks.VoiceResult, leaks []membank.SpoilerLeak) error {
	rs := store.RetrievalState{
		BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, SnapshotID: snapID,
		NStyleFlags:       cheap.Total(),
		NBanknoteLines:    bank.NLines,
		BanknoteParseFail: boolToStoreInt(bank.ParseFail),
		BanknoteTruncated: boolToStoreInt(bank.Truncated),
		BanknoteDetail:    bankProps,
	}
	if cheap.Total() > 0 {
		if b, err := json.Marshal(cheap); err == nil {
			rs.StyleDetail = string(b)
		}
	}
	setVoiceState(&rs, voice, leaks)
	for _, p := range sel.Injected {
		if p.Sticky {
			rs.NSticky++
		} else {
			rs.NExactHits++
		}
		if p.Disp == membank.Ambiguous {
			rs.NAmbiguousFlagged++
		}
	}
	rs.NSpoilerBlocked = len(sel.Rejected)
	rs.NEvicted = len(sel.Evicted)
	r.recordEvicted(sel.Evicted)
	// Loud record of the disposition-gated suppressor (D39 layer 4): a longer lower-trust key refused
	// from eating a nested higher-trust one — the term-drift code root, now visible instead of a
	// silent drop (research/13 §7). Detail carries the refused suppressor→protected pairs for a human.
	rs.NTrustGatedSuppress = len(sel.TrustGated)
	if len(sel.TrustGated) > 0 {
		if b, err := json.Marshal(sel.TrustGated); err == nil {
			rs.TrustGateDetail = string(b)
		}
	}
	if outputChecked {
		// n_postcheck_miss is the CONFIRMED-miss count (the actionable consistency-failure
		// signal, external-review #1); the detail carries ALL misses (confirmed AND the
		// ambiguous forced-post-check ones, each disp-tagged) for the human.
		rs.NPostcheckMiss = misses.ConfirmedCount()
		rs.NUnverifiedShown, rs.NUnverifiedFollowed = misses.Shown, misses.Followed
		if all := misses.All(); len(all) > 0 {
			if b, err := json.Marshal(all); err == nil {
				rs.PostcheckDetail = string(b)
			}
		}
	}
	ids := slices.Sorted(maps.Keys(sel.ActiveIDs))
	if ids == nil {
		// slices.Sorted returns NIL on an empty map → json.Marshal would render "null",
		// but the historical format of the persistent column is "[]" (a chunk with no
		// exact matches — the common case); we keep the bytes stable (self-review finding №4).
		ids = []string{}
	}
	if b, err := json.Marshal(ids); err == nil {
		rs.InjectedIDs = string(b)
	}
	return r.Store.UpsertRetrievalState(rs)
}

// recordEvicted accumulates WHICH bank rows the injection budget dropped, book-wide, so the wave can name
// them once instead of leaving n_evicted as a number nobody can act on.
//
// The list is kept in memory and reported to the LOG rather than stored beside the counter, deliberately: a
// per-row detail column would be a schema migration, and this repository has already paid for a
// non-idempotent one (backlog row 49а). The count keeps its column; the names ride the plane an operator
// reads when deciding whether to raise gates.glossary token budget or prune the seed.
func (r *Runner) recordEvicted(evicted []membank.PickedEntry) {
	if len(evicted) == 0 {
		return
	}
	r.evictMu.Lock()
	defer r.evictMu.Unlock()
	if r.evictedRows == nil {
		r.evictedRows = map[string]int{}
	}
	for _, p := range evicted {
		if src := p.Src(); src != "" {
			r.evictedRows[src]++
		}
	}
}

// evictedNameCap bounds the names one warning carries: the point is to name the rows that lose most often,
// not to reprint the bank.
const evictedNameCap = 20

// reportEvicted drains the accumulator and names the rows the budget dropped most often. Called once per
// wave, so a book that never overflows its injection budget stays silent.
//
// ⚠ Scope, stated: only the DRAFT wave feeds it, because n_evicted is a draft-wave column
// (persistRetrievalState) — the edit wave runs its own Select whose evictions this pack does not record.
func (r *Runner) reportEvicted(ctx context.Context, wave string) {
	r.evictMu.Lock()
	rows := r.evictedRows
	r.evictedRows = nil
	r.evictMu.Unlock()
	if len(rows) == 0 {
		return
	}
	type row struct {
		src string
		n   int
	}
	list := make([]row, 0, len(rows))
	for src, n := range rows {
		list = append(list, row{src, n})
	}
	sort.Slice(list, func(i, j int) bool {
		if list[i].n != list[j].n {
			return list[i].n > list[j].n
		}
		return list[i].src < list[j].src
	})
	named := make([]string, 0, evictedNameCap)
	for _, e := range list {
		if len(named) == evictedNameCap {
			break
		}
		named = append(named, fmt.Sprintf("%s ×%d", e.src, e.n))
	}
	r.Log.WarnContext(ctx, "memory: the injection token budget DROPPED bank rows before the model saw them — these terms had no canon on the wire for those units",
		"book", r.Book.BookID, "wave", wave, "rows", len(list), "shown", len(named),
		"budget_tokens", r.Pipeline.Context.GlossaryTokenBudget, "terms", strings.Join(named, ", "))
}

// boolToStoreInt maps a bool telemetry flag to the store's 0/1 integer column form.
func boolToStoreInt(b bool) int {
	if b {
		return 1
	}
	return 0
}

// cheapGateConfig builds the cheap-gate knobs from the book brief: the ё-policy and the
// lower-cased per-project interjection allowlist. Pure, no store access.
func (r *Runner) cheapGateConfig() checks.CheapGateConfig {
	allow := make(map[string]bool, len(r.Book.StyleAllowlist))
	for _, s := range r.Book.StyleAllowlist {
		allow[s] = true
	}
	return checks.CheapGateConfig{
		YoPolicy:          r.Book.YoPolicy,
		Allowlist:         allow,
		RegressionEnabled: r.Pipeline.Gates.RegressionGuard.Enabled,
		Checkers:          r.checkers,               // compiled once in openRunner (pair-14 data-out); nil-inert for a no-pack book
		RegisterBlocklist: r.Book.RegisterBlocklist, // DC6 out-of-register lexis, from THIS book (D39.79 Q4)
	}
}