textmachine/backend/internal/pipeline/mining.go

package pipeline

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"sort"
	"strings"

	"gopkg.in/yaml.v3"

	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/membank"
	"textmachine/backend/internal/miner"
	"textmachine/backend/internal/store"
	"textmachine/backend/internal/terminology"
	"textmachine/backend/internal/text"
)

// mining.go: bank-mining stop boundary (WS3 wired live, R1). Between the drafts done and the edit wave, the
// miner scores WHICH-candidates over the the draft wave source (the detector is offline — it reads the SOURCE, not the
// drafts; drafts only mark that a chapter was reached) against the general-zh contrast, emits an
// alias-clustered seed-delta (default B, WHICH-only — the dst is the owner's to attach via the banknote at
// sign time), and — on a NON-EMPTY delta — writes the owner SIGNATURE MAP and STOPS before the edit wave. The owner
// reviews it, promotes terms into the mined-delta file (approved + dst), and re-runs: seedGlossary loads
// those as Source:mined (moving only edit-wave snapshot), the draft wave resumes at $0, the bank-mining stop re-mines (the now-seeded terms are
// excluded → the delta empties), and the edit wave runs. Mining is OFF (auto-continue) unless a langpack AND a contrast
// artifact are both configured — so every $0 test / the golden fixture (no contrast) auto-continues.

// signatureMapPath is where the bank-mining stop writes the mined-delta YAML for owner sign — beside the project DB, so it
// travels with the book state (never in git, like the DB). Deterministic (no time/rand).
func (r *Runner) signatureMapPath() string {
	return r.Book.ProjectDB + ".mined-signature.yaml"
}

// runBankMiningStop executes the bank-mining stop. Returns stopped=true (with the signature map written and
// r.lastMinedCount set) when the miner proposes a non-empty delta; stopped=false (auto-continue) when mining
// is unconfigured or the delta is empty. $0 to providers (the detector is deterministic + offline).
func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, draftSnapshot string, editWave bool) (stopped bool, err error) {
	if r.pack == nil || r.Pipeline.Mining.ContrastPath == "" {
		// FAIL LOUD when the operator ASKED to verify the bank and the book cannot mine one. Silently
		// continuing would hand back a clean exit code for a verification that never happened — the worst
		// possible answer to «остановись и покажи мне банк».
		if r.VerifyBank {
			return false, fmt.Errorf("pipeline: --verify-bank was passed but this book cannot mine a bank: it needs BOTH a langpack (book.yaml `langpack_root`, currently %s) and a mining contrast artifact (pipeline.yaml `mining.contrast_path`, currently %q); without them there is no candidate list to verify",
				packStateLabel(r.pack), r.Pipeline.Mining.ContrastPath)
		}
		// A book that cannot mine has no signing table — and if one is lying beside its DB from when it
		// could (a langpack or contrast path removed from the config), it would outlive every run and keep
		// offering terms to a screen forever. Clear it, but only when it exists: a book that never mines
		// must not acquire a sidecar it has no use for.
		r.clearBankStopTableJSON(ctx, "mining is not configured for this book")
		return false, nil // mining not configured → auto-continue to the edit wave
	}
	f, err := os.Open(r.Pipeline.Mining.ContrastPath)
	if err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop open mining contrast %s: %w", r.Pipeline.Mining.ContrastPath, err)
	}
	defer f.Close()
	contrast, err := miner.LoadContrast(f)
	if err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop load mining contrast %s: %w", r.Pipeline.Mining.ContrastPath, err)
	}

	// The miner works over the memnorm-normalized SOURCE of every chunk (the candidate space matches the
	// glossary/GT space). Deterministic; the annotation/blurb rule is handled inside miner.MineBank's filters.
	minerChunks := make([]miner.Chunk, len(chunks))
	for i, ch := range chunks {
		minerChunks[i] = miner.Chunk{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, NSource: text.NormalizeSourceKey(ch.Text)}
	}
	seed, err := r.Store.GlossaryForBook(r.Book.BookID)
	if err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop read glossary for mining: %w", err)
	}
	// The owner's reject list excludes declined terms from the emission (R1-FL-B): a term the owner reviewed
	// and rejected is dropped from the delta exactly like a seed surface, so it never re-fires the stop. The
	// stop therefore clears once EVERY proposed term is EITHER promoted (into mined_delta → a seed surface)
	// OR rejected (mined_rejects) — the two owner verbs that empty the delta.
	rejects, err := r.loadMinedRejects()
	if err != nil {
		return false, err
	}
	// Risk 2 (self-exclusion): the engine's OWN unsigned rows must not read as seed surfaces, or the auto
	// mode silently switches the stop off after its first run — see unsignedEngineSurfaces.
	mined, emission := miner.MineBankStats(minerChunks, contrast, unsignedEngineSurfaces(seed), rejects, miner.FrozenConfig(), r.pack)

	// The WHAT side: the banknote proposals the draft waves collected, over every sampling of the book that
	// exists. They arrive as EVIDENCE — nothing proposed enters the bank without a signature. A read failure
	// degrades to the WHICH-only map rather than blocking the stop: the map is what the owner needs, the
	// dst is a bonus.
	// The parse rule is LOGGED with the run, which is the whole contract that lets it stay out of the
	// snapshot: it cannot change a paid byte, but it decides which candidate lines became evidence, so a
	// signature map has to be attributable to the rule that produced it. Until the fix-pack the constant was
	// declared "logged with the run" and read by nothing — the tag was a comment, not a mechanism, and two
	// artifacts produced by different rules were indistinguishable.
	observed, offLanguage, oerr := r.bankObservedForBook()
	if oerr != nil {
		r.Log.WarnContext(ctx, "bank-mining: could not read the banknote proposals; the signature map falls back to WHICH-only (bare terms)", "err", oerr)
	}
	r.Log.InfoContext(ctx, "bank-mining: draft-side proposals folded", "book", r.Book.BookID,
		"surfaces", len(observed), "parse_version", bankParseVersion, "slice_version", bankSliceVersion)
	if offLanguage > 0 {
		r.Log.WarnContext(ctx, "bank-mining: draft-side proposals were written in another script and are NOT offered for signature",
			"book", r.Book.BookID, "dropped", offLanguage, "target_script", r.Pipeline.Gates.Terminology.TargetScript)
	}

	// The TERMINOLOGIST (pack-20, D39.42): merge both channels, gather each candidate's source contexts,
	// rank the renderings the drafts produced (§C2-3), and — when the gate is on — consolidate the whole
	// bank in a handful of batched calls. With the gate off this is $0 assembly, but NOT a no-op for the
	// artifact: since the fix-pack the delta's dst comes from these ranked, target-form-folded candidates
	// rather than from the raw draft-side order, so a term whose drafts disagreed can carry a different
	// proposal than it did before — see proposalsFromCandidates. That is bank CONTENT, so it moves
	// memory_version and the edit-wave snapshot with it (bank-only → $0 re-pin for every unit the term does
	// not occur in). The draft wave is untouched: these rows are Source:"mined" and base-excluded.
	tchunks := make([]terminology.Chunk, len(minerChunks))
	for i, c := range minerChunks {
		tchunks[i] = terminology.Chunk{Chapter: c.Chapter, ChunkIdx: c.ChunkIdx, NSource: c.NSource}
	}
	cands := r.buildBankCandidates(mined, observed, tchunks)
	// The REVERSE section: surfaces only the draft side named. The miner cannot see them by construction —
	// a cluster touching a seed surface is suppressed as an alias-of-existing — so this is where the
	// measured 3% channel overlap actually leaks. They join the delta only when the role is on (an
	// unconsolidated reverse row would be a bare surface with no evidence), and only when the SOURCE
	// actually contains them: a term a draft invented is not a bank term.
	if r.Pipeline.Gates.Terminology.Enabled {
		// The SAME surface filter as the miner's (risk 2): the engine's own unsigned rows must not read as
		// "already banked" here either, or the reverse section would empty itself after its first run — and,
		// worse, the delta would differ between run 1 and run 2, moving the edit-wave snapshot and re-billing
		// a wave over nothing. Determinism across runs is what makes the auto mode resumable at all.
		reverse, eligible := reverseSectionTerms(cands, unsignedEngineSurfaces(seed), rejects)
		if eligible > len(reverse) {
			r.Log.WarnContext(ctx, "bank-mining: the reverse section is capped like the miner's own emission; the tail is NOT in this signature map and re-proposes on the next run once these are signed or declined",
				"book", r.Book.BookID, "eligible", eligible, "kept", len(reverse))
		}
		mined = append(mined, reverse...)
		sort.Slice(mined, func(i, j int) bool { return mined[i].Src < mined[j].Src })
	}
	r.lastMinedCount = len(mined)
	if len(mined) == 0 {
		// G10 (polygon package seven): an empty delta must never READ as "this book is clean" when it means
		// "the alphabet was full and the emission layer cut all of it". The funnel is printed with the
		// verdict so the two are distinguishable at a glance, and `--verify-bank` — the mode whose whole
		// promise is «остановись и покажи мне банк» — says it out loud rather than at info level.
		msg := "bank-mining: empty delta, auto-continuing to the edit wave"
		args := append([]any{"book", r.Book.BookID}, emissionArgs(emission, cands)...)
		if r.VerifyBank && emission.Ranked > 0 {
			r.Log.WarnContext(ctx, msg+": the detector RANKED candidates and the emission layer cut every one of them — this is not the same as a book with no new terms", args...)
		} else {
			r.Log.InfoContext(ctx, msg, args...)
		}
		// The MACHINE table is rewritten EMPTY here, and that is the difference between it and the human one
		// (row 101). An empty delta means "nothing awaits signature"; leaving the previous run's rows on disk
		// would let a signing screen re-offer terms the owner has just signed, with no field in the document
		// to tell that state from a live one. The text sidecar keeps its prior behaviour deliberately — it is
		// the document a human re-reads, and a human knows which run they are looking at.
		//
		// Written unconditionally here (unlike the not-configured path above): mining DID run, so "nothing
		// awaits signature" is this run's own finding and the artifact should state it.
		if werr := r.writeBankStopTableJSON(nil); werr != nil {
			r.Log.WarnContext(ctx, "bank-mining: could not clear the machine stop table sidecar; it still holds the PREVIOUS run's rows", "err", werr)
		}
		return false, nil
	}
	consolidated, classified, tres, err := r.runTerminologist(ctx, draftSnapshot, cands)
	if err != nil {
		return false, err
	}
	r.lastTerminology = tres
	mined = attachClassifiedType(mined, classified)
	mined = attachConsolidatedDst(mined, consolidated)

	// Non-empty delta → write the owner signature map (the mined seed-delta YAML) and STOP before the edit wave.
	proposals := proposalsFromCandidates(cands)
	withDst := 0
	for _, m := range mined {
		if m.Dst != "" || len(proposals[text.NormalizeSourceKey(m.Src)]) > 0 {
			withDst++
		}
	}
	yamlDelta, err := miner.DeltaYAML(mined, proposals)
	if err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop marshal mined delta: %w", err)
	}
	if err := os.WriteFile(r.signatureMapPath(), []byte(yamlDelta), 0o644); err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop write signature map %s: %w", r.signatureMapPath(), err)
	}
	// The RICH table (D39.36's «стоп с таблицей»: src · dst · frequency · variant spread · evidence). It is
	// written as a sidecar on EVERY run, signed or not, because it is also the auto mode's record of what
	// the book decided on its own — and it is capped on stdout, never in the file (emitRankCap is 200).
	rows := bankStopRows(cands, consolidated, tres)
	if werr := os.WriteFile(r.bankStopTablePath(), []byte(renderBankStopTable(rows)), 0o644); werr != nil {
		r.Log.WarnContext(ctx, "bank-mining: could not write the stop table sidecar (the signature map is unaffected)", "err", werr)
	}
	// …and the SAME table as a machine surface (backlog row 101). Same rows, same order, same fields — the
	// text sidecar stays the human document and this one is what a signing screen is built from, so the two
	// can never describe the bank differently. Same failure discipline as the text sidecar: a sidecar that
	// cannot be written must not take a paid run down with it.
	if werr := r.writeBankStopTableJSON(rows); werr != nil {
		r.Log.WarnContext(ctx, "bank-mining: could not write the machine stop table sidecar (the signature map and the text table are unaffected)", "err", werr)
	}

	// THE FLAG (D39.42 п.5, owner's words: «можно запустить перевод так, чтоб сессия не останавливалась и
	// не запрашивала верификацию банка, а просто как намайнит и закончит — шла в редактуру»). Default is
	// auto-continue; the operator asks for the pause with --verify-bank. A DRAFT-ONLY pipeline never stops
	// either way: there is no edit wave for the stop to sit before, and stopping there would discard the
	// assembled BookResult of an already-paid draft wave (S16).
	if !r.VerifyBank {
		// THE AUTO WIRE (D39.42 п.3). The unsigned rows are persisted and folded back into the bank right
		// here, before the edit-wave snapshot is computed — so this run's editor actually receives them,
		// marked ⟨проверить⟩ in its own section, and the next run's drafts do too. Writing the file and
		// re-seeding through the ordinary seedGlossary path (rather than a second, private write) is what
		// keeps ONE definition of what the bank is: every guard the seed path owns — the collision checks,
		// the reject filter, the fail-louds — applies to the engine's rows exactly as to the owner's.
		if err := r.writeAutoBank(ctx, mined, proposals, ownerHandled(unsignedEngineSurfaces(seed), rejects)); err != nil {
			return false, err
		}
		if err := r.seedGlossary(ctx); err != nil {
			return false, fmt.Errorf("pipeline: re-seed the bank with the auto rows: %w", err)
		}
		// The bank just changed (row 125): the auto rows are IN it now, so the read-out has to say so
		// before the edit wave starts translating against them.
		r.exportBank(ctx, "bank-mining/auto-continue")
		r.Log.InfoContext(ctx, "bank-mining: auto-continuing with an UNSIGNED bank (pass --verify-bank to stop and review it)",
			append([]any{"book", r.Book.BookID, "terms", len(mined), "terms_with_proposed_dst", withDst,
				"auto_bank", r.autoBankPath(), "signature_map", r.signatureMapPath(),
				"table", r.bankStopTablePath(), "table_json", r.bankStopTableJSONPath()},
				emissionArgs(emission, cands)...)...)
		return false, nil
	}
	if !editWave {
		r.Log.WarnContext(ctx, "bank-mining: --verify-bank has nothing to stop before in a draft-only pipeline (no edit wave); the signature map and table are written and the run continues",
			"book", r.Book.BookID, "terms", len(mined), "signature_map", r.signatureMapPath())
		return false, nil
	}
	r.lastBankStopRows = rows
	// The signature stop is the boundary the signing screen reads at (row 125): refresh the bank read-out
	// so the state behind the decisions is the state as of this stop, not as of the last run.
	r.exportBank(ctx, "bank-mining/signature-stop")
	r.Log.WarnContext(ctx, "bank-mining: new terms await owner signature; run STOPPED before the edit wave (review the signature map, then for EACH term either promote it into the mined-delta file OR decline it in the mined-rejects file, then resume — the stop clears once every proposed term is promoted or rejected)",
		append([]any{"book", r.Book.BookID, "terms", len(mined), "terms_with_proposed_dst", withDst,
			"signature_map", r.signatureMapPath()}, emissionArgs(emission, cands)...)...)
	return true, nil
}

// emissionArgs renders the WHICH-funnel as log fields. It rides EVERY verdict of the stop, not only the
// empty one (G10 asks for the empty case; the same numbers answer "why so few?" on a non-empty delta, and
// they are the only place the top-200 keyhole is visible at all). Deterministic, $0 — the counters are read
// off the pass that already ran.
func emissionArgs(e miner.EmissionStats, cands []terminology.Candidate) []any {
	draftSide := 0
	for _, c := range cands {
		if c.Origin != terminology.OriginMined {
			draftSide++
		}
	}
	return []any{
		"ranked_alphabet", e.Ranked, "after_rank_cap", e.AfterCap, "eligible", e.Eligible,
		"skipped_as_alias_of_seeded", e.SeedSkipped, "skipped_as_declined", e.Rejected,
		"emitted_by_miner", e.Emitted, "draft_side_candidates", draftSide,
	}
}

// packStateLabel describes the langpack state for the --verify-bank fail-loud, so the operator is told
// WHICH of the two preconditions is missing rather than "it did not work".
func packStateLabel(p *lang.Pack) string {
	if p == nil {
		return "absent"
	}
	return "loaded (" + p.Pair + ")"
}

// bankStopTablePath is the sidecar holding the FULL stop table. It sits beside the signature map (and the
// DB) so it travels with the book state and never enters git, and it is a separate artifact from the
// signature map because that map must stay a loadable seed YAML: evidence in it would be schema noise
// (§C2-7 — evidence belongs in the sign-map sidecar, not the seed schema).
func (r *Runner) bankStopTablePath() string {
	return r.Book.ProjectDB + ".bank-stop.txt"
}

// bankStopTableJSONPath is the MACHINE sidecar of the same table (backlog row 101). It sits beside the
// text one for the same reason that one sits beside the signature map: a loadable seed YAML cannot carry
// evidence, and a table meant to be parsed cannot be a column layout. Deliberately a SEPARATE file rather
// than a replacement — the text table is the document a human reads at 3 a.m. with `less`.
func (r *Runner) bankStopTableJSONPath() string {
	return r.Book.ProjectDB + ".bank-stop.json"
}

// bankStopTableVersion versions the SHAPE of the machine table, so a consumer that must tolerate a field
// set changing under it has something to branch on. It is not a snapshot input: the table is evidence
// produced from stored answers, never a paid byte (the same standing the parse rule has, see above).
const bankStopTableVersion = "tm-bank-stop-v1"

// bankStopTableFile is the machine table's document shape.
type bankStopTableFile struct {
	Version string            `json:"table_version"`
	BookID  string            `json:"book_id"`
	Terms   int               `json:"terms"`
	Rows    []bankStopRowJSON `json:"rows"`
}

// bankStopRowJSON is ONE row of the machine table — the same fields the text table prints, with the two
// places where a human-facing rendering would lose information made explicit:
//
//   - Conf is a POINTER: the role's own confidence is absent for a term it never mentioned, and "absent"
//     and "0% sure" are opposites (mining.go confOrAbsent). The text table omits the word `confidence`
//     for the first case; JSON says null, and a consumer that reads 0 for both would sort the sheet
//     exactly wrong.
//   - every list is emitted as a list, never a joined string, because the text table's separators (", "
//     / " | " / "; ") occur inside real renderings and a consumer cannot split them back safely.
//
// It is a DTO rather than JSON tags on BankStopRow on purpose: BankStopRow is the CLI's in-memory row and
// its field names are free to follow the code, while these names are a published surface.
type bankStopRowJSON struct {
	Src         string            `json:"src"`
	Dst         string            `json:"dst"` // "" when nothing was consolidated for this term
	Origin      string            `json:"origin"`
	Type        string            `json:"type"`
	Freq        int               `json:"freq"`
	Spread      int               `json:"spread"`
	Conventions int               `json:"conventions"`
	Conf        *int              `json:"conf"`
	Invented    bool              `json:"invented"`
	Signals     []string          `json:"signals"`
	Contradicts []string          `json:"contradicts"`
	Variants    []BankStopVariant `json:"variants"`
	Evidence    []string          `json:"evidence"`
	Contexts    []string          `json:"contexts"`
}

// bankStopTableJSON projects the rows into the machine document. Deterministic: the row order is the
// caller's (source-key order, the reference order the text sidecar also uses), and every empty list is
// rendered as `[]` rather than `null` so a consumer has one shape to handle instead of two.
func bankStopTableJSON(bookID string, rows []BankStopRow) bankStopTableFile {
	out := bankStopTableFile{Version: bankStopTableVersion, BookID: bookID, Terms: len(rows), Rows: make([]bankStopRowJSON, 0, len(rows))}
	for _, row := range rows {
		jr := bankStopRowJSON{
			Src: row.Src, Dst: row.Dst, Origin: row.Origin, Type: row.Type,
			Freq: row.Freq, Spread: row.Spread, Conventions: row.Conventions, Invented: row.Invented,
			Signals: emptyIfNil(row.Signals), Contradicts: emptyIfNil(row.Contradicts),
			Variants: emptyVariantsIfNil(row.Variants), Evidence: emptyIfNil(row.Evidence), Contexts: emptyIfNil(row.Contexts),
		}
		if row.Conf >= 0 {
			conf := row.Conf
			jr.Conf = &conf
		}
		out.Rows = append(out.Rows, jr)
	}
	return out
}

// emptyVariantsIfNil is emptyIfNil for the variant list — one helper per element type keeps the render
// deterministic without reaching for generics, the same trade render.go's sortedKeys pair already makes.
func emptyVariantsIfNil(v []BankStopVariant) []BankStopVariant {
	if v == nil {
		return []BankStopVariant{}
	}
	return v
}

// emptyIfNil renders a nil slice as an empty one (see bankStopRowJSON).
func emptyIfNil(s []string) []string {
	if s == nil {
		return []string{}
	}
	return s
}

// clearBankStopTableJSON empties an EXISTING machine stop table, and does nothing when there is none —
// the difference between "this book has nothing awaiting signature" (a statement worth making) and "this
// book has no signing surface at all" (nothing to say, and a file nobody asked for).
func (r *Runner) clearBankStopTableJSON(ctx context.Context, why string) {
	if _, err := os.Stat(r.bankStopTableJSONPath()); err != nil {
		return // absent (or unreadable — then rewriting it is not this path's business either)
	}
	if werr := r.writeBankStopTableJSON(nil); werr != nil {
		r.Log.WarnContext(ctx, "bank-mining: could not clear the stale machine stop table sidecar; it still offers terms for signature",
			"path", r.bankStopTableJSONPath(), "reason", why, "err", werr)
		return
	}
	r.Log.InfoContext(ctx, "bank-mining: cleared the machine stop table sidecar", "path", r.bankStopTableJSONPath(), "reason", why)
}

// writeBankStopTableJSON serializes and atomically replaces the machine table sidecar.
func (r *Runner) writeBankStopTableJSON(rows []BankStopRow) error {
	body, err := json.MarshalIndent(bankStopTableJSON(r.Book.BookID, rows), "", "  ")
	if err != nil {
		return fmt.Errorf("pipeline: marshal the machine bank-stop table: %w", err)
	}
	return writeFileAtomic(r.bankStopTableJSONPath(), append(body, '\n'))
}

// BankStopRow is one row of the bank-verification table the stop shows the operator — the table D39.36
// specified and the CLI never had (it printed a term count and a path). Exported because the CLI renders it.
type BankStopRow struct {
	Src      string
	Dst      string // the consolidated rendering, or "" when nothing was consolidated
	Origin   string // mined | banknote | both — WHICH channel found it
	Type     string
	Freq     int               // occurrences in the source
	Spread   int               // how many DISTINCT renderings the drafts produced (the disagreement signal)
	Variants []BankStopVariant // the renderings the drafts produced, best-ranked first
	Contexts []string          // source KWIC
	Evidence []string
	// The §G3 arbitration record: until this pack, «why does this term have THIS dst» was unanswerable from
	// the artifacts (research/24 §A7). Every field below is read off work the ranking already did.
	//
	// Conventions is how many genuinely different DECISIONS the drafts made, once renderings differing only
	// in target form are folded (Spread counts the raw forms). Signals are the §C2-3 factors that fired for
	// the TOP-ranked variant — the winner's audit trail, printed for the row rather than per variant,
	// because the row is what the owner signs. Invented says the consolidated rendering is NOT one the
	// drafts proposed: legitimate (the role sees the whole book, the drafts saw fragments) and exactly the
	// class to read first. Conf is the role's own stated confidence, which orders the review list and
	// nothing else (D39.102) — NEGATIVE when the reply carried none, because «the role said it was 0% sure»
	// is the most important row on the sheet and «the role said nothing» is not a row at all.
	// Contradicts names THIS RUN's other consolidations the rendering breaks (§G2).
	Conventions int
	Signals     []string
	Invented    bool
	Conf        int
	Contradicts []string
}

// BankStopVariant is ONE rendering the drafts produced, kept in its PARTS rather than as the sentence a
// table prints. The renderers format it (Label); the machine sidecar ships the parts. Before this the row
// carried the pre-rendered «<dst> ×<n> (proposed for <via>)» string and nothing else, so the machine
// surface could only be un-parsed by splitting on «×» and on a parenthesis — both of which occur inside
// real renderings.
// Its JSON tags ARE the published shape of the machine sidecar: unlike the row, whose field names in the
// file are deliberately independent of the code's, a variant's three parts are called the same thing on
// both sides, and a second type to say so would only be a conversion waiting to drift.
type BankStopVariant struct {
	Dst    string `json:"dst"`
	Chunks int    `json:"chunks"` // how many draft chunks proposed it
	// Via names the ALIAS this rendering was proposed for, "" for a direct proposal. Load-bearing rather
	// than decoration: an alias-routed rendering must never silently become the cluster owner's dst
	// (proposalsFromCandidates), so a surface that offers one has to be able to say whose it was.
	Via string `json:"via,omitempty"`
}

// Label is the one rendering of a variant every human-facing table uses, so the capped stdout view and
// the text sidecar cannot describe a variant differently.
func (v BankStopVariant) Label() string {
	s := fmt.Sprintf("%s ×%d", v.Dst, v.Chunks)
	if v.Via != "" {
		s += " (proposed for " + v.Via + ")"
	}
	return s
}

// VariantLabels renders a row's variants for a text table.
func (r BankStopRow) VariantLabels() []string {
	out := make([]string, 0, len(r.Variants))
	for _, v := range r.Variants {
		out = append(out, v.Label())
	}
	return out
}

// bankStopRows projects the merged candidates into the operator table, best-ranked variants first.
// Deterministic: cands is key-ordered and nothing here iterates a map for output.
func bankStopRows(cands []terminology.Candidate, consolidated map[string]string, tres terminologyResult) []BankStopRow {
	out := make([]BankStopRow, 0, len(cands))
	for _, c := range cands {
		dst := consolidated[c.Key]
		row := BankStopRow{
			Src: c.Src, Dst: dst, Origin: string(c.Origin), Type: c.Type,
			Freq: c.Freq, Spread: c.Spread(), Conventions: c.Conventions(),
			Contexts: c.KWIC, Evidence: c.Evidence,
			Conf: confOrAbsent(tres.Conf, c.Key), Contradicts: tres.Contradictions[c.Src],
		}
		for i, v := range c.Variants {
			row.Variants = append(row.Variants, BankStopVariant{Dst: v.Dst, Chunks: v.Chunks, Via: v.Via})
			if i == 0 {
				row.Signals = v.Signals
			}
		}
		row.Invented = dst != "" && !proposedByDrafts(dst, c.Variants)
		out = append(out, row)
	}
	return out
}

// confOrAbsent reads the role's stated confidence for a key, or -1 when the reply carried none. A plain
// zero would merge the two, and they are opposites: one is the first row to review, the other is silence.
func confOrAbsent(conf map[string]int, key string) int {
	if v, ok := conf[key]; ok {
		return v
	}
	return -1
}

// proposedByDrafts reports whether the consolidated rendering is one the drafts actually produced, compared
// under the same target-form fold the vote is counted with — so a case or ё difference is not reported as
// an invention.
func proposedByDrafts(dst string, vs []terminology.Variant) bool {
	want := text.NormalizeTargetForm(dst)
	for _, v := range vs {
		if text.NormalizeTargetForm(v.Dst) == want {
			return true
		}
	}
	return false
}

// renderBankStopTable serializes the FULL table for the sidecar. One block per term, the same shape the
// stdout banner prints — so the capped view and the file cannot describe the bank differently.
func renderBankStopTable(rows []BankStopRow) string {
	var b strings.Builder
	fmt.Fprintf(&b, "BANK VERIFICATION TABLE — %d term(s)\n", len(rows))
	b.WriteString("src · proposed dst · origin · type · freq · variant spread · conventions · confidence ·\n")
	b.WriteString("why (the ranking factors that won) · contradictions · drafts · evidence · source contexts\n\n")
	for _, r := range rows {
		fmt.Fprintf(&b, "%s\t%s\n", r.Src, dashIfEmpty(r.Dst))
		fmt.Fprintf(&b, "  origin=%s type=%s freq=%d spread=%d conventions=%d", r.Origin, dashIfEmpty(r.Type), r.Freq, r.Spread, r.Conventions)
		if r.Conf >= 0 {
			fmt.Fprintf(&b, " confidence=%d", r.Conf)
		}
		if r.Invented {
			b.WriteString(" INVENTED(no draft proposed it)")
		}
		b.WriteString("\n")
		if len(r.Signals) > 0 {
			fmt.Fprintf(&b, "  why: %s\n", strings.Join(r.Signals, ", "))
		}
		if len(r.Contradicts) > 0 {
			fmt.Fprintf(&b, "  CONTRADICTS this run's own: %s\n", strings.Join(r.Contradicts, "; "))
		}
		if len(r.Variants) > 0 {
			fmt.Fprintf(&b, "  drafts: %s\n", strings.Join(r.VariantLabels(), " | "))
		}
		if len(r.Evidence) > 0 {
			fmt.Fprintf(&b, "  evidence: %s\n", strings.Join(r.Evidence, ", "))
		}
		for _, k := range r.Contexts {
			fmt.Fprintf(&b, "  ctx: %s\n", k)
		}
		b.WriteString("\n")
	}
	return b.String()
}

func dashIfEmpty(s string) string {
	if strings.TrimSpace(s) == "" {
		return "—"
	}
	return s
}

// reverseSectionTerms turns the banknote-only candidates into emittable terms. Guards, each closing a way
// the reverse section could pollute the delta:
//   - the surface must OCCUR in the source (a KWIC context exists) — a rendering a draft invented for a
//     word that is not in the book is not a term of the book;
//   - an existing seed surface is skipped (it is already in the bank);
//   - a declined surface is skipped, so a reject stays declined through this door too (R1-FL-B).
//
// Deterministic: cands is key-ordered and nothing here iterates a map.
// Returns the capped list plus how many were ELIGIBLE before the cap, so a truncation is reported rather
// than silent.
func reverseSectionTerms(cands []terminology.Candidate, seed []store.GlossaryEntry, rejects map[string]bool) (out []miner.Term, eligible int) {
	seedSurfaces := map[string]bool{}
	for _, e := range seed {
		seedSurfaces[text.NormalizeSourceKey(e.Src)] = true
		for _, a := range e.Aliases {
			seedSurfaces[text.NormalizeSourceKey(a.Alias)] = true
		}
	}
	for _, c := range cands {
		if c.Origin != terminology.OriginBanknote || len(c.KWIC) == 0 {
			continue
		}
		if seedSurfaces[c.Key] || rejects[c.Key] {
			continue
		}
		ev := []string{"banknote-only candidate (the miner did not surface it)"}
		if len(c.Related) > 0 {
			ev = append(ev, "related to mined "+strings.Join(c.Related, ", "))
		}
		out = append(out, miner.Term{Src: c.Key, Type: c.Type, Freq: c.Freq, SinceCh: 0, Evidence: ev})
	}
	// The SAME volume cap the miner's own emission applies (miner.EmitRankCap), ranked the same way — by
	// frequency. Without it this door is uncapped: every banknote-only surface of the whole book enters the
	// delta the owner is asked to sign, the auto-bank, and the terminologist's batches, while the miner's
	// side of the same file stops at 200. The drop is reported by the caller, never silent.
	eligible = len(out)
	sort.Slice(out, func(i, j int) bool {
		if out[i].Freq != out[j].Freq {
			return out[i].Freq > out[j].Freq
		}
		return out[i].Src < out[j].Src
	})
	if cap := miner.EmitRankCap(); len(out) > cap {
		out = out[:cap]
	}
	return out, eligible
}

// proposalsFromCandidates re-shapes the MERGED candidates into the signature-map join's input.
//
// It reads the candidates rather than the raw draft-side view, and that is the fix (§G3, acceptance
// finding): a proposal that arrived under an ALIAS of a cluster is routed to the cluster's owner by
// Merge — the stop table therefore showed it — while the signature map joined on the alias's own key and
// the owner's row never mentioned it. The map the owner signs then disagreed with the table he was reading
// it against, for exactly the terms the miner clustered. One source for both removes the divergence
// structurally instead of keeping two joins in step by hand.
//
// The list arrives §C2-3-ranked and target-form folded, so the note's alternatives are the ones the table
// shows, in the order it shows them.
func proposalsFromCandidates(cands []terminology.Candidate) map[string][]miner.DstProposal {
	out := make(map[string][]miner.DstProposal, len(cands))
	for _, c := range cands {
		if len(c.Variants) == 0 {
			continue
		}
		// DIRECT proposals first, alias-routed ones after — and DeltaYAML takes the term's dst from a DIRECT
		// one only. Merge routes an alias's rendering to the cluster owner so the ranking sees all of the
		// entity's evidence; letting that rendering become the OWNER's dst is a different act entirely. It
		// would put «Малыш Фан» on 方源 with no model involved, on the $0 path, with the terminology gate OFF —
		// the exact harm Variant.Via was introduced to prevent, arriving through the fix that introduced Via.
		list := make([]miner.DstProposal, 0, len(c.Variants))
		for _, v := range c.Variants {
			if v.Via == "" {
				list = append(list, miner.DstProposal{Dst: v.Dst, Type: c.Type, Chunks: v.Chunks})
			}
		}
		for _, v := range c.Variants {
			if v.Via != "" {
				list = append(list, miner.DstProposal{Dst: v.Dst, Type: c.Type, Chunks: v.Chunks, Via: v.Via})
			}
		}
		out[c.Key] = list
	}
	return out
}

// autoBankPath is where the AUTO mode records the bank it built for itself: the terminologist's
// consolidated rows, unsigned. It sits beside the project DB like the signature map — book state, never
// git — and is deliberately a SEPARATE file from the owner's mined_delta: that file is the owner's word,
// this one is the engine's, and merging the two would make it impossible to tell later which renderings
// a human actually approved.
func (r *Runner) autoBankPath() string { return r.Book.ProjectDB + ".auto-bank.yaml" }

// loadAutoBank reads the engine's unsigned rows and returns the ones that may enter the bank, plus the
// human-readable list of those dropped. Three filters, each closing a named risk of the phase-1 design:
//
//   - the REJECT SET applies here too (risk 3). Until pack-20 rejects were consulted only at EMISSION, so
//     a term the owner declined could survive in an accumulated file and re-enter the bank through the
//     back door. «reject-set works in both modes» has to mean on the way IN, not only on the way out.
//   - a row whose UNIQUE key (src, sense, since_ch, until_ch) is already held by a SIGNED row is dropped
//     (risk 4). The flat INSERT in ReplaceGlossary would otherwise crash on the constraint and abort a
//     paid run — and the right resolution is never "the engine's guess replaces the signature".
//   - nothing here can carry `approved`: the loader is the same seed loader, and the status it reads is
//     whatever the emission wrote (auto/draft). A file hand-edited to say `approved` is refused loudly,
//     because that would be a signature nobody gave.
func (r *Runner) loadAutoBank(signed []store.GlossaryEntry) (rows []store.GlossaryEntry, dropped []string, err error) {
	// Only ABSENT means "auto mode has not run"; an unreadable file must not drop the mined rows.
	if _, serr := os.Stat(r.autoBankPath()); serr != nil {
		if errors.Is(serr, fs.ErrNotExist) {
			return nil, nil, nil // the auto mode has not run yet (or the book never uses it)
		}
		return nil, nil, fmt.Errorf("pipeline: stat auto-bank %s: %w", r.autoBankPath(), serr)
	}
	entries, err := membank.LoadGlossarySeed(r.autoBankPath())
	if err != nil {
		return nil, nil, fmt.Errorf("pipeline: load auto-bank %s: %w", r.autoBankPath(), err)
	}
	rejects, err := r.loadMinedRejects()
	if err != nil {
		return nil, nil, err
	}
	for _, e := range entries {
		if e.Status == "approved" {
			return nil, nil, fmt.Errorf("pipeline: auto-bank %s carries an `approved` row (%q → %q): this file is the ENGINE's unsigned proposals, and nothing in it may claim a signature — move the term into the owner's mined-delta file instead",
				r.autoBankPath(), e.Src, e.Dst)
		}
	}
	type ukey struct {
		src, sense   string
		since, until int
	}
	held := map[ukey]store.GlossaryEntry{}
	for _, e := range signed {
		held[ukey{e.Src, e.Sense, e.SinceCh, e.UntilCh}] = e
	}
	for _, e := range entries {
		if rejects[text.NormalizeSourceKey(e.Src)] {
			dropped = append(dropped, fmt.Sprintf("%q (declined in mined_rejects)", e.Src))
			continue
		}
		if prior, clash := held[ukey{e.Src, e.Sense, e.SinceCh, e.UntilCh}]; clash {
			dropped = append(dropped, fmt.Sprintf("%q→%q (key held by the signed %q→%q)", e.Src, e.Dst, prior.Src, prior.Dst))
			continue
		}
		e.Source = "mined" // base-excluded: an unsigned row never moves the draft wave's snapshot
		rows = append(rows, e)
	}
	return rows, dropped, nil
}

// unsignedEngineSurfaces reports the bank rows that must NOT count as mining seed surfaces: the engine's
// own unsigned proposals (Source:"mined" + not approved). Risk 2 of the phase-1 design — the
// self-exclusion trap. Once the auto mode writes its rows into the bank they would, on the next run,
// look to the miner exactly like a seeded term: the delta empties, and `--verify-bank` silently stops
// firing on terms nobody ever reviewed. Filtering them keeps the proposal list stable until the owner
// PROMOTES a term (into mined_delta, as approved → a real seed surface) or DECLINES it (mined_rejects) —
// the two verbs that are supposed to be the only way the stop clears.
func unsignedEngineSurfaces(rows []store.GlossaryEntry) []store.GlossaryEntry {
	out := rows[:0:0]
	for _, e := range rows {
		if e.Source == "mined" && e.Status != "approved" {
			continue
		}
		out = append(out, e)
	}
	return out
}

// writeAutoBank persists the unsigned rows the auto mode decided to carry forward, as the same seed-YAML
// schema everything else in this pipeline speaks (so `tmctl seed-lint` reads it, and a row can be moved
// into the owner's delta by copy-paste). Deterministic: the mined list is already sorted by src.
//
// It DIFFS the file it is about to replace, and that is the $0 minimum of backlog row 130. The file is
// rewritten WHOLE from this run's delta, and the delta is capped at the miner's top-200 (emitRankCap,
// applied BEFORE the emission filters, so seed and declined terms do not free their slots). The two
// mechanisms were ratified separately and their INTERACTION never was: as a book grows, a term that was in
// the bank for twenty chapters silently vanishes from it mid-run, with nothing in the artifacts saying so.
// Naming the losers costs nothing and moves no bytes.
//
// The BOUNDARY is deliberate and is the owner's STOP: this reports, it does not accumulate. Merging the old
// file into the new one would change what the bank CONTAINS, which moves memory_version and re-prices the
// edit wave — a decision, not a hygiene fix.
func (r *Runner) writeAutoBank(ctx context.Context, mined []miner.Term, proposals map[string][]miner.DstProposal, signed map[string]bool) error {
	before := r.autoBankSurfaces()
	body, err := miner.DeltaYAML(mined, proposals)
	if err != nil {
		return fmt.Errorf("pipeline: marshal auto-bank: %w", err)
	}
	if err := os.WriteFile(r.autoBankPath(), []byte(body), 0o644); err != nil {
		return fmt.Errorf("pipeline: write auto-bank %s: %w", r.autoBankPath(), err)
	}
	if len(before) == 0 {
		return nil
	}
	now := make(map[string]bool, len(mined))
	for _, m := range mined {
		now[text.NormalizeSourceKey(m.Src)] = true
	}
	var gone []string
	for _, src := range before {
		key := text.NormalizeSourceKey(src)
		if now[key] || signed[key] {
			// signed[] is the owner's two verbs — PROMOTED into the mined-delta or DECLINED in mined-rejects.
			// Both remove the term from this run's delta on purpose, and reporting them as a silent loss would
			// fire a false alarm on the normal signature cycle — advising the owner to do what he just did.
			continue
		}
		gone = append(gone, src)
	}
	if len(gone) > 0 {
		sort.Strings(gone)
		r.Log.WarnContext(ctx, "bank-mining: terms that were in the auto-bank are NOT in the one this run just wrote — the file is rewritten whole from a top-N-capped delta, so a term the book still uses can drop out of the bank mid-run; if one of these matters, promote it into the owner's mined-delta file (it is then a seed surface and cannot be cut again)",
			"book", r.Book.BookID, "dropped", len(gone), "kept", len(mined), "rank_cap", miner.EmitRankCap(),
			"terms", strings.Join(gone, ", "), "auto_bank", r.autoBankPath())
	}
	return nil
}

// ownerHandled is the surface set the OWNER has already decided about: every signed seed surface (a promoted
// term is one) plus every declined one. A term leaving the delta through either door is not a loss.
//
// ⚠ The caller MUST pass it through unsignedEngineSurfaces — risk 2, the self-exclusion trap this file
// documents twice and works around in two other places. From the SECOND auto-mode run the stored glossary
// also holds the engine's OWN unsigned rows (seedGlossary re-seeds the auto-bank at start), so a raw seed
// makes every term the engine ever proposed look owner-decided: the diff empties, and the row-130 warning —
// whose entire purpose is to name terms the rewrite dropped — can never fire again in production.
func ownerHandled(seed []store.GlossaryEntry, rejects map[string]bool) map[string]bool {
	out := make(map[string]bool, len(seed)+len(rejects))
	for _, e := range seed {
		out[text.NormalizeSourceKey(e.Src)] = true
		for _, a := range e.Aliases {
			out[text.NormalizeSourceKey(a.Alias)] = true
		}
	}
	for k := range rejects {
		out[k] = true
	}
	return out
}

// autoBankSurfaces reads the src surfaces of the auto-bank file as it stands BEFORE this run rewrites it,
// in file order. Absent or unreadable → nil: the diff is observability, and failing a paid run because the
// PREVIOUS artifact cannot be parsed would be the tail wagging the dog.
func (r *Runner) autoBankSurfaces() []string {
	entries, err := membank.LoadGlossarySeed(r.autoBankPath())
	if err != nil {
		return nil
	}
	out := make([]string, 0, len(entries))
	for _, e := range entries {
		if e.Src != "" {
			out = append(out, e.Src)
		}
	}
	return out
}

// minedRejectFile is the owner's mined-term reject list (Book.MinedRejects, R1-FL-B): the src surfaces the
// owner reviewed and DECLINED. It is a PROPOSAL filter only — rejects never enter the bank content, so this
// file is deliberately NOT folded into the snapshot (a reject affects the next mining PROPOSAL, not any
// checkpoint's wire/verdict).
type minedRejectFile struct {
	Rejects []minedReject `yaml:"rejects"`
}

// minedReject is one declined mined term. Note is the owner's optional reason, ignored by the miner but
// kept so a reject list stays self-documenting (six months on, "why was this declined").
type minedReject struct {
	Src  string `yaml:"src"`
	Note string `yaml:"note,omitempty"`
}

// loadMinedRejects reads Book.MinedRejects and returns the normalized src set the emission excludes (like
// the seed surfaces). Empty path → nil (no rejects). Each src is normalized via text.NormalizeSourceKey so a
// reject matches the miner's normalized candidate surface whichever orthographic form the owner pasted from
// the signature map; a blank src is skipped (a stray list entry must not silently match everything).
func (r *Runner) loadMinedRejects() (map[string]bool, error) {
	if r.Book.MinedRejects == "" {
		return nil, nil
	}
	raw, err := os.ReadFile(r.Book.MinedRejects)
	if err != nil {
		return nil, fmt.Errorf("pipeline: read mined-rejects %s: %w", r.Book.MinedRejects, err)
	}
	var f minedRejectFile
	if err := yaml.Unmarshal(raw, &f); err != nil {
		return nil, fmt.Errorf("pipeline: parse mined-rejects %s: %w", r.Book.MinedRejects, err)
	}
	rejects := map[string]bool{}
	for _, rj := range f.Rejects {
		if nk := text.NormalizeSourceKey(rj.Src); nk != "" {
			rejects[nk] = true
		}
	}
	return rejects, nil
}

// loadMinedDelta reads the owner-curated mined-delta YAML (book.MinedDelta) and stamps every entry
// Source:"mined" — NOT via membank.LoadGlossarySeed (which hardcodes Source:"seed", memseed.go, moving the base
// bank / draft-wave snapshot). This is the mined-write path (plan §1(в), F2): the mined terms land in the ENRICHED
// bank version but NOT the base, so adding them moves ONLY edit-wave snapshot ("re-paying ONCE"). Reuses
// membank.LoadGlossarySeed's parser/validation, then re-stamps the Source. Empty path → nil (no mined terms).
func (r *Runner) loadMinedDelta() ([]store.GlossaryEntry, error) {
	if r.Book.MinedDelta == "" {
		return nil, nil
	}
	entries, err := membank.LoadGlossarySeed(r.Book.MinedDelta)
	if err != nil {
		return nil, fmt.Errorf("pipeline: load mined-delta %s: %w", r.Book.MinedDelta, err)
	}
	for i := range entries {
		entries[i].Source = "mined" // override the seed loader's Source:seed → mined (base-excluded)
	}
	return entries, nil
}