textmachine/backend/internal/pipeline/mining.go

package pipeline

import (
	"context"
	"fmt"
	"os"

	"gopkg.in/yaml.v3"

	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/membank"
	"textmachine/backend/internal/miner"
	"textmachine/backend/internal/store"
	"textmachine/backend/internal/text"
)

// mining.go: bank-mining stop boundary (WS3 wired live, R1). Between the drafts done and the edit wave, the
// miner scores WHICH-candidates over the the draft wave source (the detector is offline — it reads the SOURCE, not the
// drafts; drafts only mark that a chapter was reached) against the general-zh contrast, emits an
// alias-clustered seed-delta (default B, WHICH-only — the dst is the owner's to attach via the banknote at
// sign time), and — on a NON-EMPTY delta — writes the owner SIGNATURE MAP and STOPS before the edit wave. The owner
// reviews it, promotes terms into the mined-delta file (approved + dst), and re-runs: seedGlossary loads
// those as Source:mined (moving only edit-wave snapshot), the draft wave resumes at $0, the bank-mining stop re-mines (the now-seeded terms are
// excluded → the delta empties), and the edit wave runs. Mining is OFF (auto-continue) unless a langpack AND a contrast
// artifact are both configured — so every $0 test / the golden fixture (no contrast) auto-continues.

// signatureMapPath is where the bank-mining stop writes the mined-delta YAML for owner sign — beside the project DB, so it
// travels with the book state (never in git, like the DB). Deterministic (no time/rand).
func (r *Runner) signatureMapPath() string {
	return r.Book.ProjectDB + ".mined-signature.yaml"
}

// runBankMiningStop executes the bank-mining stop. Returns stopped=true (with the signature map written and
// r.lastMinedCount set) when the miner proposes a non-empty delta; stopped=false (auto-continue) when mining
// is unconfigured or the delta is empty. $0 to providers (the detector is deterministic + offline).
func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk) (stopped bool, err error) {
	if r.pack == nil || r.Pipeline.Mining.ContrastPath == "" {
		return false, nil // mining not configured → auto-continue to the edit wave
	}
	f, err := os.Open(r.Pipeline.Mining.ContrastPath)
	if err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop open mining contrast %s: %w", r.Pipeline.Mining.ContrastPath, err)
	}
	defer f.Close()
	contrast, err := miner.LoadContrast(f)
	if err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop load mining contrast %s: %w", r.Pipeline.Mining.ContrastPath, err)
	}

	// The miner works over the memnorm-normalized SOURCE of every chunk (the candidate space matches the
	// glossary/GT space). Deterministic; the annotation/blurb rule is handled inside miner.MineBank's filters.
	minerChunks := make([]miner.Chunk, len(chunks))
	for i, ch := range chunks {
		minerChunks[i] = miner.Chunk{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, NSource: text.NormalizeSourceKey(ch.Text)}
	}
	seed, err := r.Store.GlossaryForBook(r.Book.BookID)
	if err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop read glossary for mining: %w", err)
	}
	// The owner's reject list excludes declined terms from the emission (R1-FL-B): a term the owner reviewed
	// and rejected is dropped from the delta exactly like a seed surface, so it never re-fires the stop. The
	// stop therefore clears once EVERY proposed term is EITHER promoted (into mined_delta → a seed surface)
	// OR rejected (mined_rejects) — the two owner verbs that empty the delta.
	rejects, err := r.loadMinedRejects()
	if err != nil {
		return false, err
	}
	mined := miner.MineBank(minerChunks, contrast, seed, rejects, miner.FrozenConfig(), r.pack)
	r.lastMinedCount = len(mined)
	if len(mined) == 0 {
		r.Log.InfoContext(ctx, "bank-mining: empty delta, auto-continuing to the edit wave", "book", r.Book.BookID)
		return false, nil
	}

	// Non-empty delta → write the owner signature map (the mined seed-delta YAML) and STOP before the edit wave.
	//
	// The WHAT join (D39.36 fix): the banknote proposals the draft wave collected are read off the durable
	// per-chunk rows and attached to the map, so the owner signs a term that already carries the
	// translator's proposed rendering. They arrive as EVIDENCE — every emitted term stays status:auto, so
	// nothing proposed enters the bank without a signature. A read failure degrades to the previous
	// WHICH-only map rather than blocking the stop: the map is what the owner needs, the dst is a bonus.
	proposals := map[string][]miner.DstProposal{}
	if states, serr := r.Store.RetrievalStatesForBook(r.Book.BookID); serr != nil {
		r.Log.WarnContext(ctx, "bank-mining: could not read the banknote proposals; the signature map falls back to WHICH-only (bare terms)", "err", serr)
	} else {
		proposals = bankProposalsByKey(states)
	}
	withDst := 0
	for _, m := range mined {
		if len(proposals[text.NormalizeSourceKey(m.Src)]) > 0 {
			withDst++
		}
	}
	yamlDelta, err := miner.DeltaYAML(mined, proposals)
	if err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop marshal mined delta: %w", err)
	}
	if err := os.WriteFile(r.signatureMapPath(), []byte(yamlDelta), 0o644); err != nil {
		return false, fmt.Errorf("pipeline: the bank-mining stop write signature map %s: %w", r.signatureMapPath(), err)
	}
	r.Log.WarnContext(ctx, "bank-mining: new terms await owner signature; run STOPPED before the edit wave (review the signature map, then for EACH term either promote it into the mined-delta file OR decline it in the mined-rejects file, then resume — the stop clears once every proposed term is promoted or rejected)",
		"book", r.Book.BookID, "terms", len(mined), "terms_with_proposed_dst", withDst, "signature_map", r.signatureMapPath())
	return true, nil
}

// minedRejectFile is the owner's mined-term reject list (Book.MinedRejects, R1-FL-B): the src surfaces the
// owner reviewed and DECLINED. It is a PROPOSAL filter only — rejects never enter the bank content, so this
// file is deliberately NOT folded into the snapshot (a reject affects the next mining PROPOSAL, not any
// checkpoint's wire/verdict).
type minedRejectFile struct {
	Rejects []minedReject `yaml:"rejects"`
}

// minedReject is one declined mined term. Note is the owner's optional reason, ignored by the miner but
// kept so a reject list stays self-documenting (six months on, "why was this declined").
type minedReject struct {
	Src  string `yaml:"src"`
	Note string `yaml:"note,omitempty"`
}

// loadMinedRejects reads Book.MinedRejects and returns the normalized src set the emission excludes (like
// the seed surfaces). Empty path → nil (no rejects). Each src is normalized via text.NormalizeSourceKey so a
// reject matches the miner's normalized candidate surface whichever orthographic form the owner pasted from
// the signature map; a blank src is skipped (a stray list entry must not silently match everything).
func (r *Runner) loadMinedRejects() (map[string]bool, error) {
	if r.Book.MinedRejects == "" {
		return nil, nil
	}
	raw, err := os.ReadFile(r.Book.MinedRejects)
	if err != nil {
		return nil, fmt.Errorf("pipeline: read mined-rejects %s: %w", r.Book.MinedRejects, err)
	}
	var f minedRejectFile
	if err := yaml.Unmarshal(raw, &f); err != nil {
		return nil, fmt.Errorf("pipeline: parse mined-rejects %s: %w", r.Book.MinedRejects, err)
	}
	rejects := map[string]bool{}
	for _, rj := range f.Rejects {
		if nk := text.NormalizeSourceKey(rj.Src); nk != "" {
			rejects[nk] = true
		}
	}
	return rejects, nil
}

// loadMinedDelta reads the owner-curated mined-delta YAML (book.MinedDelta) and stamps every entry
// Source:"mined" — NOT via membank.LoadGlossarySeed (which hardcodes Source:"seed", memseed.go, moving the base
// bank / draft-wave snapshot). This is the mined-write path (plan §1(в), F2): the mined terms land in the ENRICHED
// bank version but NOT the base, so adding them moves ONLY edit-wave snapshot ("re-paying ONCE"). Reuses
// membank.LoadGlossarySeed's parser/validation, then re-stamps the Source. Empty path → nil (no mined terms).
func (r *Runner) loadMinedDelta() ([]store.GlossaryEntry, error) {
	if r.Book.MinedDelta == "" {
		return nil, nil
	}
	entries, err := membank.LoadGlossarySeed(r.Book.MinedDelta)
	if err != nil {
		return nil, fmt.Errorf("pipeline: load mined-delta %s: %w", r.Book.MinedDelta, err)
	}
	for i := range entries {
		entries[i].Source = "mined" // override the seed loader's Source:seed → mined (base-excluded)
	}
	return entries, nil
}