package pipeline import ( "context" "fmt" "os" "gopkg.in/yaml.v3" "textmachine/backend/internal/chunk" "textmachine/backend/internal/membank" "textmachine/backend/internal/miner" "textmachine/backend/internal/store" "textmachine/backend/internal/text" ) // mining.go: bank-mining stop boundary (WS3 wired live, R1). Between the drafts done and the edit wave, the // miner scores WHICH-candidates over the the draft wave source (the detector is offline — it reads the SOURCE, not the // drafts; drafts only mark that a chapter was reached) against the general-zh contrast, emits an // alias-clustered seed-delta (default B, WHICH-only — the dst is the owner's to attach via the banknote at // sign time), and — on a NON-EMPTY delta — writes the owner SIGNATURE MAP and STOPS before the edit wave. The owner // reviews it, promotes terms into the mined-delta file (approved + dst), and re-runs: seedGlossary loads // those as Source:mined (moving only edit-wave snapshot), the draft wave resumes at $0, the bank-mining stop re-mines (the now-seeded terms are // excluded → the delta empties), and the edit wave runs. Mining is OFF (auto-continue) unless a langpack AND a contrast // artifact are both configured — so every $0 test / the golden fixture (no contrast) auto-continues. // signatureMapPath is where the bank-mining stop writes the mined-delta YAML for owner sign — beside the project DB, so it // travels with the book state (never in git, like the DB). Deterministic (no time/rand). func (r *Runner) signatureMapPath() string { return r.Book.ProjectDB + ".mined-signature.yaml" } // runBankMiningStop executes the bank-mining stop. Returns stopped=true (with the signature map written and // r.lastMinedCount set) when the miner proposes a non-empty delta; stopped=false (auto-continue) when mining // is unconfigured or the delta is empty. $0 to providers (the detector is deterministic + offline). func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk) (stopped bool, err error) { if r.pack == nil || r.Pipeline.Mining.ContrastPath == "" { return false, nil // mining not configured → auto-continue to the edit wave } f, err := os.Open(r.Pipeline.Mining.ContrastPath) if err != nil { return false, fmt.Errorf("pipeline: the bank-mining stop open mining contrast %s: %w", r.Pipeline.Mining.ContrastPath, err) } defer f.Close() contrast, err := miner.LoadContrast(f) if err != nil { return false, fmt.Errorf("pipeline: the bank-mining stop load mining contrast %s: %w", r.Pipeline.Mining.ContrastPath, err) } // The miner works over the memnorm-normalized SOURCE of every chunk (the candidate space matches the // glossary/GT space). Deterministic; the annotation/blurb rule is handled inside miner.MineBank's filters. minerChunks := make([]miner.Chunk, len(chunks)) for i, ch := range chunks { minerChunks[i] = miner.Chunk{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, NSource: text.NormalizeSourceKey(ch.Text)} } seed, err := r.Store.GlossaryForBook(r.Book.BookID) if err != nil { return false, fmt.Errorf("pipeline: the bank-mining stop read glossary for mining: %w", err) } // The owner's reject list excludes declined terms from the emission (R1-FL-B): a term the owner reviewed // and rejected is dropped from the delta exactly like a seed surface, so it never re-fires the stop. The // stop therefore clears once EVERY proposed term is EITHER promoted (into mined_delta → a seed surface) // OR rejected (mined_rejects) — the two owner verbs that empty the delta. rejects, err := r.loadMinedRejects() if err != nil { return false, err } mined := miner.MineBank(minerChunks, contrast, seed, rejects, miner.FrozenConfig(), r.pack) r.lastMinedCount = len(mined) if len(mined) == 0 { r.Log.InfoContext(ctx, "bank-mining: empty delta, auto-continuing to the edit wave", "book", r.Book.BookID) return false, nil } // Non-empty delta → write the owner signature map (the mined seed-delta YAML) and STOP before the edit wave. yamlDelta, err := miner.DeltaYAML(mined) if err != nil { return false, fmt.Errorf("pipeline: the bank-mining stop marshal mined delta: %w", err) } if err := os.WriteFile(r.signatureMapPath(), []byte(yamlDelta), 0o644); err != nil { return false, fmt.Errorf("pipeline: the bank-mining stop write signature map %s: %w", r.signatureMapPath(), err) } r.Log.WarnContext(ctx, "bank-mining: new terms await owner signature; run STOPPED before the edit wave (review the signature map, then for EACH term either promote it into the mined-delta file OR decline it in the mined-rejects file, then resume — the stop clears once every proposed term is promoted or rejected)", "book", r.Book.BookID, "terms", len(mined), "signature_map", r.signatureMapPath()) return true, nil } // minedRejectFile is the owner's mined-term reject list (Book.MinedRejects, R1-FL-B): the src surfaces the // owner reviewed and DECLINED. It is a PROPOSAL filter only — rejects never enter the bank content, so this // file is deliberately NOT folded into the snapshot (a reject affects the next mining PROPOSAL, not any // checkpoint's wire/verdict). type minedRejectFile struct { Rejects []minedReject `yaml:"rejects"` } // minedReject is one declined mined term. Note is the owner's optional reason, ignored by the miner but // kept so a reject list stays self-documenting (six months on, "why was this declined"). type minedReject struct { Src string `yaml:"src"` Note string `yaml:"note,omitempty"` } // loadMinedRejects reads Book.MinedRejects and returns the normalized src set the emission excludes (like // the seed surfaces). Empty path → nil (no rejects). Each src is normalized via text.NormalizeSourceKey so a // reject matches the miner's normalized candidate surface whichever orthographic form the owner pasted from // the signature map; a blank src is skipped (a stray list entry must not silently match everything). func (r *Runner) loadMinedRejects() (map[string]bool, error) { if r.Book.MinedRejects == "" { return nil, nil } raw, err := os.ReadFile(r.Book.MinedRejects) if err != nil { return nil, fmt.Errorf("pipeline: read mined-rejects %s: %w", r.Book.MinedRejects, err) } var f minedRejectFile if err := yaml.Unmarshal(raw, &f); err != nil { return nil, fmt.Errorf("pipeline: parse mined-rejects %s: %w", r.Book.MinedRejects, err) } rejects := map[string]bool{} for _, rj := range f.Rejects { if nk := text.NormalizeSourceKey(rj.Src); nk != "" { rejects[nk] = true } } return rejects, nil } // loadMinedDelta reads the owner-curated mined-delta YAML (book.MinedDelta) and stamps every entry // Source:"mined" — NOT via membank.LoadGlossarySeed (which hardcodes Source:"seed", memseed.go, moving the base // bank / draft-wave snapshot). This is the mined-write path (plan §1(в), F2): the mined terms land in the ENRICHED // bank version but NOT the base, so adding them moves ONLY edit-wave snapshot ("re-paying ONCE"). Reuses // membank.LoadGlossarySeed's parser/validation, then re-stamps the Source. Empty path → nil (no mined terms). func (r *Runner) loadMinedDelta() ([]store.GlossaryEntry, error) { if r.Book.MinedDelta == "" { return nil, nil } entries, err := membank.LoadGlossarySeed(r.Book.MinedDelta) if err != nil { return nil, fmt.Errorf("pipeline: load mined-delta %s: %w", r.Book.MinedDelta, err) } for i := range entries { entries[i].Source = "mined" // override the seed loader's Source:seed → mined (base-excluded) } return entries, nil }