textmachine/backend/internal/pipeline/bankexport.go

package pipeline

import (
	"context"
	"encoding/json"
	"sort"

	"textmachine/backend/internal/membank"
	"textmachine/backend/internal/obs"
	"textmachine/backend/internal/store"
)

// bankexport.go: the BANK EXPORT artifact (backlog row 125) — the engine's read-out of the whole memory
// bank, all three statuses, as a file beside the project DB.
//
// It exists because the bank lives in the engine's PRIVATE SQLite and the platform must not read that
// store (D39.85), so a read channel for it had no carrier at all. It is deliberately NOT the same thing
// as the bank-stop table (row 101): that one is the set of DECISIONS put in front of the owner at one
// stop — proposals, evidence, why-this-rendering — and it exists only for a run that mined something.
// This one is the STATE of the bank as it stands: every row the book has, including seed rows nobody
// ever had to decide on, and it exists for every book with a bank whether anything was mined or not.
//
// It is a projection, never a source: nothing reads it back into the engine. Deciding about terms goes
// through the `tmctl bank-apply` door into the two decision files (D39.156), because the pipeline
// REPLACES a book's whole glossary from its deterministic inputs on every run and a row edit here would
// be erased by the next one.

// bankExportVersion versions the SHAPE of the document (not its content), so a consumer that has to
// tolerate the field set changing has something to branch on.
const bankExportVersion = "tm-bank-v1"

// BankExport is the whole-bank read-out.
type BankExport struct {
	Version string `json:"bank_version"`
	BookID  string `json:"book_id"`
	// Total / Signed are whole-bank counters (not counters of some page): rows in the file, and rows of
	// those whose status is `approved` — the only status that is canon on the wire.
	Total  int `json:"total"`
	Signed int `json:"signed"`
	// AsOf / RunID are the freshness anchor (fix3 §4.6). The write discipline below is loud-but-not-fatal,
	// so the artifact on disk MAY be a previous boundary's projection — and without these fields that
	// state was knowable only from a log line. AsOf names the boundary that produced THIS document
	// (run-start/seeded · bank-mining/auto-continue · bank-mining/signature-stop · run-finished ·
	// redrive/re-seeded); RunID is the same id the run's event journal streams as engine_run_id, so a
	// consumer following events.jsonl tells «this run's projection» from «a stale one» by comparing ids.
	// Empty RunID means the write had no run identity (an in-process driver with no traced context) —
	// unknown, never «current». Additive on tm-bank-v1: absent in old documents, never invented.
	AsOf  string           `json:"as_of"`
	RunID string           `json:"run_id"`
	Terms []BankExportTerm `json:"terms"`
	// Proposed is what a signature STOP is putting in front of the owner: rows this run mined and
	// consolidated that are NOT in the bank yet. Row 224 is the reason it exists — the stop refreshes this
	// artifact precisely so the signing screen can read it, and until now the refresh projected only the
	// glossary, whose mined rows are written in the auto-continue branch. So the one boundary opened FOR
	// these rows published a document guaranteed to be empty of them.
	//
	// ⚠ A SEPARATE SECTION, NEVER MERGED INTO Terms, and the separation is the money. Terms is the bank as
	// it stands; a proposal is not in it. Getting them into Terms would mean seeding them, and the seed
	// path injects unsigned rows as law (D39.104 §2 — both are injected, with no branch on status), which
	// moves memory_version and turns the stop into an auto-continue with a re-snapshot behind it.
	//
	// Empty at every boundary that is not a stop. Order is the stop's own RANKING, not a sort: which row
	// to read first is the information here, and the bank's diffable byte-stability belongs to Terms.
	Proposed []BankExportProposal `json:"proposed,omitempty"`
	// Consolidation is how COMPLETE this bank is — and ABSENT, never zeroed, when nothing measured it.
	// Backlog row 253(б): the engine has always known that a budget cut the pass and said so in its own
	// log, while the artifact the signing screen is built from carried no field for it, so an owner signed
	// a bank the engine itself calls partial. A pointer because the read-out is written at five boundaries
	// and the pass runs at one: a zeroed section would answer «consolidated 0, unanswered 0» — which reads
	// as «nothing is missing» — at the four boundaries that never asked.
	Consolidation *BankConsolidation `json:"consolidation,omitempty"`
}

// BankConsolidation is what the paid terminology pass MANAGED, in the fields a signing decision needs. It
// is a projection of counters the pass already produced; nothing here re-derives completeness by a second
// route, because two mechanisms answering one question is how they come to disagree.
//
// ⛔ THREE FACTS THAT LOOK ALIKE AND ARE NOT, which is why this is six fields and not one number:
//   - the RENDER pass was cut → terms have no consolidated rendering → the BANK is partial;
//   - the CLASSIFY pass was cut → the bank is whole and the term TYPES are unrefined. A different budget,
//     a different remedy, and reading it as an incomplete bank is a false alarm. On the run of 08.09 the
//     engine's own warning fired on exactly this: the render pass was intact and it still said «PARTIALLY
//     consolidated»;
//   - candidates the role was NEVER ASKED about because the bank already renders them — a saving, not a gap.
type BankConsolidation struct {
	// Complete is the BANK's completeness and nothing else: every render batch the pass planned was bought.
	// It is published rather than left to the reader to derive, because deriving it means re-implementing
	// engine law on the far side of the seam — the thing the inbound law forbids (D39.156 п.6).
	Complete bool `json:"complete"`
	// RenderBatchesDropped is what makes Complete false. ClassifyBatchesDropped is the OTHER pass's cut, on
	// its OWN budget: two fields and never one sum, because an operator raises `budget_usd` or
	// `classify_budget_usd`, never «the» budget.
	RenderBatchesDropped   int `json:"render_batches_dropped"`
	ClassifyBatchesDropped int `json:"classify_batches_dropped"`
	// Consolidated came back with a rendering; Declined the role explicitly could not render.
	Consolidated int `json:"consolidated"`
	Declined     int `json:"declined"`
	// Unanswered is the role's silence AND the budget's cut TOGETHER — the engine's counter conflates them
	// and says so in the warning beside it: a term in a dropped batch was never offered, and lands here
	// looking exactly like one the role saw and skipped. It means «the role stayed silent» only where
	// Complete is true; where it is false, part of this number is the money and not the model.
	Unanswered int `json:"unanswered"`
	// NeverAsked is the opposite of a gap and must not be read as one: these candidates never reached the
	// paid role because the bank already renders the surface and every draft agreed with it.
	NeverAsked int `json:"never_asked"`
	// SettledEarlier is the OTHER way a candidate skips the paid role: this book decided its rendering in
	// an earlier purchase and the evidence has not moved since, so the settled basis answered it.
	//
	// ⛔ AN ADDITIVE FIELD RATHER THAN A WIDER `never_asked`, and the addition is the whole point. That
	// field's published meaning is «the seed holds the surface and every draft agreed» — a sentence a
	// consumer is entitled to act on — and both of its halves are FALSE for a row the basis answered. Adding
	// these rows to it would change what a shipped number means without changing its name, which is the
	// shape this engine refuses everywhere else. Consumers that do not know the new field keep reading the
	// old one correctly; that is what makes the change a minor one.
	SettledEarlier int `json:"settled_earlier"`
}

// projectBankConsolidation turns the pass's counters into the section. NIL IN, NIL OUT, and that is the
// whole distinction the section exists to publish: a pass that did not run has not found the bank complete.
func projectBankConsolidation(t *terminologyResult) *BankConsolidation {
	if t == nil {
		return nil
	}
	return &BankConsolidation{
		Complete:             t.BatchesDropped == 0,
		RenderBatchesDropped: t.BatchesDropped, ClassifyBatchesDropped: t.ClassifyBatchesDropped,
		Consolidated: t.Consolidated, Declined: t.Declined, Unanswered: t.Unanswered,
		NeverAsked: t.BankSettled, SettledEarlier: t.BasisServed,
	}
}

// BankExportProposal is one row of a stop's verification table in the fields a signing decision needs.
// The KWIC contexts and the ranking evidence are NOT here: they are a review artifact of a thousand lines
// and they already have a carrier of their own (the stop table beside the DB).
type BankExportProposal struct {
	Src  string `json:"src"`
	Dst  string `json:"dst"` // the consolidated rendering, "" when nothing was consolidated
	Kind string `json:"kind"`
	// Channel is which detector found the row: mined | banknote | both. It is deliberately NOT called
	// `origin`: terms[].origin above is the row's PROVENANCE (seed|ruby|mined) and a consumer already maps
	// that vocabulary to its own. One word meaning two things inside one document is the exact confusion
	// the field comment on BankExportTerm.Origin refuses for `source`.
	Channel string `json:"channel"`
	Freq    int    `json:"freq"`
	Spread  int    `json:"spread"` // how many DISTINCT renderings the drafts produced
	// Conventions is how many of those are genuinely different DECISIONS, after renderings differing only in
	// case or spacing fold together — and it is the length of Variants below.
	//
	// ⛔ IT IS HERE BECAUSE ITS ABSENCE COST TWO READINGS. The printed stop table has always carried both
	// numbers on one line («spread=4 conventions=3»), but this JSON carried only `spread` beside a list whose
	// length is the OTHER number, with nothing naming it. Two sessions in a row therefore compared `spread`
	// to len(variants), found them unequal on 开窍大典, and filed it as two definitions of spread disagreeing.
	// They were reading one number against a different measurement, and the document gave them no way to see
	// that. A consumer now gets both, and the mismatch it used to look like is arithmetic it can check.
	Conventions int `json:"conventions"`
	// Conf is the role's stated confidence, NEGATIVE when the reply carried none: «the role said nothing»
	// and «the role said 0 %» are different rows on a signing sheet.
	Conf int `json:"conf"`
	// Invented says the rendering is not one the drafts proposed — legitimate, and the class to read first.
	//
	// ⚠ It is NOT set on a row the basis settled, and the absence there is a decision: the flag is a finding
	// about THIS run's drafts, and a rendering carried over from an earlier purchase was never offered to
	// them. Read `settled_earlier` beside it or the missing flag reads as «the drafts proposed this».
	Invented bool `json:"invented,omitempty"`
	// SettledEarlier says this book decided the rendering in an earlier purchase and the roles were not paid
	// to decide it again. It is NOT the same as the summary's `never_asked`, whose sentence — «the bank
	// renders the surface and every draft agreed» — is false of this row in both halves.
	SettledEarlier bool `json:"settled_earlier,omitempty"`
	// Contradicts names this run's OTHER consolidations the rendering breaks; BankHolds names rows the bank
	// already carries for the same firing surface with a different rendering. Two fields because "the run
	// disagreed with itself" and "the book already calls it something else" are different decisions.
	Contradicts []string `json:"contradicts,omitempty"`
	BankHolds   []string `json:"bank_holds,omitempty"`
	// Variants are the drafts' renderings, best-ranked first, in the ONE labelling every human-facing table
	// uses (BankStopVariant.Label), so this section and the printed table cannot describe a variant
	// differently.
	Variants []string `json:"variants,omitempty"`
}

// BankExportTerm is one bank row.
//
// The field names follow the ENGINE's own vocabulary for the term's surfaces (`src`/`dst`) and call the
// provenance column `origin`, because the engine's own column name for provenance is `source` and using
// that word here would make `source` mean the term's text in one schema and its provenance in the other.
type BankExportTerm struct {
	// ID is a STABLE key derived from the row's uniqueness key (src, sense, since_ch, until_ch) — NOT
	// glossary.id. That column is a fresh autoincrement on every bank replace (store/migrate.go v5), and
	// the bank is replaced on every run, so an exported autoincrement would re-point under a consumer
	// between two reads of the same unchanged term. Derived, it survives exactly as long as the term does.
	ID  string `json:"id"`
	Src string `json:"src"`
	Dst string `json:"dst"` // "" for a candidate with no proposed rendering yet
	// Kind is the engine's `type` column and is LEGITIMATELY empty: a ruby candidate that is neither a
	// name nor a place carries none (membank/memseed.go). It is exported as the empty string rather than
	// omitted — "the engine did not decide" is a state the row still needs signing in, and a missing field
	// would invite a consumer to invent one.
	Kind string `json:"kind"`
	// Status is the three-valued signing state auto|draft|approved; only `approved` is injected as canon.
	Status string `json:"status"`
	// Origin is the provenance: which path created the row (seed|ruby|mined). It is an axis INDEPENDENT
	// of Status — a mined row can be approved, a seed row can be draft.
	Origin       string   `json:"origin"`
	Sense        string   `json:"sense"`
	SinceChapter int      `json:"since_chapter"`
	UntilChapter int      `json:"until_chapter"`
	Aliases      []string `json:"aliases"`
}

// bankExportPath is where the read-out lands: beside the project DB, like the signature map, the stop
// tables and the auto-bank. That is the book's state directory — it travels with the DB and never enters
// git — and it is the one place a reader already has to know about.
func (r *Runner) bankExportPath() string { return r.Book.ProjectDB + ".bank.json" }

// bankTermID is the stable derived id (see BankExportTerm.ID). It lives in membank because the id is a
// property of a BANK ROW, and two commands now have to agree on it byte for byte: the export that
// publishes it and `bank-apply`, which resolves a caller's decision back to the row it names. Two copies
// of the derivation would let a decision silently address a different term than the one on the screen.
func bankTermID(src, sense string, since, until int) string {
	return membank.TermID(src, sense, since, until)
}

// projectBankExport turns stored rows into the document. Deterministic: rows are sorted by the same
// uniqueness key the id is derived from, so two runs over an unchanged bank produce byte-identical files
// (a consumer can diff them, and a no-op run is visibly a no-op).
func projectBankExport(bookID string, rows []store.GlossaryEntry) BankExport {
	out := BankExport{Version: bankExportVersion, BookID: bookID, Total: len(rows), Terms: make([]BankExportTerm, 0, len(rows))}
	for _, e := range rows {
		if e.Status == "approved" {
			out.Signed++
		}
		aliases := make([]string, 0, len(e.Aliases))
		for _, a := range e.Aliases {
			aliases = append(aliases, a.Alias)
		}
		sort.Strings(aliases) // the store returns them in insert order; a read-out must not depend on it
		out.Terms = append(out.Terms, BankExportTerm{
			ID:   bankTermID(e.Src, e.Sense, e.SinceCh, e.UntilCh),
			Kind: e.Type, Src: e.Src, Dst: e.Dst, Status: e.Status, Origin: e.Source,
			Sense: e.Sense, SinceChapter: e.SinceCh, UntilChapter: e.UntilCh, Aliases: aliases,
		})
	}
	sort.SliceStable(out.Terms, func(i, j int) bool {
		a, b := out.Terms[i], out.Terms[j]
		switch {
		case a.Src != b.Src:
			return a.Src < b.Src
		case a.Sense != b.Sense:
			return a.Sense < b.Sense
		case a.SinceChapter != b.SinceChapter:
			return a.SinceChapter < b.SinceChapter
		default:
			return a.UntilChapter < b.UntilChapter
		}
	})
	return out
}

// projectBankProposals turns a stop's verification rows into the sidecar's proposed section. Order is
// preserved, not sorted: the rows arrive best-ranked first and that ranking is what a reviewer reads down.
func projectBankProposals(rows []BankStopRow) []BankExportProposal {
	if len(rows) == 0 {
		return nil
	}
	out := make([]BankExportProposal, 0, len(rows))
	for _, row := range rows {
		p := BankExportProposal{
			Src: row.Src, Dst: row.Dst, Kind: row.Type, Channel: row.Origin,
			Freq: row.Freq, Spread: row.Spread, Conventions: row.Conventions, Conf: row.Conf, Invented: row.Invented,
			Contradicts: row.Contradicts, BankHolds: row.BankHolds, SettledEarlier: row.SettledByBasis,
		}
		for _, v := range row.Variants {
			p.Variants = append(p.Variants, v.Label())
		}
		out = append(out, p)
	}
	return out
}

// exportBank writes the read-out. `at` names the boundary that triggered it and rides the log line, so a
// stale artifact can be traced to the moment it was last written rather than guessed at.
//
// FAILURE DISCIPLINE: a read-out that cannot be written is loud but never fatal. Killing a paid run
// because a projection of state the run already holds could not be serialized would trade real money for
// an artifact that the next boundary rewrites anyway.
func (r *Runner) exportBank(ctx context.Context, at string) {
	rows, err := r.Store.GlossaryForBook(r.Book.BookID)
	if err != nil {
		r.Log.WarnContext(ctx, "bank export: could not read the bank; the export artifact was NOT refreshed and may be stale",
			"at", at, "path", r.bankExportPath(), "err", err)
		return
	}
	exp := projectBankExport(r.Book.BookID, rows)
	exp.AsOf, exp.RunID = at, r.exportRunID(ctx)
	// The stop's rows ride along (row 224). r.lastBankStopRows is set in the stopping branch and nowhere
	// else, and a stopped run returns before every later boundary, so this is non-empty exactly at a
	// signature stop.
	exp.Proposed = projectBankProposals(r.lastBankStopRows)
	// The completeness of what is being signed (row 253б). r.lastTerminology is nil until the paid pass has
	// run, so the section is absent at the boundaries that never measured it rather than zero there.
	exp.Consolidation = projectBankConsolidation(r.lastTerminology)
	body, err := json.MarshalIndent(exp, "", "  ")
	if err != nil {
		r.Log.WarnContext(ctx, "bank export: could not marshal the bank; the export artifact was NOT refreshed and may be stale",
			"at", at, "path", r.bankExportPath(), "err", err)
		return
	}
	if err := writeFileAtomic(r.bankExportPath(), append(body, '\n')); err != nil {
		r.Log.WarnContext(ctx, "bank export: could not write the export artifact; it was NOT refreshed and may be stale",
			"at", at, "path", r.bankExportPath(), "err", err)
		return
	}
	// `approved` is a SUBSET of `terms`; `awaiting_signature` is disjoint from both by construction (a
	// proposal is not in the bank). Named apart so the three do not read as one proportion.
	r.Log.InfoContext(ctx, "bank export refreshed", "at", at, "path", r.bankExportPath(),
		"terms", exp.Total, "approved", exp.Signed, "awaiting_signature_outside_the_bank", len(exp.Proposed))
}

// exportRunID is the identity the freshness anchor carries: the run-event stream's own id when the
// journal is open (the id a reader of events.jsonl is already following — openEmitter may re-mint it,
// so the emitter, not the context, is authoritative), else the context's trace id, else "" — a caller
// with no traced context has no run identity, and the field must say so rather than invent one.
func (r *Runner) exportRunID(ctx context.Context) string {
	if r.events != nil {
		return r.events.runID
	}
	if ri, ok := obs.ReqInfoFromContext(ctx); ok {
		return ri.TraceID
	}
	return ""
}