textmachine/backend/internal/pipeline/bankexport.go

package pipeline

import (
	"context"
	"encoding/json"
	"sort"

	"textmachine/backend/internal/membank"
	"textmachine/backend/internal/obs"
	"textmachine/backend/internal/store"
)

// bankexport.go: the BANK EXPORT artifact (backlog row 125) — the engine's read-out of the whole memory
// bank, all three statuses, as a file beside the project DB.
//
// It exists because the bank lives in the engine's PRIVATE SQLite and the platform must not read that
// store (D39.85), so a read channel for it had no carrier at all. It is deliberately NOT the same thing
// as the bank-stop table (row 101): that one is the set of DECISIONS put in front of the owner at one
// stop — proposals, evidence, why-this-rendering — and it exists only for a run that mined something.
// This one is the STATE of the bank as it stands: every row the book has, including seed rows nobody
// ever had to decide on, and it exists for every book with a bank whether anything was mined or not.
//
// It is a projection, never a source: nothing reads it back into the engine. Deciding about terms goes
// through the `tmctl bank-apply` door into the two decision files (D39.156), because the pipeline
// REPLACES a book's whole glossary from its deterministic inputs on every run and a row edit here would
// be erased by the next one.

// bankExportVersion versions the SHAPE of the document (not its content), so a consumer that has to
// tolerate the field set changing has something to branch on.
const bankExportVersion = "tm-bank-v1"

// BankExport is the whole-bank read-out.
type BankExport struct {
	Version string `json:"bank_version"`
	BookID  string `json:"book_id"`
	// Total / Signed are whole-bank counters (not counters of some page): rows in the file, and rows of
	// those whose status is `approved` — the only status that is canon on the wire.
	Total  int `json:"total"`
	Signed int `json:"signed"`
	// AsOf / RunID are the freshness anchor (fix3 §4.6). The write discipline below is loud-but-not-fatal,
	// so the artifact on disk MAY be a previous boundary's projection — and without these fields that
	// state was knowable only from a log line. AsOf names the boundary that produced THIS document
	// (run-start/seeded · bank-mining/auto-continue · bank-mining/signature-stop · run-finished ·
	// redrive/re-seeded); RunID is the same id the run's event journal streams as engine_run_id, so a
	// consumer following events.jsonl tells «this run's projection» from «a stale one» by comparing ids.
	// Empty RunID means the write had no run identity (an in-process driver with no traced context) —
	// unknown, never «current». Additive on tm-bank-v1: absent in old documents, never invented.
	AsOf  string           `json:"as_of"`
	RunID string           `json:"run_id"`
	Terms []BankExportTerm `json:"terms"`
}

// BankExportTerm is one bank row.
//
// The field names follow the ENGINE's own vocabulary for the term's surfaces (`src`/`dst`) and call the
// provenance column `origin`, because the engine's own column name for provenance is `source` and using
// that word here would make `source` mean the term's text in one schema and its provenance in the other.
type BankExportTerm struct {
	// ID is a STABLE key derived from the row's uniqueness key (src, sense, since_ch, until_ch) — NOT
	// glossary.id. That column is a fresh autoincrement on every bank replace (store/migrate.go v5), and
	// the bank is replaced on every run, so an exported autoincrement would re-point under a consumer
	// between two reads of the same unchanged term. Derived, it survives exactly as long as the term does.
	ID  string `json:"id"`
	Src string `json:"src"`
	Dst string `json:"dst"` // "" for a candidate with no proposed rendering yet
	// Kind is the engine's `type` column and is LEGITIMATELY empty: a ruby candidate that is neither a
	// name nor a place carries none (membank/memseed.go). It is exported as the empty string rather than
	// omitted — "the engine did not decide" is a state the row still needs signing in, and a missing field
	// would invite a consumer to invent one.
	Kind string `json:"kind"`
	// Status is the three-valued signing state auto|draft|approved; only `approved` is injected as canon.
	Status string `json:"status"`
	// Origin is the provenance: which path created the row (seed|ruby|mined). It is an axis INDEPENDENT
	// of Status — a mined row can be approved, a seed row can be draft.
	Origin       string   `json:"origin"`
	Sense        string   `json:"sense"`
	SinceChapter int      `json:"since_chapter"`
	UntilChapter int      `json:"until_chapter"`
	Aliases      []string `json:"aliases"`
}

// bankExportPath is where the read-out lands: beside the project DB, like the signature map, the stop
// tables and the auto-bank. That is the book's state directory — it travels with the DB and never enters
// git — and it is the one place a reader already has to know about.
func (r *Runner) bankExportPath() string { return r.Book.ProjectDB + ".bank.json" }

// bankTermID is the stable derived id (see BankExportTerm.ID). It lives in membank because the id is a
// property of a BANK ROW, and two commands now have to agree on it byte for byte: the export that
// publishes it and `bank-apply`, which resolves a caller's decision back to the row it names. Two copies
// of the derivation would let a decision silently address a different term than the one on the screen.
func bankTermID(src, sense string, since, until int) string {
	return membank.TermID(src, sense, since, until)
}

// projectBankExport turns stored rows into the document. Deterministic: rows are sorted by the same
// uniqueness key the id is derived from, so two runs over an unchanged bank produce byte-identical files
// (a consumer can diff them, and a no-op run is visibly a no-op).
func projectBankExport(bookID string, rows []store.GlossaryEntry) BankExport {
	out := BankExport{Version: bankExportVersion, BookID: bookID, Total: len(rows), Terms: make([]BankExportTerm, 0, len(rows))}
	for _, e := range rows {
		if e.Status == "approved" {
			out.Signed++
		}
		aliases := make([]string, 0, len(e.Aliases))
		for _, a := range e.Aliases {
			aliases = append(aliases, a.Alias)
		}
		sort.Strings(aliases) // the store returns them in insert order; a read-out must not depend on it
		out.Terms = append(out.Terms, BankExportTerm{
			ID:   bankTermID(e.Src, e.Sense, e.SinceCh, e.UntilCh),
			Kind: e.Type, Src: e.Src, Dst: e.Dst, Status: e.Status, Origin: e.Source,
			Sense: e.Sense, SinceChapter: e.SinceCh, UntilChapter: e.UntilCh, Aliases: aliases,
		})
	}
	sort.SliceStable(out.Terms, func(i, j int) bool {
		a, b := out.Terms[i], out.Terms[j]
		switch {
		case a.Src != b.Src:
			return a.Src < b.Src
		case a.Sense != b.Sense:
			return a.Sense < b.Sense
		case a.SinceChapter != b.SinceChapter:
			return a.SinceChapter < b.SinceChapter
		default:
			return a.UntilChapter < b.UntilChapter
		}
	})
	return out
}

// exportBank writes the read-out. `at` names the boundary that triggered it and rides the log line, so a
// stale artifact can be traced to the moment it was last written rather than guessed at.
//
// FAILURE DISCIPLINE: a read-out that cannot be written is loud but never fatal. Killing a paid run
// because a projection of state the run already holds could not be serialized would trade real money for
// an artifact that the next boundary rewrites anyway.
func (r *Runner) exportBank(ctx context.Context, at string) {
	rows, err := r.Store.GlossaryForBook(r.Book.BookID)
	if err != nil {
		r.Log.WarnContext(ctx, "bank export: could not read the bank; the export artifact was NOT refreshed and may be stale",
			"at", at, "path", r.bankExportPath(), "err", err)
		return
	}
	exp := projectBankExport(r.Book.BookID, rows)
	exp.AsOf, exp.RunID = at, r.exportRunID(ctx)
	body, err := json.MarshalIndent(exp, "", "  ")
	if err != nil {
		r.Log.WarnContext(ctx, "bank export: could not marshal the bank; the export artifact was NOT refreshed and may be stale",
			"at", at, "path", r.bankExportPath(), "err", err)
		return
	}
	if err := writeFileAtomic(r.bankExportPath(), append(body, '\n')); err != nil {
		r.Log.WarnContext(ctx, "bank export: could not write the export artifact; it was NOT refreshed and may be stale",
			"at", at, "path", r.bankExportPath(), "err", err)
		return
	}
	r.Log.InfoContext(ctx, "bank export refreshed", "at", at, "path", r.bankExportPath(),
		"terms", exp.Total, "approved", exp.Signed)
}

// exportRunID is the identity the freshness anchor carries: the run-event stream's own id when the
// journal is open (the id a reader of events.jsonl is already following — openEmitter may re-mint it,
// so the emitter, not the context, is authoritative), else the context's trace id, else "" — a caller
// with no traced context has no run identity, and the field must say so rather than invent one.
func (r *Runner) exportRunID(ctx context.Context) string {
	if r.events != nil {
		return r.events.runID
	}
	if ri, ok := obs.ReqInfoFromContext(ctx); ok {
		return ri.TraceID
	}
	return ""
}