textmachine/backend/internal/pipeline/bankexport.go

package pipeline

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"strings"

	"textmachine/backend/internal/store"
)

// bankexport.go: the BANK EXPORT artifact (backlog row 125) — the engine's read-out of the whole memory
// bank, all three statuses, as a file beside the project DB.
//
// It exists because the bank lives in the engine's PRIVATE SQLite and the platform must not read that
// store (D39.85), so a read channel for it had no carrier at all. It is deliberately NOT the same thing
// as the bank-stop table (row 101): that one is the set of DECISIONS put in front of the owner at one
// stop — proposals, evidence, why-this-rendering — and it exists only for a run that mined something.
// This one is the STATE of the bank as it stands: every row the book has, including seed rows nobody
// ever had to decide on, and it exists for every book with a bank whether anything was mined or not.
//
// It is a projection, never a source: nothing reads it back into the engine. Signing still goes through
// the ratified stop mechanics (promote into the mined-delta file / decline in the mined-rejects file),
// because the pipeline REPLACES a book's whole glossary from its deterministic inputs on every run and a
// row edit would be erased by the next one.

// bankExportVersion versions the SHAPE of the document (not its content), so a consumer that has to
// tolerate the field set changing has something to branch on.
const bankExportVersion = "tm-bank-v1"

// BankExport is the whole-bank read-out.
type BankExport struct {
	Version string `json:"bank_version"`
	BookID  string `json:"book_id"`
	// Total / Signed are whole-bank counters (not counters of some page): rows in the file, and rows of
	// those whose status is `approved` — the only status that is canon on the wire.
	Total  int              `json:"total"`
	Signed int              `json:"signed"`
	Terms  []BankExportTerm `json:"terms"`
}

// BankExportTerm is one bank row.
//
// The field names follow the ENGINE's own vocabulary for the term's surfaces (`src`/`dst`) and call the
// provenance column `origin`, because the engine's own column name for provenance is `source` and using
// that word here would make `source` mean the term's text in one schema and its provenance in the other.
type BankExportTerm struct {
	// ID is a STABLE key derived from the row's uniqueness key (src, sense, since_ch, until_ch) — NOT
	// glossary.id. That column is a fresh autoincrement on every bank replace (store/migrate.go v5), and
	// the bank is replaced on every run, so an exported autoincrement would re-point under a consumer
	// between two reads of the same unchanged term. Derived, it survives exactly as long as the term does.
	ID  string `json:"id"`
	Src string `json:"src"`
	Dst string `json:"dst"` // "" for a candidate with no proposed rendering yet
	// Kind is the engine's `type` column and is LEGITIMATELY empty: a ruby candidate that is neither a
	// name nor a place carries none (membank/memseed.go). It is exported as the empty string rather than
	// omitted — "the engine did not decide" is a state the row still needs signing in, and a missing field
	// would invite a consumer to invent one.
	Kind string `json:"kind"`
	// Status is the three-valued signing state auto|draft|approved; only `approved` is injected as canon.
	Status string `json:"status"`
	// Origin is the provenance: which path created the row (seed|ruby|mined). It is an axis INDEPENDENT
	// of Status — a mined row can be approved, a seed row can be draft.
	Origin       string   `json:"origin"`
	Sense        string   `json:"sense"`
	SinceChapter int      `json:"since_chapter"`
	UntilChapter int      `json:"until_chapter"`
	Aliases      []string `json:"aliases"`
}

// bankExportPath is where the read-out lands: beside the project DB, like the signature map, the stop
// tables and the auto-bank. That is the book's state directory — it travels with the DB and never enters
// git — and it is the one place a reader already has to know about.
func (r *Runner) bankExportPath() string { return r.Book.ProjectDB + ".bank.json" }

// bankTermID is the stable derived id (see BankExportTerm.ID).
//
// The four parts are LENGTH-PREFIXED rather than separator-joined. The first version used U+001F and
// asserted in a comment that it "cannot occur in any of them" — an assumption about data the engine never
// validates: `src` and `sense` are free text copied from a seed YAML, and nothing on the load path
// rejects a control character. Length-prefixing makes the encoding injective by construction, so the
// claim does not have to be true (and does not have to be re-checked when a new seed source appears).
// 16 hex chars of SHA-256 is 64 bits, which for a bank of thousands of rows makes a collision an
// irrelevance rather than a risk taken on purpose.
func bankTermID(src, sense string, since, until int) string {
	var b strings.Builder
	for _, part := range []string{src, sense, strconv.Itoa(since), strconv.Itoa(until)} {
		fmt.Fprintf(&b, "%d:%s", len(part), part)
	}
	sum := sha256.Sum256([]byte(b.String()))
	return hex.EncodeToString(sum[:])[:16]
}

// projectBankExport turns stored rows into the document. Deterministic: rows are sorted by the same
// uniqueness key the id is derived from, so two runs over an unchanged bank produce byte-identical files
// (a consumer can diff them, and a no-op run is visibly a no-op).
func projectBankExport(bookID string, rows []store.GlossaryEntry) BankExport {
	out := BankExport{Version: bankExportVersion, BookID: bookID, Total: len(rows), Terms: make([]BankExportTerm, 0, len(rows))}
	for _, e := range rows {
		if e.Status == "approved" {
			out.Signed++
		}
		aliases := make([]string, 0, len(e.Aliases))
		for _, a := range e.Aliases {
			aliases = append(aliases, a.Alias)
		}
		sort.Strings(aliases) // the store returns them in insert order; a read-out must not depend on it
		out.Terms = append(out.Terms, BankExportTerm{
			ID:   bankTermID(e.Src, e.Sense, e.SinceCh, e.UntilCh),
			Kind: e.Type, Src: e.Src, Dst: e.Dst, Status: e.Status, Origin: e.Source,
			Sense: e.Sense, SinceChapter: e.SinceCh, UntilChapter: e.UntilCh, Aliases: aliases,
		})
	}
	sort.SliceStable(out.Terms, func(i, j int) bool {
		a, b := out.Terms[i], out.Terms[j]
		switch {
		case a.Src != b.Src:
			return a.Src < b.Src
		case a.Sense != b.Sense:
			return a.Sense < b.Sense
		case a.SinceChapter != b.SinceChapter:
			return a.SinceChapter < b.SinceChapter
		default:
			return a.UntilChapter < b.UntilChapter
		}
	})
	return out
}

// exportBank writes the read-out. `at` names the boundary that triggered it and rides the log line, so a
// stale artifact can be traced to the moment it was last written rather than guessed at.
//
// FAILURE DISCIPLINE: a read-out that cannot be written is loud but never fatal. Killing a paid run
// because a projection of state the run already holds could not be serialized would trade real money for
// an artifact that the next boundary rewrites anyway.
func (r *Runner) exportBank(ctx context.Context, at string) {
	rows, err := r.Store.GlossaryForBook(r.Book.BookID)
	if err != nil {
		r.Log.WarnContext(ctx, "bank export: could not read the bank; the export artifact was NOT refreshed and may be stale",
			"at", at, "path", r.bankExportPath(), "err", err)
		return
	}
	exp := projectBankExport(r.Book.BookID, rows)
	body, err := json.MarshalIndent(exp, "", "  ")
	if err != nil {
		r.Log.WarnContext(ctx, "bank export: could not marshal the bank; the export artifact was NOT refreshed and may be stale",
			"at", at, "path", r.bankExportPath(), "err", err)
		return
	}
	if err := writeFileAtomic(r.bankExportPath(), append(body, '\n')); err != nil {
		r.Log.WarnContext(ctx, "bank export: could not write the export artifact; it was NOT refreshed and may be stale",
			"at", at, "path", r.bankExportPath(), "err", err)
		return
	}
	r.Log.InfoContext(ctx, "bank export refreshed", "at", at, "path", r.bankExportPath(),
		"terms", exp.Total, "approved", exp.Signed)
}