textmachine/backend/internal/store/glossary.go

package store

import (
	"database/sql"
	"errors"
)

// glossary.go: durable storage for the memory bank v2 glossary (schema v5). Dumb
// storage, mirroring ruby.go/chunkstatus.go: the pipeline owns aggregation,
// classification and matching; this layer only replaces and reads back. A book's
// glossary is a deterministic function of its inputs (seed file + classified ruby),
// so ReplaceGlossary rewrites the WHOLE book set in one transaction — re-seeding the
// identical inputs rewrites identical rows, a seed edit converges every column, and a
// removed term disappears instead of lingering (the same full-replace idempotency the
// ruby step adopted after external-review #6).

// GlossaryEntry is one term row plus its aliases. Strings are stored raw; the
// pipeline normalizes for matching (§3.6). Decl is a JSON blob the post-check reads.
type GlossaryEntry struct {
	ID                  int64
	BookID              string
	Src                 string
	Dst                 string
	Type                string
	Sense               string
	Gender              string
	Speech              string
	Decl                string // JSON {"invariant":bool,"forms":[...]}
	TranslitPolicy      string
	FirstPerson         string
	NicknameTranslation string
	SinceCh             int
	UntilCh             int
	Status              string // auto|draft|approved
	AllowShort          bool
	Source              string // seed|ruby|auto
	RubyReading         string
	RubyClass           string
	Confidence          int
	Note                string
	Aliases             []GlossaryAlias
}

// GlossaryAlias is one alias-graph edge (an alternative source surface + its type).
type GlossaryAlias struct {
	Alias     string
	AliasType string
}

// GlossaryRevision is one append-only editorial-time record (B1).
type GlossaryRevision struct {
	Src         string
	Sense       string
	OldDst      string
	NewDst      string
	EditorialTS string
	Reason      string
}

// RetrievalState is one per-chunk observability record (registry gate #4).
type RetrievalState struct {
	BookID            string
	Chapter           int
	ChunkIdx          int
	SnapshotID        string
	NExactHits        int
	NSticky           int
	NAmbiguousFlagged int
	NSpoilerBlocked   int
	NEvicted          int
	EmbeddingTierUsed int
	NPostcheckMiss    int
	PostcheckDetail   string // JSON
	InjectedIDs       string // JSON
	NStyleFlags       int    // cheap style/number gate hits (dialogue-dash, ё, translit-interj, 万/億)
	StyleDetail       string // JSON breakdown of the style-flag hits
}

// ReplaceGlossary replaces a book's ENTIRE glossary (rows + aliases) with the given
// entries in ONE transaction, and appends a B1 revision record for every term whose
// approved rendering (dst) changed since the previous set. Scoped to book_id, so other
// books are untouched. Each entry's BookID must equal bookID. term_id linkage is
// established within the tx: a glossary row is inserted, its rowid taken, then its
// aliases inserted against that id (ids are fresh each replace — never hashed).
func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error {
	ctx, cancel := opContext()
	defer cancel()
	tx, err := s.w.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback()

	// Snapshot the prior (src,sense,window)→dst to detect editorial-time changes (B1).
	prior := map[[4]any]string{}
	rows, err := tx.QueryContext(ctx, `SELECT src, sense, since_ch, until_ch, dst FROM glossary WHERE book_id = ?`, bookID)
	if err != nil {
		return err
	}
	for rows.Next() {
		var src, sense, dst string
		var since, until int
		if err := rows.Scan(&src, &sense, &since, &until, &dst); err != nil {
			rows.Close()
			return err
		}
		prior[[4]any{src, sense, since, until}] = dst
	}
	if err := rows.Err(); err != nil {
		rows.Close()
		return err
	}
	rows.Close()

	if _, err := tx.ExecContext(ctx, `DELETE FROM glossary_aliases WHERE book_id = ?`, bookID); err != nil {
		return err
	}
	if _, err := tx.ExecContext(ctx, `DELETE FROM glossary WHERE book_id = ?`, bookID); err != nil {
		return err
	}

	for _, e := range entries {
		res, err := tx.ExecContext(ctx, `
			INSERT INTO glossary (
				book_id, src, dst, type, sense, gender, speech, decl,
				translit_policy, first_person, nickname_translation,
				since_ch, until_ch, status, allow_short, source,
				ruby_reading, ruby_class, confidence, note
			) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
			e.BookID, e.Src, e.Dst, e.Type, e.Sense, e.Gender, e.Speech, e.Decl,
			e.TranslitPolicy, e.FirstPerson, e.NicknameTranslation,
			e.SinceCh, e.UntilCh, e.Status, boolToInt(e.AllowShort), e.Source,
			e.RubyReading, e.RubyClass, e.Confidence, e.Note)
		if err != nil {
			return err
		}
		termID, err := res.LastInsertId()
		if err != nil {
			return err
		}
		for _, a := range e.Aliases {
			if _, err := tx.ExecContext(ctx, `
				INSERT INTO glossary_aliases (book_id, term_id, alias, alias_type)
				VALUES (?, ?, ?, ?)`, e.BookID, termID, a.Alias, a.AliasType); err != nil {
				return err
			}
		}
		// B1: a term that existed with a non-empty dst and now renders differently
		// (also non-empty) is an editorial change → append the revision journal row.
		if old, ok := prior[[4]any{e.Src, e.Sense, e.SinceCh, e.UntilCh}]; ok && old != "" && e.Dst != "" && old != e.Dst {
			if _, err := tx.ExecContext(ctx, `
				INSERT INTO glossary_revisions (book_id, src, sense, old_dst, new_dst, reason)
				VALUES (?, ?, ?, ?, ?, ?)`, e.BookID, e.Src, e.Sense, old, e.Dst, "seed-replace"); err != nil {
				return err
			}
		}
	}
	return tx.Commit()
}

// GlossaryForBook returns every glossary entry for a book with its aliases, in a
// DETERMINISTIC order (src, sense, since_ch, until_ch, status, dst) with aliases sorted
// — the stable materialization the hot-path matcher and the F1 memoryVersion() consume.
func (s *Store) GlossaryForBook(bookID string) ([]GlossaryEntry, error) {
	ctx, cancel := opContext()
	defer cancel()
	rows, err := s.r.QueryContext(ctx, `
		SELECT id, src, dst, type, sense, gender, speech, decl,
		       translit_policy, first_person, nickname_translation,
		       since_ch, until_ch, status, allow_short, source,
		       ruby_reading, ruby_class, confidence, note
		FROM glossary WHERE book_id = ?
		ORDER BY src, sense, since_ch, until_ch, status, dst`, bookID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var out []GlossaryEntry
	byID := map[int64]int{} // glossary.id → index in out
	for rows.Next() {
		e := GlossaryEntry{BookID: bookID}
		var allowShort int
		if err := rows.Scan(&e.ID, &e.Src, &e.Dst, &e.Type, &e.Sense, &e.Gender, &e.Speech, &e.Decl,
			&e.TranslitPolicy, &e.FirstPerson, &e.NicknameTranslation,
			&e.SinceCh, &e.UntilCh, &e.Status, &allowShort, &e.Source,
			&e.RubyReading, &e.RubyClass, &e.Confidence, &e.Note); err != nil {
			return nil, err
		}
		e.AllowShort = allowShort != 0
		byID[e.ID] = len(out)
		out = append(out, e)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	// Aliases in a second pass, ordered so each entry's alias list is deterministic.
	arows, err := s.r.QueryContext(ctx, `
		SELECT term_id, alias, alias_type FROM glossary_aliases
		WHERE book_id = ? ORDER BY term_id, alias`, bookID)
	if err != nil {
		return nil, err
	}
	defer arows.Close()
	for arows.Next() {
		var termID int64
		var a GlossaryAlias
		if err := arows.Scan(&termID, &a.Alias, &a.AliasType); err != nil {
			return nil, err
		}
		if idx, ok := byID[termID]; ok {
			out[idx].Aliases = append(out[idx].Aliases, a)
		}
	}
	return out, arows.Err()
}

// GlossaryRevisionsForBook returns the editorial-time journal (B1) for a book,
// newest first, for the report / a human audit.
func (s *Store) GlossaryRevisionsForBook(bookID string) ([]GlossaryRevision, error) {
	ctx, cancel := opContext()
	defer cancel()
	rows, err := s.r.QueryContext(ctx, `
		SELECT src, sense, old_dst, new_dst, editorial_ts, reason
		FROM glossary_revisions WHERE book_id = ?
		ORDER BY id DESC`, bookID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var out []GlossaryRevision
	for rows.Next() {
		var r GlossaryRevision
		if err := rows.Scan(&r.Src, &r.Sense, &r.OldDst, &r.NewDst, &r.EditorialTS, &r.Reason); err != nil {
			return nil, err
		}
		out = append(out, r)
	}
	return out, rows.Err()
}

// UpsertRetrievalState writes (or overwrites) the per-chunk retrieval-state record.
// Overwrite is the resolve semantics: recomputing the deterministic matcher over the
// same chunk reproduces the same row (self-heals on resume).
func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
	ctx, cancel := opContext()
	defer cancel()
	_, err := s.w.ExecContext(ctx, `
		INSERT INTO retrieval_state (
			book_id, chapter, chunk_idx, snapshot_id,
			n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted,
			embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
			n_style_flags, style_detail, updated_at
		) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
		ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET
			snapshot_id         = excluded.snapshot_id,
			n_exact_hits        = excluded.n_exact_hits,
			n_sticky            = excluded.n_sticky,
			n_ambiguous_flagged = excluded.n_ambiguous_flagged,
			n_spoiler_blocked   = excluded.n_spoiler_blocked,
			n_evicted           = excluded.n_evicted,
			embedding_tier_used = excluded.embedding_tier_used,
			n_postcheck_miss    = excluded.n_postcheck_miss,
			postcheck_detail    = excluded.postcheck_detail,
			injected_ids        = excluded.injected_ids,
			n_style_flags       = excluded.n_style_flags,
			style_detail        = excluded.style_detail,
			updated_at          = excluded.updated_at`,
		rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID,
		rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted,
		rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs,
		rs.NStyleFlags, rs.StyleDetail)
	return err
}

// GetRetrievalState returns the record for a chunk, or nil if none.
func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*RetrievalState, error) {
	ctx, cancel := opContext()
	defer cancel()
	rs := RetrievalState{BookID: bookID, Chapter: chapter, ChunkIdx: chunkIdx}
	err := s.r.QueryRowContext(ctx, `
		SELECT snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked,
		       n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
		       n_style_flags, style_detail
		FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`,
		bookID, chapter, chunkIdx).Scan(
		&rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked,
		&rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
		&rs.NStyleFlags, &rs.StyleDetail)
	if errors.Is(err, sql.ErrNoRows) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	return &rs, nil
}

// RetrievalStatesForBook returns every retrieval-state record for a book, ordered for
// a stable report (the memory section of tmctl report).
func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error) {
	ctx, cancel := opContext()
	defer cancel()
	rows, err := s.r.QueryContext(ctx, `
		SELECT chapter, chunk_idx, snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged,
		       n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
		       n_style_flags, style_detail
		FROM retrieval_state WHERE book_id = ?
		ORDER BY chapter, chunk_idx`, bookID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var out []RetrievalState
	for rows.Next() {
		rs := RetrievalState{BookID: bookID}
		if err := rows.Scan(&rs.Chapter, &rs.ChunkIdx, &rs.SnapshotID, &rs.NExactHits, &rs.NSticky,
			&rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed,
			&rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
			&rs.NStyleFlags, &rs.StyleDetail); err != nil {
			return nil, err
		}
		out = append(out, rs)
	}
	return out, rows.Err()
}

func boolToInt(b bool) int {
	if b {
		return 1
	}
	return 0
}