textmachine/backend/internal/lang/declinephrases.go

package lang

import (
	"fmt"
	"strings"
	"sync"
	"unicode"
)

// declinephrases.go: the registry of the words a bank ROLE writes when it declines a term instead of
// rendering it. Part of the BANK-ASSEMBLY data plane (bankdata.go) — read only when the terminologist's
// reply is parsed, by nothing a wave touches.
//
// WHY IT IS DATA AND NOT A LIST IN GO. «не термин» is a sentence in one target language. A Go literal
// would make the engine's parse of a reply pair-specific in the one place the project's first goal forbids
// it (an engine that branches on the pair), and a second target would need a Go edit to be understood at
// all. The DECISION the phrase encodes — a decline is not a rendering — is the algorithm and stays here.
//
// ⚠ WHY IT IS NOT IN THE LANGPACK, where reader.txt and the miner tables live. A langpack is resolved per
// BOOK from `langpack_root`, and a book may legitimately have none (the ja→ru golden runs with no pack at
// all): the same reply would then be read two ways depending on a path in book.yaml, which is exactly the
// kind of silent divergence this file exists to remove. The embedded refusal blacklist (data/refusal.txt)
// is here for the same reason and says so in the same words.

// DeclinePhrases is one target's decline vocabulary: the folded phrases plus the any-target rows. The zero
// value is INERT and that is the shipping state — Match reports false for everything, so the parser
// recognises the engine sentinel and nothing else, exactly as it did before this registry existed.
type DeclinePhrases struct {
	folded map[string]bool
}

// Empty reports whether this target has no decline phrases at all. It exists so a test can ASSERT the
// shipping emptiness (a mechanism that lands switched off in DATA is invisible to every other gate —
// backlog rows 433 and 435 are that class twice).
func (d DeclinePhrases) Empty() bool { return len(d.folded) == 0 }

// Match reports whether a role's `dst` field IS a decline phrase — the WHOLE field, never a substring.
//
// ⛔ THE STRICTNESS IS THE POINT, and it is a money argument rather than a taste. A substring match turns
// any rendering that happens to contain the phrase into a decline: the term leaves the bank, the book
// loses a canon row it PAID for, and the next purchase asks for it again — and a bank role has no retry
// and no fallback to be caught by (backlog row 438). The opposite error, a decline written in words this
// file does not carry, leaves today's behaviour exactly as it is: the answer is banked, which is the
// defect this registry narrows rather than the one it introduces.
func (d DeclinePhrases) Match(dst string) bool {
	if len(d.folded) == 0 {
		return false
	}
	return d.folded[foldDeclinePhrase(dst)]
}

// foldDeclinePhrase is the ONE normalization both sides go through — the authored phrase at load and the
// model's field at parse — so a row cannot be authored in a shape the matcher can never see. Spaces,
// surrounding quotation of every flavour a model reaches for, trailing sentence punctuation, and case.
func foldDeclinePhrase(s string) string {
	s = strings.TrimSpace(s)
	s = strings.Trim(s, `"'«»“”„‘’()[]{}`)
	s = strings.TrimFunc(s, func(r rune) bool {
		return unicode.IsSpace(r) || strings.ContainsRune(".,;:!?…—–-", r)
	})
	return strings.ToLower(strings.Join(strings.Fields(s), " "))
}

// declineAnyTarget is the row key for a phrase that fires whatever the book's target: a model that drifts
// into English declines in English even in a →ru book, and that drift is a fact about models, not about
// the pair.
const declineAnyTarget = "*"

var (
	declineOnce sync.Once
	declineBy   map[string]map[string]bool
)

// DeclinePhrasesFor resolves the decline vocabulary for a book's TARGET language: its own rows plus the
// any-target rows. A target with no rows gets the inert zero value.
func DeclinePhrasesFor(targetLang string) DeclinePhrases {
	declineOnce.Do(func() {
		m, err := parseDeclinePhrases(mustBankData("bankdata/decline-phrases.txt"))
		if err != nil {
			panic(fmt.Sprintf("lang: embedded bankdata/decline-phrases.txt is corrupt: %v", err))
		}
		declineBy = m
	})
	return declinePhrasesFrom(declineBy, targetLang)
}

// declinePhrasesFrom is the resolution itself, over rows given rather than loaded: the target's own rows
// plus the any-target ones. Split out so the matcher can be exercised on a synthetic registry without
// reaching for the package's loaded state — a test that swapped the global would leave the NEXT test
// asserting emptiness against its leftovers and calling that a measurement of the shipped file.
func declinePhrasesFrom(rows map[string]map[string]bool, targetLang string) DeclinePhrases {
	key := strings.ToLower(strings.TrimSpace(targetLang))
	own, anyTarget := rows[key], rows[declineAnyTarget]
	if len(own) == 0 && len(anyTarget) == 0 {
		return DeclinePhrases{}
	}
	folded := make(map[string]bool, len(own)+len(anyTarget))
	for p := range own {
		folded[p] = true
	}
	for p := range anyTarget {
		folded[p] = true
	}
	return DeclinePhrases{folded: folded}
}

func parseDeclinePhrases(b []byte) (map[string]map[string]bool, error) {
	out := map[string]map[string]bool{}
	for i, raw := range strings.Split(string(b), "\n") {
		line := strings.TrimSpace(strings.TrimRight(raw, "\r"))
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		f := strings.SplitN(line, "\t", 2)
		if len(f) != 2 || strings.TrimSpace(f[0]) == "" {
			return nil, fmt.Errorf("line %d: want `lang<TAB>phrase`, got %q", i+1, line)
		}
		phrase := foldDeclinePhrase(f[1])
		if phrase == "" {
			// A row that folds to nothing would match every empty field — and an empty `dst` is the
			// sentinel's own bucket, decided upstream. Refuse it rather than let it sit there matching.
			return nil, fmt.Errorf("line %d: the phrase %q folds to nothing", i+1, f[1])
		}
		key := strings.ToLower(strings.TrimSpace(f[0]))
		if out[key] == nil {
			out[key] = map[string]bool{}
		}
		out[key][phrase] = true
	}
	return out, nil
}