package text

import (
	"strings"
	"testing"
	"unicode"
)

// TestTokenizeLetters pins the script-agnostic tokenizer the bank post-check uses (fix-pack §и consolidation):
// maximal any-letter runs, lower-cased, with a combining mark dropped rather than splitting the word — the
// same combining-mark handling TokenizeScript has, now shared through one core.
func TestTokenizeLetters(t *testing.T) {
	got := TokenizeLetters("Фан Юаню, к Горе!")
	if strings.Join(got, "|") != "фан|юаню|к|горе" {
		t.Fatalf("tokens = %v", got)
	}
	// A stress accent (о + U+0301) must not split the word — the pack-21 #11 fix, now inherited by the
	// decl post-check that lang.TokenizeWords feeds.
	if got := TokenizeLetters("сло́во"); len(got) != 1 || got[0] != "слово" {
		t.Fatalf("a combining mark must not split the word, got %v", got)
	}
	// Script-agnostic: a Latin word and a Cyrillic word both tokenize (TokenizeScript would drop one).
	if got := TokenizeLetters("Гу and Gu"); strings.Join(got, "|") != "гу|and|gu" {
		t.Fatalf("any-letter tokenization must keep both scripts, got %v", got)
	}
	// It agrees with TokenizeScript on a single-script string (the shared core), so the consolidation is
	// behaviour-preserving for the target lexical checkers.
	if a, b := TokenizeLetters("Меч и Щит"), TokenizeScript("Меч и Щит", unicode.Cyrillic); strings.Join(a, "|") != strings.Join(b, "|") {
		t.Fatalf("TokenizeLetters and TokenizeScript diverged on Cyrillic-only text: %v vs %v", a, b)
	}
}