Land banknote discipline and language screen build as D39.53: output-side script screen, quote folding, checkpoint aggregation with per-chunk votes, kwic sizing to pair data
This commit is contained in:
Claude (backend session)
parent
21e7099133
commit
b05c59a843
23 changed files with 1141 additions and 101 deletions
|
|
@ -6,6 +6,7 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
|
@ -254,11 +255,12 @@ type Gates struct {
|
|||
|
||||
// TerminologyGate controls the TERMINOLOGIST role (pack-20, D39.42 п.1): between the draft wave and the
|
||||
// edit wave, one cheap model reads the WHOLE book's evidence for every bank candidate at once — the merged
|
||||
// miner∪banknote list, the source contexts of each occurrence, the renderings the drafts produced, the
|
||||
// pair's transliteration convention and its genre glossary — and returns ONE consolidated rendering per
|
||||
// term. It exists because neither existing channel answers that question: the miner is source-side and
|
||||
// emits no dst at all, and the banknote's per-chunk guesses named an overlapping entity in 3% of the
|
||||
// mini-run's terms.
|
||||
// miner∪banknote list, the source contexts of each occurrence, the renderings the drafts produced and the
|
||||
// book's own SIGNED rows — and returns ONE consolidated rendering per term. (A pair-wide genre glossary
|
||||
// was a fourth input here and was removed as a class by D39.47: prescribing one register to every book of
|
||||
// a pair overrides the only authority that exists — the owner's signature on THIS book's bank.) It exists
|
||||
// because neither existing channel answers that question: the miner is source-side and emits no dst at
|
||||
// all, and the banknote's per-chunk guesses named an overlapping entity in 3% of the mini-run's terms.
|
||||
//
|
||||
// Opt-in and OFF by default, like every other gate here. With it off the bank-mining stop takes exactly
|
||||
// the path it took before (WHICH-only terms plus whatever the banknote join attached), no provider is
|
||||
|
|
@ -274,10 +276,20 @@ type TerminologyGate struct {
|
|||
BudgetUSD float64 `yaml:"budget_usd"`
|
||||
// BatchRunes bounds ONE call's candidate block. 0 → terminologyDefaultBatchRunes.
|
||||
BatchRunes int `yaml:"batch_runes"`
|
||||
// KWICPerTerm / KWICWidth size the source contexts each candidate carries. 0 → the engine defaults.
|
||||
// They are the whole reason the role can translate at all, and the whole reason a call is not free.
|
||||
// KWICPerTerm / KWICWidth size the source contexts each candidate carries. 0 → the pair's own sizing
|
||||
// (langpack terminology.txt), else the engine defaults. They are the whole reason the role can
|
||||
// translate at all, and the whole reason a call is not free.
|
||||
KWICPerTerm int `yaml:"kwic_per_term"`
|
||||
KWICWidth int `yaml:"kwic_width"`
|
||||
// TargetScript is the Unicode SCRIPT NAME of the target language ("Cyrillic", "Latin", "Han", …) the
|
||||
// answer-language screen checks renderings against. REQUIRED when the gate is on, by the same contract
|
||||
// as the budget and the prompt path: the failure it guards is silent by construction, so a run with
|
||||
// the screen disabled would bank a foreign-language canon and report nothing.
|
||||
//
|
||||
// It lives in the gate — which is NOT snapshot-folded — rather than in the langpack, whose bytes move
|
||||
// LangpackVersion and re-buy both waves of every book of the pair. The banknote fold reads the same
|
||||
// declaration, so a book running that channel without the terminologist may set it with the gate off.
|
||||
TargetScript string `yaml:"target_script"`
|
||||
// PromptPath is the RESOLVED role prompt (`<prompts root>/<pair>/terminologist.md`), filled by
|
||||
// LoadPipeline by the ordinary role convention. Not a config key.
|
||||
PromptPath string `yaml:"-"`
|
||||
|
|
@ -992,6 +1004,13 @@ func LoadPipeline(path string, models *Models, pair string, labels []string) (*P
|
|||
if tg.BudgetUSD <= 0 {
|
||||
bad("gates.terminology.budget_usd must be > 0 when the gate is enabled (a gate that can never spend is a silent no-op)")
|
||||
}
|
||||
// Validated against the same standard-library table the runner resolves with
|
||||
// (terminology.ScriptByName), so an accepted name can never fail to resolve later.
|
||||
if tg.TargetScript == "" {
|
||||
bad("gates.terminology.target_script is required when the gate is enabled — the Unicode script name of the target language (e.g. Cyrillic | Latin | Han); without it a reply in the wrong language is banked as this book's canon in silence")
|
||||
} else if _, ok := unicode.Scripts[tg.TargetScript]; !ok {
|
||||
bad("gates.terminology.target_script %q is not a Unicode script name (exact spelling, e.g. Cyrillic | Latin | Han | Hiragana | Katakana | Hangul | Greek | Arabic)", tg.TargetScript)
|
||||
}
|
||||
if pair == "" {
|
||||
bad("gates.terminology is enabled but the book declares no language pair — the role prompt is resolved as <prompts root>/<pair>/terminologist.md")
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -427,3 +427,47 @@ func TestRepairGateValidation(t *testing.T) {
|
|||
t.Fatalf("a disabled repair gate must not be validated: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTerminologyGateValidatesItsTargetScript pins the load-time contract of the answer-language screen:
|
||||
// an enabled gate must declare the target language's script, and the name must be one Unicode defines.
|
||||
// The failure it guards is silent at runtime, so it has to be loud before any money moves.
|
||||
func TestTerminologyGateValidatesItsTargetScript(t *testing.T) {
|
||||
dir := promptProject(t, "zh-ru")
|
||||
if err := os.WriteFile(filepath.Join(dir, "prompts", "zh-ru", "terminologist.md"), []byte("role\n---USER---\n{{text}}\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gate := func(extra string) string {
|
||||
return "\ngates:\n terminology:\n enabled: true\n model: fake\n budget_usd: 1\n" + extra
|
||||
}
|
||||
cases := []struct{ name, gates, want string }{
|
||||
{"missing script", gate(""), "target_script is required"},
|
||||
{"unknown script", gate(" target_script: Cyrilic\n"), "not a Unicode script name"},
|
||||
{"case must be exact", gate(" target_script: cyrillic\n"), "not a Unicode script name"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
_, err := LoadPipeline(draftOnlyConfig(t, dir, c.gates), miniModels(t), "zh-ru", nil)
|
||||
if err == nil {
|
||||
t.Fatal("a terminology gate that cannot screen the answer language must fail loud")
|
||||
}
|
||||
if !strings.Contains(err.Error(), c.want) {
|
||||
t.Fatalf("error must explain %q, got: %v", c.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
p, err := LoadPipeline(draftOnlyConfig(t, dir, gate(" target_script: Cyrillic\n")), miniModels(t), "zh-ru", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a declared script must load: %v", err)
|
||||
}
|
||||
if p.Gates.Terminology.TargetScript != "Cyrillic" {
|
||||
t.Fatalf("the declaration must reach the gate, got %q", p.Gates.Terminology.TargetScript)
|
||||
}
|
||||
// A pair that is not in the repo declares another script the same way — no Go edit involved.
|
||||
if _, err := LoadPipeline(draftOnlyConfig(t, dir, gate(" target_script: Hiragana\n")), miniModels(t), "zh-ru", nil); err != nil {
|
||||
t.Fatalf("any Unicode script must be declarable: %v", err)
|
||||
}
|
||||
// A DISABLED gate is never validated.
|
||||
if _, err := LoadPipeline(draftOnlyConfig(t, dir, "\ngates:\n terminology:\n enabled: false\n"), miniModels(t), "zh-ru", nil); err != nil {
|
||||
t.Fatalf("a disabled terminology gate must not be validated: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,11 @@ import (
|
|||
// you change the data, because the data's bytes ARE the version (the text/norm.go drift-proofing).
|
||||
// v2 (pair-14): new pair/source files (title-formant, sentence-terminator, palladius-phonotactics, dc-checkers)
|
||||
// + new formats (the pattern/category rows, the generic Palladius parser) — a schema change, so the tag bumps.
|
||||
//
|
||||
// "Schema change" means the READING of bytes that already exist changes. Adding a file that every current
|
||||
// pair lacks does NOT: an absent optional file writes nothing into the hash, so those packs load to the
|
||||
// identical version and nobody is re-billed for a mechanism they do not use. Bump the tag when a REQUIRED
|
||||
// file joins the manifest, or when any existing file starts parsing differently.
|
||||
const packAlgoVersion = "langpack-v2"
|
||||
|
||||
// Pack is a loaded, versioned language-data pack for one source→target pair. Fields are the DATA the
|
||||
|
|
@ -77,9 +82,22 @@ type Pack struct {
|
|||
// data/algorithm boundary the miner tables keep.
|
||||
Heading *HeadingRule
|
||||
|
||||
// Terminology is the OPTIONAL pair sizing of the terminologist's source contexts
|
||||
// (configs/langpacks/<pair>/terminology.txt); nil when the pair ships no file. It exists because the
|
||||
// KWIC width is measured in RUNES, and N runes buy a different amount of context in every script, so
|
||||
// the number is a fact about the pair and belongs in pair data rather than in a Go constant.
|
||||
Terminology *TerminologySizing
|
||||
|
||||
version string
|
||||
}
|
||||
|
||||
// TerminologySizing is the pair's own "how much source context is one context". A zero field means
|
||||
// unstated and falls through to the engine default; the order lives in pipeline.terminologyOpts.
|
||||
type TerminologySizing struct {
|
||||
KWICPerTerm int
|
||||
KWICWidth int
|
||||
}
|
||||
|
||||
// Palladius is the pair transliteration table: the pinyin→Cyrillic maps and the syllable-generator's
|
||||
// phonotactic constraint sets (owner addendum 24.07 — one typed struct instead of four+three parallel Pack
|
||||
// fields). Populated from palladius.txt (Initials/Finals/YW/SpecialI) + palladius-phonotactics.txt
|
||||
|
|
@ -223,6 +241,22 @@ func Load(root, sourceLang, targetLang string) (*Pack, error) {
|
|||
p.DCCheckers = dc
|
||||
}
|
||||
|
||||
// Optional per-pair TERMINOLOGIST sizing. Same contract as heading.txt: ABSENT → nil, the engine
|
||||
// defaults apply and Version() is byte-stable, so shipping the mechanism re-bills nobody. A pair whose
|
||||
// measured value equals the default deliberately ships no file — the bytes would move the pack version
|
||||
// (and with it both waves of every book of the pair) to change nothing.
|
||||
if tb, ok, terr := readOptional(root, pair, "terminology.txt"); terr != nil {
|
||||
return nil, fmt.Errorf("langpack %q terminology.txt: %w", pair, terr)
|
||||
} else if ok {
|
||||
h.Write([]byte("\x00" + pair + "/terminology.txt\x00"))
|
||||
h.Write(tb)
|
||||
ts, perr := parseTerminology(tb)
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("langpack %q terminology.txt: %w", pair, perr)
|
||||
}
|
||||
p.Terminology = ts
|
||||
}
|
||||
|
||||
// A per-pair GENRE glossary was read here (pack-20 / D39.42 п.1) and was removed by D39.47: a pair-wide
|
||||
// file of "the industry rendering" prescribes ONE register to every book of the pair, and the choice
|
||||
// between «культивация» and «совершенствование» belongs to the owner's signature on a book's bank —
|
||||
|
|
@ -554,6 +588,42 @@ func parseHeading(b []byte) (*HeadingRule, error) {
|
|||
return hr, nil
|
||||
}
|
||||
|
||||
// parseTerminology reads terminology.txt into a TerminologySizing. Format: `key<TAB>value`, keys
|
||||
// kwic_per_term | kwic_width, each optional but positive when present. Fail-loud like parseHeading; an
|
||||
// empty file is refused too, because its bytes move Version() while changing nothing.
|
||||
func parseTerminology(b []byte) (*TerminologySizing, error) {
|
||||
ts := &TerminologySizing{}
|
||||
seen := false
|
||||
for i, raw := range strings.Split(string(b), "\n") {
|
||||
t := strings.TrimRight(raw, "\r")
|
||||
if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") {
|
||||
continue
|
||||
}
|
||||
f := strings.SplitN(t, "\t", 2)
|
||||
if len(f) != 2 || strings.TrimSpace(f[1]) == "" {
|
||||
return nil, fmt.Errorf("line %d: want `key<TAB>value` with a non-empty value (%q)", i+1, t)
|
||||
}
|
||||
key, val := strings.TrimSpace(f[0]), strings.TrimSpace(f[1])
|
||||
n, err := strconv.Atoi(val)
|
||||
if err != nil || n <= 0 {
|
||||
return nil, fmt.Errorf("line %d: %s must be a positive integer (%q)", i+1, key, val)
|
||||
}
|
||||
switch key {
|
||||
case "kwic_per_term":
|
||||
ts.KWICPerTerm = n
|
||||
case "kwic_width":
|
||||
ts.KWICWidth = n
|
||||
default:
|
||||
return nil, fmt.Errorf("line %d: unknown key %q (want kwic_per_term|kwic_width)", i+1, key)
|
||||
}
|
||||
seen = true
|
||||
}
|
||||
if !seen {
|
||||
return nil, fmt.Errorf("the file states nothing (want at least one of kwic_per_term|kwic_width); delete it instead — an empty table moves the pack version without changing behaviour")
|
||||
}
|
||||
return ts, nil
|
||||
}
|
||||
|
||||
// runeSet reads a rune SET: every non-whitespace rune of every non-comment line is a member (order-free).
|
||||
func runeSet(b []byte) map[rune]bool {
|
||||
m := map[rune]bool{}
|
||||
|
|
|
|||
|
|
@ -439,6 +439,58 @@ func TestPairDataEditMovesThePackVersion(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestTerminologySizingIsOptionalPairData pins both halves of the optional contract: a pair that ships no
|
||||
// file is byte-stable (so shipping the mechanism re-bills nobody, which is why zh-ru deliberately ships
|
||||
// none), and a pair that does ship one folds its bytes like every other authored file.
|
||||
func TestTerminologySizingIsOptionalPairData(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, dir := range []string{"zh", "zh-ru"} {
|
||||
copyPackDir(t, filepath.Join("../../configs/langpacks", dir), filepath.Join(root, dir))
|
||||
}
|
||||
p1, err := Load(root, "zh", "ru")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p1.Terminology != nil {
|
||||
t.Fatal("the shipped pair ships no terminology.txt: its measured sizing equals the engine default, and the bytes would move the pack version for nothing")
|
||||
}
|
||||
shipped, err := Load("../../configs/langpacks", "zh", "ru")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if shipped.Version() != p1.Version() {
|
||||
t.Fatalf("the copy and the shipped pack must hash identically: %s vs %s", shipped.Version(), p1.Version())
|
||||
}
|
||||
// The tag itself is the other half of "nobody is re-billed": adding an optional file changes no
|
||||
// existing pack's bytes, so bumping the tag here would move every pair's version for nothing.
|
||||
if packAlgoVersion != "langpack-v2" {
|
||||
t.Fatalf("the pack algorithm tag moved to %q — that re-bills both waves of every book of every pair, so it must be a deliberate decision, not a side effect of adding an optional file", packAlgoVersion)
|
||||
}
|
||||
|
||||
path := filepath.Join(root, "zh-ru", "terminology.txt")
|
||||
if err := os.WriteFile(path, []byte("# pair sizing\nkwic_per_term\t5\nkwic_width\t120\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p2, err := Load(root, "zh", "ru")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p2.Terminology == nil || p2.Terminology.KWICPerTerm != 5 || p2.Terminology.KWICWidth != 120 {
|
||||
t.Fatalf("the pair sizing must be parsed, got %+v", p2.Terminology)
|
||||
}
|
||||
if p2.Version() == p1.Version() {
|
||||
t.Fatal("a present pair file MUST fold into the version, like every other authored file")
|
||||
}
|
||||
for _, bad := range []string{"", "# only a comment\n", "kwic_width\t0\n", "kwic_width\twide\n", "kwick\t3\n"} {
|
||||
if err := os.WriteFile(path, []byte(bad), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Load(root, "zh", "ru"); err == nil {
|
||||
t.Fatalf("a corrupt or empty table must fail loud at load, %q did not", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func copyPackDir(t *testing.T, from, to string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(to, 0o755); err != nil {
|
||||
|
|
|
|||
101
backend/internal/pipeline/bankfold_test.go
Normal file
101
backend/internal/pipeline/bankfold_test.go
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"unicode"
|
||||
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
)
|
||||
|
||||
// stateWithProposals builds one per-chunk telemetry row carrying the given banknote lines.
|
||||
func stateWithProposals(chapter, chunk int, lines ...bankEntry) store.RetrievalState {
|
||||
return store.RetrievalState{Chapter: chapter, ChunkIdx: chunk, BanknoteDetail: bankProposalsJSON(lines)}
|
||||
}
|
||||
|
||||
// TestBankFoldJoinsEnclosedAndBareSurfaces pins the measured defect: one draft wave proposed the same
|
||||
// title twice, once wrapped in paired marks and once bare, with two different renderings. Unfolded that is
|
||||
// two rows of the owner's signature map, and neither joins the miner's candidate space.
|
||||
func TestBankFoldJoinsEnclosedAndBareSurfaces(t *testing.T) {
|
||||
obs := bankObservedByKey([]store.RetrievalState{
|
||||
stateWithProposals(1, 0, bankEntry{Src: "《咏梅》", Dst: "Ода сливе", Type: "title"}),
|
||||
stateWithProposals(1, 1, bankEntry{Src: "咏梅", Dst: "Воспевая сливу", Type: "title"}),
|
||||
}, nil)
|
||||
if len(obs) != 1 {
|
||||
t.Fatalf("the two surfaces must fold into ONE candidate, got %d: %+v", len(obs), obs)
|
||||
}
|
||||
if obs[0].Key != "咏梅" || obs[0].Src != "咏梅" {
|
||||
t.Fatalf("key and displayed surface must both be the bare form, got key=%q src=%q", obs[0].Key, obs[0].Src)
|
||||
}
|
||||
if len(obs[0].Proposals) != 2 {
|
||||
t.Fatalf("both renderings must survive as alternatives for the owner, got %+v", obs[0].Proposals)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBankFoldScreensTheAnswerLanguage pins the draft side of the answer-language screen: a rendering in
|
||||
// another script never becomes evidence, and the same rows pass untouched when no script is declared.
|
||||
func TestBankFoldScreensTheAnswerLanguage(t *testing.T) {
|
||||
states := []store.RetrievalState{stateWithProposals(1, 0,
|
||||
bankEntry{Src: "元海", Dst: "Primordial Sea", Type: "term"},
|
||||
bankEntry{Src: "方源", Dst: "Фан Юань", Type: "name"},
|
||||
)}
|
||||
f := newBankFold(unicode.Scripts["Cyrillic"])
|
||||
addRetrievalStates(f, states)
|
||||
obs := f.observed()
|
||||
if len(obs) != 1 || obs[0].Key != "方源" {
|
||||
t.Fatalf("only the target-language proposal may become evidence, got %+v", obs)
|
||||
}
|
||||
if f.OffLanguage != 1 {
|
||||
t.Fatalf("the dropped proposal must be counted, got %d", f.OffLanguage)
|
||||
}
|
||||
if inert := bankObservedByKey(states, nil); len(inert) != 2 {
|
||||
t.Fatalf("with no declared script the fold must be inert, got %+v", inert)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBankFoldCountsChunksNotRows is the aggregation invariant in miniature: the same chunk seen twice
|
||||
// (a re-drafted chunk keeps both checkpoints) votes ONCE, so a union never inflates the evidence a
|
||||
// consolidation is ranked on, while a genuinely different chunk does add its vote.
|
||||
func TestBankFoldCountsChunksNotRows(t *testing.T) {
|
||||
prop := []bankProposal{{SrcKey: "方源", Src: "方源", Dst: "Фан Юань", Type: "name"}}
|
||||
f := newBankFold(nil)
|
||||
f.add(1, 0, prop)
|
||||
f.add(1, 0, prop) // the same chunk, drafted a second time
|
||||
if got := f.observed()[0].Proposals[0].Chunks; got != 1 {
|
||||
t.Fatalf("one chunk must count once however many stored answers mention it, got %d", got)
|
||||
}
|
||||
f.add(1, 1, prop)
|
||||
if got := f.observed()[0].Proposals[0].Chunks; got != 2 {
|
||||
t.Fatalf("a second chunk must add its own vote, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBankFoldIsDeterministic pins that neither the key order nor the rendering order depends on map
|
||||
// iteration — the signature map must be byte-identical between two runs over the same evidence.
|
||||
func TestBankFoldIsDeterministic(t *testing.T) {
|
||||
states := []store.RetrievalState{
|
||||
stateWithProposals(1, 0, bankEntry{Src: "花家", Dst: "Дом Хуа", Type: "name"}),
|
||||
stateWithProposals(1, 1, bankEntry{Src: "方源", Dst: "Фан Юань", Type: "name"}),
|
||||
stateWithProposals(1, 2, bankEntry{Src: "方源", Dst: "Фан Юань", Type: "name"}),
|
||||
stateWithProposals(1, 3, bankEntry{Src: "方源", Dst: "Фан-Юань", Type: "name"}),
|
||||
}
|
||||
var first []terminology.Observed
|
||||
for i := 0; i < 8; i++ {
|
||||
got := bankObservedByKey(states, nil)
|
||||
if i == 0 {
|
||||
first = got
|
||||
continue
|
||||
}
|
||||
if len(got) != len(first) {
|
||||
t.Fatalf("unstable fold size: %d vs %d", len(got), len(first))
|
||||
}
|
||||
for j := range got {
|
||||
if got[j].Key != first[j].Key || got[j].Proposals[0].Dst != first[j].Proposals[0].Dst {
|
||||
t.Fatalf("unstable order at %d: %+v vs %+v", j, got[j], first[j])
|
||||
}
|
||||
}
|
||||
}
|
||||
if first[0].Key != "方源" || first[0].Proposals[0].Dst != "Фан Юань" {
|
||||
t.Fatalf("keys sort, renderings rank most-proposed-first: %+v", first)
|
||||
}
|
||||
}
|
||||
|
|
@ -12,7 +12,6 @@ import (
|
|||
|
||||
"textmachine/backend/internal/chunk"
|
||||
"textmachine/backend/internal/config"
|
||||
"textmachine/backend/internal/miner"
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
"textmachine/backend/internal/text"
|
||||
|
|
@ -276,13 +275,8 @@ type bankProposal struct {
|
|||
Type string `json:"t"`
|
||||
}
|
||||
|
||||
// bankProposalsJSON serializes a chunk's parsed entries for retrieval_state.banknote_detail. Empty
|
||||
// entries → "" (the column stays empty on every channel-off chunk, so a banknote-free book's row bytes
|
||||
// are unchanged). Deterministic: the parser's line order is preserved, nothing is sorted by map order.
|
||||
func bankProposalsJSON(entries []bankEntry) string {
|
||||
if len(entries) == 0 {
|
||||
return ""
|
||||
}
|
||||
// bankProposalsOf keys a chunk's parsed entries. Deterministic: the parser's line order is preserved.
|
||||
func bankProposalsOf(entries []bankEntry) []bankProposal {
|
||||
out := make([]bankProposal, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
key := text.NormalizeSourceKey(e.Src)
|
||||
|
|
@ -291,6 +285,13 @@ func bankProposalsJSON(entries []bankEntry) string {
|
|||
}
|
||||
out = append(out, bankProposal{SrcKey: key, Src: e.Src, Dst: strings.TrimSpace(e.Dst), Type: e.Type})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// bankProposalsJSON serializes them for retrieval_state.banknote_detail. Empty → "" (the column stays
|
||||
// empty on every channel-off chunk, so a banknote-free book's row bytes are unchanged).
|
||||
func bankProposalsJSON(entries []bankEntry) string {
|
||||
out := bankProposalsOf(entries)
|
||||
if len(out) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
|
@ -301,68 +302,75 @@ func bankProposalsJSON(entries []bankEntry) string {
|
|||
return string(b)
|
||||
}
|
||||
|
||||
// bankProposalsByKey folds every chunk's stored proposals into src-key → proposals, counting how many
|
||||
// chunks proposed each rendering. The COUNT is the signal the mini-run surfaced: the same term came back
|
||||
// with three different renderings across chunks (学堂家老 → «старейшина школы» / «старейшина-наставник» /
|
||||
// «учитель-старейшина»), which is precisely the drift a canon exists to close — so the owner sees the
|
||||
// alternatives and their weight, not one arbitrary winner.
|
||||
func bankProposalsByKey(states []store.RetrievalState) map[string][]miner.DstProposal {
|
||||
obs := bankObservedByKey(states)
|
||||
out := make(map[string][]miner.DstProposal, len(obs))
|
||||
for _, o := range obs {
|
||||
list := make([]miner.DstProposal, 0, len(o.Proposals))
|
||||
for _, p := range o.Proposals {
|
||||
list = append(list, miner.DstProposal{Dst: p.Dst, Type: p.Type, Chunks: p.Chunks})
|
||||
}
|
||||
out[o.Key] = list
|
||||
}
|
||||
return out
|
||||
// bankFold accumulates the draft side of the bank over every sampling of the book that exists. It is the
|
||||
// one place a proposal becomes evidence, so the three disciplines apply here: the enclosure trim, the
|
||||
// answer-language screen, and per-CHUNK counting.
|
||||
type bankFold struct {
|
||||
target *unicode.RangeTable // nil → the language screen is inert
|
||||
votes map[string]*bankAgg
|
||||
seen map[bankVote]bool
|
||||
OffLanguage int
|
||||
}
|
||||
|
||||
// bankObservedByKey is the ONE fold of the durable per-chunk banknote rows into the draft-side view of
|
||||
// each source surface: its key, the surface as the model wrote it, and every rendering with the number of
|
||||
// chunks that proposed it. Both consumers read it — the signature-map join (through bankProposalsByKey)
|
||||
// and the terminologist's merge, which additionally needs the SURFACE, so that a proposal for a term the
|
||||
// miner never found can appear in the reverse section instead of being silently dropped on the floor.
|
||||
//
|
||||
// Deterministic throughout: the output slice is key-ordered and each key's renderings are ordered
|
||||
// most-proposed-first with ties broken on the rendering itself — never on map order.
|
||||
func bankObservedByKey(states []store.RetrievalState) []terminology.Observed {
|
||||
type agg struct {
|
||||
src string
|
||||
typ string
|
||||
byDst map[string]*terminology.Proposal
|
||||
}
|
||||
votes := map[string]*agg{}
|
||||
for _, rs := range states {
|
||||
if rs.BanknoteDetail == "" {
|
||||
type bankAgg struct {
|
||||
src string
|
||||
typ string
|
||||
byDst map[string]*terminology.Proposal
|
||||
}
|
||||
|
||||
// bankVote is one chunk's vote for one rendering, so a count means "how many chunks proposed this" and
|
||||
// not "how many stored rows mention it" — they diverge the moment a chunk is re-drafted.
|
||||
type bankVote struct {
|
||||
chapter, chunk int
|
||||
key, dst string
|
||||
}
|
||||
|
||||
func newBankFold(target *unicode.RangeTable) *bankFold {
|
||||
return &bankFold{target: target, votes: map[string]*bankAgg{}, seen: map[bankVote]bool{}}
|
||||
}
|
||||
|
||||
// add folds one chunk's parsed proposals; src/dst arrive as the model wrote them.
|
||||
func (f *bankFold) add(chapter, chunk int, props []bankProposal) {
|
||||
for _, p := range props {
|
||||
// Key and displayed surface are trimmed together: a decorated surface beside a bare key would be
|
||||
// one thing described two ways, and no matcher will ever see the decoration in the source.
|
||||
key, src := text.TrimEnclosure(p.SrcKey), text.TrimEnclosure(p.Src)
|
||||
if key == "" || strings.TrimSpace(p.Dst) == "" {
|
||||
continue
|
||||
}
|
||||
var props []bankProposal
|
||||
if json.Unmarshal([]byte(rs.BanknoteDetail), &props) != nil {
|
||||
continue // a corrupt detail blob is observability, never a reason to fail a run
|
||||
if terminology.OffLanguage(p.Dst, f.target) {
|
||||
f.OffLanguage++
|
||||
continue
|
||||
}
|
||||
for _, p := range props {
|
||||
a := votes[p.SrcKey]
|
||||
if a == nil {
|
||||
a = &agg{src: p.Src, typ: p.Type, byDst: map[string]*terminology.Proposal{}}
|
||||
votes[p.SrcKey] = a
|
||||
}
|
||||
if cur := a.byDst[p.Dst]; cur != nil {
|
||||
cur.Chunks++
|
||||
continue
|
||||
}
|
||||
a.byDst[p.Dst] = &terminology.Proposal{Dst: p.Dst, Type: p.Type, Chunks: 1}
|
||||
v := bankVote{chapter, chunk, key, p.Dst}
|
||||
if f.seen[v] {
|
||||
continue
|
||||
}
|
||||
f.seen[v] = true
|
||||
a := f.votes[key]
|
||||
if a == nil {
|
||||
a = &bankAgg{src: src, typ: p.Type, byDst: map[string]*terminology.Proposal{}}
|
||||
f.votes[key] = a
|
||||
}
|
||||
if cur := a.byDst[p.Dst]; cur != nil {
|
||||
cur.Chunks++
|
||||
continue
|
||||
}
|
||||
a.byDst[p.Dst] = &terminology.Proposal{Dst: p.Dst, Type: p.Type, Chunks: 1}
|
||||
}
|
||||
keys := make([]string, 0, len(votes))
|
||||
for k := range votes {
|
||||
}
|
||||
|
||||
// observed materializes the fold: key-ordered, each key's renderings most-proposed-first with ties broken
|
||||
// on the rendering itself — never on map order.
|
||||
func (f *bankFold) observed() []terminology.Observed {
|
||||
keys := make([]string, 0, len(f.votes))
|
||||
for k := range f.votes {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]terminology.Observed, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
a := votes[key]
|
||||
a := f.votes[key]
|
||||
list := make([]terminology.Proposal, 0, len(a.byDst))
|
||||
for _, p := range a.byDst {
|
||||
list = append(list, *p)
|
||||
|
|
@ -378,6 +386,57 @@ func bankObservedByKey(states []store.RetrievalState) []terminology.Observed {
|
|||
return out
|
||||
}
|
||||
|
||||
// bankObservedByKey folds the durable per-chunk banknote rows into the draft-side view of each source
|
||||
// surface. Both consumers read it: the signature-map join (proposalsFromObserved) and the terminologist's
|
||||
// merge, which also needs the SURFACE so a proposal for a term the miner never found is not lost.
|
||||
func bankObservedByKey(states []store.RetrievalState, target *unicode.RangeTable) []terminology.Observed {
|
||||
f := newBankFold(target)
|
||||
addRetrievalStates(f, states)
|
||||
return f.observed()
|
||||
}
|
||||
|
||||
// bankObservedForBook is the draft side over EVERY sampling of the book that exists: the per-chunk
|
||||
// telemetry rows plus every stored translator answer, including the ones a later run superseded. Both
|
||||
// sources are folded into one accumulator, and votes are counted per chunk, so a chunk present in both
|
||||
// (the normal case) counts once and the union only ever grows when new drafts were actually bought.
|
||||
//
|
||||
// Superseded attempts are included deliberately: a draft rejected for a defect in its TEXT does not make
|
||||
// the terms it declared untrue, and nothing here enters the bank without a signature anyway.
|
||||
func (r *Runner) bankObservedForBook() ([]terminology.Observed, int, error) {
|
||||
f := newBankFold(r.targetScript)
|
||||
states, err := r.Store.RetrievalStatesForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
addRetrievalStates(f, states)
|
||||
// Only answers that carry the channel's own marker can contribute, and a book's answer history is
|
||||
// megabytes: the marker fragment (which a malformed separator still contains) bounds the read.
|
||||
answers, err := r.Store.RoleResponsesForBook(r.Book.BookID, roleTranslator, bankMarkerFragment)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
for _, a := range answers {
|
||||
// The same slice/parse the live path runs, so the trust rule (a complete generation only) and the
|
||||
// derived $0 rows are handled by one definition rather than by a second copy here.
|
||||
_, _, _, entries := r.applyBanknoteWithEntries(roleTranslator, a.ResponseText, a.FinishReason)
|
||||
f.add(a.Chapter, a.ChunkIdx, bankProposalsOf(entries))
|
||||
}
|
||||
return f.observed(), f.OffLanguage, nil
|
||||
}
|
||||
|
||||
func addRetrievalStates(f *bankFold, states []store.RetrievalState) {
|
||||
for _, rs := range states {
|
||||
if rs.BanknoteDetail == "" {
|
||||
continue
|
||||
}
|
||||
var props []bankProposal
|
||||
if json.Unmarshal([]byte(rs.BanknoteDetail), &props) != nil {
|
||||
continue // a corrupt detail blob is observability, never a reason to fail a run
|
||||
}
|
||||
f.add(rs.Chapter, rs.ChunkIdx, props)
|
||||
}
|
||||
}
|
||||
|
||||
// bankDerivedHash is the CONTENT-ADDRESSED id of a banknote-stripped export checkpoint (§4б EXACT
|
||||
// formula, mirroring commitSanitizedExport's namespacing): sha256("tm-banknote-v1\x00"+reqHash+"\x00"+
|
||||
// stripped), prefixed "tm-banknote-v1:" so it can never collide with a real hex attempt hash and a
|
||||
|
|
|
|||
110
backend/internal/pipeline/bankunion_test.go
Normal file
110
backend/internal/pipeline/bankunion_test.go
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/terminology"
|
||||
)
|
||||
|
||||
// TestBankUnionRecoversASupersededSampling is the aggregation contract end to end. The per-chunk
|
||||
// telemetry row is keyed (book, chapter, chunk) and upserted, so a chunk drafted a second time OVERWRITES
|
||||
// the proposals of the first — and that channel proposes a substantially different set on every sampling,
|
||||
// so the overwrite loses real coverage. The checkpoints keep every answer, so the union must hold both.
|
||||
//
|
||||
// The second answer is written through the ordinary money path rather than simulated in the fold, so the
|
||||
// test would catch a query that misses re-drafted attempts as surely as a fold that drops them.
|
||||
func TestBankUnionRecoversASupersededSampling(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isEditBody(body) {
|
||||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||||
}
|
||||
return "Фан Юань пришёл к горе Цинмао.\n" + bankBlockForMining, "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
r := newRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{}))
|
||||
defer r.Close()
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
before, _, err := r.bankObservedForBook()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(before) == 0 {
|
||||
t.Fatal("setup: the first sampling must have produced proposals")
|
||||
}
|
||||
if got := keysOf(before); contains(got, "花家") {
|
||||
t.Fatalf("setup: 花家 must not be proposed yet, got %v", got)
|
||||
}
|
||||
// Every proposal of this run came from one chunk, so the union must not have inflated any count.
|
||||
for _, o := range before {
|
||||
for _, p := range o.Proposals {
|
||||
if p.Chunks != 1 {
|
||||
t.Fatalf("a single sampling must count once per chunk, %s→%s has %d", o.Key, p.Dst, p.Chunks)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A SECOND answer for the same chunk, as a re-purchase produces: the telemetry row still holds the
|
||||
// first one, the checkpoint store holds both.
|
||||
snap, _, err := r.snapshotIDForWave(waveDraft)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, err := r.Store.EnsureJob(r.Book.BookID, 1, "draft", snap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, verdict, err := r.Store.Reserve(r.Book.BookID, 0.01, store.Ceilings{BookUSD: 100, DayUSD: 100})
|
||||
if err != nil || verdict != store.ReserveOK {
|
||||
t.Fatalf("reserve: %v %v", verdict, err)
|
||||
}
|
||||
second := "Фан Юань пришёл к горе Цинмао.\n" + bankSeparator + "\n花家\tДом Хуа\tname\n"
|
||||
if err := r.Store.SettleWithCheckpoint(res, 0.01, store.Checkpoint{
|
||||
RequestHash: "second-sampling", JobID: job.ID, ChunkIdx: 0, Attempt: 1,
|
||||
Stage: "draft", Role: roleTranslator, ModelRequested: "fake-model", ModelActual: "fake-model",
|
||||
ResponseText: second, UsageJSON: "{}", CostUSD: 0.01, FinishReason: "stop",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
after, _, err := r.bankObservedForBook()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !contains(keysOf(after), "花家") {
|
||||
t.Fatalf("the union must recover the superseded sampling, got %v", keysOf(after))
|
||||
}
|
||||
if !contains(keysOf(after), "方源") {
|
||||
t.Fatalf("the union must keep the surviving sampling too, got %v", keysOf(after))
|
||||
}
|
||||
// The re-drafted chunk still votes once per rendering: two answers from ONE chunk are one chunk.
|
||||
for _, o := range after {
|
||||
for _, p := range o.Proposals {
|
||||
if p.Chunks != 1 {
|
||||
t.Fatalf("a re-drafted chunk must not vote twice, %s→%s has %d", o.Key, p.Dst, p.Chunks)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func keysOf(obs []terminology.Observed) []string {
|
||||
out := make([]string, 0, len(obs))
|
||||
for _, o := range obs {
|
||||
out = append(out, o.Key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contains(ss []string, want string) bool {
|
||||
for _, s := range ss {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -80,15 +80,17 @@ func (r *Runner) runBankMiningStop(ctx context.Context, chunks []chunk.Chunk, dr
|
|||
// mode silently switches the stop off after its first run — see unsignedEngineSurfaces.
|
||||
mined, emission := miner.MineBankStats(minerChunks, contrast, unsignedEngineSurfaces(seed), rejects, miner.FrozenConfig(), r.pack)
|
||||
|
||||
// The WHAT side (D39.36 fix): the banknote proposals the draft wave collected are read off the durable
|
||||
// per-chunk rows. They arrive as EVIDENCE — nothing proposed enters the bank without a signature. A read
|
||||
// failure degrades to the previous WHICH-only map rather than blocking the stop: the map is what the
|
||||
// owner needs, the dst is a bonus.
|
||||
var observed []terminology.Observed
|
||||
if states, serr := r.Store.RetrievalStatesForBook(r.Book.BookID); serr != nil {
|
||||
r.Log.WarnContext(ctx, "bank-mining: could not read the banknote proposals; the signature map falls back to WHICH-only (bare terms)", "err", serr)
|
||||
} else {
|
||||
observed = bankObservedByKey(states)
|
||||
// The WHAT side: the banknote proposals the draft waves collected, over every sampling of the book that
|
||||
// exists. They arrive as EVIDENCE — nothing proposed enters the bank without a signature. A read failure
|
||||
// degrades to the WHICH-only map rather than blocking the stop: the map is what the owner needs, the
|
||||
// dst is a bonus.
|
||||
observed, offLanguage, oerr := r.bankObservedForBook()
|
||||
if oerr != nil {
|
||||
r.Log.WarnContext(ctx, "bank-mining: could not read the banknote proposals; the signature map falls back to WHICH-only (bare terms)", "err", oerr)
|
||||
}
|
||||
if offLanguage > 0 {
|
||||
r.Log.WarnContext(ctx, "bank-mining: draft-side proposals were written in another script and are NOT offered for signature",
|
||||
"book", r.Book.BookID, "dropped", offLanguage, "target_script", r.Pipeline.Gates.Terminology.TargetScript)
|
||||
}
|
||||
|
||||
// The TERMINOLOGIST (pack-20, D39.42): merge both channels, gather each candidate's source contexts,
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ type miningStopOpts struct {
|
|||
terminology bool // enable the terminologist role (pack-20) with a fixture prompt
|
||||
batchRunes int // gates.terminology.batch_runes; 0 = engine default (one batch for this corpus)
|
||||
budgetUSD float64 // gates.terminology.budget_usd; 0 = 1.0 (effectively unbounded here)
|
||||
targetScript string // gates.terminology.target_script; "" = Cyrillic (the fixtures' target)
|
||||
}
|
||||
|
||||
// bankBlockForMining is the banknote a translator emits over this corpus: one line whose src IS a mined
|
||||
|
|
@ -83,7 +84,11 @@ func setupMiningStopProject(t *testing.T, providerURL string, o miningStopOpts)
|
|||
if budget <= 0 {
|
||||
budget = 1.0
|
||||
}
|
||||
gates += fmt.Sprintf(" terminology:\n enabled: true\n model: fake-model\n budget_usd: %g\n", budget)
|
||||
script := o.targetScript
|
||||
if script == "" {
|
||||
script = "Cyrillic" // the fixtures translate into Russian; an enabled gate must declare a script
|
||||
}
|
||||
gates += fmt.Sprintf(" terminology:\n enabled: true\n model: fake-model\n budget_usd: %g\n target_script: %s\n", budget, script)
|
||||
if o.batchRunes > 0 {
|
||||
gates += fmt.Sprintf(" batch_runes: %d\n", o.batchRunes)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -165,7 +165,7 @@ func TestBanknoteProposalsArePersistedAndJoined(t *testing.T) {
|
|||
if len(props) != 2 {
|
||||
t.Fatalf("want 2 stored proposals, got %d: %s", len(props), rs.BanknoteDetail)
|
||||
}
|
||||
byKey := bankProposalsByKey([]store.RetrievalState{*rs})
|
||||
byKey := proposalsFromObserved(bankObservedByKey([]store.RetrievalState{*rs}, nil))
|
||||
if got := byKey[props[0].SrcKey]; len(got) == 0 || got[0].Dst != props[0].Dst {
|
||||
t.Fatalf("the fold must key proposals by the miner's normalized surface, got %+v", byKey)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
|
||||
"textmachine/backend/internal/checks"
|
||||
"textmachine/backend/internal/chunk"
|
||||
|
|
@ -69,6 +70,10 @@ type Runner struct {
|
|||
// nil otherwise, which is what makes the whole role a no-op for every book that does not enable it. It is
|
||||
// NOT snapshot-folded: the role writes a signature map, not a checkpoint (config.TerminologyGate).
|
||||
terminologyTemplate *PromptTemplate
|
||||
// targetScript is the target language's Unicode script, resolved once from gates.terminology
|
||||
// .target_script (config validates the name). nil when the book declares none — the answer-language
|
||||
// screen is then inert, which loadTargetScript says out loud if any channel could have used it.
|
||||
targetScript *unicode.RangeTable
|
||||
// rateGuards is the per-model wave-concurrency guard set (WS1 §1б), built once in the precompute pass and
|
||||
// read-only in the waves — a transport axis, never snapshot-folded. nil until buildRateGuards.
|
||||
rateGuards map[string]*rateGuard
|
||||
|
|
@ -242,6 +247,7 @@ func openRunner(bookPath string, logger *slog.Logger, forWrite bool) (*Runner, e
|
|||
st.Close()
|
||||
return nil, err
|
||||
}
|
||||
r.loadTargetScript()
|
||||
// The echo-exposure half of the reasoning-off hole (D39.26 добор B): loud, not fatal — see
|
||||
// sourceEchoExposure. Needs the templates, so it runs after they load.
|
||||
if exposed := r.sourceEchoExposure(); len(exposed) > 0 {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
|
||||
"textmachine/backend/internal/chunk"
|
||||
"textmachine/backend/internal/config"
|
||||
"textmachine/backend/internal/lang"
|
||||
"textmachine/backend/internal/ledger"
|
||||
"textmachine/backend/internal/llm"
|
||||
"textmachine/backend/internal/miner"
|
||||
|
|
@ -70,6 +71,9 @@ type terminologyResult struct {
|
|||
Declined int // terms the role explicitly could not render → status:auto
|
||||
Unanswered int // terms no reply line covered (also status:auto — silence is not a decision)
|
||||
BadLines int // reply lines the parser refused
|
||||
// OffLanguage counts refused lines whose rendering was not in the target's script — separate from
|
||||
// BadLines because it means the model answered in another language, not that it broke the format.
|
||||
OffLanguage int
|
||||
// CanonConflicts counts consolidated renderings that contradict a row the owner already signed. It is
|
||||
// OBSERVABILITY, never a gate: the rows stay unverified either way, and this is what tells the owner
|
||||
// which of them to look at first.
|
||||
|
|
@ -94,11 +98,32 @@ func (r *Runner) loadTerminologyTemplate() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// terminologyOpts resolves the sizing knobs (config value, else the engine default) in ONE place, so the
|
||||
// batching and the request rendering can never disagree about what a batch is.
|
||||
// loadTargetScript resolves the answer-language screen's script once. config refuses an unknown name for
|
||||
// an enabled gate, so an unresolved script here means the book simply declared none — inert, and said out
|
||||
// loud when a channel that would have used it is on.
|
||||
func (r *Runner) loadTargetScript() {
|
||||
if name := r.Pipeline.Gates.Terminology.TargetScript; name != "" {
|
||||
r.targetScript, _ = terminology.ScriptByName(name)
|
||||
}
|
||||
if r.targetScript == nil && r.Pipeline.Gates.Banknote.Enabled {
|
||||
r.Log.Warn("the banknote channel is on but no gates.terminology.target_script is declared: draft-side proposals are NOT screened for the answer language, so a rendering in another script can enter the signature map and the auto-bank",
|
||||
"book", r.Book.BookID)
|
||||
}
|
||||
}
|
||||
|
||||
// terminologyOpts resolves the sizing knobs in ONE place — explicit config, else the pair's own data,
|
||||
// else the engine default — so batching and rendering can never disagree about what a batch is.
|
||||
func (r *Runner) terminologyOpts() (batchRunes, kwicPer, kwicWidth int) {
|
||||
g := r.Pipeline.Gates.Terminology
|
||||
batchRunes, kwicPer, kwicWidth = g.BatchRunes, g.KWICPerTerm, g.KWICWidth
|
||||
if pair := r.packTerminology(); pair != nil {
|
||||
if kwicPer <= 0 {
|
||||
kwicPer = pair.KWICPerTerm
|
||||
}
|
||||
if kwicWidth <= 0 {
|
||||
kwicWidth = pair.KWICWidth
|
||||
}
|
||||
}
|
||||
if batchRunes <= 0 {
|
||||
batchRunes = terminologyDefaultBatchRunes
|
||||
}
|
||||
|
|
@ -111,6 +136,13 @@ func (r *Runner) terminologyOpts() (batchRunes, kwicPer, kwicWidth int) {
|
|||
return batchRunes, kwicPer, kwicWidth
|
||||
}
|
||||
|
||||
func (r *Runner) packTerminology() *lang.TerminologySizing {
|
||||
if r.pack == nil {
|
||||
return nil
|
||||
}
|
||||
return r.pack.Terminology
|
||||
}
|
||||
|
||||
// buildBankCandidates is the $0 half of the role: the two-way miner∪banknote merge, the source contexts,
|
||||
// and the §C2-3 ranking of whatever the drafts already produced. It runs whether or not the gate is on —
|
||||
// with the gate off nothing consumes the ranking, but building it costs nothing and it is what the stop's
|
||||
|
|
@ -250,8 +282,16 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
for _, c := range b {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
got, bad := terminology.ParseReply(att.text, keys, text.NormalizeSourceKey)
|
||||
res.BadLines += bad
|
||||
got, st := terminology.ParseReply(att.text, keys, text.NormalizeSourceKey, r.targetScript)
|
||||
res.BadLines += st.Bad
|
||||
res.OffLanguage += st.OffLanguage
|
||||
// A batch answered largely in another language is the measured cold-start failure, not noise: say
|
||||
// it while the run is happening, naming the lines, because the terms themselves just stay auto.
|
||||
if st.OffLanguage > 0 {
|
||||
r.Log.WarnContext(ctx, "terminology: the model answered in another script; those lines are REFUSED (the terms stay unconsolidated) — a book with few signed rows gives the model no target-language anchor",
|
||||
"book", r.Book.BookID, "batch", i, "terms", len(b), "off_language", st.OffLanguage,
|
||||
"target_script", r.Pipeline.Gates.Terminology.TargetScript, "lines", strings.Join(st.OffLanguageSamples, "; "))
|
||||
}
|
||||
// A batch that came back with NOTHING usable is a paid call that bought no terminology: an empty
|
||||
// completion, a truncation, a refusal, a reply in prose. Silence here would spend the money, mark
|
||||
// every term of the batch "the role declined" (§C2-7's auto mode, which is a DECISION) and exit 0.
|
||||
|
|
@ -259,7 +299,7 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
// this step started from, not a broken one.
|
||||
if len(got) == 0 {
|
||||
r.Log.WarnContext(ctx, "terminology: a paid batch returned nothing the parser could use; its terms stay unconsolidated",
|
||||
"book", r.Book.BookID, "batch", i, "terms", len(b), "bad_lines", bad,
|
||||
"book", r.Book.BookID, "batch", i, "terms", len(b), "bad_lines", st.Bad,
|
||||
"reply_chars", len(att.text), "cost_usd", fmt.Sprintf("%.6f", att.runCost))
|
||||
}
|
||||
for k, v := range got {
|
||||
|
|
@ -291,7 +331,8 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te
|
|||
}
|
||||
r.Log.InfoContext(ctx, "terminology finished", "book", r.Book.BookID,
|
||||
"consolidated", res.Consolidated, "declined", res.Declined, "unanswered", res.Unanswered,
|
||||
"bad_lines", res.BadLines, "canon_conflicts", res.CanonConflicts, "cost_usd", fmt.Sprintf("%.6f", res.CostUSD))
|
||||
"bad_lines", res.BadLines, "off_language", res.OffLanguage,
|
||||
"canon_conflicts", res.CanonConflicts, "cost_usd", fmt.Sprintf("%.6f", res.CostUSD))
|
||||
return out, res, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -262,6 +262,55 @@ func (s *Store) GetCheckpoint(requestHash string) (*Checkpoint, error) {
|
|||
return &cp, nil
|
||||
}
|
||||
|
||||
// RoleResponse is one persisted model answer with the chunk coordinates it was produced for — enough to
|
||||
// re-derive anything the raw text carried, and nothing more (no usage, no money, no hashes).
|
||||
type RoleResponse struct {
|
||||
Chapter int
|
||||
ChunkIdx int
|
||||
Attempt int
|
||||
ResponseText string
|
||||
FinishReason string
|
||||
}
|
||||
|
||||
// RoleResponsesForBook returns EVERY persisted answer a role ever produced for a book, oldest job first —
|
||||
// including the attempts a later run superseded.
|
||||
//
|
||||
// It exists for the banknote union. The per-chunk telemetry row is keyed (book, chapter, chunk) and
|
||||
// UPSERTED, so when a chunk is genuinely re-drafted the new proposals overwrite the old ones — and that
|
||||
// channel was measured to propose a substantially different set on every sampling, so the overwrite loses
|
||||
// real coverage. Checkpoints keep every answer the book ever paid for, addressed by its own request hash,
|
||||
// so the union is DERIVED rather than stored: no new table, no migration, self-healing like the row it
|
||||
// supplements.
|
||||
//
|
||||
// It returns the $0 DERIVED rows too (the stripped export, the sanitized export), deliberately: they carry
|
||||
// a non-provider finish_reason, so the caller's ordinary "trust a complete generation only" gate already
|
||||
// excludes them, and filtering them here would put a second, silently drifting copy of that rule in SQL.
|
||||
//
|
||||
// mustContain is an optional VOLUME filter on the response text (empty = no filter). A book's whole
|
||||
// answer history is megabytes and grows with every re-purchase, so a caller that only cares about answers
|
||||
// carrying a marker passes it here instead of reading everything into memory. It is never a semantic
|
||||
// rule — the caller still parses whatever it gets — and the marker stays a Go constant, passed as a
|
||||
// value, so SQL holds no second copy of it.
|
||||
//
|
||||
// Ordering is (job, chunk, attempt) so a fold over the result is deterministic without sorting in Go.
|
||||
func (s *Store) RoleResponsesForBook(bookID, role, mustContain string) ([]RoleResponse, error) {
|
||||
q := `SELECT j.chapter, c.chunk_idx, c.attempt, c.response_text, c.finish_reason
|
||||
FROM checkpoints c JOIN jobs j ON j.id = c.job_id
|
||||
WHERE j.book_id = ? AND c.role = ?`
|
||||
args := []any{bookID, role}
|
||||
if mustContain != "" {
|
||||
q += ` AND instr(c.response_text, ?) > 0`
|
||||
args = append(args, mustContain)
|
||||
}
|
||||
q += ` ORDER BY j.id, c.chunk_idx, c.attempt`
|
||||
return queryAll(s.r, q,
|
||||
func(rows *sql.Rows) (RoleResponse, error) {
|
||||
var r RoleResponse
|
||||
err := rows.Scan(&r.Chapter, &r.ChunkIdx, &r.Attempt, &r.ResponseText, &r.FinishReason)
|
||||
return r, err
|
||||
}, args...)
|
||||
}
|
||||
|
||||
// SpentUSD reports (committed, reserved) for a book across all days.
|
||||
func (s *Store) SpentUSD(bookID string) (committed, reserved float64, err error) {
|
||||
ctx, cancel := opContext()
|
||||
|
|
|
|||
46
backend/internal/terminology/script.go
Normal file
46
backend/internal/terminology/script.go
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
package terminology
|
||||
|
||||
import "unicode"
|
||||
|
||||
// script.go: the answer-language guarantee. Measured on live replies: with no signed rows to show the
|
||||
// model, the role answers in a foreign script — and every existing guard admits it, because a foreign
|
||||
// script is legitimate for foreign proper names, the answer is not an echo of the source, and an empty
|
||||
// canon anchor has nothing to contradict. Input-side fixes (the anchor, the drafts evidence, a format
|
||||
// example in the prompt) lower the rate; a guarantee may not depend on the content of a request, so it
|
||||
// lives on the output side.
|
||||
|
||||
// ScriptByName resolves a Unicode script NAME ("Cyrillic", "Latin", "Han", …) to its range table. The
|
||||
// table set is the standard library's, so the accepted spellings are Unicode's — this engine keeps no
|
||||
// list of languages to fall out of date. config validates a declared name against the same map.
|
||||
func ScriptByName(name string) (*unicode.RangeTable, bool) {
|
||||
t, ok := unicode.Scripts[name]
|
||||
return t, ok
|
||||
}
|
||||
|
||||
// OffLanguage reports whether dst carries letters yet not one of them belongs to the target script.
|
||||
//
|
||||
// The weakest rule that catches the measured failure: one target-script letter is enough to pass, so a
|
||||
// mixed rendering is untouched, and a rendering with no letters at all is not judged here.
|
||||
//
|
||||
// Direction of error, declared: it can refuse a legitimate rendering a pair wants kept in foreign letters.
|
||||
// That costs one proposal the owner can still sign by hand; the opposite mistake puts a foreign word in a
|
||||
// book's canon.
|
||||
//
|
||||
// target == nil → inert, mirroring a nil ScoreOpts.Conformance. The loudness for that case belongs to
|
||||
// config and to the runner, not here.
|
||||
func OffLanguage(dst string, target *unicode.RangeTable) bool {
|
||||
if target == nil {
|
||||
return false
|
||||
}
|
||||
letters := false
|
||||
for _, r := range dst {
|
||||
if !unicode.IsLetter(r) {
|
||||
continue
|
||||
}
|
||||
if unicode.Is(target, r) {
|
||||
return false
|
||||
}
|
||||
letters = true
|
||||
}
|
||||
return letters
|
||||
}
|
||||
105
backend/internal/terminology/script_test.go
Normal file
105
backend/internal/terminology/script_test.go
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
package terminology
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// offLanguageReplies are renderings a live model actually returned when it lost the target language, and
|
||||
// signedRenderings are renderings from a real signed bank plus the shapes most likely to be false
|
||||
// positives (two letters, digits only, quoted, doubled space). Both lists are the acceptance material for
|
||||
// the screen: every line of the first must be refused, every line of the second must pass.
|
||||
var (
|
||||
offLanguageReplies = []string{"cultivation", "Primordial Sea", "Rank 1", "Grade C", "talent",
|
||||
"Wine Bug", "realm", "Qingmao Mountain", "Gu Yue", "is"}
|
||||
signedRenderings = []string{"культивация", "море истинной ци", "гу-мастер", "Фан Юань", "гу",
|
||||
"талант", "винный червь", "старейшина", "Гуюэ", "апертура", "третья гу", "гу Силы",
|
||||
"Река времени", "9", "«Ода сливе»", "Фан Юань"}
|
||||
)
|
||||
|
||||
func TestOffLanguageSeparatesForeignRenderingsFromTargetOnes(t *testing.T) {
|
||||
cyr := unicode.Scripts["Cyrillic"]
|
||||
for _, s := range offLanguageReplies {
|
||||
if !OffLanguage(s, cyr) {
|
||||
t.Errorf("%q must be refused: it carries letters and none of them are in the target script", s)
|
||||
}
|
||||
}
|
||||
for _, s := range signedRenderings {
|
||||
if OffLanguage(s, cyr) {
|
||||
t.Errorf("%q is a legitimate rendering and must pass", s)
|
||||
}
|
||||
}
|
||||
// A mixed rendering passes deliberately: one target-script letter is enough, so the screen never
|
||||
// touches a rendering that carries a foreign name beside a target word.
|
||||
if OffLanguage("Фан Юань (Fang Yuan)", cyr) {
|
||||
t.Error("a mixed rendering must pass — the screen judges language, not spelling")
|
||||
}
|
||||
if OffLanguage("anything", nil) {
|
||||
t.Error("with no declared script the screen must be inert")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOffLanguageIsTargetBlind is the generality pin: nothing about a language lives in this package, so
|
||||
// swapping the declared script swaps every verdict and no Go edit is involved.
|
||||
func TestOffLanguageIsTargetBlind(t *testing.T) {
|
||||
lat := unicode.Scripts["Latin"]
|
||||
if !OffLanguage("культивация", lat) || OffLanguage("cultivation", lat) {
|
||||
t.Fatal("under a Latin target the verdicts must be exactly reversed")
|
||||
}
|
||||
// A pair with neither script: Han as a target refuses both of the above and accepts a Han rendering.
|
||||
han := unicode.Scripts["Han"]
|
||||
if !OffLanguage("культивация", han) || !OffLanguage("cultivation", han) || OffLanguage("元海", han) {
|
||||
t.Fatal("a third target must work on the same code path")
|
||||
}
|
||||
if _, ok := ScriptByName("Cyrillic"); !ok {
|
||||
t.Fatal("a valid script name must resolve")
|
||||
}
|
||||
if _, ok := ScriptByName("cyrillic"); ok {
|
||||
t.Fatal("resolution must be exact, so a typo fails loudly at config load instead of disabling the screen")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseReplyRefusesForeignLanguageRenderings pins the screen where it actually protects the bank: a
|
||||
// reply mixing correct and foreign-language lines banks only the correct ones, counts the rest, and names
|
||||
// them. Without the screen every line here is accepted and reaches the editor as this book's canon.
|
||||
func TestParseReplyRefusesForeignLanguageRenderings(t *testing.T) {
|
||||
id := func(s string) string { return s }
|
||||
keys := []string{"修行", "元海", "转", "是为", "资质"}
|
||||
reply := "修行\tcultivation\n元海\tPrimordial Sea\n转\trealm\n是为\tis\n资质\tталант"
|
||||
|
||||
got, st := ParseReply(reply, keys, id, unicode.Scripts["Cyrillic"])
|
||||
if len(got) != 1 || got["资质"] != "талант" {
|
||||
t.Fatalf("only the target-language line may be banked, got %#v", got)
|
||||
}
|
||||
if st.OffLanguage != 4 || st.Bad != 4 {
|
||||
t.Fatalf("all four foreign lines must be refused AND counted, got off_language=%d bad=%d", st.OffLanguage, st.Bad)
|
||||
}
|
||||
if len(st.OffLanguageSamples) != 4 || !strings.Contains(st.OffLanguageSamples[0], "cultivation") {
|
||||
t.Fatalf("the refused lines must be named for the operator, got %q", st.OffLanguageSamples)
|
||||
}
|
||||
// Without a declared script the same reply is banked whole — the state the engine was in before, kept
|
||||
// visible so the cost of an undeclared script is a test, not a surprise.
|
||||
if inert, st := ParseReply(reply, keys, id, nil); len(inert) != 5 || st.OffLanguage != 0 {
|
||||
t.Fatalf("with no script the screen must be inert (that is what config refuses for an enabled gate), got %#v", inert)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOffLanguageSamplesAreBounded keeps a whole batch answered in the wrong language from turning one
|
||||
// warning into the reply itself.
|
||||
func TestOffLanguageSamplesAreBounded(t *testing.T) {
|
||||
id := func(s string) string { return s }
|
||||
var lines []string
|
||||
var keys []string
|
||||
for _, k := range []string{"a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9", "b1", "b2", "b3"} {
|
||||
keys = append(keys, k)
|
||||
lines = append(lines, k+"\tforeign")
|
||||
}
|
||||
_, st := ParseReply(strings.Join(lines, "\n"), keys, id, unicode.Scripts["Cyrillic"])
|
||||
if st.OffLanguage != len(keys) {
|
||||
t.Fatalf("every foreign line must be counted, got %d of %d", st.OffLanguage, len(keys))
|
||||
}
|
||||
if len(st.OffLanguageSamples) != offLanguageSampleCap {
|
||||
t.Fatalf("the samples must stop at %d, got %d", offLanguageSampleCap, len(st.OffLanguageSamples))
|
||||
}
|
||||
}
|
||||
|
|
@ -831,33 +831,46 @@ func sharedRunes(a, b string) int {
|
|||
return n
|
||||
}
|
||||
|
||||
// ReplyStats is what one reply cost in unusable lines. Bad is the TOTAL refused; OffLanguage is the
|
||||
// subset refused by the answer-language screen, kept apart because the two mean different things to an
|
||||
// operator — a broken format against a run producing canon in the wrong language.
|
||||
type ReplyStats struct {
|
||||
Bad int
|
||||
OffLanguage int
|
||||
OffLanguageSamples []string // first few refused lines verbatim, so the warning names them
|
||||
}
|
||||
|
||||
const offLanguageSampleCap = 8
|
||||
|
||||
// ParseReply turns the terminologist's reply into key → consolidated rendering. Lines are
|
||||
// `<src><TAB><dst>`; the src is matched through `normalize` against the batch's EXPECTED keys, so a line
|
||||
// for a term that was not asked about is ignored (a model inventing rows must not inject terms into the
|
||||
// bank) and a rendering written for a differently-spelled src still lands. NoDst yields an explicit empty
|
||||
// rendering — the two-mode emission's "no dst" branch, which is a decision, not a failure.
|
||||
//
|
||||
// Returns the accepted map plus the count of unusable lines, which the caller logs: silence about a reply
|
||||
// `target` is the target language's script (nil → the answer-language check is inert; see OffLanguage).
|
||||
//
|
||||
// Returns the accepted map plus the tally of unusable lines, which the caller logs: silence about a reply
|
||||
// the parser could not read is how a paid call turns into an empty bank with no signal.
|
||||
func ParseReply(reply string, expected []string, normalize func(string) string) (map[string]string, int) {
|
||||
func ParseReply(reply string, expected []string, normalize func(string) string, target *unicode.RangeTable) (map[string]string, ReplyStats) {
|
||||
want := make(map[string]bool, len(expected))
|
||||
for _, k := range expected {
|
||||
want[k] = true
|
||||
}
|
||||
out := map[string]string{}
|
||||
bad := 0
|
||||
var st ReplyStats
|
||||
for _, ln := range strings.Split(reply, "\n") {
|
||||
if strings.TrimSpace(ln) == "" || strings.HasPrefix(strings.TrimSpace(ln), "#") {
|
||||
continue
|
||||
}
|
||||
f := splitFields(ln)
|
||||
if len(f) < 2 {
|
||||
bad++
|
||||
st.Bad++
|
||||
continue
|
||||
}
|
||||
key := normalize(f[0])
|
||||
if !want[key] {
|
||||
bad++
|
||||
st.Bad++
|
||||
continue
|
||||
}
|
||||
if _, dup := out[key]; dup {
|
||||
|
|
@ -875,7 +888,7 @@ func ParseReply(reply string, expected []string, normalize func(string) string)
|
|||
continue
|
||||
}
|
||||
if !wellFormedLemma(dst) {
|
||||
bad++
|
||||
st.Bad++
|
||||
continue
|
||||
}
|
||||
// A rendering that IS the source is not a rendering. Models answer this under pressure (a term they
|
||||
|
|
@ -883,12 +896,22 @@ func ParseReply(reply string, expected []string, normalize func(string) string)
|
|||
// it is stamped status:draft, and it reaches the editor as «方源 ⟨проверить⟩» — an untranslated
|
||||
// surface presented as this book's canon. Counted as a bad line, so a reply full of echoes is loud.
|
||||
if normalize(dst) == key {
|
||||
bad++
|
||||
st.Bad++
|
||||
continue
|
||||
}
|
||||
// Nor is a rendering written in another script — the same argument as the echo check above, at the
|
||||
// one place a reply becomes bank content. Past here it is indistinguishable from a correct answer.
|
||||
if OffLanguage(dst, target) {
|
||||
st.Bad++
|
||||
st.OffLanguage++
|
||||
if len(st.OffLanguageSamples) < offLanguageSampleCap {
|
||||
st.OffLanguageSamples = append(st.OffLanguageSamples, f[0]+" → "+dst)
|
||||
}
|
||||
continue
|
||||
}
|
||||
out[key] = dst
|
||||
}
|
||||
return out, bad
|
||||
return out, st
|
||||
}
|
||||
|
||||
// Batch splits candidates into groups whose rendered size stays under maxRunes, preserving key order. A
|
||||
|
|
|
|||
|
|
@ -340,15 +340,15 @@ func TestCanonConflictsFlagsOnlyRealContradictions(t *testing.T) {
|
|||
// stray double space inside a rendering must not truncate it (the splitter is tolerant BY DESIGN, and that
|
||||
// tolerance cuts both ways), and a batch must actually render within the budget it was split for.
|
||||
func TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds(t *testing.T) {
|
||||
got, bad := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s })
|
||||
if bad != 0 || got["方源"] != "Фан Юань" {
|
||||
t.Fatalf("a double space inside a rendering must not bank half a name: %v (bad=%d)", got, bad)
|
||||
got, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
|
||||
if st.Bad != 0 || got["方源"] != "Фан Юань" {
|
||||
t.Fatalf("a double space inside a rendering must not bank half a name: %v (bad=%d)", got, st.Bad)
|
||||
}
|
||||
// A rendering that IS the source is not a rendering: a model echoing the term back would otherwise be
|
||||
// banked as this book's canon and shown to the editor as «方源 ⟨проверить⟩».
|
||||
echo, bad := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s })
|
||||
if _, banked := echo["方源"]; banked || bad != 1 {
|
||||
t.Fatalf("an echoed source must be refused and counted: %v (bad=%d)", echo, bad)
|
||||
echo, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
|
||||
if _, banked := echo["方源"]; banked || st.Bad != 1 {
|
||||
t.Fatalf("an echoed source must be refused and counted: %v (bad=%d)", echo, st.Bad)
|
||||
}
|
||||
cands := make([]Candidate, 200)
|
||||
for i := range cands {
|
||||
|
|
@ -380,13 +380,13 @@ func TestParseReplyAcceptsOnlyAskedTerms(t *testing.T) {
|
|||
"忘却\t" + NoDst,
|
||||
"мусор",
|
||||
}, "\n")
|
||||
got, bad := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id)
|
||||
got, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil)
|
||||
want := map[string]string{"方源": "Фан Юань", "花家": "Дом Хуа", "青茅山": "гора Цинмао", "忘却": ""}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("parse = %#v, want %#v", got, want)
|
||||
}
|
||||
if bad != 2 {
|
||||
t.Fatalf("the unusable lines must be COUNTED (silence about them is how a paid call turns into an empty bank), got %d", bad)
|
||||
if st.Bad != 2 {
|
||||
t.Fatalf("the unusable lines must be COUNTED (silence about them is how a paid call turns into an empty bank), got %d", st.Bad)
|
||||
}
|
||||
// The declined term is present with an EMPTY rendering — a decision, distinct from "no line at all".
|
||||
if v, ok := got["忘却"]; !ok || v != "" {
|
||||
|
|
@ -398,7 +398,7 @@ func TestParseReplyNormalizesTheKey(t *testing.T) {
|
|||
// The caller passes the engine's source-key normalizer; a reply written in another orthography must
|
||||
// still land on its term.
|
||||
lower := strings.ToLower
|
||||
got, _ := ParseReply("FANG\tФан", []string{"fang"}, lower)
|
||||
got, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil)
|
||||
if got["fang"] != "Фан" {
|
||||
t.Fatalf("the reply key must be normalized by the caller's function, got %#v", got)
|
||||
}
|
||||
|
|
|
|||
32
backend/internal/text/enclosure.go
Normal file
32
backend/internal/text/enclosure.go
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
package text
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// TrimEnclosure strips MATCHED enclosing punctuation from a surface, repeatedly. The ends are classified
|
||||
// by Unicode category (Ps/Pi open, Pe/Pf close) plus the two ASCII quotes, which are Po and invisible to
|
||||
// those categories — so no table of any language's quotation marks is held here.
|
||||
//
|
||||
// Requiring BOTH ends to pair keeps it from mangling a surface that merely touches punctuation, and an
|
||||
// empty pair is returned as-is: a key must never be trimmed to nothing.
|
||||
//
|
||||
// NOT part of the normalization artifact (norm.go / NormVersion): it is applied by one consumer to one
|
||||
// class of string (a surface a model wrote), and folding it into NormalizeSourceKey would re-verdict every
|
||||
// match the bank ever made. TestNormVersionUnmovedByEnclosureTrim pins that separation.
|
||||
func TrimEnclosure(s string) string {
|
||||
rs := []rune(s)
|
||||
for len(rs) >= 3 && isOpeningMark(rs[0]) && isClosingMark(rs[len(rs)-1]) {
|
||||
rs = rs[1 : len(rs)-1]
|
||||
}
|
||||
return strings.TrimSpace(string(rs))
|
||||
}
|
||||
|
||||
func isOpeningMark(r rune) bool {
|
||||
return unicode.Is(unicode.Ps, r) || unicode.Is(unicode.Pi, r) || r == '"' || r == '\''
|
||||
}
|
||||
|
||||
func isClosingMark(r rune) bool {
|
||||
return unicode.Is(unicode.Pe, r) || unicode.Is(unicode.Pf, r) || r == '"' || r == '\''
|
||||
}
|
||||
38
backend/internal/text/enclosure_test.go
Normal file
38
backend/internal/text/enclosure_test.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
package text
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTrimEnclosureFoldsPairedMarksInAnyScript(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"《咏梅》", "咏梅"}, // CJK book-title marks: the case measured in a live draft wave
|
||||
{"「名前」", "名前"}, // CJK corner brackets
|
||||
{"«Название»", "Название"}, // guillemets
|
||||
{"“Title”", "Title"}, // curly quotes
|
||||
{`"Title"`, "Title"}, // ASCII quotes (category Po — not covered by Ps/Pe)
|
||||
{"(термин)", "термин"},
|
||||
{"《《двойное》》", "двойное"}, // repeats until the ends stop pairing
|
||||
{"咏梅", "咏梅"}, // nothing to trim
|
||||
{"O'Brien", "O'Brien"}, // an apostrophe inside a word is not an enclosure
|
||||
{"咏梅》", "咏梅》"}, // one-sided: left visibly wrong rather than silently repaired
|
||||
{"「」", "「」"}, // an empty pair is never trimmed to nothing
|
||||
{"", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := TrimEnclosure(c.in); got != c.want {
|
||||
t.Errorf("TrimEnclosure(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormVersionUnmovedByEnclosureTrim is the axis pin: the trim lives outside the normalization
|
||||
// artifact, so shipping it moves no snapshot and re-bills no book. If someone folds it into
|
||||
// NormalizeSourceKey, this literal must be updated deliberately — which is a --resnapshot decision.
|
||||
func TestNormVersionUnmovedByEnclosureTrim(t *testing.T) {
|
||||
if NormalizeSourceKey("《咏梅》") == NormalizeSourceKey("咏梅") {
|
||||
t.Fatal("the normalizer must NOT trim enclosures: that would re-verdict every match the bank ever made")
|
||||
}
|
||||
const want = "memnorm-v3-nfkc+apos+stripignorable+trad+kana+lower/nfc+dash+apos+stripignorable+lower+yofold+u15.0.0+xtextv0.38.0"
|
||||
if got := NormVersion(); len(got) < len(want) || got[:len(want)] != want {
|
||||
t.Fatalf("the normalization artifact version moved: %q", got)
|
||||
}
|
||||
}
|
||||
|
|
@ -4,6 +4,9 @@
|
|||
(люди и места — транскрипция; предметы, приёмы, существа — по смыслу): широкая форма опровергнута
|
||||
замером T (P2−P5 = +0.013 при пороге 0.05), а полное снятие правила ломает 月光蛊 и 王婆.
|
||||
Блок ⟦TM-GENRE⟧ снят по D39.47 — жанрового словаря нет как класса.
|
||||
Строка примера формата добавлена по замеру D39.52: без якоря ⟦TM-CANON⟧ роль срывалась в английский
|
||||
(22 строки из 120), с примером на целевом письме — 0 из 120. Пример-терм 师父 не входит ни в сид, ни в
|
||||
якорь: он показывает ПИСЬМО ответа, а не перевод какого-либо термина книги.
|
||||
Файл остаётся ДАННЫМИ: резолвится конвенцией prompts/<пара>/<роль>.md, грузится как шаблон, в снапшот
|
||||
НЕ фолдится и ни один тест не пинит его текст. Смена текста стоит переоплаты только батчей терминолога
|
||||
(их адрес включает байты запроса), Go не трогается. -->
|
||||
|
|
@ -44,6 +47,10 @@
|
|||
|
||||
термин_исходника<TAB>перевод
|
||||
|
||||
Пример строки ответа:
|
||||
|
||||
师父 наставник
|
||||
|
||||
Никаких заголовков, нумерации, комментариев и markdown. Термины, которых нет в списке, не добавляй.
|
||||
|
||||
---USER---
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1162,3 +1162,9 @@ API-529-долг закрыт: 8-осевой refute-by-default воркфлоу
|
|||
## D39.52 — Замеры D39.51 закрыты ($0.00960; пакет-8 итого $0.12273): (а) строка примера формата ЧИНИТ срыв языка (22/120 → 0/120) — ИДЁТ В ПРОМПТ РОЛИ данными вместе со стройкой пп.1–7; (б) P1 consistency НЕ разделяет (вырожденность 0.740, критерий провален, частота — ноль) — права на порядок подачи нет, пере-замер на холодном прогоне $0 (26.07, оркестратор №8). ✅
|
||||
|
||||
Приёмка воспроизведением из сырья: (а) 0/120 не-целевых строк в арме с примером — независимый пересчёт; фактор один (дифф запросов — только три строки примера; пример-терм 师父 не входит ни в выборку, ни в якорь, ни в сид); мощность 0.104 названа до прогона, удвоение повторов не берём (страховка = экран + OffLanguage живьём). (б) ре-ран скорера: 54/73 вырождено, критерий ≥80/≥60 не выполнен нигде, частотный арм — независимое подтверждение §5.1 на другом наборе и другой разметке (три слепых модельных разметчика, согласие 0.973–0.986; оговорка «разметчики — модели» названа). **Хвост в пак-21: утечка алфавита** — «甲等 → категория A» (7 строк): языковой предикат пропускает (кириллица есть), подписанные dst латиницы не содержат вовсе; кандидаты — банк-линт латиницы в dst или строгая форма предиката (§2.2-Б дизайна). **Вопросы холодного мини-прогона пополнены:** + пере-замер P1 на популяции без цензуры ($0 из чекпойнтов). Стройка бэкенда: пп.1–7 §8 + п.8 «строка примера в промпте роли» — санкционирована, ждёт релея.
|
||||
|
||||
## D39.53 — Стройка «дисциплина банкноты + языковой экран» ПРИНЯТА И ЗАЛЕНДЕНА (пп.1–8 D39.51/52, $0, +454/−100): экран на выходе роли и черновой стороне · кавычки в свёртке · агрегация из чекпойнтов с голосами по ЧАНКАМ · kwic_width в пар-данные (zh-ru файл не шипуется) · строка примера побайтно та, что мерил полигон (26.07, оркестратор №8). ✅
|
||||
|
||||
**Приёмка:** M-Я1/M-Я2 красные + моя мутация (дедуп голосов) красная · предикат по живому сырью: 27 латинских строк §3.4 побайтно + **два улова сверх слепоты старой метрики** (китайские пересказы 第四代族长, 长老 — эхо-гейт и wellFormedLemma их пропускали) · регресс 16 живых строк чист, смешанная `Фан Юань (Fang Yuan)` проходит (экран судит язык, не написание) · пины общности/NormVersion/инварианта агрегации зелёные · vet/gofmt/golden чисты. Отчёт: `docs/archive/reports/PACK20_BANKNOTE_BUILD_2026-07-26.md` (ревью-шапка). Осей нет: снапшоты не двигаются; редакторская волна перекупается один раз только у книги, где банк реально изменится (иноязычная строка/кавычечные дубли).
|
||||
|
||||
**Находка стройки, решение задефолчено:** `hasBankSrcHan` — канал банкноты МЁРТВ для не-CJK исходника (каждая строка отвергается с parse_fail; утечка языковой семьи в общий слой, существующая, не этой стройки). Дефолт: чинить пар-слепым правилом «исходная строка непуста и встречается в тексте книги» (заодно отсекает выдуманные строки) **на холодном мини-прогоне** — версия парсера фолдится в снапшот, а холодная база платит ноль. **Холодный мини-прогон теперь несёт:** свои пять вопросов (D39.51) + пере-замер P1 + веса §C2-3 + allow_short 蛊/转 + фикс парсера банкноты + счёт «утечки алфавита» (категория A) живьём. Очередь: пак-19 → холодный мини-прогон → пак-21.
|
||||
|
|
|
|||
225
docs/archive/reports/PACK20_BANKNOTE_BUILD_2026-07-26.md
Normal file
225
docs/archive/reports/PACK20_BANKNOTE_BUILD_2026-07-26.md
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
# Пак-20, продолжение: СТРОЙКА дисциплины банкноты и языкового экрана
|
||||
|
||||
**База:** D39.51 (дизайн ратифицирован, пп.1–7 санкционированы) + D39.52 (п.8 — строка примера в промпте).
|
||||
**Зона:** `backend/`. **Деньги сессии: $0** — ни одного платного вызова, вся приёмка на сохранённом сырье.
|
||||
**Статус:** построено и проверено. Сессия НЕ коммитит, лендинг — оркестратора.
|
||||
|
||||
> **Ревью-шапка (оркестратор №8, 26.07): ПРИНЯТО И ЗАЛЕНДЕНО, D39.53.** Приёмка исполнением: vet/gofmt
|
||||
> чисты (llm.go — до-паковый), свежие прогоны terminology/text `-race -count=1` и golden зелёные; моя
|
||||
> адверсариальная мутация вне списка сессии (дедуп голосов по чанкам снят) — КРАСНАЯ, восстановление
|
||||
> побайтно. Прогон предиката по живому сырью — 27 латинских строк совпали побайтно + два честных улова
|
||||
> сверх метрики полигона (китайские пересказы 第四代族长/长老) — принято как улучшение, не расхождение.
|
||||
> §5 (парсер банкноты мёртв для не-CJK исходника, parse_fail на каждой строке) — дефолт решения:
|
||||
> чинить пар-слепым правилом «строка непуста и встречается в тексте книги» НА холодном мини-прогоне
|
||||
> (версия парсера фолдится → перекупка; холодная база платит ноль). Вето владельца — одной строкой.
|
||||
|
||||
---
|
||||
|
||||
## §1 Что построено
|
||||
|
||||
| № | Пункт | Где |
|
||||
|---|---|---|
|
||||
| 1 | Языковой экран на выходе роли | `terminology/script.go` (`OffLanguage`, `ScriptByName`) + отказ строки в `ParseReply` там же, где эхо-гейт |
|
||||
| 2 | Счётчик `OffLanguage` + громкий лог батча | `terminology.ReplyStats`, `terminologyResult.OffLanguage`, warn в `runTerminologist` |
|
||||
| 3 | Языковой фильтр черновой стороны | `pipeline/banknote.go` — `bankFold`, счётчик в стоп-логе |
|
||||
| 4 | Нормализация обрамляющих кавычек | `text/enclosure.go` (`TrimEnclosure`), применяется в свёртке |
|
||||
| 5 | Агрегация из чекпойнтов | `store.RoleResponsesForBook` + `Runner.bankObservedForBook` |
|
||||
| 6 | `kwic_width` в пар-данные | `lang.TerminologySizing` + необязательный `terminology.txt`; порядок резолвинга в `terminologyOpts` |
|
||||
| 7 | Хвост доки про жанровый словарь | `config.TerminologyGate` |
|
||||
| 8 | Строка примера формата | `prompts/zh-ru/terminologist.md` — ровно три строки, которые мерил полигон |
|
||||
|
||||
**Объявление письма цели:** `gates.terminology.target_script` (имя скрипта Unicode). Обязателен при
|
||||
`enabled: true` — гейт, который не может отработать, отвергается на ЗАГРУЗКЕ, той же идиомой, что бюджет
|
||||
и путь промпта. Резолвится через `unicode.Scripts`, поэтому набор допустимых значений — Unicode'овский, а
|
||||
не список языков, который движку пришлось бы вести.
|
||||
|
||||
**Свёртка банкноты переписана в один накопитель.** Раньше это были две функции над строками телеметрии;
|
||||
теперь `bankFold` — единственное место, где предложение становится уликой, и потому единственное место,
|
||||
где применяются все три дисциплины. Побочно удалена `bankProposalsByKey` — она дублировала
|
||||
`proposalsFromObserved` и оставалась только ради теста.
|
||||
|
||||
**Голоса считаются по ЧАНКАМ, а не по строкам хранилища.** Это то, что делает объединение безопасным:
|
||||
чанк, перекупленный дважды, голосует один раз, поэтому union не раздувает частотный фактор ранжирования.
|
||||
|
||||
---
|
||||
|
||||
## §2 Приёмка
|
||||
|
||||
### §2.1 Мутации
|
||||
|
||||
| Мутация | Что сделано | Результат |
|
||||
|---|---|---|
|
||||
| **M-Я1** | экран снят из `ParseReply` целиком | **КРАСНАЯ**: `TestParseReplyRefusesForeignLanguageRenderings`, `TestOffLanguageSamplesAreBounded` |
|
||||
| **M-Я2** | предикат ослаблен порогом «строка короче 3 рун — не проверять» | **КРАСНАЯ**: обе плюс `TestOffLanguageSeparatesForeignRenderingsFromTargetOnes` (ловится на `is`) |
|
||||
|
||||
Оба раза файл восстановлен из копии, сьюта после восстановления зелёная.
|
||||
|
||||
### §2.2 Прогон предиката по ЖИВОМУ сырью пакета-8
|
||||
|
||||
126 сохранённых вызовов замера 3 (сетка + арм без якоря), **827 принятых строк ответа**. Разбор — боевым
|
||||
`ParseReply` без объявленного письма (чтобы увидеть все принятые строки), затем предикат.
|
||||
|
||||
| Арм | Помечено | из них латиница | из них ханьцзы |
|
||||
|---|---|---|---|
|
||||
| 0×0 · 3×40 · 3×120 · 5×80 · 8×40 | **0** | 0 | 0 |
|
||||
| 8×120 | 7 | **5** | 2 |
|
||||
| без якоря | 22 | **22** | 0 |
|
||||
| **Итого** | **29** | **27** | **2** |
|
||||
|
||||
**Критерий «ровно 27 строк §3.4 и ни одной сверх» выполнен на своих условиях: 27 латинских — те самые
|
||||
27** (5 в 8×120 + 22 в арме без якоря, побайтно совпадает с таблицей полигона), и ни одного срабатывания
|
||||
в четырёх армах, где отчёт даёт 0/40.
|
||||
|
||||
**Расхождение 29 против 27 объясняю, а не сглаживаю.** Полигон считал «латиница без единой кириллицы»;
|
||||
предикат считает «есть буквы, и ни одна не в письме цели». Ханьцзы — тоже буквы, поэтому предикат видит
|
||||
два ответа, которых метрика полигона не могла увидеть по построению:
|
||||
|
||||
```
|
||||
四代族长 → 第四代族长 家老 → 长老
|
||||
```
|
||||
|
||||
Модель ответила китайским пересказом вместо русского перевода. Эхо-гейт их пропускает (ответ не равен
|
||||
исходнику побайтно), `wellFormedLemma` пропускает. Это не ложные срабатывания — это два настоящих улова
|
||||
в классе, к которому исходный замер был слеп. Ошибок в другую сторону нет: 798 корректных строк не
|
||||
задеты.
|
||||
|
||||
**Независимые подтверждения на том же сырье:**
|
||||
|
||||
- **арм со строкой примера (замер D39.51):** 9 вызовов, 120 принятых строк, **0 помечено** — боевой код
|
||||
воспроизводит объявленные полигоном 0/120;
|
||||
- **проба хвоста (замер 2):** 21 вызов, 312 принятых строк, **18 помечено**, все латиница — включая
|
||||
названный в отчёте `是为 → is` и пиньинь-передачи `方源 → Fangyuan`, `方正 → Fangzheng`, которые для
|
||||
zh→ru переводом на целевой язык не являются.
|
||||
|
||||
### §2.3 Регресс, общность, инварианты
|
||||
|
||||
| Проверка | Итог |
|
||||
|---|---|
|
||||
| 16 живых целевых строк (подписанные владельцем + пограничные: 2 буквы, только цифры, в кавычках, двойной пробел) | зелёные, ни одного ложного срабатывания |
|
||||
| Смешанная строка `Фан Юань (Fang Yuan)` | проходит — экран судит язык, а не написание |
|
||||
| **Пин общности** | при латинской цели вердикты переворачиваются; при ханьской работают обе; `ScriptByName` строгий, опечатка падает на загрузке, а не выключает экран молча |
|
||||
| Пин отсутствия объявления | без письма экран инертен — состояние, которое конфиг отвергает для включённого гейта, зафиксировано тестом, а не оставлено сюрпризом |
|
||||
| **Инвариант агрегации** | `TestBankUnionRecoversASupersededSampling`: второй ответ на тот же чанк пишется через обычный денежный путь, объединение достаёт затёртую выборку, счётчик чанков остаётся 1 |
|
||||
| Байт-стабильность карты при резюме | держит существующий `TestAutoWireIsDeterministicAndDoesNotMoveTheDraftWave` (авто-банк побайтно равен, черновая волна не двигается, резюм $0) |
|
||||
| Детерминизм свёртки | 8 прогонов подряд — порядок ключей и рендерингов идентичен |
|
||||
| Кавычки | `《咏梅》` и `咏梅` сходятся в один кандидат с двумя вариантами; односторонний огрызок не «чинится» молча; пустая пара не срезается в ничто |
|
||||
| `NormVersion` не сдвинут | пин-страж: нормализатор кавычки НЕ снимает, версия артефакта прежняя |
|
||||
| Пар-данные | пак zh-ru без файла хешируется как раньше; файл добавили — версия сдвинулась; пустой/битый файл падает на загрузке |
|
||||
| Конфиг | пропущенное/неизвестное/не в том регистре имя письма — громкая ошибка загрузки; `Hiragana` объявляется так же, как `Cyrillic` |
|
||||
|
||||
### §2.4 Сьюта
|
||||
|
||||
`go test ./...` зелёная · `-race` зелёная на pipeline (67 c), terminology, lang, text, store, config ·
|
||||
`TestGoldenDeterminism` PASS · `go vet` чисто · `gofmt -l` — только предсуществующий `internal/llm/llm.go`.
|
||||
Диффстат: 20 файлов, +454 / −100.
|
||||
|
||||
---
|
||||
|
||||
## §3 Оси
|
||||
|
||||
| Изменение | Снапшот | Кто перекупается |
|
||||
|---|---|---|
|
||||
| Языковой экран, счётчик, фильтр свёртки, кавычки, агрегация | не двигается | никто напрямую. У книги, где в банке лежала иноязычная строка или пара кавычечных дублей, содержимое банка изменится → обогащённая версия сдвинется → редакторская волна оплатится ОДИН раз. Книга без таких строк — байт-в-байт прежняя |
|
||||
| `target_script` в конфиге | не двигается (гейт не фолдится) | никто |
|
||||
| `kwic_width` в пар-данные | не двигается: файл для zh-ru не шипуется, `Version()` побайтно прежний (проверено тестом) | никто |
|
||||
| Строка примера в промпте | волны не двигаются | только батчи терминолога (их адрес включает байты запроса) |
|
||||
| Агрегация | — | у книги с ОДНОЙ покупкой черновиков поведение прежнее; у книги, где черновики покупались повторно, карта подписи вырастет один раз |
|
||||
|
||||
Прод после стройки не дорожает ни на цент: всё детерминированное и $0.
|
||||
|
||||
---
|
||||
|
||||
### §2.5 Синк с полигоном: промпт побайтно тот, что мерили
|
||||
|
||||
Строка примера взята не по описанию, а сверена с артефактом. Скопированный полигоном промпт лежит в его
|
||||
скретчпаде (`d51/prompt/terminologist-example.md`); диффом от «Ты —» до конца файла **тело промпта,
|
||||
который дал 0 из 120, и тело промпта в репозитории совпадают побайтно** (различается только шапка-
|
||||
комментарий, которую модель не видит). Табуляция в строке примера — настоящая, проверено `cat -A`.
|
||||
|
||||
Прочие пункты отчётов полигона сверены и разведены по местам:
|
||||
|
||||
| Находка полигона | Где в стройке |
|
||||
|---|---|
|
||||
| Срыв языка без якоря (22/120) | п.1–3, приёмка §2.2 |
|
||||
| `是为 → is` в пробе хвоста | ловится, §2.2 |
|
||||
| Кавычки `《咏梅》` против `咏梅` | п.4 |
|
||||
| Невоспроизводимость канала между прогонами | п.5 |
|
||||
| `kwic_width` в рунах — утечка пары | п.6 |
|
||||
| P1 consistency не разделяет | НЕ строил ничего: права на порядок подачи метрика не заработала |
|
||||
| Утечка алфавита («категория A» с латинской буквой) | НЕ строил: отнесено в пак-21. Экран её не ловит по построению (кириллица в строке есть) — и стройка этого не меняет ни в одну сторону. Замечу только, что арм с примером дал таких строк больше (7 против 3), так что в холодном прогоне их стоит посчитать |
|
||||
| `allow_short`, холодный старт, второй детектор | не эта стройка |
|
||||
|
||||
---
|
||||
|
||||
## §3.1 Селф-ревью по диффу (после стройки, до отчёта)
|
||||
|
||||
Прошёл диффом по всем файлам; три вещи нашёл и починил, одну назвал.
|
||||
|
||||
1. **Висячая ссылка в доке.** Комментарий `bankObservedByKey` ссылался на `bankProposalsByKey` — функцию,
|
||||
которую я в этой же стройке удалил. Исправлено на реального потребителя.
|
||||
2. **Объединение читало ВСЮ историю ответов книги в память.** `RoleResponsesForBook` тянула каждый
|
||||
сохранённый черновик; на десяти главах это мегабайты, на тысяче глав с повторами — сотни. Добавил
|
||||
необязательный фильтр объёма по подстроке: маркер канала передаётся ПАРАМЕТРОМ запроса, поэтому в SQL
|
||||
не появилось второй копии константы, а читаются только те ответы, которые в принципе могут что-то дать.
|
||||
Семантику это не трогает — разбор по-прежнему один, в Go.
|
||||
3. **Лишний параметр.** `loadTargetScript` принимал логгер, хотя `r.Log` к этому моменту уже установлен.
|
||||
4. **Назвал, менять не стал:** комментарий у `packAlgoVersion` требует бампать тег «на новый файл», а я
|
||||
тег не двигал. Расписал правило точнее прямо в коде: бампается тег, когда меняется ЧТЕНИЕ уже
|
||||
существующих байт; необязательный файл, которого нет ни у одной пары, ничего не читает иначе и потому
|
||||
версию двигать не должен — иначе перепокупаются обе волны каждой книги ради механизма, которым никто не
|
||||
пользуется. Добавлен пин на сам тег, чтобы будущий бамп был решением, а не побочным эффектом.
|
||||
|
||||
**Изменение поведения, которое надо назвать явно.** Голоса теперь считаются по чанкам, поэтому
|
||||
повторённая В ОДНОМ блоке строка `src→dst` даёт один голос, а не два, как раньше. Это исправление (один
|
||||
чанк — одно свидетельство, а не «модель дважды написала одно и то же»), но на уже накопленных данных оно
|
||||
может слегка сдвинуть ранжирование вариантов §C2-3. Осей это не двигает: ранжирование — вход роли и
|
||||
таблицы стопа, не байты запроса волн.
|
||||
|
||||
---
|
||||
|
||||
## §4 Отклонения и решения, принятые по ходу
|
||||
|
||||
1. **Кавычки нормализуются ТОЛЬКО в свёртке, запись сырой строки не тронута.** В дизайне допускались оба
|
||||
места. Одно место — одно определение; сохранённая строка остаётся записью того, что модель написала на
|
||||
самом деле, а сведение ключей — свойство join'а, а не наблюдения.
|
||||
2. **Запрос чекпойнтов не фильтрует производные $0-строки в SQL.** Они несут не-провайдерский
|
||||
`finish_reason`, и обычное правило «доверяем только завершённой генерации» отсекает их само. Второй
|
||||
копии этого правила в SQL быть не должно.
|
||||
3. **Предложения отвергнутых попыток входят в объединение** — как и объявлено в дизайне: черновик,
|
||||
забракованный за дефект ТЕКСТА, не делает объявленные в нём термины неправдой, а подписи всё равно нет.
|
||||
4. **Тестовые фикстуры терминолога теперь объявляют письмо.** Это не «правка под тест»: гейт без
|
||||
объявления перестал грузиться — ровно то поведение, которое строилось.
|
||||
|
||||
---
|
||||
|
||||
## §5 Найдено при стройке, НЕ починено (нужно решение)
|
||||
|
||||
**Канал банкноты мёртв для любого не-CJK исходника.** `parseBanknote` отвергает строку, в исходной части
|
||||
которой нет ханьского иероглифа (`hasBankSrcHan`, диапазон U+3400–U+9FFF). Проверено исполнением:
|
||||
|
||||
```
|
||||
исходник не-CJK: Silversaint→Серебряный святой, San Michon→Сан-Мишон → принято 0, parse_fail=true
|
||||
исходник CJK: 方源→Фан Юань → принято 1, parse_fail=false
|
||||
```
|
||||
|
||||
То есть на английской или любой другой не-иероглифической паре канал, который несёт 5/6 кандидатов банка,
|
||||
не просто молчит — он отвергает каждую строку и поднимает флаг ошибки разбора. Это прямая утечка
|
||||
конкретной языковой семьи в общий слой, и она из существующего кода, не из этой стройки.
|
||||
|
||||
Чинить в этом паке не стал: правило живёт в ПАРСЕРЕ, чья версия фолдится в снапшот, поэтому правка =
|
||||
`--resnapshot` и перепокупка черновой волны каждой книги с включённым каналом. Это отдельное решение об
|
||||
оси. Содержательно правильное правило, по-моему, не «в исходнике есть иероглиф», а «исходная строка
|
||||
непуста и встречается в тексте книги» — проверка, которая пар-слепа по построению и вдобавок отсекает
|
||||
выдуманные строки, чего нынешняя не делает.
|
||||
|
||||
---
|
||||
|
||||
## §6 Что на владельце
|
||||
|
||||
1. **Лендинг стройки** (оркестратора) — пп.1–8 готовы, приёмка §2 исполнена.
|
||||
2. **Решение по §5** — чинить ли пар-зависимость парсера банкноты и когда: правка дешёвая, ось дорогая
|
||||
(перепокупка черновой волны). Естественный момент — холодный мини-прогон, где книга покупается заново.
|
||||
3. Ранее отложенное и не изменившееся: судья-с-декоем ждёт холодного старта; `allow_short` для `蛊`/`转` —
|
||||
данными на холодном прогоне; пере-замер P1 — оттуда же, $0 из чекпойнтов.
|
||||
|
||||
**СТОП.** Сессия не коммитила.
|
||||
Loading…
Add table
Reference in a new issue