diff --git a/backend/cmd/tmmutate/mutations.json b/backend/cmd/tmmutate/mutations.json index 6ba5463c..9e8ea586 100644 --- a/backend/cmd/tmmutate/mutations.json +++ b/backend/cmd/tmmutate/mutations.json @@ -7198,8 +7198,8 @@ "edits": [ { "file": "internal/chunk/inlinemarkup.go", - "find": "\t\tif ir.classes[c] {", - "replace": "\t\tif false && ir.classes[c] {" + "find": "\t\tif len(elems) == 0 || elems[name] {", + "replace": "\t\tif false && (len(elems) == 0 || elems[name]) {" } ] }, @@ -7267,5 +7267,135 @@ "replace": "\t\t\t\t\t_ = v" } ] + }, + { + "id": "SP2-a-lost-marker-inverts-the-paragraph", + "battery": true, + "package": "./internal/bookfile/", + "why": "with one glyph on both sides, pairing left to right without asking which marker may OPEN turns a single marker lost by the model into emphasis on the words between it and the next one: the reader's book emphasises text nobody marked, the EPUB parses, and nothing reports a fault. The same arithmetic mangles a scene break the author typed", + "edits": [ + { + "file": "internal/bookfile/model.go", + "find": "\tsame := open == closeMark", + "replace": "\tsame := false" + } + ] + }, + { + "id": "SP2-carrier-count-includes-excluded-service-pages", + "battery": true, + "package": "./internal/chunk/", + "why": "the carrier count is the operator's ONLY signal for a book whose emphasis the scanner does not recognise, so a cover page with one italic word hides the zero the count exists to show", + "edits": [ + { + "file": "internal/chunk/ingest.go", + "find": "\t\t\tif body, _, xerr := extractXHTML(data, nil); xerr == nil {", + "replace": "\t\t\tif body, _, xerr := extractXHTML(data, ir); xerr == nil {" + } + ] + }, + { + "id": "SP2-declared-markup-over-a-text-source-goes-silent", + "battery": true, + "package": "./internal/chunk/", + "why": "the declaration is per PAIR and the dispatch is per FILE EXTENSION, so a pair that ships the glyphs and a .txt book is ordinary — and the production Chinese book is exactly that. Without the honest zero the pair believes its markers are flowing", + "edits": [ + { + "file": "internal/chunk/ingest.go", + "find": "\tif err == nil && im != nil {\n\t\tdoc.InlineMarkupDeclared = true\n\t}", + "replace": "\tif err == nil && im != nil && false {\n\t\tdoc.InlineMarkupDeclared = true\n\t}" + } + ] + }, + { + "id": "SP2-a-block-italic-wrapper-loses-every-run", + "battery": true, + "package": "./internal/chunk/", + "why": "a
around a verse or an epigraph is how that content is actually marked up, and dropping block-level emphasis at a paragraph break loses all of it while the carrier count stays non-zero from other runs", + "edits": [ + { + "file": "internal/chunk/ingest.go", + "find": "\t\tfor _, t := range open {\n\t\t\tif t.block {\n\t\t\t\tkept = append(kept, t)\n\t\t\t}\n\t\t}", + "replace": "\t\tfor range open {\n\t\t}" + } + ] + }, + { + "id": "SP2-an-inner-element-closes-the-run-around-it", + "battery": true, + "package": "./internal/chunk/", + "why": "closing by NAME among the emphasising elements alone lets a plain inner close the that contains it, so the closing marker lands mid-run and the rest of the emphasis is lost", + "edits": [ + { + "file": "internal/chunk/ingest.go", + "find": "\t\tif skip == \"\" && !voidTags[name] {", + "replace": "\t\tif skip == \"\" && !voidTags[name] && ir.isItalicTag(name, class, style) {" + } + ] + }, + { + "id": "SP2-a-class-rule-loses-the-element-it-was-written-for", + "battery": true, + "package": "./internal/chunk/", + "why": "a stylesheet that italicises h1.prologue did not say the class is italic anywhere; a second English book in this tree declares 26 such heading classes, and dropping the qualifier turns every chapter's first line into an emphasised run", + "edits": [ + { + "file": "internal/chunk/inlinemarkup.go", + "find": "if len(elems) == 0 || elems[name] {", + "replace": "if len(elems) >= 0 {" + } + ] + }, + { + "id": "SP2-the-emphasis-detector-is-not-versioned", + "battery": true, + "package": "./internal/lang/", + "why": "the pair's GLYPHS fold through the file's bytes and the DETECTION CODE folded through nothing: widening the scanner would change the text of every chunk under an unchanged snapshot, which is a silent re-buy in the direction that does not even ask", + "edits": [ + { + "file": "internal/lang/langpack.go", + "find": "\t\tif o.name == \"inline-markup.txt\" {\n\t\t\th.Write([]byte(\"\\x00\" + inlineMarkupAlgoVersion))\n\t\t}", + "replace": "\t\tif o.name == \"inline-markup.txt\" && false {\n\t\t\th.Write([]byte(\"\\x00\" + inlineMarkupAlgoVersion))\n\t\t}" + } + ] + }, + { + "id": "SP2-brief-fields-and-their-wiring-drift-apart", + "battery": true, + "package": "./internal/config/", + "why": "the list a pair's declaration is judged against and the list that judges a book were two spellings of one set: a field in the first and not the second loads clean and enforces nothing, so the pair believes it declared a policy and the book violates it in silence", + "edits": [ + { + "file": "internal/config/pipeline.go", + "find": "func briefTargets(b *Book) map[string]*string {\n\treturn map[string]*string{\"transcription\": &b.Transcription, \"honorifics\": &b.Honorifics}", + "replace": "func briefTargets(b *Book) map[string]*string {\n\treturn map[string]*string{\"transcription\": &b.Transcription}" + } + ] + }, + { + "id": "SP2-a-pair-label-may-name-no-prompt", + "battery": true, + "package": "./internal/config/", + "why": "a typo in a prompt_versions key is well-shaped and therefore invisible at load, and a dead key silently leaves the SHARED snapshot-folded label in force — the exact state the pair-scoped label exists to end", + "edits": [ + { + "file": "configs/pairs/en-ru.yaml", + "find": " editor: v5-discourse-reflow-nogenre-italics", + "replace": " editer: v5-discourse-reflow-nogenre-italics" + } + ] + }, + { + "id": "SP2-the-hidden-gender-directive-defaults-to-a-gender", + "battery": true, + "package": "./internal/membank/", + "why": "the owner's decision is that a concealed gender is handled by RECASTING the sentence, never by defaulting to one. The directive sat unexecuted in the data while a note in the journal declared it done, and the pin that read the line only checked its opening words", + "edits": [ + { + "file": "internal/lang/data/injection.txt", + "find": "не получается — перестрой фразу", + "replace": "при неизбежности — мужские" + } + ] } ] diff --git a/backend/configs/langpacks/en-ru/inline-markup.txt b/backend/configs/langpacks/en-ru/inline-markup.txt index e9c8a8f5..f49a5df7 100644 --- a/backend/configs/langpacks/en-ru/inline-markup.txt +++ b/backend/configs/langpacks/en-ru/inline-markup.txt @@ -1,28 +1,22 @@ -# How en→ru carries the SOURCE's inline emphasis to the model. `keyvalue` per line; both keys are -# required, because an opener with no closer would mark the rest of the chapter. +# ⛔ THE BYTES OF THIS FILE — COMMENTS INCLUDED — ARE PART OF Pack.Version(), which folds into the +# snapshot, the manifest key and the cut tag. Editing a comment here re-buys and re-cuts every book of +# this pair. Prompts are the opposite (their comments are stripped before hashing); langpack files are not. # -# ⛔ THIS FILE IS ONE HALF OF A CONTRACT, and the other half is prompts/en-ru/. The ingest wraps an italic -# run in these glyphs; the pair's prompts say what they mean and require them to come back. A pair whose -# prompts have not learned the glyph must not ship this file — the model would be handed punctuation -# nobody explained, in the middle of prose, and would rightly invent a meaning for it. +# How en→ru carries the source's inline emphasis to the model. Both keys required: an opener with no +# closer would mark the rest of the chapter. The glyphs are one half of a contract whose other half is +# prompts/en-ru/ — a pair whose prompts have not learned the glyph must not ship this file. The engine +# renders no verdict about what emphasis MEANS; it carries the distinction and the model reads it. # -# WHY THE GLYPH IS AN ASTERISK, measured rather than chosen by taste: -# - it must not occur in the source, or the engine would be marking text the author wrote. Measured on -# the production book with python3+zipfile: 126 documents, 1 588 694 characters of prose read, -# literal `*` found — 0. The same count over the project's four Chinese texts is 0 at 38.9 MB. -# - it must survive normalization, sentence splitting and paragraph packing untouched: one ASCII -# character, not whitespace, not a letter — it is carried through as ordinary punctuation. -# - it must be legible to a model without a lexicon, which `*слово*` is by convention and `⟦i⟧` is not. +# WHY THE ASTERISK, and what the measurement does and does not cover: it must not collide with the +# source's own text. Measured through the engine's own reader on the books in this tree — +# Empire_of_the_Dawn 0 literal `*` in 1 626 362 runes; isekai_majutsushi_jp 0; **fifty_shades_1_en 12** +# (SMS messages the author typed with asterisks). So the premise holds for the production book and NOT +# for every English book: where a source carries its own asterisks the engine will read them as markers. +# The ingest prints their count beside italic_carriers so the collision is a number, not a surprise. # -# WHAT IT COSTS, so the number is on the page rather than in a report: two characters per carrier. The -# production book has 4842 italic carriers, i.e. +9684 characters on 1.6M — +0.6 % of prose volume, which -# moves the token estimate, the length corridor and the source-script share by the same fraction. -# -# ⚠ WHAT THE ENGINE DOES NOT DO WITH THIS, and the reason there is no second key for it: it renders no -# verdict about what emphasis MEANS. A run of italics is a thought, a stage whisper, a foreign word, a -# line of verse, an epigraph — the source does not distinguish them, and three candidate detectors for -# "this is verse" were measured on this very book and rejected (block nesting caught 37 of 88, the -# epub:type/role semantics were dead on all four books of the project, length statistics caught 5 of 88). -# The engine carries the distinction to the wire; the reading is the model's. +# COST: two characters per carrier — 4842 carriers on the production book, +0.6 % of prose volume, which +# moves the token estimate, the length corridor and the source-script share by that fraction. It also +# shifts SENTENCE splitting slightly (the splitter cuts on a terminator before a space, and a marker is +# neither): measured 17 190 → 17 159 sentences and 385 → 386 chunks on the production book. italic_open * italic_close * diff --git a/backend/configs/langpacks/en-ru/terminology.txt b/backend/configs/langpacks/en-ru/terminology.txt index c2c58701..8d1c6268 100644 --- a/backend/configs/langpacks/en-ru/terminology.txt +++ b/backend/configs/langpacks/en-ru/terminology.txt @@ -1,46 +1,26 @@ -# Terminologist sizing for en→ru. `keyvalue` per line; every key is a count of RUNES except -# kwic_per_term, which is a count of contexts. +# ⛔ THE BYTES OF THIS FILE — COMMENTS INCLUDED — ARE PART OF Pack.Version(), which folds into the +# snapshot, the manifest key and the cut tag. Editing a comment here re-buys and re-cuts every book of +# this pair. Prompts are the opposite (their comments are stripped before hashing); langpack files are not. # -# ⛔ WHY ANY OF THESE ARE HERE, when the engine already has defaults: the defaults are counted in RUNES and -# were chosen on a Han source. A rune of Han and a rune of Latin do not buy the same amount of language, so -# the same number means a different thing in this pair — which is the whole reason this file is pair data -# (D39.50 п.4). +# Terminologist sizing for en→ru. `keyvalue`; every key but kwic_per_term counts RUNES. # -# THE MULTIPLIER IS MEASURED, NOT ASSUMED — 2.74, and here is the measurement. The engine's own sizing -# taxonomy (text.DenseScript, D39.64) values a dense rune at ~1 model token and a sparse one at ~⅓. Run -# through `tmctl manifest --json` on both production books, summing the per-unit source_chars_dense / -# _sparse the engine itself reports: -# zh (蛊真人, 2283 chapters): dense 6 191 523 + sparse 937 761 → 0.9123 tokens per rune -# en (Empire of the Dawn, 71 chapters): dense 0 + sparse 1 310 510 → 0.3333 tokens per rune -# ratio 0.9123 / 0.3333 = 2.7369 -# Control that the instrument was asked about an existing subject: the English book's dense count is 0 of -# 1 310 510 runes, i.e. the measurement is of a genuinely non-dense source and not of an empty read. +# The engine defaults are counted in runes and were chosen on a Han source, so the same number buys a +# different amount of language here. MULTIPLIER 2.74, measured with `tmctl manifest --json` over both +# production books (per-unit source_chars_dense/_sparse): zh 0.9123 tokens per rune, en 0.3333 (dense 0 of +# 1 310 510 runes — the control that the measurement is of a genuinely non-dense source). The tokenizer +# calibration (experiments/01) gives 3.1–3.2; the smaller number is taken, which errs toward a narrower +# window and a smaller batch. # -# ⚠ AN INDEPENDENT INSTRUMENT SAYS 3.1–3.2, AND THE SMALLER NUMBER IS THE ONE TAKEN. The tokenizer -# calibration (experiments/01) measured DeepSeek at 0.86 tokens per hanzi and 3.64 English characters per -# token → a ratio near 3.2. The engine taxonomy is used anyway, for two reasons: it is the project's ONE -# sizing taxonomy and a second arithmetic in the data layer is how two copies come to disagree; and its -# answer is the SMALLER one, which errs toward a narrower window and a smaller batch — the safe direction -# for both context overflow and cost per call. The same pair file already takes that side for fertility. +# All four keys are written out, including the two that equal the engine default: a file carrying one key +# leaves its neighbours on Han-sized defaults while looking decided, and for basis_width the first later +# correction is not a tuning but a re-purchase of the bank out of a lifetime role budget. # -# ⛔ ALL FOUR KEYS ARE WRITTEN OUT, including the two that equal the engine default, and that is deliberate. -# A file carrying one key leaves its neighbours on the engine's Han-sized defaults while LOOKING like the -# pair has decided its sizing — and for basis_width the first later correction of that oversight is not a -# tuning but a RE-PURCHASE: the evidence width is part of every settled-term fingerprint, so moving it -# invalidates them all and re-buys the bank out of a LIFETIME role budget. The cost of naming a key that -# happens to match the default is zero here: this pack is new, so no book has ever been priced under it. -# -# WHAT EACH ONE DOES -# kwic_per_term — how many contexts per candidate. A COUNT, not a width: it does not scale with the script, -# so it stays at the engine default. -# kwic_width — ±runes around each occurrence shown to the model (AttachKWIC takes the window BOTH ways, so -# the model sees 2× this). 40 × 2.74 ≈ 110: ±110 Latin runes ≈ ±37 tokens, the same context a Han source -# gets from ±40. -# basis_width — ±runes that DEFINE an occurrence for the settled-basis fingerprint. Scaled by the same -# number for the same reason, and kept a separate key so that tuning the prompt above cannot re-buy a bank. -# batch_runes — the bound on ONE call's candidate block. 6000 × 2.74 ≈ 16 400. Left at 6000 an English pass -# would pack a third of the intended call and split into ~3× as many calls, all drawn from the role's -# LIFETIME budget — the difference between a consolidated bank and one that stops halfway. +# kwic_per_term — contexts per candidate; a count, not a width, so it does not scale with the script. +# kwic_width — ±runes around each occurrence shown to the model (AttachKWIC takes the window both ways). +# basis_width — ±runes that define an occurrence for the settled-basis fingerprint. Separate key so that +# tuning the prompt cannot re-buy a bank. +# batch_runes — bound on ONE call's candidate block. At 6000 an English pass packs a third of the +# intended call and splits into ~3× the calls, all from a LIFETIME role budget. kwic_per_term 3 kwic_width 110 basis_width 110 diff --git a/backend/configs/langpacks/en/manifest.txt b/backend/configs/langpacks/en/manifest.txt index 8397a18d..e32b5c40 100644 --- a/backend/configs/langpacks/en/manifest.txt +++ b/backend/configs/langpacks/en/manifest.txt @@ -1,19 +1,15 @@ -# Data CHANNELS the source language `en` ships to the engine. `channelname` per line. +# ⛔ THE BYTES OF THIS FILE — COMMENTS INCLUDED — ARE PART OF Pack.Version(), which folds into the +# snapshot, the manifest key and the cut tag. Editing a comment here re-buys and re-cuts every book of +# this pair. Prompts are the opposite (their comments are stripped before hashing); langpack files are not. # -# It ships NONE, and that is a statement about the writing system rather than a gap in this directory. -# The two channels the engine defines are the name-miner's morphology schema (百家姓 surnames, 甲乙丙 grade -# prefixes, CJK numerals …) and the pinyin→Cyrillic transliteration table. Neither exists for a Latin -# source: there is no pinyin to transliterate, and the miner's typed-candidate tables are the zh family's -# schema, not a universal one. Declaring them and shipping invented files would be worse than declaring -# nothing — the miner would run on fiction. +# Data CHANNELS the source `en` ships. `channelname` per line. # -# WHY THE FILE EXISTS AT ALL, since it declares nothing: without it a manifest-less pack means EVERY -# channel, so loading `en` demands eleven Chinese morphology tables and refuses the book. With it the pack -# LOADS, and what the pair needs — the terminologist sizing next door, and the bank contour itself — hangs -# off a pack existing. The bank of a non-Han source is not mined; it arrives through the banknote channel, -# which is measured to work on this book (backlog row 347). +# It ships none: both channels the engine defines are the zh family's schema — the name-miner's morphology +# tables and the pinyin→Cyrillic table — and a Latin source has neither. Without this file a manifest-less +# pack means EVERY channel, so loading `en` demands eleven Chinese tables and the bank contour refuses the +# book before its first cent. The pair's own data (sizing, emphasis glyphs) hangs off the pack existing; +# a non-Han source's bank arrives through the banknote channel, not the miner (backlog row 347). # -# `none` is written out instead of leaving the file empty on purpose: an empty file and a TRUNCATED one are -# the same bytes, and inferring "ships nothing" from emptiness would turn a half-written manifest into a -# silently inert pack. A file that states nothing is still refused. +# `none` is spelled out rather than inferred from an empty file: an empty file and a truncated one have the +# same bytes, and inferring would make a half-written manifest a silently inert pack. channel none diff --git a/backend/internal/bookfile/bookfile_test.go b/backend/internal/bookfile/bookfile_test.go index bf10c199..7be009e8 100644 --- a/backend/internal/bookfile/bookfile_test.go +++ b/backend/internal/bookfile/bookfile_test.go @@ -342,3 +342,72 @@ func TestOneChapterBook(t *testing.T) { t.Fatalf("one-chapter book: spine %d, blocks %v", len(doc.Spine), blocksOf(entries["OEBPS/ch1.xhtml"])) } } + +// TestALostMarkerCannotInvertAParagraph is the reader-facing half of the emphasis contract, and the one +// that cost the most to find: with one glyph on both sides, pairing left to right without asking WHICH +// marker may open turns a single marker lost by the model into emphasis on the words between it and the +// next one. The file parses, the run reports success, and the book emphasises text nobody marked. +// +// ⛔ THE FIRST VERSION OF THIS TEST HELD CONSTANT THE EXACT FORM THAT BREAKS — in every case the opening +// marker stood after a space or at the start of the line — and it went green over a rule that lost 68 of +// the production book's 4842 runs. The cases below therefore LEAD with the shapes measured in that book: +// emphasis that lands on a full stop, on a comma, or inside a word, so the opener follows a letter. +func TestALostMarkerCannotInvertAParagraph(t *testing.T) { + b := &Book{ItalicOpen: "*", ItalicClose: "*"} + render := func(s string) string { + var sb strings.Builder + b.emphasisSpans(s, func(text string, italic bool) { + if text == "" { + return + } + if italic { + sb.WriteString("" + text + "") + return + } + sb.WriteString(text) + }) + return sb.String() + } + for _, tc := range []struct{ name, in, want string }{ + // MEASURED IN THE PRODUCTION BOOK: 68 of its 4842 runs open right after a letter or a mark. + {"emphasis on a full stop after a word", "HIM*.*", "HIM."}, + {"emphasis inside a word", "Fucks*sakes*, Baptiste", "Fuckssakes, Baptiste"}, + {"emphasis on a comma", "heat*,* that blessed", "heat, that blessed"}, + {"several runs in one line, one of them mid-word", + "*Word* all*.*’ and then *more* text.", "Word all.’ and then more text."}, + // AND THE DEFECT THE RULE EXISTS FOR: a marker the model dropped must not move the emphasis. + {"a marker the model lost does not move the emphasis", + "Он поставил сноску* и сказал *важно* потом.", "Он поставил сноску* и сказал важно потом."}, + {"a scene break is the author's, not a marker pair", "* * *", "* * *"}, + {"the model's doubled markers stay literal", "**слово**", "**слово**"}, + {"and ordinary emphasis still works", "Она *почти* ушла, но *вернулась*.", + "Она почти ушла, но вернулась."}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := render(tc.in); got != tc.want { + t.Errorf("got %q\nwant %q", got, tc.want) + } + }) + } + // ⚠ THE LIMIT, ASSERTED RATHER THAN LEFT TO BE DISCOVERED: when BOTH sides of the author's own glyph + // are text, nothing here can tell it from ours. The answer is the ingest's `source_marker_glyphs` + // count (0 on the production book, 12 on the other English one), not a cleverer predicate — the + // cleverer predicate is what lost the 68 runs above. + if got := render("5*5=25 и 3*3=9"); got != "55=25 и 33=9" { + t.Errorf("the known limit changed shape (%q) — if this is now handled, the comment above is stale", got) + } + // The premise the rule rests on: with DIFFERENT glyphs there is no ambiguity to resolve, and the + // pairing must stay exactly as permissive as before. + d := &Book{ItalicOpen: "⟦i⟧", ItalicClose: "⟦/i⟧"} + var sb strings.Builder + d.emphasisSpans("слово⟦i⟧выделено⟦/i⟧", func(text string, italic bool) { + if italic { + sb.WriteString("" + text + "") + return + } + sb.WriteString(text) + }) + if got := sb.String(); got != "слововыделено" { + t.Errorf("distinct glyphs need no flanking rule, got %q", got) + } +} diff --git a/backend/internal/bookfile/model.go b/backend/internal/bookfile/model.go index bd22161a..8ad55c3f 100644 --- a/backend/internal/bookfile/model.go +++ b/backend/internal/bookfile/model.go @@ -21,6 +21,8 @@ import ( "fmt" "strings" "time" + "unicode" + "unicode/utf8" ) // Formats are the file formats this package writes, in the order a build produces them. A format name @@ -65,11 +67,12 @@ type Book struct { // means the text carries no markers and every writer below is byte-identical to one that never knew // about them. // - // ⛔ BOTH FORMATS HAVE TO ANSWER FOR THEM, and the reason is worth stating where the field is: the - // EPUB can render emphasis and the text file cannot, so a marker mapped in one and forgotten in the - // other reaches a READER as punctuation — after the run, after the money, in the file they open. The - // EPUB turns the pair into ; the text file re-spells it as *…*, which is what plain text has always - // meant by emphasis. A pair whose glyphs are already the asterisk sees the second mapping do nothing. + // ⛔ BOTH FORMATS ANSWER FOR THEM, but they do not answer equally, and the asymmetry is worth knowing + // before trusting a green run: the EPUB turns the pair into , while the text file re-spells it as + // *…* — and for a pair whose glyphs ARE the asterisk that second mapping is a byte no-op. So a + // mis-paired marker is visible in the EPUB and invisible in the .txt, and the shipping pair is exactly + // that case. A marker forgotten by one writer reaches a READER as punctuation, after the run and after + // the money, in the file they open. ItalicOpen, ItalicClose string } @@ -83,17 +86,80 @@ func (b *Book) emphasisSpans(s string, emit func(text string, italic bool)) { emit(s, false) return } + // ⛔ WITH ONE GLYPH ON BOTH SIDES, POSITION IS THE ONLY WAY TO TELL AN OPENER FROM A CLOSER, and + // pairing left to right without asking is how a paragraph comes out inverted: one marker lost by the + // model turns `сноску* и сказал *важно*` into emphasis on « и сказал » — the reader's book then + // emphasises words the model did not, and drops the ones it did, in a file that parses perfectly. + // + // The test is the ONE thing the ingest guarantees BY CONSTRUCTION (inlineRules.wrap trims the run, then + // puts the markers against the text): the INNER side of a marker is never whitespace. An opener is + // therefore followed by a non-space and a closer preceded by one; a glyph that fails its side is the + // author's own punctuation or a marker the model lost, and it stays literal. + // + // ⚠ WHAT THIS DELIBERATELY DOES NOT TEST, because the first version of this rule did and it cost 68 of + // this book's 4842 runs: the OUTER side. «An opener never follows a letter» reads plausible and is + // false — italics land on a full stop (`all*.*`), on a comma, inside a word (`Fucks*sakes*`) — and + // rejecting those openers made their closers open instead, inverting everything after them. + // + // ⚠ AND WHAT IT CANNOT DO: tell the author's asterisk from ours when both sides happen to be text + // (`5*5=25 и 3*3=9`). That is answered by MEASUREMENT, not by cleverness — the ingest counts the pair's + // glyphs in the source and prints them as `source_marker_glyphs` (0 on this book), and a pair whose + // books carry the glyph should choose another one. + same := open == closeMark + opensHere := func(s string, i int) bool { + if !same { + return true + } + rest := s[i+len(open):] + if rest == "" { + return false + } + r, _ := utf8.DecodeRuneInString(rest) + return !unicode.IsSpace(r) + } + closesHere := func(s string, i int) bool { + if !same || i == 0 { + return true + } + r, _ := utf8.DecodeLastRuneInString(s[:i]) + return !unicode.IsSpace(r) + } for { i := strings.Index(s, open) + for i >= 0 && !opensHere(s, i) { + if next := strings.Index(s[i+len(open):], open); next >= 0 { + i += len(open) + next + } else { + i = -1 + } + } if i < 0 { break } - j := strings.Index(s[i+len(open):], closeMark) + j := -1 + for at := i + len(open); ; { + k := strings.Index(s[at:], closeMark) + if k < 0 { + break + } + if closesHere(s, at+k) { + j = at + k - (i + len(open)) + break + } + at += k + len(closeMark) + } if j < 0 { break } emit(s[:i], false) - emit(s[i+len(open):i+len(open)+j], true) + // An EMPTY span is not emphasis, and swallowing it loses what the model wrote: with one glyph on + // both sides `**word**` pairs into two empty spans and the word arrives unmarked and unmarkable. + // The glyphs go back as literal text, so the reader sees what came out of the model. + if j == 0 { + emit(open+closeMark, false) + } else { + emit(s[i+len(open):i+len(open)+j], true) + } s = s[i+len(open)+j+len(closeMark):] } emit(s, false) @@ -229,6 +295,8 @@ func (b *Book) xmlEmphasis(s string) string { // ⚠ The alternative — dropping the markers — was rejected because it is the one choice that loses // something the reader could have had: plain text cannot show italics, but `*слово*` is legible as // emphasis to every reader of a .txt file, and an unmarked word is simply the author's distinction gone. +func (b *Book) PlainEmphasis(s string) string { return b.txtEmphasis(s) } + func (b *Book) txtEmphasis(s string) string { if b.ItalicOpen == "" || b.ItalicClose == "" { return CleanText(s) diff --git a/backend/internal/chunk/ingest.go b/backend/internal/chunk/ingest.go index 83a02be8..72db9e22 100644 --- a/backend/internal/chunk/ingest.go +++ b/backend/internal/chunk/ingest.go @@ -108,6 +108,11 @@ type Document struct { // lose it in silence. InlineMarkupDeclared bool ItalicCarriers int + // SourceMarkerGlyphs counts the pair's own marker glyphs found in the SOURCE text. It is reported beside the + // carriers because the glyph is chosen to be absent from the source, and that choice is a measurement + // on particular books: a second English book in this tree carries twelve author-typed asterisks. A + // non-zero here means those characters will be read as emphasis and will not reach the reader as typed. + SourceMarkerGlyphs int } // ExcludedDocument is one spine document this reader declined to read as book text, and why. @@ -139,6 +144,7 @@ func (d *Document) IngestNotes() []any { // NOT through add(): a zero here is the number that matters, and add() prints only positives. kv = append(kv, "italic_carriers", d.ItalicCarriers) } + add("source_marker_glyphs", d.SourceMarkerGlyphs) add("toc_unreadable", d.TOCUnreadable) add("toc_unresolved", d.TOCUnresolved) add("targets_collapsed", d.TargetsCollapsed) @@ -202,7 +208,23 @@ func IngestEncoded(p, encoding, sourceLang string, st *lang.SourceStructure, im if strings.EqualFold(extOf(p), ".epub") { return ingestEPUB(p, im) } - return ingestTXT(p, encoding, sourceLang, st) + doc, err := ingestTXT(p, encoding, sourceLang, st) + // A plain-text source carries no markup, so the pair's emphasis glyphs cannot apply — but the pair PAID + // for declaring them (its pack version moved and its prompts teach the glyph), so the answer it gets is + // a zero it can see rather than silence. The declaration is per PAIR and the dispatch is per FILE + // EXTENSION, so this combination is ordinary, not a misconfiguration. + if err == nil && im != nil { + doc.InlineMarkupDeclared = true + } + return doc, err +} + +// CarriesInlineMarkup reports whether a source of this kind can carry inline emphasis into the chunks — +// today, whether it is an epub. It exists so that the book WRITER arms its markers by the same answer the +// ingest used, instead of a second reading of the same question: a plain-text source is never marked, and +// a writer that believed otherwise would turn an author's own asterisks into italics in the reader's file. +func CarriesInlineMarkup(sourcePath string) bool { + return strings.EqualFold(extOf(sourcePath), ".epub") } // extOf returns the lower-cased file extension incl. the dot ("" if none). @@ -805,10 +827,11 @@ func ingestEPUB(p string, im *lang.InlineMarkup) (*Document, error) { continue } // Read it — only to SIZE it. An exclusion no gate can second-guess should at least say how much text - // it took with it. + // it took with it. Read WITHOUT the emphasis rules: this text never reaches a chunk, so counting its + // carriers would inflate the book's count and its markers would inflate this very size. runes := 0 if data, rerr := readZipEntry(files, docs[i].entry); rerr == nil { - if body, _, xerr := extractXHTML(data, ir); xerr == nil { + if body, _, xerr := extractXHTML(data, nil); xerr == nil { runes = utf8.RuneCountInString(text.NormalizeSource(body)) } } @@ -951,7 +974,7 @@ func ingestEPUB(p string, im *lang.InlineMarkup) (*Document, error) { } doc.Structure = structureOf(doc.Chapters, witness) if ir != nil { - doc.InlineMarkupDeclared, doc.ItalicCarriers = true, ir.marked + doc.InlineMarkupDeclared, doc.ItalicCarriers, doc.SourceMarkerGlyphs = true, ir.marked, ir.sourceGlyphs } return doc, nil } @@ -1155,31 +1178,71 @@ func extractXHTML(data []byte, ir *inlineRules) (string, []RubyReading, error) { skip, skipDepth := "", 0 // An italic run is buffered rather than written through, because its markers must hug the TEXT and - // leave the surrounding whitespace outside (inlineRules.wrap). `italOpen` is the stack of element - // names that opened the run — names and not a counter, for the reason the ruby state above is a mode: - // real epub markup mis-nests, and an end tag that matches nothing must close nothing. + // leave the surrounding whitespace outside (inlineRules.wrap). The stack holds the OPEN elements — + // every element, not only the emphasising ones, so that a closing tag is matched by POSITION and an + // inner cannot close the that contains it. var ital strings.Builder - var italOpen []string + type openTag struct { + name string + italic bool // this element opened the run + block bool // …and it is block-level, so its run legitimately spans paragraphs + } + var open []openTag + inItalic := func() bool { + for _, t := range open { + if t.italic { + return true + } + } + return false + } // out writes prose to whichever sink is current. Every body write goes through it, so an italic run // cannot leak past the buffer by way of a path somebody forgot. out := func(s string) { - if len(italOpen) > 0 { + if ir != nil { + // Counted HERE and not on the finished text: what passes through this sink is the source's own + // characters, while the engine's markers are written straight to the body by wrap(). + ir.sourceGlyphs += strings.Count(s, ir.open) + if ir.close != ir.open { + ir.sourceGlyphs += strings.Count(s, ir.close) + } + } + if inItalic() { ital.WriteString(s) return } body.WriteString(s) } - // flushItalic ends the run and writes it, marked. Called on the matching end tag AND at every block - // boundary: an unclosed inline element must not outlive its paragraph, which is the lesson the ruby - // mode beside it was written from — left open, it would mark the whole rest of the chapter. - flushItalic := func() { - if len(italOpen) == 0 { - return - } - italOpen = italOpen[:0] + // emitItalic writes what is buffered, marked, and empties the buffer. It does NOT change the stack: + // a run is CLOSED by its own end tag, and it is merely CUT here — at a paragraph break — because a + // marker pair that spans one would reach the model as two unpaired halves in two different chunks. + emitItalic := func() { run := ital.String() ital.Reset() - body.WriteString(ir.wrap(run)) + if run != "" { + body.WriteString(ir.wrap(run)) + } + } + // paragraphBreak cuts the run at a block boundary and then drops every INLINE element still open: + // an inline element that crosses a block boundary is malformed markup, and left standing it would + // mark the rest of the chapter. A BLOCK element that emphasises (a
around + // a verse) is the opposite case — spanning paragraphs is what it is for — so it stays, and each of + // its paragraphs gets its own marker pair. + // + // ⚠ AND THE CONDITION UNDER WHICH THAT STOPS BEING RIGHT: the block is trusted to close itself. One + // that never does marks every remaining paragraph of the document — which is what a browser would + // render too, so it is a faithful reading of broken markup rather than a leak, but it is a reading + // nobody chose. Measured on the three books in this tree: no such element (the count that would show + // it is italic_carriers against the document's own carrier count). + paragraphBreak := func() { + emitItalic() + kept := open[:0] + for _, t := range open { + if t.block { + kept = append(kept, t) + } + } + open = kept } // flushRuby closes the current ruby, emitting its (base, reading) if both are non-empty, and @@ -1224,16 +1287,22 @@ func extractXHTML(data []byte, ir *inlineRules) (string, []RubyReading, error) { case name == "rb": rubyPart = rubyBase case name == "br" && rubyDepth == 0: + // A second newline in a row IS a paragraph break for the chunker, so a run has to be cut here + // exactly as at a block tag — otherwise `one

two` reaches the model as two chunks + // carrying one marker each. + if inItalic() && strings.HasSuffix(ital.String(), "\n") { + emitItalic() + } out("\n") case blockTags[name]: if rubyDepth > 0 { flushRuby() } - flushItalic() + paragraphBreak() body.WriteString("\n\n") } - if skip == "" && ir.isItalicTag(name, class, style) { - italOpen = append(italOpen, name) + if skip == "" && !voidTags[name] { + open = append(open, openTag{name: name, italic: ir.isItalicTag(name, class, style), block: blockTags[name]}) } } @@ -1261,21 +1330,29 @@ func extractXHTML(data []byte, ir *inlineRules) (string, []RubyReading, error) { if rubyDepth > 0 { flushRuby() } - flushItalic() + paragraphBreak() body.WriteString("\n\n") } - // Close the run this end tag opened, searching from the inside out. A tag that opened nothing - // closes nothing: mis-nested inline markup is ordinary in real epubs, and a counter would let one - // stray end a run its opener still owns. - for i := len(italOpen) - 1; i >= 0; i-- { - if italOpen[i] != name { + // Close the innermost element of this name, and with it everything opened inside it — the shape a + // browser gives mis-nested markup. Searching the WHOLE stack rather than the emphasising elements + // alone is what stops an inner plain from closing the around it. A tag + // that opened nothing closes nothing. + for i := len(open) - 1; i >= 0; i-- { + if open[i].name != name { continue } - if i == 0 { - flushItalic() - } else { - italOpen = append(italOpen[:i], italOpen[i+1:]...) + // Cut the run before the stack shrinks: if this element (or one inside it) was emphasising, + // what is buffered belongs to it. + closesItalic := false + for _, t := range open[i:] { + if t.italic { + closesItalic = true + } } + if closesItalic { + emitItalic() + } + open = open[:i] break } } @@ -1329,9 +1406,9 @@ func extractXHTML(data []byte, ir *inlineRules) (string, []RubyReading, error) { } // CommentToken / DoctypeToken carry no prose — dropped. } - // A document that ends inside an unclosed inline element still owns its text: without this the - // buffered run would be DROPPED, which is a silent deletion of prose rather than a lost marker. - flushItalic() + // A document that ends inside an unclosed element still owns its text: without this the buffered run + // would be DROPPED, which is a silent deletion of prose rather than a lost marker. + emitItalic() return body.String(), ruby, nil } diff --git a/backend/internal/chunk/inlinemarkup.go b/backend/internal/chunk/inlinemarkup.go index 19c11e75..1f0434ed 100644 --- a/backend/internal/chunk/inlinemarkup.go +++ b/backend/internal/chunk/inlinemarkup.go @@ -28,15 +28,23 @@ import ( // nil means the pair declared nothing: no detection runs and no byte moves. type inlineRules struct { open, close string - // classes are the CSS class names whose rule declares an italic font-style. It is the only live - // carrier in the production book: of 4842 italic carriers there, tags and account for 0 — - // every one is a with a class, and the stylesheet holds exactly one italic rule. - classes map[string]bool + // classes maps an italicising CSS class to the element names its rule was written for — empty set + // meaning "any element". It is the only live carrier in the production book: of 4842 italic carriers + // there, tags and account for 0 — every one is a with a class. + // + // ⚠ THE QUALIFIER IS KEPT because dropping it marks text the stylesheet never italicised: a second + // English book in this tree declares 26 heading classes (`h1.prologue`, `h1.epilogue`, …) italic, and + // a class-only reading turns every chapter's first line into an emphasised run. + classes map[string]map[string]bool // marked counts the runs actually wrapped. It is the ONE honest signal for a book whose emphasis this // scanner does not see: the CSS shapes it accepts are narrow on purpose, and a book that expresses // italics through a shorthand or a descendant rule would otherwise lose them in silence. A number an // operator can read turns that into a visible zero. marked int + // sourceGlyphs counts the pair's glyphs found in the SOURCE's own text. The glyph is chosen so that a + // book does not use it, and that choice is a measurement on the books at hand — a non-zero here says + // this book breaks the assumption and its own punctuation is about to be read as emphasis. + sourceGlyphs int } // newInlineRules resolves the pair's declaration against this book's stylesheets. It returns nil when the @@ -45,10 +53,21 @@ func newInlineRules(im *lang.InlineMarkup, stylesheets [][]byte) *inlineRules { if im == nil || im.ItalicOpen == "" || im.ItalicClose == "" { return nil } - ir := &inlineRules{open: im.ItalicOpen, close: im.ItalicClose, classes: map[string]bool{}} + ir := &inlineRules{open: im.ItalicOpen, close: im.ItalicClose, classes: map[string]map[string]bool{}} for _, css := range stylesheets { - for name := range italicClasses(css) { - ir.classes[name] = true + for name, elems := range italicClasses(css) { + cur, seen := ir.classes[name] + if !seen { + ir.classes[name] = elems + continue + } + if len(cur) == 0 || len(elems) == 0 { + ir.classes[name] = map[string]bool{} // one unqualified rule wins: it applies anywhere + continue + } + for e := range elems { + cur[e] = true + } } } return ir @@ -59,7 +78,7 @@ var cssComment = regexp.MustCompile(`(?s)/\*.*?\*/`) // cssSimpleClassSelector matches the selectors this scanner accepts: an optional element name followed by // ONE class, and nothing else — `.italic`, `span.italic`, `p.i1`. -var cssSimpleClassSelector = regexp.MustCompile(`^[A-Za-z0-9]*\.([A-Za-z0-9_-]+)$`) +var cssSimpleClassSelector = regexp.MustCompile(`^([\p{L}\p{N}]*)\.([\p{L}\p{N}_-]+)$`) // italicClasses scans a stylesheet for the class names whose rule declares an italic font-style. // @@ -73,25 +92,79 @@ var cssSimpleClassSelector = regexp.MustCompile(`^[A-Za-z0-9]*\.([A-Za-z0-9_-]+) // an inline style on every span is NOT covered by the class channel. The first two are invisible here; the // third is caught by the walker's style-attribute check. The honest signal is the carrier COUNT — a book // whose italics this misses reports zero of them, which is a number an operator can see. -func italicClasses(css []byte) map[string]bool { - out := map[string]bool{} - body := cssComment.ReplaceAllString(string(css), " ") - for _, block := range strings.Split(body, "}") { - open := strings.IndexByte(block, '{') +func italicClasses(css []byte) map[string]map[string]bool { + out := map[string]map[string]bool{} + scanCSSRules(cssComment.ReplaceAllString(string(css), " "), out) + return out +} + +// scanCSSRules walks `selectors { body }` in order. A selector starting with `@` is a conditional group +// (@media, @supports) whose body holds more rules — it is recursed into, because a rule inside one styles +// the book exactly as an unwrapped rule does. Unbalanced braces stop the walk rather than guess. +func scanCSSRules(s string, out map[string]map[string]bool) { + for { + open := strings.IndexByte(s, '{') if open < 0 { - continue + return } - if !declaresItalic(block[open+1:]) { - continue - } - for _, sel := range strings.Split(block[:open], ",") { - m := cssSimpleClassSelector.FindStringSubmatch(strings.TrimSpace(sel)) - if m != nil { - out[m[1]] = true + // Braces inside a string literal are text, not structure (`content: "}"`), and counting them as + // structure ends the rule early and takes every rule after it with it. A stylesheet that never + // closes its last rule is read to the end rather than dropped, for the same reason: losing the + // remainder is a silent zero, and a silent zero here is a book whose emphasis nobody sees. + depth, end, quote := 1, -1, byte(0) + for i := open + 1; i < len(s) && end < 0; i++ { + c := s[i] + switch { + case quote != 0: + switch c { + case '\\': + i++ + case quote: + quote = 0 + } + case c == '"' || c == '\'': + quote = c + case c == '{': + depth++ + case c == '}': + if depth--; depth == 0 { + end = i + } } } + unclosed := end < 0 + if unclosed { + end = len(s) // a stylesheet that never closes its last rule still states it + } + sel, body := strings.TrimSpace(s[:open]), s[open+1:end] + switch { + case strings.HasPrefix(sel, "@"): + scanCSSRules(body, out) + case declaresItalic(body): + for _, one := range strings.Split(sel, ",") { + m := cssSimpleClassSelector.FindStringSubmatch(strings.TrimSpace(one)) + if m == nil { + continue + } + elem, name := strings.ToLower(m[1]), m[2] + cur, seen := out[name] + switch { + case !seen && elem == "": + out[name] = map[string]bool{} + case !seen: + out[name] = map[string]bool{elem: true} + case len(cur) == 0 || elem == "": + out[name] = map[string]bool{} // an unqualified rule makes the class apply anywhere + default: + cur[elem] = true + } + } + } + if unclosed { + return + } + s = s[end+1:] } - return out } // declaresItalic reports whether a declaration block sets font-style to italic (or oblique, which is the @@ -134,7 +207,11 @@ func (ir *inlineRules) isItalicTag(name, class, style string) bool { return true } for _, c := range strings.Fields(class) { - if ir.classes[c] { + elems, ok := ir.classes[c] + if !ok { + continue + } + if len(elems) == 0 || elems[name] { return true } } diff --git a/backend/internal/chunk/inlinemarkup_test.go b/backend/internal/chunk/inlinemarkup_test.go index d537b682..d2f76f51 100644 --- a/backend/internal/chunk/inlinemarkup_test.go +++ b/backend/internal/chunk/inlinemarkup_test.go @@ -91,7 +91,7 @@ func TestADeclaredPairCarriesTheEmphasis(t *testing.T) { // paragraph boundary. The last two are the dangerous ones — the same class as the ruby mode beside them, // where one unclosed element used to swallow the rest of a chapter. func TestTheWalkerSurvivesTheMarkupRealBooksShip(t *testing.T) { - ir := &inlineRules{open: "*", close: "*", classes: map[string]bool{"ital": true}} + ir := &inlineRules{open: "*", close: "*", classes: map[string]map[string]bool{"ital": {}}} for _, tc := range []struct{ name, doc, want string }{ {"the markers hug the word, not the spaces", `

a word b

`, "a *word* b"}, @@ -136,12 +136,12 @@ span.i2, .i3 { color: red; font-style: oblique } #n4 { font-style: italic } `)) for _, want := range []string{"i1", "i2", "i3", "i4"} { - if !got[want] { + if _, ok := got[want]; !ok { t.Errorf("the class %q declares an italic font-style and must be found", want) } } for _, refused := range []string{"commented", "n1", "n2", "inner", "n3", "n4"} { - if got[refused] { + if _, ok := got[refused]; ok { t.Errorf("%q must NOT be read as italic — a false marker instructs the model to emphasise ordinary prose", refused) } } @@ -256,3 +256,180 @@ func epubChapterDocuments(t *testing.T, zipped []byte) string { } return sb.String() } + +// TestTheScannerReadsTheShapesRealBooksUse extends the accepted set to the two shapes a converted book +// commonly uses and the first scanner missed: a rule inside a conditional group, and a class name outside +// ASCII. Both were measured on fixtures as silent zeroes — the failure mode this file's own counter exists +// to make visible, arriving one level earlier where the counter cannot see it. +func TestTheScannerReadsTheShapesRealBooksUse(t *testing.T) { + got := italicClasses([]byte(` +@media screen { .inmedia { font-style: italic } } +@supports (font-style: italic) { .insupports { font-style: italic } } +.kursiv-ü { font-style: italic } +.пример { font-style: italic } +@font-face { font-family: X; font-style: italic } +@media print { .notitalic { font-weight: bold } } +`)) + for _, want := range []string{"inmedia", "insupports", "kursiv-ü", "пример"} { + if _, ok := got[want]; !ok { + t.Errorf("%q declares an italic font-style and must be found — a book that wraps its rules in a media query or names its classes outside ASCII would otherwise report zero carriers", want) + } + } + if _, ok := got["notitalic"]; ok { + t.Error("a rule that does not declare italic must not be collected, wrapper or no wrapper") + } + if len(got) != 4 { + t.Errorf("exactly the four accepted classes, got %v (an @font-face block declares a FONT, not a class)", got) + } + + // ⛔ AND THE SHAPES THAT USED TO END THE WALK. A brace inside a string literal is text, and a rule the + // file never closes is still a rule; counting either as structure dropped every rule that followed — + // a total, silent loss that reaches an operator only as a zero carrier count. + for _, tc := range []struct { + name, css string + want []string + }{ + {"a brace inside a string literal", `.a { content: "}"; font-style: italic } .later { font-style: italic }`, []string{"a", "later"}}, + {"a rule the stylesheet never closes", `@media screen { .m { font-style: italic } } .tail { font-style: italic`, []string{"m", "tail"}}, + } { + t.Run(tc.name, func(t *testing.T) { + got := italicClasses([]byte(tc.css)) + for _, w := range tc.want { + if _, ok := got[w]; !ok { + t.Errorf("%q must survive: got %v", w, got) + } + } + }) + } +} + +// TestTheCarrierCountIsAboutTheBookAndNotItsServicePages: the count is the operator's only signal for a +// book whose emphasis this reader does not recognise, so a carrier in a document the reader EXCLUDED must +// not enter it — a cover page with one italic word would otherwise hide the zero the count exists to show. +func TestTheCarrierCountIsAboutTheBookAndNotItsServicePages(t *testing.T) { + book := chunktest.EPUB{ + Chapters: []chunktest.Chapter{ + {ID: "css", Href: "style.css", MType: "text/css", Raw: true, Body: ".ital { font-style: italic }"}, + {ID: "cover", Href: "cover.xhtml", Body: `

Vampyr

`}, + {ID: "c1", Href: "c1.xhtml", Body: `

He said Vampyr.

`}, + }, + Spine: []chunktest.SpineRef{{ID: "cover"}, {ID: "c1"}}, + Nav: &chunktest.Nav{TOC: []string{"c1.xhtml"}, Landmarks: map[string]string{"cover.xhtml": "cover"}}, + }.Build(t) + doc, err := IngestEncoded(book, "", "", nil, &lang.InlineMarkup{ItalicOpen: "*", ItalicClose: "*"}) + if err != nil { + t.Fatal(err) + } + if len(doc.Excluded) != 1 { + t.Fatalf("premise broken: the cover must be EXCLUDED, else this test measures an ordinary two-chapter book (%v)", doc.Excluded) + } + if doc.ItalicCarriers != 1 { + t.Errorf("the book carries ONE italic run; the excluded cover's own must not be counted, got %d", doc.ItalicCarriers) + } +} + +// TestADeclaredPairOverATextSourceReportsAnHonestZero: the declaration is per PAIR and the ingest dispatches +// per FILE EXTENSION, so a pair that ships the glyphs and a book that is a .txt is an ordinary combination — +// and the pair has already paid for the declaration (its pack version moved, its prompts teach the glyph). +// Silence would let it believe the markers are flowing. +func TestADeclaredPairOverATextSourceReportsAnHonestZero(t *testing.T) { + p := filepath.Join(t.TempDir(), "book.txt") + if err := os.WriteFile(p, []byte("Обычный текст без разметки."), 0o644); err != nil { + t.Fatal(err) + } + doc, err := IngestEncoded(p, "", "", nil, &lang.InlineMarkup{ItalicOpen: "*", ItalicClose: "*"}) + if err != nil { + t.Fatal(err) + } + if !doc.InlineMarkupDeclared || doc.ItalicCarriers != 0 { + t.Fatalf("declared=%v carriers=%d — a text source carries no markup, and the pair must be told so with a number", doc.InlineMarkupDeclared, doc.ItalicCarriers) + } + var named bool + for i, kv := range doc.IngestNotes() { + if i%2 == 0 && kv == "italic_carriers" { + named = true + } + } + if !named { + t.Errorf("the zero must REACH the operator, got notes %v", doc.IngestNotes()) + } +} + +// TestAnEmptySpanKeepsItsGlyphs: with one glyph on both sides `**word**` pairs into two EMPTY spans, and +// swallowing them loses the model's own markup without a trace. The glyphs come back as literal text. +func TestAnEmptySpanKeepsItsGlyphs(t *testing.T) { + book := &bookfile.Book{ + Identifier: "urn:t", Title: "Т", Language: "ru", + Modified: time.Date(2026, 9, 18, 0, 0, 0, 0, time.UTC), + Chapters: []bookfile.Chapter{{Title: "Глава 1", Paragraphs: []string{"Он сказал **слово** и ушёл."}}}, + ItalicOpen: "*", + ItalicClose: "*", + } + var txt bytes.Buffer + if err := bookfile.WriteTXT(&txt, book); err != nil { + t.Fatal(err) + } + if !strings.Contains(txt.String(), "**слово**") { + t.Errorf("the model's own doubled markers must survive to the reader rather than vanish:\n%s", txt.String()) + } +} + +// TestAClassRuleKeepsTheElementItWasWrittenFor: a stylesheet that italicises `h1.prologue` did not say +// "this class is italic anywhere". Dropping the qualifier turns every chapter's first line into an +// emphasised run — measured on a second English book in this tree, which declares 26 such heading classes. +func TestAClassRuleKeepsTheElementItWasWrittenFor(t *testing.T) { + ir := newInlineRules(&lang.InlineMarkup{ItalicOpen: "*", ItalicClose: "*"}, [][]byte{ + []byte("h1.prologue { font-style: italic }\n.anywhere { font-style: italic }\nspan.q, i.q { font-style: italic }"), + }) + for _, tc := range []struct { + doc, want string + }{ + {`

PROLOGUE

text

`, "*PROLOGUE*\n\n\n\ntext"}, + {`

not a heading

`, "not a heading"}, + {`

marked

`, "*marked*"}, + {`

also marked

`, "*also marked*"}, + // `b` is not in the rule's element list — and unlike it is not an italic ELEMENT either, so + // nothing else can mark it. A case built on would have measured the tag channel instead. + {`

a b

`, "*a* b"}, + } { + t.Run(tc.doc, func(t *testing.T) { + body, _, err := extractXHTML([]byte(``+tc.doc+``), ir) + if err != nil { + t.Fatal(err) + } + if got := strings.TrimSpace(body); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } +} + +// TestARunNeverSpansAParagraph is the invariant the chunker needs: a marker pair inside one chunk. A run +// cut by a paragraph break gets its own pair on each side, so neither half reaches the model unpaired — +// and a BLOCK element that emphasises keeps emphasising across the paragraphs it contains, which is how +// verse and epigraphs are actually marked up. +func TestARunNeverSpansAParagraph(t *testing.T) { + ir := &inlineRules{open: "*", close: "*", classes: map[string]map[string]bool{"ital": {}}} + for _, tc := range []struct{ name, doc, want string }{ + {"a blank line inside an inline run", `

one

two

`, "*one*\n\n*two*"}, + {"a single line break does not cut it", `

one
two

`, "*one\ntwo*"}, + {"a block wrapper emphasises each of its paragraphs", `

verse one

verse two

`, "*verse one*\n\n\n\n*verse two*"}, + {"an inner element of the same name does not close the outer run", `

abcde

`, "a*bcd*e"}, + } { + t.Run(tc.name, func(t *testing.T) { + body, _, err := extractXHTML([]byte(``+tc.doc+``), ir) + if err != nil { + t.Fatal(err) + } + got := strings.TrimSpace(body) + if got != tc.want { + t.Fatalf("got %q, want %q", got, tc.want) + } + for _, para := range strings.Split(got, "\n\n") { + if n := strings.Count(para, "*"); n%2 != 0 { + t.Errorf("paragraph %q carries %d markers — an odd count means one half of a pair travels to the model alone", para, n) + } + } + }) + } +} diff --git a/backend/internal/config/pipeline.go b/backend/internal/config/pipeline.go index 61737bad..a5bd5e18 100644 --- a/backend/internal/config/pipeline.go +++ b/backend/internal/config/pipeline.go @@ -70,15 +70,26 @@ func (p *Pipeline) ApplyBriefPolicies(b *Book) error { if len(p.Brief) == 0 || b == nil { return nil } - fields := map[string]*string{"transcription": &b.Transcription, "honorifics": &b.Honorifics} + fields := briefTargets(b) var problems []string for _, name := range BriefPolicyFields { pol, declared := p.Brief[name] - target, known := fields[name] - if !declared || !known { + if !declared { continue } - if strings.TrimSpace(*target) == "" { + target, wired := fields[name] + if !wired { + // The list LoadPair judges a declaration against and the wiring here are two spellings of one + // set; a name in the first and not the second would let a pair declare a policy that judges + // nothing. TestBriefPolicyFieldsAndTheirWiringAreOneList keeps them equal, and this keeps the + // disagreement loud if it ever stops. + problems = append(problems, fmt.Sprintf("%s: the pair declares a policy for this field and the engine wires none", name)) + continue + } + // Trimmed on BOTH sides of the question: without it `" "` silently takes the default while + // `" practical "` is refused by name, which is one function answering one question two ways. + *target = strings.TrimSpace(*target) + if *target == "" { *target = pol.Default continue } @@ -93,6 +104,13 @@ func (p *Pipeline) ApplyBriefPolicies(b *Book) error { return nil } +// briefTargets maps each brief field a pair may constrain to the book field that carries it. It is the +// ONE place the set is spelled for the engine; BriefPolicyFields is what a pair's declaration is judged +// against, and the two are held equal by a test. +func briefTargets(b *Book) map[string]*string { + return map[string]*string{"transcription": &b.Transcription, "honorifics": &b.Honorifics} +} + // LabelPolicy is one entry of the ordered content-policy registry: what to do with content carrying // this label. `action: route` sends the labelled stages to their label_models and takes the single // escalation hop from `chain`; `action: terminal` refuses to process such a book at all — a book-level diff --git a/backend/internal/config/promptlabel_pair_test.go b/backend/internal/config/promptlabel_pair_test.go index 7989bfbd..7e5ba15d 100644 --- a/backend/internal/config/promptlabel_pair_test.go +++ b/backend/internal/config/promptlabel_pair_test.go @@ -95,3 +95,114 @@ func TestAPairLabelKeyHasToNameAPromptFile(t *testing.T) { }) } } + +// TestBriefPolicyFieldsAndTheirWiringAreOneList closes the gap between the list a pair's declaration is +// JUDGED against (BriefPolicyFields, checked in LoadPair) and the list that actually judges a book +// (briefTargets). A field in the first and not the second loads clean and enforces nothing: the pair +// believes it declared a policy, the book violates it, and the engine says nothing. +func TestBriefPolicyFieldsAndTheirWiringAreOneList(t *testing.T) { + wired := briefTargets(&Book{}) + if len(wired) != len(BriefPolicyFields) { + t.Fatalf("the two lists differ in size: judged %v, wired %d entries", BriefPolicyFields, len(wired)) + } + for _, name := range BriefPolicyFields { + if _, ok := wired[name]; !ok { + t.Errorf("%q is a field a pair may declare and nothing in the engine reads it", name) + } + } +} + +// TestAPairsPromptLabelNamesAPromptItHas is the guarantee this file's neighbour only claims: a key of +// prompt_versions is the BASE NAME of a prompt file, and a typo in it is a dead key that silently leaves +// the shared, snapshot-folded label in force — the exact state the pair-scoped label exists to end. The +// shape check at load cannot see this (a typo is well-shaped); only the pair's own directory can. +func TestAPairsPromptLabelNamesAPromptItHas(t *testing.T) { + dir := filepath.Join("..", "..", "configs") + pairs, err := filepath.Glob(filepath.Join(dir, "pairs", "*.yaml")) + if err != nil || len(pairs) == 0 { + t.Fatalf("premise broken: no pair files to walk (%v)", err) + } + checked := 0 + for _, path := range pairs { + pair := strings.TrimSuffix(filepath.Base(path), ".yaml") + pc, err := LoadPair(dir, pair) + if err != nil || pc == nil { + t.Fatalf("pair %s must load: %v", pair, err) + } + for name := range pc.PromptVersions { + checked++ + p := filepath.Join(pc.PromptsRoot, pair, name+".md") + if _, err := os.Stat(p); err != nil { + t.Errorf("pair %s labels %q, which names no prompt: %s does not exist — the label is dead and the run keeps the shared one", pair, name, p) + } + } + } + if checked == 0 { + t.Skip("no pair declares its own prompt labels yet; nothing to check") + } +} + +// TestAPairThatDeclaresAGlyphTeachesItToItsPrompts is the second half of a contract both the data file and +// the loader's doc comment state and nothing enforced: the ingest wraps emphasis in the pair's glyphs, and +// only that pair's PROMPTS can tell the model what they mean. A pair changing its glyph, or copying the +// mechanism without the instruction, would hand a model unexplained punctuation mid-prose — and the label +// ledger would not see it, because it watches prompt bytes against labels, not data against prompts. +func TestAPairThatDeclaresAGlyphTeachesItToItsPrompts(t *testing.T) { + dir := filepath.Join("..", "..", "configs") + packs, err := filepath.Glob(filepath.Join(dir, "langpacks", "*", "inline-markup.txt")) + if err != nil { + t.Fatal(err) + } + if len(packs) == 0 { + t.Skip("no pair declares inline markup yet") + } + for _, path := range packs { + pair := filepath.Base(filepath.Dir(path)) + pc, err := LoadPair(dir, pair) + if err != nil || pc == nil { + t.Fatalf("pair %s declares inline markup but has no pair config: %v", pair, err) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var glyphs []string + for _, ln := range strings.Split(string(body), "\n") { + if f := strings.SplitN(strings.TrimSpace(ln), "\t", 2); len(f) == 2 && strings.HasPrefix(f[0], "italic_") { + glyphs = append(glyphs, strings.TrimSpace(f[1])) + } + } + if len(glyphs) == 0 { + t.Fatalf("premise broken: %s declares no glyph, so this test would judge nothing", path) + } + prompts, err := filepath.Glob(filepath.Join(pc.PromptsRoot, pair, "*.md")) + if err != nil || len(prompts) == 0 { + t.Fatalf("pair %s has no prompt pack to teach the glyph to: %v", pair, err) + } + // ⛔ A BULLET IS NOT AN EXPLANATION. With `*` as the glyph, `strings.Contains` is satisfied by every + // markdown list in the file: cutting every sentence that actually teaches the glyph out of the + // pair's prompts left this gate green, which was measured, not imagined. Lines that are list items + // are therefore not evidence, and the glyph must appear where prose can explain it. + taught := 0 + for _, pr := range prompts { + b, err := os.ReadFile(pr) + if err != nil { + t.Fatal(err) + } + for _, ln := range strings.Split(string(b), "\n") { + t := strings.TrimSpace(ln) + if strings.HasPrefix(t, glyphs[0]+" ") || strings.HasPrefix(t, "- ") && !strings.Contains(t[2:], glyphs[0]) { + continue // a list item: its leading glyph says "bullet", not "emphasis" + } + if strings.Contains(t, glyphs[0]) { + taught++ + break + } + } + } + if taught == 0 { + t.Errorf("pair %s ships the glyph %q and not one of its %d prompts mentions it: the model would meet it with nothing said", + pair, glyphs[0], len(prompts)) + } + } +} diff --git a/backend/internal/lang/data/injection.txt b/backend/internal/lang/data/injection.txt index 1d2ef0eb..572e4ba5 100644 --- a/backend/internal/lang/data/injection.txt +++ b/backend/internal/lang/data/injection.txt @@ -9,5 +9,5 @@ ru editor_unverified_header НЕПОДТВЕРЖДЁННЫЕ РАБОЧИЕ ВА ru unverified_marker ⟨проверить⟩ ru gender_male (муж. — мужские родовые формы) ru gender_female (жен. — женские родовые формы) -ru gender_hidden (пол СКРЫТ до раскрытия — избегай родовых форм; при неизбежности — мужские) +ru gender_hidden (пол СКРЫТ до раскрытия — избегай родовых форм; не получается — перестрой фразу; если и это не читается, пиши как лучше звучит) ru gender_neuter (ср. — средний род: используй родовые формы среднего рода) diff --git a/backend/internal/lang/langpack.go b/backend/internal/lang/langpack.go index c675ceaa..326d627d 100644 --- a/backend/internal/lang/langpack.go +++ b/backend/internal/lang/langpack.go @@ -39,6 +39,19 @@ import ( // file joins the manifest, or when any existing file starts parsing differently. const packAlgoVersion = "langpack-v2" +// inlineMarkupAlgoVersion versions the CODE that decides what counts as emphasis in a book — +// internal/chunk/inlinemarkup.go: which CSS shapes are read, which elements and attributes open a run. +// The pair's file says which GLYPHS to use and its bytes already fold; this tag covers the other half, +// because widening the detector changes the text of the chunks and therefore the wire. +// +// It is folded ONLY when a pair ships inline-markup.txt, which is what makes it affordable: a pair that +// does not use the feature is byte-identical and is never re-billed for a detector it does not run. +// Bump it whenever inlinemarkup.go starts reading a shape it did not read before — that is a loud +// --resnapshot for the pairs that carry emphasis, and nothing at all for the rest. +// A var and not a const so that a test can swap it and SEE the fold: the file's own bytes already move +// the version, so nothing else can tell "the tag is folded" from "the file is present". +var inlineMarkupAlgoVersion = "inline-v2-mediagroups+unicodeclass" + // Pack is a loaded, versioned language-data pack for one source→target pair. Fields are the DATA the // pipeline algorithms read; the zero value is unusable (load via Load). Maps are membership sets / lookup // tables (order-free); slices preserve their authored order. @@ -174,10 +187,11 @@ type TerminologySizing struct { // they are drawn from a LIFETIME role budget, which is where "a few more calls" becomes a bank that // stops halfway. A pair that states nothing keeps the engine default, as before. // - // ⚠ ADDING THIS KEY DID NOT BUMP packAlgoVersion, and here is the condition under which that stays - // true: no pack in the tree ships terminology.txt at all, so no existing byte starts parsing - // differently and no pack's Version() moves. The moment a shipped pack carries this file, changing how - // ANY key in it is read is a schema change and the tag must bump with it. + // ⚠ ADDING THIS KEY DID NOT BUMP packAlgoVersion, and the reason is the tag's own rule: a bump is for + // a REQUIRED file joining the manifest or for existing bytes starting to parse differently. A new + // OPTIONAL key changes neither — every pack that lacks it hashes exactly as before. What WOULD require + // the bump is changing how an existing key is read, and that now costs every pair at once, because + // packAlgoVersion sits in every pack's Version(). BatchRunes int } @@ -353,7 +367,9 @@ func Load(root, sourceLang, targetLang string) (*Pack, error) { // terminologist sizing (a pair whose value equals the default ships no file — moving the version to // change nothing would re-bill every book of the pair). {pair, "terminology.txt", func(b []byte) error { ts, e := parseTerminology(b); p.Terminology = ts; return e }}, - // inline emphasis carriage — the pair's half of the ingest↔prompt contract. + // inline emphasis carriage — the pair's half of the ingest↔prompt contract. Its presence also folds + // inlineMarkupAlgoVersion (below), so the ALGORITHM that reads a book's emphasis is versioned for + // the pairs that use it and invisible to the pairs that do not. {pair, "inline-markup.txt", func(b []byte) error { im, e := parseInlineMarkup(b); p.Inline = im; return e }}, } for _, o := range optional { @@ -366,6 +382,9 @@ func Load(root, sourceLang, targetLang string) (*Pack, error) { } h.Write([]byte("\x00" + o.dir + "/" + o.name + "\x00")) h.Write(b) + if o.name == "inline-markup.txt" { + h.Write([]byte("\x00" + inlineMarkupAlgoVersion)) + } if perr := o.apply(b); perr != nil { return nil, fmt.Errorf("langpack %q %s: %w", pair, o.name, perr) } diff --git a/backend/internal/lang/langpack_test.go b/backend/internal/lang/langpack_test.go index 0b85d7a0..44eba2e8 100644 --- a/backend/internal/lang/langpack_test.go +++ b/backend/internal/lang/langpack_test.go @@ -734,3 +734,142 @@ func TestTheShippedSecondPairLoads(t *testing.T) { t.Fatalf("the shipped zh→ru pack must keep loading: %v", err) } } + +// TestEveryFileInAPackIsOneThePackReads: the base pack directory reads files BY NAME, so a typo, a wrong +// extension or a right name in the wrong directory is not an error — it is an inert file. The overlay +// loader already refuses unexpected names (overlayDirFiles); the base never did, and the pack that added a +// fifth optional name is the one that has to notice, because the third pair is the reader who types these +// names for the first time. +func TestEveryFileInAPackIsOneThePackReads(t *testing.T) { + // Keyed BY DIRECTORY KIND, because a flat list cannot see the third case its own name promises: a file + // whose name is right and whose directory is wrong (`zh/inline-markup.txt` is read only from a PAIR + // directory, `zh-ru/reader.txt` only from a TARGET one). Both are inert data that looks shipped. + pairNames := map[string]bool{"heading.txt": true, "dc-checkers.txt": true, "terminology.txt": true, "inline-markup.txt": true} + for _, n := range pairFiles { + pairNames[n] = true + } + langNames := map[string]bool{"manifest.txt": true, "speech-cue.txt": true, "structure.txt": true, "reader.txt": true} + for _, n := range srcFiles { + langNames[n] = true + } + knownIn := func(dir, name string) bool { + if strings.Contains(dir, "-") { + return pairNames[name] + } + return langNames[name] + } + dirs, err := os.ReadDir(realRoot) + if err != nil { + t.Fatal(err) + } + seen := 0 + for _, d := range dirs { + if !d.IsDir() { + continue + } + files, err := os.ReadDir(filepath.Join(realRoot, d.Name())) + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if f.IsDir() { + continue + } + seen++ + if !knownIn(d.Name(), f.Name()) { + t.Errorf("%s/%s is not a name any loader reads FROM THAT DIRECTORY — it is inert data that looks shipped", d.Name(), f.Name()) + } + } + } + if seen == 0 { + t.Fatal("premise broken: no pack files were walked, so this test judged nothing") + } + t.Logf("pack files judged: %d", seen) +} + +// TestACommentInAPackFileIsPartOfItsVersion states a property that costs money and is written down +// nowhere else: a langpack file is hashed RAW, so editing a comment in it moves Version() and re-buys +// every book of that pair. Prompts are the opposite — their comments are stripped before hashing, which +// is what makes tidying one free — and the asymmetry is exactly the kind a reader assumes away. +// +// The test does not argue for the behaviour; it makes the next person meet it before a book does. +func TestACommentInAPackFileIsPartOfItsVersion(t *testing.T) { + root := t.TempDir() + write := func(rel, body string) { + t.Helper() + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("xx/manifest.txt", "channel\tnone\n") + write("xx-yy/terminology.txt", "# first wording\nkwic_width\t110\n") + before, err := Load(root, "xx", "yy") + if err != nil { + t.Fatal(err) + } + write("xx-yy/terminology.txt", "# second wording, same data\nkwic_width\t110\n") + after, err := Load(root, "xx", "yy") + if err != nil { + t.Fatal(err) + } + if before.Version() == after.Version() { + t.Fatal("a comment edit left the version alone — if that is now true, the ⛔ headers in configs/langpacks/*.txt are lying and must be removed") + } + if before.Terminology.KWICWidth != after.Terminology.KWICWidth { + t.Fatal("premise broken: the data changed too, so this measured an edit rather than a comment") + } +} + +// TestTheEmphasisDetectorIsVersionedOnlyForPairsThatUseIt pins the conditional fold, and it pins the TAG +// rather than the file: the file's own bytes move the version by themselves, so an assertion that only +// watched "declaring it changes the version" would pass on a build where the detector's version is folded +// nowhere — which is the state this was written to end. The tag is swapped to make the fold visible. +// +// Without the first half, widening the detector changes chunk text under an unchanged snapshot; without +// the second, every pair is re-billed for a detector it never runs. +func TestTheEmphasisDetectorIsVersionedOnlyForPairsThatUseIt(t *testing.T) { + root := t.TempDir() + write := func(rel, body string) { + t.Helper() + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + version := func(what string) string { + t.Helper() + p, err := Load(root, "xx", "yy") + if err != nil { + t.Fatalf("%s: %v", what, err) + } + return p.Version() + } + write("xx/manifest.txt", "channel\tnone\n") + write("xx-yy/terminology.txt", "kwic_width\t110\n") + plainBefore := version("a pack without the emphasis file") + + restore := inlineMarkupAlgoVersion + defer func() { inlineMarkupAlgoVersion = restore }() + inlineMarkupAlgoVersion = restore + "-probe" + if got := version("the same pack under a different detector version"); got != plainBefore { + t.Errorf("a pair that ships no emphasis file must not see the detector's version at all: %s vs %s", got, plainBefore) + } + inlineMarkupAlgoVersion = restore + + write("xx-yy/inline-markup.txt", "italic_open\t*\nitalic_close\t*\n") + withFile := version("a pack that declares the glyphs") + if withFile == plainBefore { + t.Fatal("premise broken: declaring the file did not move the version, so the comparison below is about nothing") + } + inlineMarkupAlgoVersion = restore + "-probe" + if got := version("the declaring pack under a different detector version"); got == withFile { + t.Error("the detector's version is folded NOWHERE: widening the scanner would change every chunk's text under an unchanged snapshot") + } +} diff --git a/backend/internal/membank/memory.go b/backend/internal/membank/memory.go index a5fb4fb9..014c46a2 100644 --- a/backend/internal/membank/memory.go +++ b/backend/internal/membank/memory.go @@ -944,7 +944,8 @@ func editorLines(injected []PickedEntry, tx lang.InjectionTexts) []string { // genderConstraintNote is the DC3 gender directive appended to a CONFIRMED editor-constraint line (WS5 // §5(б)): the injection DIRECTS the editor to the character's grammatical gender, which is the fix for // the Bai Ninbing class — no coreference needed. male/female ⇒ hard gender forms; hidden ⇒ a mandate to -// AVOID gender-marking constructions until the reveal (a masculine default when unavoidable, D19.3). "" +// AVOID gender-marking constructions until the reveal — and when that cannot be done, to RECAST the +// sentence rather than fall back to a gender (the wording is the owner's, carried in injection.txt). "" // for a term with no gender datum (the common case → the line is unchanged, byte-identical to before). func genderConstraintNote(gender string, tx lang.InjectionTexts) string { switch gender { diff --git a/backend/internal/membank/render_test.go b/backend/internal/membank/render_test.go index 9f96f4c8..23aa488c 100644 --- a/backend/internal/membank/render_test.go +++ b/backend/internal/membank/render_test.go @@ -59,6 +59,16 @@ func TestDC3GenderInjection(t *testing.T) { if !strings.Contains(hidden, "пол СКРЫТ") { t.Fatalf("hidden term must carry the gender-avoidance mandate, got:\n%s", hidden) } + // ⛔ AND WHAT THE MANDATE MUST SAY, which the header words alone cannot tell: when a genderless + // construction will not work, the editor RECASTS the sentence — it does not fall back to a gender. + // The line used to prescribe a masculine default, and the assertion above passed over it unchanged, + // which is how the ratified decision sat unexecuted in the data while a note declared it done. + if strings.Contains(hidden, "мужские") { + t.Errorf("the hidden directive must not prescribe a default gender, got:\n%s", hidden) + } + if !strings.Contains(hidden, "перестрой") { + t.Errorf("the hidden directive must tell the editor to recast the sentence, got:\n%s", hidden) + } none := RenderEditorConstraintBlock(mk(""), ruTX()) if !strings.HasSuffix(none, "«Фан Юань»") { t.Fatalf("a genderless term must have NO gender note (line ends at the dst), got:\n%s", none) diff --git a/backend/internal/pipeline/bookbuild.go b/backend/internal/pipeline/bookbuild.go index e5e18381..fc376630 100644 --- a/backend/internal/pipeline/bookbuild.go +++ b/backend/internal/pipeline/bookbuild.go @@ -12,6 +12,7 @@ import ( "time" "textmachine/backend/internal/bookfile" + "textmachine/backend/internal/chunk" "textmachine/backend/internal/lang" ) @@ -313,9 +314,15 @@ func (r *Runner) buildFromExport(exp *BookExport, stale map[UnitRef]bool, staleU // The markers the text carries, from the same pair datum the ingest wrote them with. Read HERE rather // than inside the assembler because it is the pack that knows them, and the assembler is deliberately // a pure function of the export — one place reads the pair, both formats render from one model. - if im := r.packInline(); im != nil { + // …and only for a source the ingest actually marks: a .txt book's asterisks are the author's, and + // rendering them as italics would put emphasis in the reader's file that nobody wrote. + if im := r.packInline(); im != nil && chunk.CarriesInlineMarkup(r.Book.SourceFile) { book.ItalicOpen, book.ItalicClose = im.ItalicOpen, im.ItalicClose } + // ⚠ AFTER the glyphs and not before: the shelf description is built from the SAME strings the chapters + // carry, so it has to be rendered by a book that knows what a marker is. Built inside the assembler — + // where it used to be — it saw an empty glyph pair and the rendering did nothing at all. + setShelfDescription(book) complete := len(holes) == 0 && exp.GhostRows == 0 if !complete && !opts.Partial { return nil, refuse(RefusalBookIncomplete, describeHoles(r.Book.BookID, exp, holes, staleUnknown)) @@ -638,10 +645,25 @@ func assembleBook(exp *BookExport, stale map[UnitRef]bool, title, language strin if exp.GhostRows > 0 { book.Notice = append(book.Notice, lang.FillReaderTemplate(words.NoticeGhost, "ghost", strconv.Itoa(exp.GhostRows))) } - book.Description = strings.Join(book.Notice, " ") + // The assembler owes its callers a complete book, description included — several tests build one + // directly. It is built again by BuildBook once the pair's glyphs are on the book, because only then + // can a marker be rendered rather than escaped; the function is a pure projection of Notice, so + // running it twice is the same answer with more information the second time. + setShelfDescription(book) return book, holes, nil } +// setShelfDescription fills dc:description from the book's notice. Metadata admits no markup, so the +// markers are rendered into the plain-text convention rather than escaped as engine glyphs: a pair whose +// glyphs are ⟦i⟧ would otherwise put them on a library shelf. +func setShelfDescription(book *bookfile.Book) { + flat := make([]string, 0, len(book.Notice)) + for _, p := range book.Notice { + flat = append(flat, book.PlainEmphasis(p)) + } + book.Description = strings.Join(flat, " ") +} + // textParagraphs splits export text into paragraphs: one per non-blank line (CR, LF and CRLF all end a // line — what text.NormalizeSource does on ingest, so a bare CR cannot make the two formats disagree), // trimmed and cleaned of the control characters no reader's format admits (bookfile.CleanText — applied diff --git a/backend/internal/pipeline/bookbuild_test.go b/backend/internal/pipeline/bookbuild_test.go index 88cf156f..1d4e8a93 100644 --- a/backend/internal/pipeline/bookbuild_test.go +++ b/backend/internal/pipeline/bookbuild_test.go @@ -26,6 +26,9 @@ import ( // chunker does (blank lines, trimmed, empties dropped — chunk.splitParagraphs). func ingestParagraphs(t *testing.T, epubPath string) [][]string { t.Helper() + // ⚠ Read with NO emphasis rules, which is right only while the fixtures below carry none: the writer + // would render and this reader would hand back bare text, so the circle would «fail» on a book + // that is in fact whole. A fixture that arms Book.ItalicOpen has to pass the same glyphs here. doc, err := chunk.IngestEncoded(epubPath, "", "", nil, nil) if err != nil { t.Fatalf("ingest of the written EPUB failed: %v", err) diff --git a/backend/internal/pipeline/secondpair_test.go b/backend/internal/pipeline/secondpair_test.go index 0772c345..e65d0219 100644 --- a/backend/internal/pipeline/secondpair_test.go +++ b/backend/internal/pipeline/secondpair_test.go @@ -8,6 +8,8 @@ import ( "path/filepath" "strings" "testing" + + "textmachine/backend/internal/lang" ) // secondpair_test.go: the money-path half of "a second language pair is DATA" (backlog row 347). The @@ -140,10 +142,21 @@ func TestTheSecondPairSizesItsOwnTerminologyCalls(t *testing.T) { if want := r.pack.Terminology; want == nil || batch != want.BatchRunes || width != want.KWICWidth || per != want.KWICPerTerm { t.Errorf("the engine must size the call from the pair's own file, got batch=%d per=%d width=%d for %+v", batch, per, width, want) } - if got := r.basisWidth(); r.pack.Terminology == nil || got != r.pack.Terminology.BasisWidth { - t.Errorf("the evidence width must come from the pair too (it decides which settled terms survive), got %d", got) + // ⚠ WHAT THE LINES ABOVE CANNOT SAY. A key whose pair value EQUALS the engine default is invisible to + // them: agreement proves nothing about which rung supplied it. kwic_per_term is such a key by design (a + // count of contexts does not scale with the script), so the rung is exercised HERE, on values no + // default can be confused with — without this the test would report for four knobs and measure three. + if r.pack.Terminology.KWICPerTerm != terminologyDefaultKWICPer { + t.Fatalf("premise broken: kwic_per_term (%d) no longer equals the engine default (%d), so the check below is testing something else", + r.pack.Terminology.KWICPerTerm, terminologyDefaultKWICPer) + } + r.pack.Terminology = &lang.TerminologySizing{KWICPerTerm: 7, KWICWidth: 123, BasisWidth: 321, BatchRunes: 4567} + if b, p, w := r.terminologyOpts(); b != 4567 || p != 7 || w != 123 { + t.Errorf("every sizing knob must take the pair rung, got batch=%d per=%d width=%d", b, p, w) + } + if got := r.basisWidth(); got != 321 { + t.Errorf("the evidence width must take the pair rung, got %d", got) } - // An explicit run-config value still wins: the pair states what is true of the pair, a run may state // what is true of the run, and the order between them is the whole reason terminologyOpts exists. r.Pipeline.Gates.Terminology.BatchRunes = 777 @@ -247,3 +260,40 @@ func TestTheBriefPolicyOfAPairRefusesAnotherPairsName(t *testing.T) { t.Errorf("an undeclared pair must leave the book's value untouched, got %q", kept.Book.Transcription) } } + +// TestBothPathsSettleTheBriefTheSameWay is the property the pair default silently depends on: the default +// is written INTO the book, the book's brief_hash folds into every snapshot, and the $0 projections +// re-render that snapshot to tell an operator what a run would cost and what it already bought. If only +// the paid path applied the default, status/export/report would compute a different snapshot than the run +// they describe — and nothing else in the suite asks. +func TestBothPathsSettleTheBriefTheSameWay(t *testing.T) { + shipped, err := filepath.Abs("../../configs/langpacks") + if err != nil { + t.Fatal(err) + } + quiet := slog.New(slog.NewTextHandler(io.Discard, nil)) + const vocab = "brief:\n transcription:\n default: practical\n allowed: [practical]\n" + book := setupPairProject(t, shipped, "en", "", vocab) + + w, err := NewRunner(book, quiet) + if err != nil { + t.Fatal(err) + } + writeHash, writeValue := w.Book.BriefHash(), w.Book.Transcription + w.Close() + + ro, err := NewReadOnlyRunner(book, quiet) + if err != nil { + t.Fatal(err) + } + defer ro.Close() + if writeValue != "practical" { + t.Fatalf("premise broken: the write path did not settle the default (%q), so the two paths could agree by both doing nothing", writeValue) + } + if got := ro.Book.Transcription; got != writeValue { + t.Errorf("the $0 path settled %q where the paid path settled %q", got, writeValue) + } + if got := ro.Book.BriefHash(); got != writeHash { + t.Errorf("the two paths hash the brief differently (%s vs %s) — every projection would then describe a snapshot the run never used", got[:12], writeHash[:12]) + } +} diff --git a/backend/internal/pipeline/testdata/golden/capture.golden b/backend/internal/pipeline/testdata/golden/capture.golden index e639893a..4c3f7476 100644 --- a/backend/internal/pipeline/testdata/golden/capture.golden +++ b/backend/internal/pipeline/testdata/golden/capture.golden @@ -1,8 +1,8 @@ ==== run 1 (fresh) ==== -snapshot_draft: 3db58425badf1f09ccdb9861f881c298c8471ff1235c4a6cdc0182de841bd9a5 -snapshot_draft_payload: {"brief_hash":"4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63","chunker_version":"chunker-v7-toc-cut+competition+srcabbrev+u15.0.0","estimator_version":"estimator-v0+u15.0.0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v3-refusal+srcscript-echo015+langscreen-v1-target-script-share+floor0.50+min200+loop+u15.0.0","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"cache_ttl":""},"segmentation":{"draft_budget_out":1797,"edit_ceiling_out":3200,"fertility_cjk":1.1978,"fertility_other":0.3852},"render_format_version":"renderfmt-v4-one-law-block+no-unverified-marker+draft-gender-on-match","embedded_version":"embed-v1-3df4329e3b9e","memory_version":"3fdc75361ef97fbc105bace256645fb10ab8a3ce807c0a4a6863f5ab73546600","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v7-checkerpack-attribution+latincaps+p2split+combmark+u15.0.0","sanitizer":{"enabled":true,"version":"sanitizer-v7-hangul-leak"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"b0e0777e281beea5aa59ecccd052f12a00d7a6e244b081ef735f2259c0e94d5d","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}}]} -snapshot_edit: 5335a2f28de54809e28056a79c67f243fbb05a7277ad82afb43211979e247156 -snapshot_edit_payload: {"brief_hash":"4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63","chunker_version":"chunker-v7-toc-cut+competition+srcabbrev+u15.0.0","estimator_version":"estimator-v0+u15.0.0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v3-refusal+srcscript-echo015+langscreen-v1-target-script-share+floor0.50+min200+loop+u15.0.0","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"cache_ttl":""},"segmentation":{"draft_budget_out":1797,"edit_ceiling_out":3200,"fertility_cjk":1.1978,"fertility_other":0.3852},"render_format_version":"renderfmt-v4-one-law-block+no-unverified-marker+draft-gender-on-match","embedded_version":"embed-v1-3df4329e3b9e","memory_version":"da2e51866d07f947d0d99a99f1a386720ee3756001cc2fda1925fa98b9809a47","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v7-checkerpack-attribution+latincaps+p2split+combmark+u15.0.0","sanitizer":{"enabled":true,"version":"sanitizer-v7-hangul-leak"},"stages":[{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]} +snapshot_draft: 06598803d28c4d040b72aa0e5a86bf495420a59518ceab2ed0ddfd631c69f332 +snapshot_draft_payload: {"brief_hash":"4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63","chunker_version":"chunker-v7-toc-cut+competition+srcabbrev+u15.0.0","estimator_version":"estimator-v0+u15.0.0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v3-refusal+srcscript-echo015+langscreen-v1-target-script-share+floor0.50+min200+loop+u15.0.0","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"cache_ttl":""},"segmentation":{"draft_budget_out":1797,"edit_ceiling_out":3200,"fertility_cjk":1.1978,"fertility_other":0.3852},"render_format_version":"renderfmt-v4-one-law-block+no-unverified-marker+draft-gender-on-match","embedded_version":"embed-v1-6cde006fc1ba","memory_version":"3fdc75361ef97fbc105bace256645fb10ab8a3ce807c0a4a6863f5ab73546600","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v7-checkerpack-attribution+latincaps+p2split+combmark+u15.0.0","sanitizer":{"enabled":true,"version":"sanitizer-v7-hangul-leak"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"b0e0777e281beea5aa59ecccd052f12a00d7a6e244b081ef735f2259c0e94d5d","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}}]} +snapshot_edit: a6cbcc322fabf79ab2f733040485d18beb34b286d1df0a865d245cb73ecb9152 +snapshot_edit_payload: {"brief_hash":"4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63","chunker_version":"chunker-v7-toc-cut+competition+srcabbrev+u15.0.0","estimator_version":"estimator-v0+u15.0.0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v3-refusal+srcscript-echo015+langscreen-v1-target-script-share+floor0.50+min200+loop+u15.0.0","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"cache_ttl":""},"segmentation":{"draft_budget_out":1797,"edit_ceiling_out":3200,"fertility_cjk":1.1978,"fertility_other":0.3852},"render_format_version":"renderfmt-v4-one-law-block+no-unverified-marker+draft-gender-on-match","embedded_version":"embed-v1-6cde006fc1ba","memory_version":"da2e51866d07f947d0d99a99f1a386720ee3756001cc2fda1925fa98b9809a47","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v7-checkerpack-attribution+latincaps+p2split+combmark+u15.0.0","sanitizer":{"enabled":true,"version":"sanitizer-v7-hangul-leak"},"stages":[{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]} brief_hash: 4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63 memory_version: da2e51866d07f947d0d99a99f1a386720ee3756001cc2fda1925fa98b9809a47 base_memory_version: 3fdc75361ef97fbc105bace256645fb10ab8a3ce807c0a4a6863f5ab73546600 @@ -51,42 +51,42 @@ chunk ch8/0 disposition=flagged flag="sanitizer_stripped" final_text="Судзу stage=edit role=editor model=fake-model resume=false disp=flagged flag="sanitizer_stripped" attempts=1 escalated=false esc_model="" finish="stop" cum_usd=0.00182 detail="CJK leak in the ru output: 特产" stage_text="" recovered="Судзуки нашёл древний камень на каменном алтаре." -- chunk_status -- -ch1/0 draft snap_match=true content_hash=9357e9264f5fbd7fa131b5bc4b7dc651d6a9fa0705a2a623674861d97d82a260 disp=ok flag="" attempts=1 final_hash=9188f58cbc206194bbefe32fe474f32c5a28d9f0eb1a8c74c673b654183143df cost=0.00182 escalated=false esc_model="" detail="" -ch1/0 edit snap_match=true content_hash=57b9e42899ec3947ad4113689d27d9de40a968d3cc9f699d649b43d985c94827 disp=ok flag="" attempts=1 final_hash=a185d245727422d182dbcbcf4b6417bcb48b1a3ae698e4b7e474567edf02c2a2 cost=0.00182 escalated=false esc_model="" detail="" -ch1/1 draft snap_match=true content_hash=d9f313e1ca7f8c7d3d6212cca8d82e79d1cf407be097dd76aa51aaada96a6c0f disp=ok flag="" attempts=1 final_hash=817fce6b23f2a560ee97f6cf8fc0a2b3d4ca9773168e0489911f2d0ab4a7a2b6 cost=0.00182 escalated=false esc_model="" detail="" -ch2/0 draft snap_match=true content_hash=bc436ccbaab13c4b8ba3aa3f68473e8c2adebea2ca9f2fbcbc6adbc68b36eda5 disp=ok flag="" attempts=1 final_hash=f35ba81640203756b0d41f1b90a2bf22255c8e357c0114e394b9847bbafc476d cost=0.00728 escalated=true esc_model="fake-fallback" detail="" -ch2/0 edit snap_match=true content_hash=ab1c286d0ae58110c4998fbb0c395906e2acf63e5db5ff68dcf4ae3f7078cb3a disp=ok flag="" attempts=1 final_hash=8b804da55dd6db91150169fb6b7988de8d437a042a74b2b23474dff47351da68 cost=0.00182 escalated=false esc_model="" detail="" +ch1/0 draft snap_match=true content_hash=9357e9264f5fbd7fa131b5bc4b7dc651d6a9fa0705a2a623674861d97d82a260 disp=ok flag="" attempts=1 final_hash=65b92b8cd320b964d70ce3a45074d57a71c759e552ae25fedfac24d6c93ee86f cost=0.00182 escalated=false esc_model="" detail="" +ch1/0 edit snap_match=true content_hash=57b9e42899ec3947ad4113689d27d9de40a968d3cc9f699d649b43d985c94827 disp=ok flag="" attempts=1 final_hash=67868ef95c722d32e58d19e7bc32c3ff7da3f13713f5b98431f97a8183ac92b0 cost=0.00182 escalated=false esc_model="" detail="" +ch1/1 draft snap_match=true content_hash=d9f313e1ca7f8c7d3d6212cca8d82e79d1cf407be097dd76aa51aaada96a6c0f disp=ok flag="" attempts=1 final_hash=992dd12f00faff07127eb4ebf753236a99577b2197551b4213938943fd6bb361 cost=0.00182 escalated=false esc_model="" detail="" +ch2/0 draft snap_match=true content_hash=bc436ccbaab13c4b8ba3aa3f68473e8c2adebea2ca9f2fbcbc6adbc68b36eda5 disp=ok flag="" attempts=1 final_hash=4483ce58dde1e6438259ad13d84fd75b7e4f53b80289731550dbdc760d5d2ee9 cost=0.00728 escalated=true esc_model="fake-fallback" detail="" +ch2/0 edit snap_match=true content_hash=ab1c286d0ae58110c4998fbb0c395906e2acf63e5db5ff68dcf4ae3f7078cb3a disp=ok flag="" attempts=1 final_hash=ecb9e90d49517f91b3c5662f6f9bbc0ee70e26fa6fb77e83a18b21dcf440e69f cost=0.00182 escalated=false esc_model="" detail="" ch3/0 draft snap_match=true content_hash=31027623851f1c0fe66d22e584141ac299f4b007badd57e6a0458275db424333 disp=flagged flag="hard_refusal" attempts=1 final_hash= cost=0.00728 escalated=true esc_model="" detail="provider finish_reason=refusal" ch3/0 edit snap_match=true content_hash= disp=skipped flag="hard_refusal" attempts=0 final_hash= cost=0 escalated=false esc_model="" detail="skipped: a member draft chunk of this edit unit was flagged (hard_refusal)" -ch4/0 draft snap_match=true content_hash=26dc8bb1a2837c31f988f93b99437e6fe8fb26ad7a3a347b9837343d6483fc7c disp=ok flag="" attempts=1 final_hash=9c6f57b88dbf33e6f0246714fcb72a73a9dff28eb0cef14dc7080080cabd6763 cost=0.00182 escalated=false esc_model="" detail="" -ch4/0 edit snap_match=true content_hash=b4bcdb493ead3d4b61cac56ff3f75faff6e04d34c3411b79abef3ec6d5b3c3f1 disp=ok flag="" attempts=1 final_hash=f1dffcea2dd3c98b175970a77b6158d0c3dd6c63567e274329435f490b93f278 cost=0.00182 escalated=false esc_model="" detail="" -ch5/0 draft snap_match=true content_hash=fa95a55fdf1fe43edc9aae59b30dffa957fa64a5ce362b48651a72282b6d95aa disp=ok flag="" attempts=1 final_hash=050a557af05ef85f88402e9bcd92b3dffb3b670c4179243988217457bfb65249 cost=0.00182 escalated=false esc_model="" detail="" -ch5/0 edit snap_match=true content_hash=228d7b5d47e8f1871cbd4347618d755248cee25c63c724bb884273ce4db21377 disp=ok flag="" attempts=1 final_hash=a26ed79bc55f7d0081127d3d049a7cc54e8ba8411af8f39814a56b976d7f1bc9 cost=0.00182 escalated=false esc_model="" detail="" -ch6/0 draft snap_match=true content_hash=66d0e00f66f0ab514a02e95080da54c3ffd356f1266c948e4fe7ae08a705f2b1 disp=ok flag="" attempts=1 final_hash=ee9921ee4123ef34e2f7452af3b12377cd43be64d2a4074e532b77e34ae32791 cost=0.00182 escalated=false esc_model="" detail="" +ch4/0 draft snap_match=true content_hash=26dc8bb1a2837c31f988f93b99437e6fe8fb26ad7a3a347b9837343d6483fc7c disp=ok flag="" attempts=1 final_hash=fe679464cf11c90c2c770c504ac57893c6b64a1e4520422214c12954b08d7ebd cost=0.00182 escalated=false esc_model="" detail="" +ch4/0 edit snap_match=true content_hash=b4bcdb493ead3d4b61cac56ff3f75faff6e04d34c3411b79abef3ec6d5b3c3f1 disp=ok flag="" attempts=1 final_hash=1ce71147cca670b80ec1e7ad0eb3b84cfaf60cef3e2e2c5d48ba85a1858d9f04 cost=0.00182 escalated=false esc_model="" detail="" +ch5/0 draft snap_match=true content_hash=fa95a55fdf1fe43edc9aae59b30dffa957fa64a5ce362b48651a72282b6d95aa disp=ok flag="" attempts=1 final_hash=a7b3084613c82738d768ea19841082d19e5314e1addbb079bf011fe6cd19f2a5 cost=0.00182 escalated=false esc_model="" detail="" +ch5/0 edit snap_match=true content_hash=228d7b5d47e8f1871cbd4347618d755248cee25c63c724bb884273ce4db21377 disp=ok flag="" attempts=1 final_hash=f09c2dc782a9c0517ed5de4ca456b2f701fe409363c9ebae1028b8c86ccd7ea8 cost=0.00182 escalated=false esc_model="" detail="" +ch6/0 draft snap_match=true content_hash=66d0e00f66f0ab514a02e95080da54c3ffd356f1266c948e4fe7ae08a705f2b1 disp=ok flag="" attempts=1 final_hash=8185f4555a6ff0386fb3141e9ea9de015455b3890e1349ce1071148ab29c3828 cost=0.00182 escalated=false esc_model="" detail="" ch6/0 edit snap_match=true content_hash=592353321cc8000e792e0897446e4dd4f88a0280b9368b62cbd5c82b0c433193 disp=flagged flag="sanitizer_defect" attempts=1 final_hash= cost=0.00182 escalated=false esc_model="" detail="leading service preamble: Вот перевод фрагмента:" -ch7/0 draft snap_match=true content_hash=9e79ad881bfa17a48b4a73691eaf18d57323adcd8b62c15b737ab0718dd1ac85 disp=ok flag="" attempts=1 final_hash=aae737a0dca760f62367824f85adb487866cbecc85454c8e3a156bb77af5929a cost=0.00182 escalated=false esc_model="" detail="" -ch7/0 edit snap_match=true content_hash=ae9a74bd0e51aec72fe81140675710dcfb8f76a542678b08e0e3f45ff43396b1 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:54a2105bb56a8b04c2b805ce29f7475082ac6949b013097eef4a7d9f7d100451 cost=0.00182 escalated=false esc_model="" detail="markdown header in the output: ### Глава 7" -ch8/0 draft snap_match=true content_hash=ab7031c53bdff2d66898262d25097bb9e35d0bf9dcfad3f28a1a38314fbe2cb6 disp=ok flag="" attempts=1 final_hash=9ef56bc3cee71fa857173bb497297dcf7ab5f057889197c4f04dfaf5c966dd4a cost=0.00182 escalated=false esc_model="" detail="" -ch8/0 edit snap_match=true content_hash=5fdfcb19354338d05c570c00be4085fc3e3cc3c186439f931406b70c312c9549 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:623c6917f8a8ef481ffef060f501ff322e8ac27739cddb8cf97c9069b3ccbf62 cost=0.00182 escalated=false esc_model="" detail="CJK leak in the ru output: 特产" +ch7/0 draft snap_match=true content_hash=9e79ad881bfa17a48b4a73691eaf18d57323adcd8b62c15b737ab0718dd1ac85 disp=ok flag="" attempts=1 final_hash=fa84bd858a964529c6cc9c8f7bd55e59f0d51d5e0952899185fb93464aafe34d cost=0.00182 escalated=false esc_model="" detail="" +ch7/0 edit snap_match=true content_hash=ae9a74bd0e51aec72fe81140675710dcfb8f76a542678b08e0e3f45ff43396b1 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:ae94867c92ce13548536b4bafb87865adddc55d119a962d05155d61c4bb315be cost=0.00182 escalated=false esc_model="" detail="markdown header in the output: ### Глава 7" +ch8/0 draft snap_match=true content_hash=ab7031c53bdff2d66898262d25097bb9e35d0bf9dcfad3f28a1a38314fbe2cb6 disp=ok flag="" attempts=1 final_hash=906333b369dea730603b5b0a3618d8f6870b7d3408df22b8650a7ad8a3ec0c8b cost=0.00182 escalated=false esc_model="" detail="" +ch8/0 edit snap_match=true content_hash=5fdfcb19354338d05c570c00be4085fc3e3cc3c186439f931406b70c312c9549 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:12dd4643c710e6b2db5c4be51c06002f80711366a6ad4e307667d057f0a49c87 cost=0.00182 escalated=false esc_model="" detail="CJK leak in the ru output: 特产" -- request_log (insertion order) -- -ch1/0 draft role=translator req=fake-model actual=fake-model hash=9188f58cbc206194bbefe32fe474f32c5a28d9f0eb1a8c74c673b654183143df tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/1 draft role=translator req=fake-model actual=fake-model hash=817fce6b23f2a560ee97f6cf8fc0a2b3d4ca9773168e0489911f2d0ab4a7a2b6 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 draft role=translator req=fake-model actual=fake-model hash=92f39bc45bfce2e0351ba3fc88e9ee38a4245561cb9707b93b2bbecc10dd8507 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=f35ba81640203756b0d41f1b90a2bf22255c8e357c0114e394b9847bbafc476d tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" -ch3/0 draft role=translator req=fake-model actual=fake-model hash=b9490e6d956d1ae91e7f3f68c423cb0d3c7f04a4ffbb4bb0e29dd3d3120706b9 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=966840c468d9b5455b455ee0fbf73a19e0a63793f712dd6710a393914b2be28e tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" -ch4/0 draft role=translator req=fake-model actual=fake-model hash=9c6f57b88dbf33e6f0246714fcb72a73a9dff28eb0cef14dc7080080cabd6763 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch5/0 draft role=translator req=fake-model actual=fake-model hash=050a557af05ef85f88402e9bcd92b3dffb3b670c4179243988217457bfb65249 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch6/0 draft role=translator req=fake-model actual=fake-model hash=ee9921ee4123ef34e2f7452af3b12377cd43be64d2a4074e532b77e34ae32791 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch7/0 draft role=translator req=fake-model actual=fake-model hash=aae737a0dca760f62367824f85adb487866cbecc85454c8e3a156bb77af5929a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch8/0 draft role=translator req=fake-model actual=fake-model hash=9ef56bc3cee71fa857173bb497297dcf7ab5f057889197c4f04dfaf5c966dd4a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/0 edit role=editor req=fake-model actual=fake-model hash=a185d245727422d182dbcbcf4b6417bcb48b1a3ae698e4b7e474567edf02c2a2 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 edit role=editor req=fake-model actual=fake-model hash=8b804da55dd6db91150169fb6b7988de8d437a042a74b2b23474dff47351da68 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch4/0 edit role=editor req=fake-model actual=fake-model hash=f1dffcea2dd3c98b175970a77b6158d0c3dd6c63567e274329435f490b93f278 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch5/0 edit role=editor req=fake-model actual=fake-model hash=a26ed79bc55f7d0081127d3d049a7cc54e8ba8411af8f39814a56b976d7f1bc9 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch6/0 edit role=editor req=fake-model actual=fake-model hash=84a676c941fa19db61796bb578fc71dc229b0121018e43cc24d84b645833281b tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch7/0 edit role=editor req=fake-model actual=fake-model hash=492913d55307d1389e94debcd4c2e8a7b4a003f0fb7fab30fc13ae7467c0027c tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch8/0 edit role=editor req=fake-model actual=fake-model hash=71863425fa8e04e406061563152ded585e62854520e5e92b8b06f9ab48f469c1 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/0 draft role=translator req=fake-model actual=fake-model hash=65b92b8cd320b964d70ce3a45074d57a71c759e552ae25fedfac24d6c93ee86f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/1 draft role=translator req=fake-model actual=fake-model hash=992dd12f00faff07127eb4ebf753236a99577b2197551b4213938943fd6bb361 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 draft role=translator req=fake-model actual=fake-model hash=da74942e0901b4a258364e08773eedb636a4868792ba96078a80080c0c6ed778 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=4483ce58dde1e6438259ad13d84fd75b7e4f53b80289731550dbdc760d5d2ee9 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" +ch3/0 draft role=translator req=fake-model actual=fake-model hash=e3c7d5bd46e3ddc2df2613077d940491ce95582d44d02ac8c1d40d68a7f0e182 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=cf8e4ae2b5b34b13b693f281b000e3a76093cf7d69dd282d078ef1e007166c51 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" +ch4/0 draft role=translator req=fake-model actual=fake-model hash=fe679464cf11c90c2c770c504ac57893c6b64a1e4520422214c12954b08d7ebd tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch5/0 draft role=translator req=fake-model actual=fake-model hash=a7b3084613c82738d768ea19841082d19e5314e1addbb079bf011fe6cd19f2a5 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch6/0 draft role=translator req=fake-model actual=fake-model hash=8185f4555a6ff0386fb3141e9ea9de015455b3890e1349ce1071148ab29c3828 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch7/0 draft role=translator req=fake-model actual=fake-model hash=fa84bd858a964529c6cc9c8f7bd55e59f0d51d5e0952899185fb93464aafe34d tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch8/0 draft role=translator req=fake-model actual=fake-model hash=906333b369dea730603b5b0a3618d8f6870b7d3408df22b8650a7ad8a3ec0c8b tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/0 edit role=editor req=fake-model actual=fake-model hash=67868ef95c722d32e58d19e7bc32c3ff7da3f13713f5b98431f97a8183ac92b0 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 edit role=editor req=fake-model actual=fake-model hash=ecb9e90d49517f91b3c5662f6f9bbc0ee70e26fa6fb77e83a18b21dcf440e69f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch4/0 edit role=editor req=fake-model actual=fake-model hash=1ce71147cca670b80ec1e7ad0eb3b84cfaf60cef3e2e2c5d48ba85a1858d9f04 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch5/0 edit role=editor req=fake-model actual=fake-model hash=f09c2dc782a9c0517ed5de4ca456b2f701fe409363c9ebae1028b8c86ccd7ea8 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch6/0 edit role=editor req=fake-model actual=fake-model hash=ba228e860af845b69ba511f3c494bc6dbf5aa193042e66e7075ad37005427171 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch7/0 edit role=editor req=fake-model actual=fake-model hash=a0df0677cb9429b4c1fce9775ac351f91bea4c6908863acd4cc9bbba602e0661 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch8/0 edit role=editor req=fake-model actual=fake-model hash=cb9f38e3635460bfadda519f74b1a1b8279b7031fa66ca40022d07f3a8259da6 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" -- retrieval_state -- ch1/0 snap_match=true exact=3 sticky=0 ambiguous=0 spoiler=1 evicted=0 postcheck_miss=1 style_flags=0 trust_gated=0 injected_ids=["紋章\u001f\u001f0\u001f0","鈴木\u001f\u001f0\u001f0","魔法学院\u001f\u001f0\u001f0"] postcheck_detail=[{"src":"紋章","dst":"герб","disp":"confirmed"}] style_detail= trust_gate_detail= @@ -126,10 +126,10 @@ ch8/0 snap_match=true exact=1 sticky=0 ambiguous=0 spoiler=0 evicted=0 postcheck [16] {"max_tokens":4000,"messages":[{"role":"system","content":"Ты — монолингвальный редактор русского текста. Правь стиль, не меняя смысла. Книга: Золотая книга."},{"role":"system","content":"КАНОНИЧЕСКИЕ ПЕРЕВОДЫ имён и терминов (в черновике термин исходника слева ДОЛЖЕН быть передан именно указанной формой справа — приводи к ней любые расхождения, склоняя по контексту; не вводи иных вариантов и не меняй ничего другого):\n- 鈴木 → «Судзуки» (муж. — мужские родовые формы)"},{"role":"user","content":"Черновик перевода для редактуры: ЧЕРНОВИК ПЕРЕВОДА a579eecf9f7f. МАРКДАУН-СЦЕНА Судзуки открыл седьмую дверь."}],"model":"fake-model","stream":false,"temperature":0.4} [17] {"max_tokens":4000,"messages":[{"role":"system","content":"Ты — монолингвальный редактор русского текста. Правь стиль, не меняя смысла. Книга: Золотая книга."},{"role":"system","content":"КАНОНИЧЕСКИЕ ПЕРЕВОДЫ имён и терминов (в черновике термин исходника слева ДОЛЖЕН быть передан именно указанной формой справа — приводи к ней любые расхождения, склоняя по контексту; не вводи иных вариантов и не меняй ничего другого):\n- 鈴木 → «Судзуки» (муж. — мужские родовые формы)"},{"role":"user","content":"Черновик перевода для редактуры: ЧЕРНОВИК ПЕРЕВОДА 2b4a836568f7. ИЕРОГЛИФ-СЦЕНА Судзуки нашёл камень."}],"model":"fake-model","stream":false,"temperature":0.4} ==== run 2 (resume) ==== -snapshot_draft: 3db58425badf1f09ccdb9861f881c298c8471ff1235c4a6cdc0182de841bd9a5 -snapshot_draft_payload: {"brief_hash":"4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63","chunker_version":"chunker-v7-toc-cut+competition+srcabbrev+u15.0.0","estimator_version":"estimator-v0+u15.0.0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v3-refusal+srcscript-echo015+langscreen-v1-target-script-share+floor0.50+min200+loop+u15.0.0","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"cache_ttl":""},"segmentation":{"draft_budget_out":1797,"edit_ceiling_out":3200,"fertility_cjk":1.1978,"fertility_other":0.3852},"render_format_version":"renderfmt-v4-one-law-block+no-unverified-marker+draft-gender-on-match","embedded_version":"embed-v1-3df4329e3b9e","memory_version":"3fdc75361ef97fbc105bace256645fb10ab8a3ce807c0a4a6863f5ab73546600","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v7-checkerpack-attribution+latincaps+p2split+combmark+u15.0.0","sanitizer":{"enabled":true,"version":"sanitizer-v7-hangul-leak"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"b0e0777e281beea5aa59ecccd052f12a00d7a6e244b081ef735f2259c0e94d5d","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}}]} -snapshot_edit: 5335a2f28de54809e28056a79c67f243fbb05a7277ad82afb43211979e247156 -snapshot_edit_payload: {"brief_hash":"4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63","chunker_version":"chunker-v7-toc-cut+competition+srcabbrev+u15.0.0","estimator_version":"estimator-v0+u15.0.0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v3-refusal+srcscript-echo015+langscreen-v1-target-script-share+floor0.50+min200+loop+u15.0.0","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"cache_ttl":""},"segmentation":{"draft_budget_out":1797,"edit_ceiling_out":3200,"fertility_cjk":1.1978,"fertility_other":0.3852},"render_format_version":"renderfmt-v4-one-law-block+no-unverified-marker+draft-gender-on-match","embedded_version":"embed-v1-3df4329e3b9e","memory_version":"da2e51866d07f947d0d99a99f1a386720ee3756001cc2fda1925fa98b9809a47","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v7-checkerpack-attribution+latincaps+p2split+combmark+u15.0.0","sanitizer":{"enabled":true,"version":"sanitizer-v7-hangul-leak"},"stages":[{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]} +snapshot_draft: 06598803d28c4d040b72aa0e5a86bf495420a59518ceab2ed0ddfd631c69f332 +snapshot_draft_payload: {"brief_hash":"4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63","chunker_version":"chunker-v7-toc-cut+competition+srcabbrev+u15.0.0","estimator_version":"estimator-v0+u15.0.0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v3-refusal+srcscript-echo015+langscreen-v1-target-script-share+floor0.50+min200+loop+u15.0.0","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"cache_ttl":""},"segmentation":{"draft_budget_out":1797,"edit_ceiling_out":3200,"fertility_cjk":1.1978,"fertility_other":0.3852},"render_format_version":"renderfmt-v4-one-law-block+no-unverified-marker+draft-gender-on-match","embedded_version":"embed-v1-6cde006fc1ba","memory_version":"3fdc75361ef97fbc105bace256645fb10ab8a3ce807c0a4a6863f5ab73546600","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v7-checkerpack-attribution+latincaps+p2split+combmark+u15.0.0","sanitizer":{"enabled":true,"version":"sanitizer-v7-hangul-leak"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"b0e0777e281beea5aa59ecccd052f12a00d7a6e244b081ef735f2259c0e94d5d","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}}]} +snapshot_edit: a6cbcc322fabf79ab2f733040485d18beb34b286d1df0a865d245cb73ecb9152 +snapshot_edit_payload: {"brief_hash":"4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63","chunker_version":"chunker-v7-toc-cut+competition+srcabbrev+u15.0.0","estimator_version":"estimator-v0+u15.0.0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v3-refusal+srcscript-echo015+langscreen-v1-target-script-share+floor0.50+min200+loop+u15.0.0","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"cache_ttl":""},"segmentation":{"draft_budget_out":1797,"edit_ceiling_out":3200,"fertility_cjk":1.1978,"fertility_other":0.3852},"render_format_version":"renderfmt-v4-one-law-block+no-unverified-marker+draft-gender-on-match","embedded_version":"embed-v1-6cde006fc1ba","memory_version":"da2e51866d07f947d0d99a99f1a386720ee3756001cc2fda1925fa98b9809a47","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v7-checkerpack-attribution+latincaps+p2split+combmark+u15.0.0","sanitizer":{"enabled":true,"version":"sanitizer-v7-hangul-leak"},"stages":[{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]} brief_hash: 4624529b63da1da48ca210155f8c0f193f7d13e107d9d9740f08a7c73809bb63 memory_version: da2e51866d07f947d0d99a99f1a386720ee3756001cc2fda1925fa98b9809a47 base_memory_version: 3fdc75361ef97fbc105bace256645fb10ab8a3ce807c0a4a6863f5ab73546600 @@ -178,58 +178,58 @@ chunk ch8/0 disposition=flagged flag="sanitizer_stripped" final_text="Судзу stage=edit role=editor model=fake-model resume=true disp=flagged flag="sanitizer_stripped" attempts=1 escalated=false esc_model="" finish="sanitized_export" cum_usd=0.00182 detail="CJK leak in the ru output: 特产" stage_text="" recovered="Судзуки нашёл древний камень на каменном алтаре." -- chunk_status -- -ch1/0 draft snap_match=true content_hash=9357e9264f5fbd7fa131b5bc4b7dc651d6a9fa0705a2a623674861d97d82a260 disp=ok flag="" attempts=1 final_hash=9188f58cbc206194bbefe32fe474f32c5a28d9f0eb1a8c74c673b654183143df cost=0.00182 escalated=false esc_model="" detail="" -ch1/0 edit snap_match=true content_hash=57b9e42899ec3947ad4113689d27d9de40a968d3cc9f699d649b43d985c94827 disp=ok flag="" attempts=1 final_hash=a185d245727422d182dbcbcf4b6417bcb48b1a3ae698e4b7e474567edf02c2a2 cost=0.00182 escalated=false esc_model="" detail="" -ch1/1 draft snap_match=true content_hash=d9f313e1ca7f8c7d3d6212cca8d82e79d1cf407be097dd76aa51aaada96a6c0f disp=ok flag="" attempts=1 final_hash=817fce6b23f2a560ee97f6cf8fc0a2b3d4ca9773168e0489911f2d0ab4a7a2b6 cost=0.00182 escalated=false esc_model="" detail="" -ch2/0 draft snap_match=true content_hash=bc436ccbaab13c4b8ba3aa3f68473e8c2adebea2ca9f2fbcbc6adbc68b36eda5 disp=ok flag="" attempts=1 final_hash=f35ba81640203756b0d41f1b90a2bf22255c8e357c0114e394b9847bbafc476d cost=0.00728 escalated=true esc_model="fake-fallback" detail="" -ch2/0 edit snap_match=true content_hash=ab1c286d0ae58110c4998fbb0c395906e2acf63e5db5ff68dcf4ae3f7078cb3a disp=ok flag="" attempts=1 final_hash=8b804da55dd6db91150169fb6b7988de8d437a042a74b2b23474dff47351da68 cost=0.00182 escalated=false esc_model="" detail="" +ch1/0 draft snap_match=true content_hash=9357e9264f5fbd7fa131b5bc4b7dc651d6a9fa0705a2a623674861d97d82a260 disp=ok flag="" attempts=1 final_hash=65b92b8cd320b964d70ce3a45074d57a71c759e552ae25fedfac24d6c93ee86f cost=0.00182 escalated=false esc_model="" detail="" +ch1/0 edit snap_match=true content_hash=57b9e42899ec3947ad4113689d27d9de40a968d3cc9f699d649b43d985c94827 disp=ok flag="" attempts=1 final_hash=67868ef95c722d32e58d19e7bc32c3ff7da3f13713f5b98431f97a8183ac92b0 cost=0.00182 escalated=false esc_model="" detail="" +ch1/1 draft snap_match=true content_hash=d9f313e1ca7f8c7d3d6212cca8d82e79d1cf407be097dd76aa51aaada96a6c0f disp=ok flag="" attempts=1 final_hash=992dd12f00faff07127eb4ebf753236a99577b2197551b4213938943fd6bb361 cost=0.00182 escalated=false esc_model="" detail="" +ch2/0 draft snap_match=true content_hash=bc436ccbaab13c4b8ba3aa3f68473e8c2adebea2ca9f2fbcbc6adbc68b36eda5 disp=ok flag="" attempts=1 final_hash=4483ce58dde1e6438259ad13d84fd75b7e4f53b80289731550dbdc760d5d2ee9 cost=0.00728 escalated=true esc_model="fake-fallback" detail="" +ch2/0 edit snap_match=true content_hash=ab1c286d0ae58110c4998fbb0c395906e2acf63e5db5ff68dcf4ae3f7078cb3a disp=ok flag="" attempts=1 final_hash=ecb9e90d49517f91b3c5662f6f9bbc0ee70e26fa6fb77e83a18b21dcf440e69f cost=0.00182 escalated=false esc_model="" detail="" ch3/0 draft snap_match=true content_hash=31027623851f1c0fe66d22e584141ac299f4b007badd57e6a0458275db424333 disp=flagged flag="hard_refusal" attempts=1 final_hash= cost=0.00728 escalated=true esc_model="" detail="provider finish_reason=refusal" ch3/0 edit snap_match=true content_hash= disp=skipped flag="hard_refusal" attempts=0 final_hash= cost=0 escalated=false esc_model="" detail="skipped: a member draft chunk of this edit unit was flagged (hard_refusal)" -ch4/0 draft snap_match=true content_hash=26dc8bb1a2837c31f988f93b99437e6fe8fb26ad7a3a347b9837343d6483fc7c disp=ok flag="" attempts=1 final_hash=9c6f57b88dbf33e6f0246714fcb72a73a9dff28eb0cef14dc7080080cabd6763 cost=0.00182 escalated=false esc_model="" detail="" -ch4/0 edit snap_match=true content_hash=b4bcdb493ead3d4b61cac56ff3f75faff6e04d34c3411b79abef3ec6d5b3c3f1 disp=ok flag="" attempts=1 final_hash=f1dffcea2dd3c98b175970a77b6158d0c3dd6c63567e274329435f490b93f278 cost=0.00182 escalated=false esc_model="" detail="" -ch5/0 draft snap_match=true content_hash=fa95a55fdf1fe43edc9aae59b30dffa957fa64a5ce362b48651a72282b6d95aa disp=ok flag="" attempts=1 final_hash=050a557af05ef85f88402e9bcd92b3dffb3b670c4179243988217457bfb65249 cost=0.00182 escalated=false esc_model="" detail="" -ch5/0 edit snap_match=true content_hash=228d7b5d47e8f1871cbd4347618d755248cee25c63c724bb884273ce4db21377 disp=ok flag="" attempts=1 final_hash=a26ed79bc55f7d0081127d3d049a7cc54e8ba8411af8f39814a56b976d7f1bc9 cost=0.00182 escalated=false esc_model="" detail="" -ch6/0 draft snap_match=true content_hash=66d0e00f66f0ab514a02e95080da54c3ffd356f1266c948e4fe7ae08a705f2b1 disp=ok flag="" attempts=1 final_hash=ee9921ee4123ef34e2f7452af3b12377cd43be64d2a4074e532b77e34ae32791 cost=0.00182 escalated=false esc_model="" detail="" +ch4/0 draft snap_match=true content_hash=26dc8bb1a2837c31f988f93b99437e6fe8fb26ad7a3a347b9837343d6483fc7c disp=ok flag="" attempts=1 final_hash=fe679464cf11c90c2c770c504ac57893c6b64a1e4520422214c12954b08d7ebd cost=0.00182 escalated=false esc_model="" detail="" +ch4/0 edit snap_match=true content_hash=b4bcdb493ead3d4b61cac56ff3f75faff6e04d34c3411b79abef3ec6d5b3c3f1 disp=ok flag="" attempts=1 final_hash=1ce71147cca670b80ec1e7ad0eb3b84cfaf60cef3e2e2c5d48ba85a1858d9f04 cost=0.00182 escalated=false esc_model="" detail="" +ch5/0 draft snap_match=true content_hash=fa95a55fdf1fe43edc9aae59b30dffa957fa64a5ce362b48651a72282b6d95aa disp=ok flag="" attempts=1 final_hash=a7b3084613c82738d768ea19841082d19e5314e1addbb079bf011fe6cd19f2a5 cost=0.00182 escalated=false esc_model="" detail="" +ch5/0 edit snap_match=true content_hash=228d7b5d47e8f1871cbd4347618d755248cee25c63c724bb884273ce4db21377 disp=ok flag="" attempts=1 final_hash=f09c2dc782a9c0517ed5de4ca456b2f701fe409363c9ebae1028b8c86ccd7ea8 cost=0.00182 escalated=false esc_model="" detail="" +ch6/0 draft snap_match=true content_hash=66d0e00f66f0ab514a02e95080da54c3ffd356f1266c948e4fe7ae08a705f2b1 disp=ok flag="" attempts=1 final_hash=8185f4555a6ff0386fb3141e9ea9de015455b3890e1349ce1071148ab29c3828 cost=0.00182 escalated=false esc_model="" detail="" ch6/0 edit snap_match=true content_hash=592353321cc8000e792e0897446e4dd4f88a0280b9368b62cbd5c82b0c433193 disp=flagged flag="sanitizer_defect" attempts=1 final_hash= cost=0.00182 escalated=false esc_model="" detail="leading service preamble: Вот перевод фрагмента:" -ch7/0 draft snap_match=true content_hash=9e79ad881bfa17a48b4a73691eaf18d57323adcd8b62c15b737ab0718dd1ac85 disp=ok flag="" attempts=1 final_hash=aae737a0dca760f62367824f85adb487866cbecc85454c8e3a156bb77af5929a cost=0.00182 escalated=false esc_model="" detail="" -ch7/0 edit snap_match=true content_hash=ae9a74bd0e51aec72fe81140675710dcfb8f76a542678b08e0e3f45ff43396b1 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:54a2105bb56a8b04c2b805ce29f7475082ac6949b013097eef4a7d9f7d100451 cost=0.00182 escalated=false esc_model="" detail="markdown header in the output: ### Глава 7" -ch8/0 draft snap_match=true content_hash=ab7031c53bdff2d66898262d25097bb9e35d0bf9dcfad3f28a1a38314fbe2cb6 disp=ok flag="" attempts=1 final_hash=9ef56bc3cee71fa857173bb497297dcf7ab5f057889197c4f04dfaf5c966dd4a cost=0.00182 escalated=false esc_model="" detail="" -ch8/0 edit snap_match=true content_hash=5fdfcb19354338d05c570c00be4085fc3e3cc3c186439f931406b70c312c9549 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:623c6917f8a8ef481ffef060f501ff322e8ac27739cddb8cf97c9069b3ccbf62 cost=0.00182 escalated=false esc_model="" detail="CJK leak in the ru output: 特产" +ch7/0 draft snap_match=true content_hash=9e79ad881bfa17a48b4a73691eaf18d57323adcd8b62c15b737ab0718dd1ac85 disp=ok flag="" attempts=1 final_hash=fa84bd858a964529c6cc9c8f7bd55e59f0d51d5e0952899185fb93464aafe34d cost=0.00182 escalated=false esc_model="" detail="" +ch7/0 edit snap_match=true content_hash=ae9a74bd0e51aec72fe81140675710dcfb8f76a542678b08e0e3f45ff43396b1 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:ae94867c92ce13548536b4bafb87865adddc55d119a962d05155d61c4bb315be cost=0.00182 escalated=false esc_model="" detail="markdown header in the output: ### Глава 7" +ch8/0 draft snap_match=true content_hash=ab7031c53bdff2d66898262d25097bb9e35d0bf9dcfad3f28a1a38314fbe2cb6 disp=ok flag="" attempts=1 final_hash=906333b369dea730603b5b0a3618d8f6870b7d3408df22b8650a7ad8a3ec0c8b cost=0.00182 escalated=false esc_model="" detail="" +ch8/0 edit snap_match=true content_hash=5fdfcb19354338d05c570c00be4085fc3e3cc3c186439f931406b70c312c9549 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:12dd4643c710e6b2db5c4be51c06002f80711366a6ad4e307667d057f0a49c87 cost=0.00182 escalated=false esc_model="" detail="CJK leak in the ru output: 特产" -- request_log (insertion order) -- -ch1/0 draft role=translator req=fake-model actual=fake-model hash=9188f58cbc206194bbefe32fe474f32c5a28d9f0eb1a8c74c673b654183143df tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/1 draft role=translator req=fake-model actual=fake-model hash=817fce6b23f2a560ee97f6cf8fc0a2b3d4ca9773168e0489911f2d0ab4a7a2b6 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 draft role=translator req=fake-model actual=fake-model hash=92f39bc45bfce2e0351ba3fc88e9ee38a4245561cb9707b93b2bbecc10dd8507 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=f35ba81640203756b0d41f1b90a2bf22255c8e357c0114e394b9847bbafc476d tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" -ch3/0 draft role=translator req=fake-model actual=fake-model hash=b9490e6d956d1ae91e7f3f68c423cb0d3c7f04a4ffbb4bb0e29dd3d3120706b9 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=966840c468d9b5455b455ee0fbf73a19e0a63793f712dd6710a393914b2be28e tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" -ch4/0 draft role=translator req=fake-model actual=fake-model hash=9c6f57b88dbf33e6f0246714fcb72a73a9dff28eb0cef14dc7080080cabd6763 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch5/0 draft role=translator req=fake-model actual=fake-model hash=050a557af05ef85f88402e9bcd92b3dffb3b670c4179243988217457bfb65249 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch6/0 draft role=translator req=fake-model actual=fake-model hash=ee9921ee4123ef34e2f7452af3b12377cd43be64d2a4074e532b77e34ae32791 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch7/0 draft role=translator req=fake-model actual=fake-model hash=aae737a0dca760f62367824f85adb487866cbecc85454c8e3a156bb77af5929a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch8/0 draft role=translator req=fake-model actual=fake-model hash=9ef56bc3cee71fa857173bb497297dcf7ab5f057889197c4f04dfaf5c966dd4a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/0 edit role=editor req=fake-model actual=fake-model hash=a185d245727422d182dbcbcf4b6417bcb48b1a3ae698e4b7e474567edf02c2a2 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 edit role=editor req=fake-model actual=fake-model hash=8b804da55dd6db91150169fb6b7988de8d437a042a74b2b23474dff47351da68 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch4/0 edit role=editor req=fake-model actual=fake-model hash=f1dffcea2dd3c98b175970a77b6158d0c3dd6c63567e274329435f490b93f278 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch5/0 edit role=editor req=fake-model actual=fake-model hash=a26ed79bc55f7d0081127d3d049a7cc54e8ba8411af8f39814a56b976d7f1bc9 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch6/0 edit role=editor req=fake-model actual=fake-model hash=84a676c941fa19db61796bb578fc71dc229b0121018e43cc24d84b645833281b tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch7/0 edit role=editor req=fake-model actual=fake-model hash=492913d55307d1389e94debcd4c2e8a7b4a003f0fb7fab30fc13ae7467c0027c tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch8/0 edit role=editor req=fake-model actual=fake-model hash=71863425fa8e04e406061563152ded585e62854520e5e92b8b06f9ab48f469c1 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/0 draft role=translator req=fake-model actual=fake-model hash=9188f58cbc206194bbefe32fe474f32c5a28d9f0eb1a8c74c673b654183143df tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch1/1 draft role=translator req=fake-model actual=fake-model hash=817fce6b23f2a560ee97f6cf8fc0a2b3d4ca9773168e0489911f2d0ab4a7a2b6 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch2/0 draft role=translator req=fake-model actual=fake-fallback hash=f35ba81640203756b0d41f1b90a2bf22255c8e357c0114e394b9847bbafc476d tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch1/0 draft role=translator req=fake-model actual=fake-model hash=65b92b8cd320b964d70ce3a45074d57a71c759e552ae25fedfac24d6c93ee86f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/1 draft role=translator req=fake-model actual=fake-model hash=992dd12f00faff07127eb4ebf753236a99577b2197551b4213938943fd6bb361 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 draft role=translator req=fake-model actual=fake-model hash=da74942e0901b4a258364e08773eedb636a4868792ba96078a80080c0c6ed778 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=4483ce58dde1e6438259ad13d84fd75b7e4f53b80289731550dbdc760d5d2ee9 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" +ch3/0 draft role=translator req=fake-model actual=fake-model hash=e3c7d5bd46e3ddc2df2613077d940491ce95582d44d02ac8c1d40d68a7f0e182 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=cf8e4ae2b5b34b13b693f281b000e3a76093cf7d69dd282d078ef1e007166c51 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" +ch4/0 draft role=translator req=fake-model actual=fake-model hash=fe679464cf11c90c2c770c504ac57893c6b64a1e4520422214c12954b08d7ebd tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch5/0 draft role=translator req=fake-model actual=fake-model hash=a7b3084613c82738d768ea19841082d19e5314e1addbb079bf011fe6cd19f2a5 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch6/0 draft role=translator req=fake-model actual=fake-model hash=8185f4555a6ff0386fb3141e9ea9de015455b3890e1349ce1071148ab29c3828 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch7/0 draft role=translator req=fake-model actual=fake-model hash=fa84bd858a964529c6cc9c8f7bd55e59f0d51d5e0952899185fb93464aafe34d tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch8/0 draft role=translator req=fake-model actual=fake-model hash=906333b369dea730603b5b0a3618d8f6870b7d3408df22b8650a7ad8a3ec0c8b tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/0 edit role=editor req=fake-model actual=fake-model hash=67868ef95c722d32e58d19e7bc32c3ff7da3f13713f5b98431f97a8183ac92b0 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 edit role=editor req=fake-model actual=fake-model hash=ecb9e90d49517f91b3c5662f6f9bbc0ee70e26fa6fb77e83a18b21dcf440e69f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch4/0 edit role=editor req=fake-model actual=fake-model hash=1ce71147cca670b80ec1e7ad0eb3b84cfaf60cef3e2e2c5d48ba85a1858d9f04 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch5/0 edit role=editor req=fake-model actual=fake-model hash=f09c2dc782a9c0517ed5de4ca456b2f701fe409363c9ebae1028b8c86ccd7ea8 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch6/0 edit role=editor req=fake-model actual=fake-model hash=ba228e860af845b69ba511f3c494bc6dbf5aa193042e66e7075ad37005427171 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch7/0 edit role=editor req=fake-model actual=fake-model hash=a0df0677cb9429b4c1fce9775ac351f91bea4c6908863acd4cc9bbba602e0661 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch8/0 edit role=editor req=fake-model actual=fake-model hash=cb9f38e3635460bfadda519f74b1a1b8279b7031fa66ca40022d07f3a8259da6 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/0 draft role=translator req=fake-model actual=fake-model hash=65b92b8cd320b964d70ce3a45074d57a71c759e552ae25fedfac24d6c93ee86f tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch1/1 draft role=translator req=fake-model actual=fake-model hash=992dd12f00faff07127eb4ebf753236a99577b2197551b4213938943fd6bb361 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch2/0 draft role=translator req=fake-model actual=fake-fallback hash=4483ce58dde1e6438259ad13d84fd75b7e4f53b80289731550dbdc760d5d2ee9 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" ch3/0 draft role=translator req=fake-model actual=fake-model hash= tm_hit=1 ok=0 finish="" degraded="hard_refusal" cost=0 tokens=0/0/0/0/0 err="" -ch4/0 draft role=translator req=fake-model actual=fake-model hash=9c6f57b88dbf33e6f0246714fcb72a73a9dff28eb0cef14dc7080080cabd6763 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch5/0 draft role=translator req=fake-model actual=fake-model hash=050a557af05ef85f88402e9bcd92b3dffb3b670c4179243988217457bfb65249 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch6/0 draft role=translator req=fake-model actual=fake-model hash=ee9921ee4123ef34e2f7452af3b12377cd43be64d2a4074e532b77e34ae32791 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch7/0 draft role=translator req=fake-model actual=fake-model hash=aae737a0dca760f62367824f85adb487866cbecc85454c8e3a156bb77af5929a tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch8/0 draft role=translator req=fake-model actual=fake-model hash=9ef56bc3cee71fa857173bb497297dcf7ab5f057889197c4f04dfaf5c966dd4a tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch1/0 edit role=editor req=fake-model actual=fake-model hash=a185d245727422d182dbcbcf4b6417bcb48b1a3ae698e4b7e474567edf02c2a2 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch2/0 edit role=editor req=fake-model actual=fake-model hash=8b804da55dd6db91150169fb6b7988de8d437a042a74b2b23474dff47351da68 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch4/0 edit role=editor req=fake-model actual=fake-model hash=f1dffcea2dd3c98b175970a77b6158d0c3dd6c63567e274329435f490b93f278 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch5/0 edit role=editor req=fake-model actual=fake-model hash=a26ed79bc55f7d0081127d3d049a7cc54e8ba8411af8f39814a56b976d7f1bc9 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch4/0 draft role=translator req=fake-model actual=fake-model hash=fe679464cf11c90c2c770c504ac57893c6b64a1e4520422214c12954b08d7ebd tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch5/0 draft role=translator req=fake-model actual=fake-model hash=a7b3084613c82738d768ea19841082d19e5314e1addbb079bf011fe6cd19f2a5 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch6/0 draft role=translator req=fake-model actual=fake-model hash=8185f4555a6ff0386fb3141e9ea9de015455b3890e1349ce1071148ab29c3828 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch7/0 draft role=translator req=fake-model actual=fake-model hash=fa84bd858a964529c6cc9c8f7bd55e59f0d51d5e0952899185fb93464aafe34d tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch8/0 draft role=translator req=fake-model actual=fake-model hash=906333b369dea730603b5b0a3618d8f6870b7d3408df22b8650a7ad8a3ec0c8b tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch1/0 edit role=editor req=fake-model actual=fake-model hash=67868ef95c722d32e58d19e7bc32c3ff7da3f13713f5b98431f97a8183ac92b0 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch2/0 edit role=editor req=fake-model actual=fake-model hash=ecb9e90d49517f91b3c5662f6f9bbc0ee70e26fa6fb77e83a18b21dcf440e69f tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch4/0 edit role=editor req=fake-model actual=fake-model hash=1ce71147cca670b80ec1e7ad0eb3b84cfaf60cef3e2e2c5d48ba85a1858d9f04 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch5/0 edit role=editor req=fake-model actual=fake-model hash=f09c2dc782a9c0517ed5de4ca456b2f701fe409363c9ebae1028b8c86ccd7ea8 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" ch6/0 edit role=editor req=fake-model actual=fake-model hash= tm_hit=1 ok=0 finish="" degraded="sanitizer_defect" cost=0 tokens=0/0/0/0/0 err="" -ch7/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:54a2105bb56a8b04c2b805ce29f7475082ac6949b013097eef4a7d9f7d100451 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err="" -ch8/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:623c6917f8a8ef481ffef060f501ff322e8ac27739cddb8cf97c9069b3ccbf62 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err="" +ch7/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:ae94867c92ce13548536b4bafb87865adddc55d119a962d05155d61c4bb315be tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err="" +ch8/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:12dd4643c710e6b2db5c4be51c06002f80711366a6ad4e307667d057f0a49c87 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err="" -- retrieval_state -- ch1/0 snap_match=true exact=3 sticky=0 ambiguous=0 spoiler=1 evicted=0 postcheck_miss=1 style_flags=0 trust_gated=0 injected_ids=["紋章\u001f\u001f0\u001f0","鈴木\u001f\u001f0\u001f0","魔法学院\u001f\u001f0\u001f0"] postcheck_detail=[{"src":"紋章","dst":"герб","disp":"confirmed"}] style_detail= trust_gate_detail=