Land the discourse editor: P1a reflow prompt with the D38.4 chengyu atom and a per-stage few_shot toggle (zero-shot for the deepseek-pro swap arm), snapshot-folded and golden-stable
This commit is contained in:
Claude (backend session)
parent
eb409f2b77
commit
0ac5f7a27c
8 changed files with 284 additions and 16 deletions
|
|
@ -55,11 +55,11 @@ stages:
|
|||
# gemini-3.1-pro — премиум-эскалация редактора ТОЛЬКО за санитайзером (утечка преамбул 6/6).
|
||||
model: glm-5
|
||||
prompt: ../prompts/editor.md
|
||||
# Bump v1-monolingual→v2-bilingual-reflow: editor.md стал билингвальным (D30.1) + reflow
|
||||
# (D30.2, убрана «Сохраняй разбивку на абзацы»). Оба меняют SHA промпта → осознанный
|
||||
# --resnapshot. Моно-вариант сохранён отдельно (prompts/editor-mono.md) — подтверждающий
|
||||
# пилот-арм D13.1.
|
||||
prompt_version: v2-bilingual-reflow
|
||||
# Bump v2-bilingual-reflow→v3-discourse-reflow: в editor.md вшито валидированное P1a-ядро
|
||||
# ДИСКУРС-ПЕРЕВЁРСТКИ + few-shot (exp14/D37 §2а — претензия-1 «рубленые абзацы» = рычаг ПРОМПТА,
|
||||
# не модель; P1a реверстает дефект у всех семей). Меняет SHA промпта → осознанный --resnapshot.
|
||||
# Билингв-каркас (D30.1) и omission-осторожность (D34.3) сохранены. Моно-вариант — prompts/editor-mono.md.
|
||||
prompt_version: v3-discourse-reflow
|
||||
temperature: 0.4
|
||||
reasoning: "off"
|
||||
|
||||
|
|
|
|||
|
|
@ -46,8 +46,9 @@ stages:
|
|||
# выше — judge/glm-5, Gemini-слот Ф2 — НЕ трогаем.)
|
||||
model: glm-5
|
||||
prompt: ../prompts/editor.md
|
||||
# Бамп v1-monolingual→v2-bilingual-reflow (label-SHA дисциплина: тот же editor.md, что в C1).
|
||||
prompt_version: v2-bilingual-reflow
|
||||
# Бамп v2-bilingual-reflow→v3-discourse-reflow (label-SHA дисциплина: тот же editor.md, что в C1 —
|
||||
# вшито P1a-ядро ДИСКУРС-ПЕРЕВЁРСТКИ, D37 §2а; лейбл обязан следовать за новым SHA файла).
|
||||
prompt_version: v3-discourse-reflow
|
||||
temperature: 0.4
|
||||
reasoning: "off"
|
||||
|
||||
|
|
|
|||
|
|
@ -82,6 +82,14 @@ type Stage struct {
|
|||
// on permissive providers, enforced at load — never a fall-through to a refusing
|
||||
// SFW provider.
|
||||
Channel string `yaml:"channel"`
|
||||
// FewShot switches the prompt's optional ---FEWSHOT--- example section on/off (D38.4).
|
||||
// nil (absent) = ON: the examples are appended to the system prompt (the P1a discourse
|
||||
// default). false = zero-shot: the ---FEWSHOT--- block is dropped, leaving only the core
|
||||
// instructions — for a reasoning model whose own CoT is disrupted by hand-written examples
|
||||
// (deepseek-thinking swap-arm, exp14 §2а). A no-op for a prompt with no ---FEWSHOT--- section.
|
||||
// Wire-affecting (it changes the system message), so it is folded into the snapshot: a flip
|
||||
// is a loud --resnapshot, never a silent false-hit.
|
||||
FewShot *bool `yaml:"few_shot"`
|
||||
}
|
||||
|
||||
// Gates is the QA-gate config skeleton (пороги полигона; исполнение — Фаза 1).
|
||||
|
|
|
|||
194
backend/internal/pipeline/fewshot_test.go
Normal file
194
backend/internal/pipeline/fewshot_test.go
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"textmachine/backend/internal/config"
|
||||
"textmachine/backend/internal/obs"
|
||||
)
|
||||
|
||||
// TestFewShotSplitAndAssembly pins the D38.4 few-shot toggle at the render layer: a template
|
||||
// splits into a CORE system + a ---FEWSHOT--- block, SystemFor(true) appends the examples (the
|
||||
// P1a discourse default), SystemFor(false) is core-only (zero-shot, for a reasoning model whose
|
||||
// CoT is disrupted by hand examples — deepseek-thinking swap-arm).
|
||||
func TestFewShotSplitAndAssembly(t *testing.T) {
|
||||
tpl := writeTemplate(t, "CORE-система.\n---FEWSHOT---\nПРИМЕР-один\nПРИМЕР-два\n---USER---\n{{text}}")
|
||||
if tpl.System != "CORE-система." {
|
||||
t.Fatalf("System must be the core only, got %q", tpl.System)
|
||||
}
|
||||
if tpl.FewShot != "ПРИМЕР-один\nПРИМЕР-два" {
|
||||
t.Fatalf("FewShot must be the example block, got %q", tpl.FewShot)
|
||||
}
|
||||
if on := tpl.SystemFor(true); on != "CORE-система.\n\nПРИМЕР-один\nПРИМЕР-два" {
|
||||
t.Fatalf("few_shot ON must append the examples after a blank line, got %q", on)
|
||||
}
|
||||
if off := tpl.SystemFor(false); off != "CORE-система." || strings.Contains(off, "ПРИМЕР") {
|
||||
t.Fatalf("few_shot OFF must be core-only, got %q", off)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFewShotAbsentIsNoop: a template with no ---FEWSHOT--- block leaves FewShot empty and the
|
||||
// toggle a no-op, so every existing prompt (translator, analyst, golden stub) is untouched.
|
||||
func TestFewShotAbsentIsNoop(t *testing.T) {
|
||||
tpl := writeTemplate(t, "Только система.\n---USER---\n{{text}}")
|
||||
if tpl.FewShot != "" {
|
||||
t.Fatalf("no ---FEWSHOT--- must leave FewShot empty, got %q", tpl.FewShot)
|
||||
}
|
||||
if tpl.SystemFor(true) != tpl.System || tpl.SystemFor(false) != tpl.System {
|
||||
t.Fatal("the few_shot toggle must be a no-op without a ---FEWSHOT--- block")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFewShotEnabledDefault: an absent flag defaults to ON (preserve current behaviour); an
|
||||
// explicit false/true resolves as written.
|
||||
func TestFewShotEnabledDefault(t *testing.T) {
|
||||
if !fewShotEnabled(config.Stage{}) {
|
||||
t.Fatal("absent few_shot must default to ON")
|
||||
}
|
||||
off := false
|
||||
if fewShotEnabled(config.Stage{FewShot: &off}) {
|
||||
t.Fatal("few_shot:false must disable the examples")
|
||||
}
|
||||
on := true
|
||||
if !fewShotEnabled(config.Stage{FewShot: &on}) {
|
||||
t.Fatal("few_shot:true must enable the examples")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEditorFewShotToggleRealFile guards the boevoy editor.md (D38.4): it carries a
|
||||
// ---FEWSHOT--- block, few_shot ON reproduces the ratified P1a discourse (core + examples +
|
||||
// the 物是人非 chengyu-atom), and few_shot OFF keeps the discourse core (incl. the chengyu-atom)
|
||||
// but drops the examples — never losing the meaning-preservation instructions.
|
||||
func TestEditorFewShotToggleRealFile(t *testing.T) {
|
||||
tpl, err := LoadPromptTemplate(filepath.Join("..", "..", "prompts", "editor.md"))
|
||||
if err != nil {
|
||||
t.Fatalf("load editor.md: %v", err)
|
||||
}
|
||||
if tpl.FewShot == "" {
|
||||
t.Fatal("editor.md must carry a ---FEWSHOT--- block (D38.4 P1a examples)")
|
||||
}
|
||||
on, off := tpl.SystemFor(true), tpl.SystemFor(false)
|
||||
// Core discourse + chengyu-atom present in BOTH modes.
|
||||
for _, mode := range []struct {
|
||||
name string
|
||||
s string
|
||||
}{{"few_shot ON", on}, {"few_shot OFF", off}} {
|
||||
if !strings.Contains(mode.s, "ДИСКУРС-ПЕРЕВЁРСТКА") {
|
||||
t.Errorf("%s must keep the discourse core", mode.name)
|
||||
}
|
||||
if !strings.Contains(mode.s, "物是人非") {
|
||||
t.Errorf("%s must keep the chengyu-atom (物是人非 — both semantic components)", mode.name)
|
||||
}
|
||||
}
|
||||
// Examples ONLY in the few-shot mode.
|
||||
if !strings.Contains(on, "Примеры требуемой") || !strings.Contains(on, "[focal-shift]") {
|
||||
t.Error("few_shot ON must include the P1a examples")
|
||||
}
|
||||
if strings.Contains(off, "Примеры требуемой") || strings.Contains(off, "[narrative") {
|
||||
t.Error("few_shot OFF (zero-shot) must drop the examples")
|
||||
}
|
||||
}
|
||||
|
||||
// setupFewShotProject builds a minimal project whose EDITOR prompt has a ---FEWSHOT--- block
|
||||
// and whose editor stage carries editStageExtra (e.g. ", few_shot: false"). It returns the
|
||||
// resolved snapshot id and payload. The translator prompt has NO block (control).
|
||||
func setupFewShotProject(t *testing.T, editStageExtra string) (string, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"),
|
||||
"CORE-редактор {{genre}}.\n---FEWSHOT---\n[ex] пример перевёрстки.\n---USER---\nИсходник: {{text}}\nЧерновик: {{draft}}")
|
||||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||||
prices_checked: %q
|
||||
default_model: fake-model
|
||||
providers:
|
||||
fake:
|
||||
kind: openai
|
||||
base_url: http://x
|
||||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||||
models:
|
||||
fake-model:
|
||||
provider: fake
|
||||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||||
`, time.Now().UTC().Format("2006-01-02")))
|
||||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
|
||||
core: C1
|
||||
version: 1
|
||||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||||
retries: { regenerate_before_escalate: 0 }
|
||||
context: { glossary_injection: selective, glossary_token_budget: 800 }
|
||||
stages:
|
||||
- { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
|
||||
- { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off"%s }
|
||||
`, editStageExtra))
|
||||
writeFile(t, filepath.Join(dir, "source.txt"), "静かな朝。")
|
||||
writeFile(t, filepath.Join(dir, "book.yaml"), `
|
||||
book_id: test-book
|
||||
title: Тест
|
||||
source_lang: ja
|
||||
target_lang: ru
|
||||
genre: ранобэ
|
||||
audience: тест
|
||||
venuti: 0.5
|
||||
honorifics: keep
|
||||
transcription: polivanov
|
||||
footnotes: minimal
|
||||
pipeline: pipeline.yaml
|
||||
models: models.yaml
|
||||
source_file: source.txt
|
||||
ceilings: { book_usd: 1.0, day_usd: 2.0 }
|
||||
`)
|
||||
r, err := NewRunner(filepath.Join(dir, "book.yaml"), obs.NewLogger())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
id, payload, err := r.snapshotID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id, payload
|
||||
}
|
||||
|
||||
// TestFewShotSnapshotFold: flipping the editor stage's few_shot must be a LOUD --resnapshot (the
|
||||
// snapshot id changes and the resolved flag appears in the payload), never a silent false-hit —
|
||||
// the ---FEWSHOT--- block is inside PromptSHA256 but dropping it at render leaves that hash
|
||||
// unchanged while the wire changes. The translator stage (no block) must NOT fold the field.
|
||||
func TestFewShotSnapshotFold(t *testing.T) {
|
||||
idOn, payloadOn := setupFewShotProject(t, "") // default → ON
|
||||
idOff, payloadOff := setupFewShotProject(t, ", few_shot: false") // zero-shot
|
||||
|
||||
if idOn == idOff {
|
||||
t.Fatal("flipping few_shot must change the snapshot id (loud --resnapshot)")
|
||||
}
|
||||
if !strings.Contains(payloadOn, `"few_shot":true`) {
|
||||
t.Fatalf("default few_shot must fold as true in the snapshot, got: %s", payloadOn)
|
||||
}
|
||||
if !strings.Contains(payloadOff, `"few_shot":false`) {
|
||||
t.Fatalf("few_shot:false must fold as false in the snapshot, got: %s", payloadOff)
|
||||
}
|
||||
// Only the editor stage carries a ---FEWSHOT--- block, so the field must appear exactly once
|
||||
// (the translator stage stays byte-identical in the snapshot).
|
||||
if n := strings.Count(payloadOn, `"few_shot"`); n != 1 {
|
||||
t.Fatalf("only the block-carrying stage must fold few_shot, want 1 occurrence, got %d: %s", n, payloadOn)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFewShotOffRendersCoreOnlyWire: a template with a block but few_shot resolved off must send
|
||||
// a system message WITHOUT the examples — the wire-level guarantee behind the snapshot fold.
|
||||
func TestFewShotOffRendersCoreOnlyWire(t *testing.T) {
|
||||
tpl := writeTemplate(t, "CORE.\n---FEWSHOT---\nПРИМЕР.\n---USER---\n{{text}}")
|
||||
tpl.System = tpl.SystemFor(false) // mirror loadTemplates' per-stage resolution
|
||||
msgs, err := Messages(tpl, RenderVars{Book: testBook(), Text: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := msgs[0].Content; got != "CORE." {
|
||||
t.Fatalf("zero-shot system message must be core-only, got %q", got)
|
||||
}
|
||||
}
|
||||
|
|
@ -65,14 +65,22 @@ const maxTokensPolicyVersion = "maxtok-v1-double-per-attempt"
|
|||
// правит файлы, не код).
|
||||
const userSeparator = "\n---USER---\n"
|
||||
|
||||
// PromptTemplate is one loaded stage template.
|
||||
// fewShotSeparator optionally splits the SYSTEM part into a core prefix and a
|
||||
// trailing few-shot example block (D38.4). A stage may drop the block (few_shot:false)
|
||||
// for a model whose own CoT is disrupted by hand examples (deepseek-thinking, exp14 §2а).
|
||||
// Absent → the whole system part is core; the toggle is a no-op.
|
||||
const fewShotSeparator = "\n---FEWSHOT---\n"
|
||||
|
||||
// PromptTemplate is one loaded stage template. System is the core system prefix;
|
||||
// FewShot is the optional example block (empty when the file has no ---FEWSHOT---).
|
||||
type PromptTemplate struct {
|
||||
System string
|
||||
FewShot string
|
||||
User string
|
||||
SHA256 string // hash of the raw file — участвует в snapshot
|
||||
}
|
||||
|
||||
// LoadPromptTemplate reads and splits a template file.
|
||||
// LoadPromptTemplate reads and splits a template file into core-system / few-shot / user.
|
||||
func LoadPromptTemplate(path string) (*PromptTemplate, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
|
|
@ -80,15 +88,36 @@ func LoadPromptTemplate(path string) (*PromptTemplate, error) {
|
|||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
parts := strings.SplitN(string(raw), userSeparator, 2)
|
||||
t := &PromptTemplate{System: strings.TrimSpace(parts[0]), SHA256: hex.EncodeToString(sum[:])}
|
||||
if len(parts) == 2 {
|
||||
t.User = strings.TrimSpace(parts[1])
|
||||
} else {
|
||||
if len(parts) != 2 {
|
||||
return nil, fmt.Errorf("pipeline: prompt %s lacks the %q separator between system and user parts", path, strings.TrimSpace(userSeparator))
|
||||
}
|
||||
t := &PromptTemplate{User: strings.TrimSpace(parts[1]), SHA256: hex.EncodeToString(sum[:])}
|
||||
// The few-shot block, if present, is the tail of the SYSTEM part (before ---USER---).
|
||||
head := strings.SplitN(parts[0], fewShotSeparator, 2)
|
||||
t.System = strings.TrimSpace(head[0])
|
||||
if len(head) == 2 {
|
||||
t.FewShot = strings.TrimSpace(head[1])
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// SystemFor returns the effective system prompt for a stage: the core prefix, plus the
|
||||
// few-shot block when the stage keeps it on (the default). The whole raw file is still
|
||||
// snapshot-pinned via SHA256, and the few_shot on/off state is folded separately, so
|
||||
// dropping the block is a loud --resnapshot, not a silent divergence.
|
||||
func (t *PromptTemplate) SystemFor(fewShotOn bool) string {
|
||||
if fewShotOn && t.FewShot != "" {
|
||||
return t.System + "\n\n" + t.FewShot
|
||||
}
|
||||
return t.System
|
||||
}
|
||||
|
||||
// fewShotEnabled resolves a stage's few_shot toggle: absent (nil) defaults to ON, so
|
||||
// every existing config keeps the few-shot examples without touching its yaml.
|
||||
func fewShotEnabled(st config.Stage) bool {
|
||||
return st.FewShot == nil || *st.FewShot
|
||||
}
|
||||
|
||||
// RenderVars are the ONLY placeholders a template may use. A fixed struct, not
|
||||
// a map: map iteration order would randomize the render.
|
||||
type RenderVars struct {
|
||||
|
|
|
|||
|
|
@ -150,6 +150,10 @@ func (r *Runner) loadTemplates() error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Resolve the effective system once per stage: fold the few-shot block into
|
||||
// System unless the stage switched it off (D38.4). FewShot is retained so the
|
||||
// snapshot can fold the on/off state only for stages that actually have a block.
|
||||
tpl.System = tpl.SystemFor(fewShotEnabled(st))
|
||||
r.templates[st.Name] = tpl
|
||||
}
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -107,6 +107,13 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
|||
PromptSHA256 string `json:"prompt_sha256"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
Reasoning string `json:"reasoning"`
|
||||
// FewShot folds the stage's few_shot on/off state (D38.4) — but ONLY for a prompt
|
||||
// that HAS a ---FEWSHOT--- block (nil/omitted otherwise, so stages without examples
|
||||
// keep a byte-identical snapshot). The block IS inside the raw file already folded via
|
||||
// PromptSHA256, yet dropping it at render (few_shot:false) leaves PromptSHA256 unchanged
|
||||
// while the wire changes, so the resolved flag must be folded to keep a flip a loud
|
||||
// --resnapshot rather than a silent false-hit — same discipline as Temperature/Reasoning.
|
||||
FewShot *bool `json:"few_shot,omitempty"`
|
||||
// ExtraBody модели меняет ТЕЛО запроса (GLM thinking-off и т.п.) —
|
||||
// без него правка ручки в models.yaml молча не инвалидировала бы
|
||||
// чекпоинты (находка ревью). json.Marshal карты сортирует ключи →
|
||||
|
|
@ -231,6 +238,12 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
|||
PromptVersion: st.PromptVersion, PromptSHA256: r.templates[st.Name].SHA256,
|
||||
Temperature: st.Temperature, Reasoning: st.Reasoning,
|
||||
}
|
||||
// Fold few_shot only where the prompt actually carries a ---FEWSHOT--- block, so
|
||||
// stages without examples stay byte-identical in the snapshot (D38.4).
|
||||
if r.templates[st.Name].FewShot != "" {
|
||||
on := fewShotEnabled(st)
|
||||
ss.FewShot = &on
|
||||
}
|
||||
if prov, ok := r.Models.Providers[r.Models.Models[st.Model].Provider]; ok {
|
||||
ss.ProviderTemp, ss.ProviderMaxTok, ss.ProviderModel = prov.Temperature, prov.MaxTokens, prov.Model
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,25 @@
|
|||
|
||||
Выведи ТОЛЬКО отредактированный текст перевода — без служебных преамбул, комментариев, пояснений, заметок о правках и markdown-заголовков.
|
||||
|
||||
ДИСКУРС-ПЕРЕВЁРСТКА (обязательная операция передачи китайского паратаксиса в русскую гипотактическую прозу — это КОНВЕНЦИЯ русского языка, а не вольность):
|
||||
1. Молча определи смысловые ходы фрагмента: единый фокал/наблюдатель; действие и его непосредственная реакция; одна тема рассуждения; границы — смена говорящего, времени, места или фокала.
|
||||
2. Один повествовательный ход оформляй ОДНИМ русским абзацем, ДАЖЕ если китайский исходник разбит на однофразовые строки. Инструмент слияния — причастные и деепричастные обороты, подчинительные союзы и связки (построй иерархию, сегментируй по смыслу, промаркируй вид/время предикатов, варьируй лексику, добавляй связки). Не сливай ТОЛЬКО ради длины и не теряй смысловые атомы.
|
||||
3. Каждую новую реплику начинай с нового абзаца через тире «—»; слова автора при той же реплике оставляй в её абзаце.
|
||||
4. Разрешено объединять и делить русские предложения; число предложений и абзацев совпадать с исходником НЕ обязано. Запрещено терять смысловые атомы (участников, действия, числа, полярность, заголовки, а также идиомы и чэнъюй: их передавай ОБОИМИ смысловыми компонентами, не сворачивая в общий смысл — 物是人非 = «вещи те же, люди не те», а не «всё изменилось»).
|
||||
|
||||
---FEWSHOT---
|
||||
Примеры требуемой ПЕРЕВЁРСТКИ (показана структура вывода, не стиль):
|
||||
|
||||
[narrative N:1] Исходник построчно — «Он вышел за дверь.» / «Небо было серым.» / «Он вздохнул.» / «Дорога уходила вдаль.» — становится ОДНИМ абзацем:
|
||||
Он вышел за дверь. Небо было серым; вздохнув, он смотрел, как дорога уходит вдаль.
|
||||
|
||||
[dialogue 1:N] Исходник — «Дядя нахмурился и сказал: „Время летит. Садись.“ Племянник ответил: „Спасибо.“» — становится:
|
||||
Дядя нахмурился.
|
||||
— Время летит. Садись, — сказал он.
|
||||
— Спасибо, — ответил племянник.
|
||||
|
||||
[focal-shift] При смене наблюдателя/места/времени — новый абзац, даже если в исходнике это продолжение той же строки.
|
||||
|
||||
---USER---
|
||||
Исходный текст (язык «{{source_lang}}»):
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue