textmachine/eval/adaptive_incontext.py

#!/usr/bin/env python3
"""INDICATIVE in-context experiment for the adaptive-memory verdict (docs/research/14).

Tests THE central BAR question of the "free win" (in-context demonstrations) strand:
does adding the book's own DEMONSTRATIONS (prior translated passages) beat the
deterministic baseline the bank ALREADY ships — soft-glossary injection — for keeping a
recurring entity's rendering canonical and consistent on a REAL zh→ru literary chunk;
and do the learning layer's failure modes actively HARM (Power-of-Noise near-miss
distractor, 2401.14887; over-trust of a poisoned entry, 2510.00829)?

Testbed: 魯迅《阿Q正传》ch5-9 (real PD zh) — a dense chunk where А-Q fantasizes the
revolution and names 8 recurring characters at once. Canonical dst = В. Рогов's published
translation (eval/data/samples/ru/luxun-ah-q-ru.txt); demonstrations are AUTHENTIC Rogov
sentences. Metric = the bank's OWN deterministic post-check (canonical stem present in
output), scored only over entities whose zh key is in the source chunk.

Arms:
  A_bare      no memory (raw model)
  B_glossary  soft-glossary "src → dst" block           <- THE deterministic baseline
  C_demos     glossary + 3 authentic prior passages     <- the learning layer (free win)
  D_noise     glossary + 3 IRRELEVANT prior passages    <- Power-of-Noise near-miss arm
  E_poison    glossary with ONE wrong dst (赵太爷→"Ли")   <- over-trust / silent-injection arm

Model: deepseek-v4-flash (the real draft model; thinking ON by default → avoids the zh
echo). N reps (temp 0.3) for stability. INDICATIVE: small N, hand-built accept-regexes
(self-confirmation risk), one model, one chunk. Not the Phase-2.5 in-house eval — a probe.

Usage: eval/.venv/bin/python eval/adaptive_incontext.py [--reps 3] [--provider deepseek]
Output: stdout table + eval/data/adaptive_incontext.json
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
import refusal_bench as rb  # reuse call_provider + providers quirks + env load

ROOT = Path(__file__).resolve().parent
SRC = ROOT / "data" / "samples" / "zh" / "luxun-ah-q-ch5-9.txt"
OUT = ROOT / "data" / "adaptive_incontext.json"

# Dense multi-entity window (offset 6360, 650 chars): А-Q's revolution fantasy.
CHUNK_OFF, CHUNK_LEN = 6360, 650

# Recurring entities: zh key -> (canonical Rogov dst shown in glossary, accept-regex on
# the canonical STEM allowing Russian declension). Stem match = the bank's post-check (E1).
ENTITIES = {
    "阿Q":   ("А-кью",                    r"А[-\s]?кью"),
    "未庄":  ("Вэйчжуан (деревня)",        r"Вэйчжуан"),
    "赵太爷": ("почтенный Чжао",            r"Чжао"),
    "王胡":  ("Бородатый Ван",             r"Ван\b"),
    "小D":   ("Маленький Дэн",             r"Дэн"),
    "假洋鬼子": ("Поддельный заморский чёрт", r"заморск\w+ чёрт|поддельн"),
    "吴妈":  ("У-ма",                       r"У[-\s]?ма"),
    "邹七嫂": ("тётушка Цзоу Седьмая",      r"Цзоу"),
}

# Arm C: AUTHENTIC prior translated passages (Rogov), each carrying a recurring entity in
# natural inflected Russian — the "demonstration" the learning layer would retrieve.
DEMOS_RELEVANT = [
    "Он знал, что вместо него нанимают Маленького Дэна, этого худого и слабосильного бедняка.",
    "Сын почтенного Чжао добился, наконец, учёной степени сюцая, и об этом оповестили всю деревню.",
    "Жители деревни Вэйчжуан требовали от него только подённой работы да насмехались над ним.",
]

# Arm D: IRRELEVANT prior passages (near-miss distractors) — plausible book prose, but
# about material NOT in this chunk. Tests "better nothing than garbage" on the real wire.
DEMOS_NOISE = [
    "Стены храма Тудигун почернели от времени, и ветер свистел в щелях по ночам.",
    "На реке скрипели вёсла лодочников, а торговцы раскладывали на пристани сушёную рыбу.",
    "В тот год урожай риса выдался скудным, и цены на масло поднялись по всей округе.",
]

SYSTEM = ("Ты профессиональный литературный переводчик. Переведи фрагмент художественного "
          "произведения на русский язык. Это перевод существующего текста, предоставленного "
          "правообладателем. Сохрани все реплики и детали без пропусков; стиль — живой "
          "литературный русский. Выведи ТОЛЬКО перевод, без комментариев.")


def glossary_block(poison=False):
    lines = []
    for zh, (dst, _) in ENTITIES.items():
        d = "господин Ли (важный чиновник)" if (poison and zh == "赵太爷") else dst
        lines.append(f"{zh} → {d}")
    return "ГЛОССАРИЙ (используй эти утверждённые переводы имён/терминов последовательно):\n" + "\n".join(lines)


def demos_block(demos):
    return ("РАНЕЕ ПЕРЕВЕДЁННЫЕ ФРАГМЕНТЫ ЭТОЙ КНИГИ (держи ту же терминологию и стиль):\n"
            + "\n".join(f"— {d}" for d in demos))


def build_user(chunk, arm):
    parts = []
    if arm in ("B_glossary", "C_demos", "D_noise"):
        parts.append(glossary_block())
    if arm == "E_poison":
        parts.append(glossary_block(poison=True))
    if arm == "C_demos":
        parts.append(demos_block(DEMOS_RELEVANT))
    if arm == "D_noise":
        parts.append(demos_block(DEMOS_NOISE))
    parts.append("ФРАГМЕНТ ДЛЯ ПЕРЕВОДА:\n" + chunk)
    return "\n\n".join(parts)


def cjk_ratio(s):
    cjk = len(re.findall(r"[㐀-鿿]", s))
    return round(cjk / max(1, len(s)), 3)


def score(output, present):
    """Bank post-check: fraction of source-present entities whose canonical stem is in output."""
    matched = [zh for zh in present if re.search(ENTITIES[zh][1], output)]
    return round(len(matched) / max(1, len(present)), 3), matched


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--reps", type=int, default=3)
    ap.add_argument("--provider", default="deepseek")
    ap.add_argument("--providers", default=str(ROOT / "providers.json"))
    args = ap.parse_args()

    prov = next(p for p in json.load(open(args.providers))["providers"] if p["name"] == args.provider)
    chunk = SRC.read_text(encoding="utf-8")[CHUNK_OFF:CHUNK_OFF + CHUNK_LEN]
    present = [zh for zh in ENTITIES if zh in chunk]
    print(f"provider={prov['model']}  chunk={len(chunk)}ch  present entities ({len(present)}): {present}\n")

    arms = ["A_bare", "B_glossary", "C_demos", "D_noise", "E_poison"]
    results = {"provider": prov["model"], "present": present, "reps": args.reps, "arms": {}}
    print(f"{'arm':<12} {'canon-consist':<14} {'cjk-echo':<9} {'poison-followed':<16} note")
    print("-" * 78)
    for arm in arms:
        user = build_user(chunk, arm)
        recs = []
        for _ in range(args.reps):
            text, err, usage = rb.call_provider(prov, SYSTEM, user, timeout=180)
            if not text:
                recs.append({"err": err}); continue
            cons, matched = score(text, present)
            rec = {"consist": cons, "matched": matched, "cjk": cjk_ratio(text),
                   "poison_followed": bool(re.search(r"господин Ли|\bЛи\b", text)) and not re.search(r"Чжао", text),
                   "len": len(text), "sample": text[:0]}
            recs.append(rec)
        ok = [r for r in recs if "err" not in r]
        if not ok:
            print(f"{arm:<12} {'ALL-FAILED':<14} {'':<9} {'':<16} {[r['err'] for r in recs][:1]}")
            results["arms"][arm] = {"records": recs}; continue
        mc = round(sum(r["consist"] for r in ok) / len(ok), 3)
        me = round(sum(r["cjk"] for r in ok) / len(ok), 3)
        pf = sum(r["poison_followed"] for r in ok)
        echo = " ECHO!" if me > 0.15 else ""
        results["arms"][arm] = {"mean_consist": mc, "mean_cjk": me, "poison_followed": pf,
                                "n_ok": len(ok), "records": recs}
        print(f"{arm:<12} {mc:<14} {me:<9} {str(pf)+'/'+str(len(ok)):<16} n={len(ok)}{echo}")

    OUT.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"\nsaved -> {OUT}")
    print("Reading: B−A = does the deterministic soft-glossary baseline help; C−B = does the\n"
          "learning layer (demonstrations) beat that baseline; D vs C = Power-of-Noise harm;\n"
          "E poison-followed>0 = over-trust (why the learning layer must NOT touch approved names).")


if __name__ == "__main__":
    main()