171 lines
8.8 KiB
Python
171 lines
8.8 KiB
Python
#!/usr/bin/env python3
|
||
"""Армы задачи B (анти-эхо, D21 п.6) на СЫРОМ проводе — в той конфигурации, в которой боевой канал
|
||
31.07 вообще отвечает.
|
||
|
||
Почему не через движок. После вендор-обновления `deepseek-v4-flash` → `DeepSeek-V4-Flash-0731`
|
||
боевая форма (max_tokens 8000, thinking по умолчанию) отдаёт ПУСТОЙ `content` (§2 отчёта). Единственная
|
||
ручка, которая её чинит, — `reasoning_effort:"low"`, а её движок для deepseek слать НЕ МОЖЕТ и не
|
||
должен: `config.echoMineViolation` (`models.go:276-281`) запрещает эхо-склонному провайдеру любой
|
||
reasoning-контроль, кроме `none|mandatory`. Гейт прав — проба показала, что `low` эхо-мину
|
||
пере-вооружает. Значит замер идёт мимо движка, как шли пробы P4, и это оговаривается вслух.
|
||
|
||
Инварианты замера: фрагменты и параметры БАЙТ-ОДИНАКОВЫ во всех армах, различается ровно промпт.
|
||
Рендер — реплика движкового (`wire_probe.render`), сверенная по `prompt_tokens` с движковым вызовом
|
||
на общем чанке (1679 = 1679).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
import unicodedata
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from pathlib import Path
|
||
|
||
from dotenv import load_dotenv
|
||
from openai import OpenAI
|
||
|
||
REPO = Path("/home/ubuntu/projects/textmachine")
|
||
PL = Path.home() / "books" / "gu-zhenren" / "promptlang"
|
||
sys.path.insert(0, str(REPO / "eval" / "promptlang"))
|
||
sys.path.insert(0, str(REPO / "eval"))
|
||
from wire_probe import render # тот же рендер, что уже сверен с движком по prompt_tokens
|
||
from refusal_bench import split_sentences
|
||
|
||
load_dotenv(REPO / "eval" / ".env")
|
||
PRICE_IN, PRICE_CACHED, PRICE_OUT = 0.14, 0.0028, 0.28
|
||
|
||
|
||
def is_cjk(ch: str) -> bool:
|
||
n = unicodedata.name(ch, "")
|
||
return ("CJK UNIFIED IDEOGRAPH" in n or "CJK COMPATIBILITY IDEOGRAPH" in n
|
||
or n.startswith("HIRAGANA ") or n.startswith("KATAKANA "))
|
||
|
||
|
||
def cjk_share(s: str) -> float:
|
||
return sum(1 for c in s if is_cjk(c)) / len(s) if s else 0.0
|
||
|
||
|
||
def fragments(path: Path, n: int, target: int) -> list[dict]:
|
||
"""Детерминированная нарезка свежего среза на n фрагментов по границам абзацев.
|
||
Одна и та же для всех армов по построению — арму нечего выбирать."""
|
||
text = path.read_text(encoding="gb18030")
|
||
paras = [p for p in text.split("\n") if p.strip()]
|
||
out, cur = [], []
|
||
for p in paras:
|
||
cur.append(p)
|
||
if sum(len(x) for x in cur) >= target:
|
||
out.append("\n".join(cur))
|
||
cur = []
|
||
if len(out) == n:
|
||
break
|
||
if cur and len(out) < n:
|
||
out.append("\n".join(cur))
|
||
return [{"id": f"f{i:02d}", "text": t} for i, t in enumerate(out[:n])]
|
||
|
||
|
||
def call(client, arm: str, frag: dict, cap: int, temp: float, effort: str, raw_dir: Path) -> dict:
|
||
tag = f"{arm}-{frag['id']}"
|
||
fp = raw_dir / f"{tag}.json"
|
||
if fp.exists():
|
||
return json.loads(fp.read_text(encoding="utf-8"))
|
||
system, user = render(arm, frag["text"])
|
||
t0 = time.time()
|
||
kw = dict(model="deepseek-v4-flash", max_tokens=cap, temperature=temp,
|
||
messages=[{"role": "system", "content": system}, {"role": "user", "content": user}])
|
||
if effort:
|
||
kw["extra_body"] = {"reasoning_effort": effort}
|
||
try:
|
||
r = client.chat.completions.create(**kw)
|
||
except Exception as e: # аномалия печатается и персистится, диагноз — по вендор-доке
|
||
rec = {"tag": tag, "arm": arm, "frag": frag["id"], "error": str(e)[:400], "cost_usd": 0.0}
|
||
fp.write_text(json.dumps(rec, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
return rec
|
||
ch = r.choices[0]
|
||
content = ch.message.content or ""
|
||
reasoning = getattr(ch.message, "reasoning_content", None) or ""
|
||
u = r.usage
|
||
pt, ct = u.prompt_tokens or 0, u.completion_tokens or 0
|
||
det = getattr(u, "completion_tokens_details", None)
|
||
rt = (getattr(det, "reasoning_tokens", 0) if det else 0) or 0
|
||
cached = getattr(getattr(u, "prompt_tokens_details", None), "cached_tokens", 0) or 0
|
||
cost = (pt - cached) / 1e6 * PRICE_IN + cached / 1e6 * PRICE_CACHED + ct / 1e6 * PRICE_OUT
|
||
src_ns = len(re.sub(r"\s", "", frag["text"]))
|
||
out_ns = len(re.sub(r"\s", "", content))
|
||
rec = {"tag": tag, "arm": arm, "frag": frag["id"], "model_returned": r.model,
|
||
"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
"max_tokens": cap, "temperature": temp, "reasoning_effort": effort or None,
|
||
"finish": ch.finish_reason, "prompt_tokens": pt, "cached_tokens": cached,
|
||
"completion_tokens": ct, "reasoning_tokens": rt, "cost_usd": round(cost, 6),
|
||
"latency_s": round(time.time() - t0, 1),
|
||
"cjk_share": round(cjk_share(content), 4), "echo": cjk_share(content) > 0.15,
|
||
"src_chars": len(frag["text"]), "content_chars": len(content),
|
||
"len_ratio": round(out_ns / src_ns, 3) if src_ns else None,
|
||
"sent_src": len(split_sentences(frag["text"])), "sent_out": len(split_sentences(content)),
|
||
"system": system, "user": user, "content": content, "reasoning_content": reasoning}
|
||
fp.write_text(json.dumps(rec, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print(f"[{tag}] {ch.finish_reason} out={ct} rt={rt} cjk={rec['cjk_share']:.2f} "
|
||
f"lr={rec['len_ratio']} ${cost:.6f} {rec['latency_s']}с", flush=True)
|
||
return rec
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--arms", default="a0,a1,a2")
|
||
ap.add_argument("--n", type=int, default=12)
|
||
ap.add_argument("--target", type=int, default=1400)
|
||
ap.add_argument("--cap", type=int, default=8000)
|
||
ap.add_argument("--temp", type=float, default=0.3)
|
||
ap.add_argument("--effort", default="low")
|
||
ap.add_argument("--cost-cap", type=float, required=True)
|
||
ap.add_argument("--workers", type=int, default=3)
|
||
ap.add_argument("--raw", type=Path, default=PL / "armsraw")
|
||
a = ap.parse_args()
|
||
|
||
a.raw.mkdir(parents=True, exist_ok=True)
|
||
frags = fragments(PL / "slice" / "guzhenren-s11-16.gb18030.txt", a.n, a.target)
|
||
print(f"фрагментов {len(frags)}: " + ", ".join(f"{f['id']}={len(f['text'])}симв" for f in frags))
|
||
(a.raw / "fragments.json").write_text(json.dumps(frags, ensure_ascii=False, indent=1),
|
||
encoding="utf-8")
|
||
arms = [x for x in a.arms.split(",") if x]
|
||
plan = [(arm, f) for arm in arms for f in frags]
|
||
todo = [(arm, f) for arm, f in plan if not (a.raw / f"{arm}-{f['id']}.json").exists()]
|
||
print(f"план {len(plan)} вызовов, к покупке {len(todo)}; смета ≈ ${len(todo)*0.0008:.4f} "
|
||
f"(из наблюдённых $0.00036–0.00091 на вызов при effort={a.effort})")
|
||
if len(todo) * 0.0008 > a.cost_cap:
|
||
print(f"СТОП: смета выше капа ${a.cost_cap}", file=sys.stderr)
|
||
return 2
|
||
|
||
key = os.environ.get("DEEPSEEK_API_KEY")
|
||
if not key:
|
||
return 2
|
||
client = OpenAI(api_key=key, base_url="https://api.deepseek.com/v1", timeout=600)
|
||
|
||
with ThreadPoolExecutor(max_workers=a.workers) as ex:
|
||
list(ex.map(lambda t: call(client, t[0], t[1], a.cap, a.temp, a.effort, a.raw), todo))
|
||
|
||
recs = [json.loads(p.read_text(encoding="utf-8")) for p in sorted(a.raw.glob("a*-f*.json"))]
|
||
total = sum(r.get("cost_usd", 0) for r in recs)
|
||
print(f"\n{'арм':<5}{'n':>4}{'эхо':>5}{'пусто':>7}{'finish≠stop':>12}{'ср.rt':>8}"
|
||
f"{'ср.len_ratio':>13}{'ср.$':>10}{'сумма $':>10}")
|
||
for arm in arms:
|
||
rs = [r for r in recs if r.get("arm") == arm and "error" not in r]
|
||
if not rs:
|
||
continue
|
||
lr = [r["len_ratio"] for r in rs if r["len_ratio"] is not None and not r["echo"]]
|
||
print(f"{arm:<5}{len(rs):>4}{sum(r['echo'] for r in rs):>5}"
|
||
f"{sum(1 for r in rs if not r['content_chars']):>7}"
|
||
f"{sum(1 for r in rs if r['finish'] != 'stop'):>12}"
|
||
f"{sum(r['reasoning_tokens'] for r in rs)/len(rs):>8.0f}"
|
||
f"{(sum(lr)/len(lr) if lr else 0):>13.2f}"
|
||
f"{sum(r['cost_usd'] for r in rs)/len(rs):>10.6f}"
|
||
f"{sum(r['cost_usd'] for r in rs):>10.6f}")
|
||
print(f"\nВСЕГО ПОТРАЧЕНО АРМАМИ: ${total:.6f}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|