182 lines
8.1 KiB
Python
182 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
||
"""§F: кросс-пары. F1 — zh→en терминолог-проход по голд-ростеру (близость стратегий к ru-банку
|
||
владельца). F2 — en→zh на Empire of the Dawn (Кристофф): майнинг термов + терминолог-проход + перевод
|
||
двух окон (deepseek-low и glm-5). Сырьё → ~/books/gu-zhenren/bank-arbitration/ (общий durable пака).
|
||
"""
|
||
from __future__ import annotations
|
||
import argparse
|
||
import collections
|
||
import json
|
||
import os
|
||
import re
|
||
import time
|
||
import zipfile
|
||
from pathlib import Path
|
||
|
||
from dotenv import load_dotenv
|
||
from openai import OpenAI
|
||
|
||
REPO = Path("/home/ubuntu/projects/textmachine")
|
||
HERE = Path(__file__).resolve().parent
|
||
RAW = Path.home() / "books" / "gu-zhenren" / "bank-arbitration"
|
||
KRISTOFF = Path.home() / "books" / "Kristoff_Jay_-_Empire_of_the_Vampire_3_-_Empire_of_the_Dawn.epub"
|
||
load_dotenv(REPO / "eval" / ".env")
|
||
|
||
import sys # noqa: E402
|
||
sys.path.insert(0, str(HERE))
|
||
import consilium_probe as cp # noqa: E402 (ростер, батчер, call, цены, клиенты)
|
||
|
||
TERM_EN_SYS = """You are the terminologist of a publishing-grade translation from Chinese into English.
|
||
Book: «蛊真人» (a cultivation webnovel). You are given the book's term list; for each term — contexts
|
||
from the SOURCE and renderings the draft translators already proposed (in Russian — a parallel edition;
|
||
treat them as evidence of meaning, not as English candidates). You see the whole book at once; pick ONE
|
||
English rendering per term, consistent across the whole book. Transcribe personal and place names in
|
||
pinyin; translate titles, ranks and realia by MEANING. Keep families of related terms sharing a source
|
||
morpheme consistent in English. Dictionary form, no explanations.
|
||
Answer format: one line per term, exactly two fields separated by a TAB: source term, English rendering."""
|
||
|
||
TERM_ZH_SYS = """你是一部出版级奇幻小说英译中项目的术语专家。小说:《Empire of the Dawn》(Jay Kristoff,
|
||
吸血鬼史诗奇幻)。给你这本书的术语表:每个术语附有原文语境。你能看到整本书,请为每个术语选定ONE个
|
||
统一的中文译名,全书一致。人名用约定俗成的音译;头衔、组织、事物按意义翻译,要求文学性、符合奇幻
|
||
小说的中文语感。同源术语的译名要保持词根呼应。只给词典形式,不要解释。
|
||
回答格式:每行一个术语,两个字段,用制表符分隔:原文术语、中文译名。"""
|
||
|
||
TRANSLATE_ZH_SYS = """你是一位专业文学翻译家,把英文小说译成中文。这是 Jay Kristoff 的吸血鬼史诗奇幻
|
||
《Empire of the Dawn》。要求:逐句翻译,不删不增;中文要有文学性,句式自然,避免翻译腔;对话用中文
|
||
小说的标点习惯。人名音译。只输出译文,不要任何前言、注释或 markdown。"""
|
||
|
||
|
||
def epub_text() -> str:
|
||
out = []
|
||
with zipfile.ZipFile(KRISTOFF) as z:
|
||
for n in sorted(z.namelist()):
|
||
if n.endswith((".xhtml", ".html")):
|
||
html = z.read(n).decode("utf-8", "ignore")
|
||
txt = re.sub(r"<[^>]+>", " ", html)
|
||
txt = re.sub(r"&[a-z]+;", " ", txt)
|
||
out.append(re.sub(r"\s+", " ", txt))
|
||
return "\n".join(out)
|
||
|
||
|
||
def mine_en_terms(text: str, top=14):
|
||
"""Рекуррентные капитализированные термы: имена + двусловные (The X / X Y)."""
|
||
stop = {"The", "But", "And", "She", "His", "Her", "They", "That", "This", "What", "When", "With",
|
||
"For", "Not", "You", "All", "Was", "Had", "Chapter", "Then", "Now", "There", "One", "It's",
|
||
"Book", "Part", "From", "Like", "Still", "Yet", "So", "In", "On", "At", "He", "We", "If",
|
||
"God", "Oui", "Non", "Mademoiselle", "Monsieur"}
|
||
words = collections.Counter(re.findall(r"\b[A-Z][a-zà-ÿé]+(?:\s[A-Z][a-zà-ÿé]+)?\b", text))
|
||
cand = [(w, n) for w, n in words.most_common(400)
|
||
if n >= 15 and w.split()[0] not in stop and len(w) > 3]
|
||
# предпочесть двусловные и явные имена/реалии
|
||
seen, out = set(), []
|
||
for w, n in cand:
|
||
head = w.split()[0]
|
||
if head in seen:
|
||
continue
|
||
seen.add(head)
|
||
out.append((w, n))
|
||
if len(out) == top:
|
||
break
|
||
return out
|
||
|
||
|
||
def kwic(text, term, per=3, width=60):
|
||
out, frm = [], 0
|
||
while len(out) < per:
|
||
i = text.find(term, frm)
|
||
if i < 0:
|
||
break
|
||
out.append(text[max(0, i - width):i + len(term) + width])
|
||
frm = i + len(term)
|
||
return out
|
||
|
||
|
||
def run_f1(models=("deepseek-v4-flash",)):
|
||
roster = cp.build_roster()
|
||
batches = cp.batches_of(roster)
|
||
total = 0.0
|
||
for model in models:
|
||
cl = cp.client_for(model)
|
||
short = {"deepseek-v4-flash": "ds", "glm-5": "glm"}[model]
|
||
for i, b in enumerate(batches):
|
||
user = "Термины книги:\n\n" + "\n\n".join(cp.block_of(c) for c in b)
|
||
rec = cp.call(cl, model, TERM_EN_SYS, user, f"xen-{short}-b{i}",
|
||
"low" if model.startswith("deepseek") else None, model == "glm-5")
|
||
total += rec["cost_usd"]
|
||
time.sleep(1)
|
||
return total
|
||
|
||
|
||
# F2-ростер: майнер дал имена; реалии добраны РУКАМИ по частотам (раскрыто в отчёте). Семья
|
||
# *-blood (coldblood/highblood/paleblood) — намеренный тест корневой связности в zh.
|
||
TERMS_F2 = ["Dior", "Gabriel", "Phoebe", "Celene", "Voss", "silversaint", "coldblood", "highblood",
|
||
"paleblood", "Forever King", "Grail", "San Michon", "aegis", "duskdancer", "famille"]
|
||
|
||
|
||
def run_f2():
|
||
text = epub_text()
|
||
print(f"epub text: {len(text)} chars")
|
||
terms = [(t, text.count(t)) for t in TERMS_F2]
|
||
print("terms:", terms)
|
||
blocks = []
|
||
for t, n in terms:
|
||
blocks.append(f"### {t}\nfreq: {n}\n" + "\n".join(f"ctx: {c}" for c in kwic(text, t)))
|
||
user = "Terms:\n\n" + "\n\n".join(blocks)
|
||
total = 0.0
|
||
for model, short in [("deepseek-v4-flash", "ds"), ("glm-5", "glm")]:
|
||
cl = cp.client_for(model)
|
||
rec = cp.call(cl, model, TERM_ZH_SYS, user, f"xzh-term-{short}",
|
||
"low" if model.startswith("deepseek") else None, model == "glm-5")
|
||
total += rec["cost_usd"]
|
||
time.sleep(1)
|
||
# два окна прозы (~900 симв.), разнесённые
|
||
paras = [p for p in text.split("\n") if len(p) > 400]
|
||
wins = [paras[3][:1200], paras[len(paras) // 2][:1200]]
|
||
for k, w in enumerate(wins):
|
||
for model, short in [("deepseek-v4-flash", "ds"), ("glm-5", "glm")]:
|
||
cl = cp.client_for(model)
|
||
rec = cp.call(cl, model, TRANSLATE_ZH_SYS, "翻译下面的段落:\n\n" + w, f"xzh-prose-w{k}-{short}",
|
||
"low" if model.startswith("deepseek") else None, model == "glm-5")
|
||
total += rec["cost_usd"]
|
||
time.sleep(1)
|
||
return total
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--part", default="all", choices=["f1", "f2", "all", "mine"])
|
||
a = ap.parse_args()
|
||
if a.part == "mine":
|
||
print(mine_en_terms(epub_text()))
|
||
return
|
||
total = 0.0
|
||
if a.part in ("f1", "all"):
|
||
total += run_f1()
|
||
if a.part in ("f2", "all"):
|
||
total += run_f2()
|
||
print(f"TOTAL crosslang: ${total:.6f}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|
||
|
||
|
||
def score_f1():
|
||
"""Джойн en-консолидаций с голдом: печать пар (src, ru владельца, en свежий) для таблицы §F."""
|
||
import glob
|
||
en = {}
|
||
for f in sorted(glob.glob(str(RAW / "xen-ds-b*.json"))):
|
||
for ln in json.load(open(f, encoding="utf-8"))["content"].split("\n"):
|
||
if "\t" in ln:
|
||
s, d = ln.split("\t", 1)
|
||
en[s.strip()] = d.strip()
|
||
gold = [json.loads(l) for l in open(HERE / "gold" / "gold.jsonl", encoding="utf-8")]
|
||
n = 0
|
||
for r in gold:
|
||
if not r.get("in_bank"):
|
||
continue
|
||
s = r["bank_src"]
|
||
if s in en:
|
||
n += 1
|
||
print(f"{s}\t{r['gold_dst']}\t{en[s]}")
|
||
print(f"# joined: {n}")
|