textmachine/eval/bank_autonomy/m36b_grade.py

94 lines
6.2 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""АУДИТ ТИПОВ 36а — grader 0 (author). Rubric-based adjudication of the branch-relevant rows.
name/place = proper noun -> transliterate; title = office/rank/honorific -> translate;
term = common noun/concept/realia -> translate (or established realia translit); nickname = epithet.
BRANCH-HARM (36а) = realia/common-noun mis-typed name/place AND dst came out transliterated."""
from parse_common import parse_bankstop
bs=parse_bankstop()
# adjudicated true class for rows where I judge type WRONG (high confidence). Others assumed OK.
# format: src -> (adjudicated_class, note)
ADJ = {
# ---- 36а BRANCH-HARM: realia typed name -> transliterated ("too Sinicized") ----
"元石": ("term","realia: mined mineral (灵泉产出大量元石); name->юаньши = 36а harm"),
"元海": ("term","realia: internal energy-sea (元海未开); name->Юаньхай = 36а harm; inconsistent w/ 真元海=term"),
# ---- name mis-typed, but branch benign (terminologist translated anyway) ----
"古月山寨": ("place","山寨=settlement; name->place; dst translated поселение"),
"古月族长": ("title","族长=clan chief=title; name->title; dst translated глава клана"),
# ---- place mis-typed onto common nouns/concepts (benign: translated) ----
"地下溶洞": ("term","common noun 'underground cave'"),
"宗祖祠堂": ("term","common noun 'ancestral hall'"),
"山寨": ("term","common noun 'mountain stronghold'"),
"池塘": ("term","common noun 'pond'"),
"灵泉": ("term","common-noun realia 'spirit spring'"),
"竹楼": ("term","common noun 'bamboo house'"),
"蛊室": ("term","common noun 'gu chamber'"),
"酒肆": ("term","common noun 'tavern'"),
"光阴之河": ("term","poetic concept 'River of Time', not a geographic place"),
"赤脉": ("term","clan sub-lineage 'Red lineage', not a place"),
"赤之一脉": ("term","clan sub-lineage, not a place"),
# ---- title mis-typed onto idiom/kinship ----
"物是人非": ("term","chengyu/idiom, not a title"),
"舅母": ("term","kinship term 'maternal aunt', not a title"),
"舅父": ("term","kinship term 'maternal uncle', not a title"),
}
# BORDERLINE (grader-disagreement plausible; EXCLUDED from mis-type numerator, reported separately)
BORDER = {
"人祖": "myth 'Human Ancestor' — name vs title/term; TL waffled Прародитель<->Жэньцзу (near-harm)",
"古月一族":"clan collective — name vs term",
"古月师": "古月 clan + 师 — person-name vs role",
"沈嬷嬷": "嬷嬷=matron title + surname — name vs title",
"家主阁": "'clan-head pavilion' — place vs term",
"花海": "'sea of flowers' — descriptive place vs term",
"熊家寨": "named settlement — place ok-ish",
"白家寨": "named settlement — place ok-ish",
"咏梅": "poem title (work-title subtype of title)",
"江城子": "cí tune-name; work-title; TL transliterated Цзянчэнцзы (near-harm)",
"将敬酒": "poem title (work-title)",
"管家": "steward — title vs term",
"魔道巨擘":"'titan of demonic path' — title vs nickname",
"花酒行者":"villain epithet — nickname; TL transliterated (near-harm)",
"采花大盗":"euphemism epithet — nickname vs term",
}
def is_translit(dst):
core=dst.replace("«","").replace("»","").replace('"',"").strip().lower()
RU=set("камень море река гора гу мастер уровень ранг класс клан вождь глава поселение храм точка город "
"бронзовый бронзовое зелёный красный чёрный белый винный червь дом талант культивация океан истинной "
"сущности серийный насильник великий титан демонического пути жизни воспеваю сливу призыв матушка "
"камера зал апертура очистка сокровище клад чудесных редчайших десять ветвь странствующего сущность "
"энергия дух душа первый второй третий четвёртый пятый шестой седьмой восьмой девятый старейшина "
"наставник управляющий наложница ученик мешок церемония открытие прародитель дядя тётя брат отец мать "
"сын дочь господин госпожа поколения разряда способности качество отверстий пустотных изначального "
"подземная пещера крепость пруд духовный источник бамбуковый терем таверна павильон обитель пурпурная "
"все лица изменилось непредсказуемы ветер облака".split())
import re
toks=re.findall(r"[а-яё]+",core)
return bool(toks) and not any(t in RU for t in toks)
rows=list(bs.items())
N=len(rows)
mistyped=[(s,bs[s]["type"],ADJ[s][0],ADJ[s][1]) for s in ADJ]
harm=[]
for s,(cls,note) in ADJ.items():
r=bs[s]
if r["type"] in ("name","place") and cls in ("term","title") and is_translit(r["dst"]):
harm.append((s,r["type"],cls,r["dst"]))
print(f"=== TYPE AUDIT (grader 0 / author) over N={N} ===")
print(f" clear mis-types: {len(mistyped)} ({len(mistyped)/N*100:.1f}%)")
print(f" borderline (excluded from numerator): {len(BORDER)}")
print(f"\n === 36а BRANCH-HARM (realia/title typed name/place AND transliterated) : {len(harm)} ===")
for s,t,c,d in harm:
print(f" {s} type={t} -> true={c} dst='{d}'")
print(f"\n === all clear mis-types by recorded->true ===")
from collections import Counter
dirs=Counter((bs[s]['type'],ADJ[s][0]) for s in ADJ)
for (a,b),n in sorted(dirs.items(),key=lambda x:-x[1]):
print(f" {a:6s} -> {b:6s}: {n}")
print(f"\n mis-type rate by class (recorded):")
for cls in ("name","place","title","term","nickname"):
tot=sum(1 for s,r in rows if r['type']==cls)
mis=sum(1 for s in ADJ if bs[s]['type']==cls)
bor=sum(1 for s in BORDER if bs[s]['type']==cls)
print(f" {cls:8s}: {mis}/{tot} clear-mis ({mis/tot*100:.0f}%) +{bor} borderline")