229 lines
11 KiB
Python
229 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""How many renderings did one canonical term get in the SHIPPED text — a mechanical count.
|
||
|
||
Why it exists: the engine measures term spread on the DRAFT, at the bank-mining stop, before the
|
||
editor wave, and only the human signing table ever reads it. Over the text a reader is finally
|
||
handed, no such measure exists at all (unified backlog row 406), so «consistent terms across a whole
|
||
book» — the owner's first priority — is today unfalsifiable by construction.
|
||
|
||
This is a COUNT, not a judgement. It says «term X left the door in three shapes, in chapters 1, 2
|
||
and 3»; whether any of the three is good translation is not its question.
|
||
|
||
THE RULE IT USES, stated so a reader can disagree with it rather than guess:
|
||
|
||
* Input is `tmctl export --json --pairs` — the sanctioned extraction of shipped text (invariant 8,
|
||
D39.5) — plus the book's known renderings (bank sidecar `.bank.json`, signature map, auto-bank).
|
||
* For a source term T, the units considered are those whose SOURCE contains T. Only there can a
|
||
rendering of T be expected, and only there is its absence meaningful.
|
||
* A rendering found in a unit's target is any known rendering of T, or a NEAR-VARIANT of one: the
|
||
same first `stem` characters with a tail differing by at most `--tail` characters. Russian
|
||
inflects, so «Ли Чанфэна» and «Ли Чанфэн» are ONE rendering; «Ли Чанфын» is another.
|
||
* Spread(T) = the number of distinct renderings, after that folding, observed across those units.
|
||
|
||
Usage:
|
||
spread.py --pairs pairs.json --bank project.db.bank.json [--signature map.yaml] [--tail 2] [--stem 4]
|
||
spread.py --selftest
|
||
"""
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
from collections import defaultdict
|
||
|
||
CAP = re.compile(r"[А-ЯЁ][а-яёА-ЯЁ\-]+(?:\s+[А-ЯЁ][а-яёА-ЯЁ\-]+)*")
|
||
|
||
|
||
def lcp(a, b):
|
||
n = 0
|
||
for x, y in zip(a, b):
|
||
if x != y:
|
||
break
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def same_rendering(a, b, stem, tail):
|
||
"""One rendering inflected, or two different renderings?
|
||
|
||
⚠ THE RULE IS ABOUT WHERE THE DIFFERENCE IS, not how big it is, and the self-test is what forced
|
||
that: Russian inflection only ever changes the END of a word, while a different transliteration
|
||
changes something INSIDE it. So two strings are the same rendering when their common prefix
|
||
reaches all but `tail` characters of the shorter one — «Чжао Сяомань»/«Чжао Сяоманя» agree on
|
||
eleven of twelve — and are different when it does not, which is what separates «Ли Чанфэн» from
|
||
«Ли Чанфына» (they part at the seventh character of nine). The length allowance is separate and
|
||
generous, because an inflection may add two or three letters and a transliteration rarely does.
|
||
"""
|
||
a, b = a.strip(), b.strip()
|
||
if a.lower() == b.lower():
|
||
return True
|
||
short = min(len(a), len(b))
|
||
if short < stem:
|
||
return a.lower() == b.lower()
|
||
return lcp(a.lower(), b.lower()) >= short - tail and abs(len(a) - len(b)) <= 4
|
||
|
||
|
||
def known_renderings(bank_path, signature_path):
|
||
"""Every rendering this book has ever recorded for a source term, from the engine's own
|
||
sidecars. Both the signed bank and the proposals: a term signed late may have shipped under its
|
||
proposal earlier, which is exactly the case this instrument exists to find."""
|
||
out = defaultdict(set)
|
||
if bank_path:
|
||
doc = json.load(open(bank_path, encoding="utf-8"))
|
||
for section in ("terms", "proposed"):
|
||
for t in doc.get(section) or []:
|
||
if t.get("src") and t.get("dst"):
|
||
out[t["src"]].add(t["dst"])
|
||
if signature_path:
|
||
src = dst = None
|
||
for line in open(signature_path, encoding="utf-8"):
|
||
m = re.match(r"\s*-?\s*src:\s*(\S+)", line)
|
||
if m:
|
||
src = m.group(1).strip('"\'')
|
||
m = re.match(r"\s*dst:\s*(.+)", line)
|
||
if m and src:
|
||
dst = m.group(1).strip().strip('"\'')
|
||
if dst:
|
||
out[src].add(dst)
|
||
return out
|
||
|
||
|
||
def near(cap, known, stem):
|
||
"""Is this capitalised token PLAUSIBLY a rendering of the same term — the same name spelled
|
||
another way, or the same spelling inflected?
|
||
|
||
⚠ Deliberately looser than `same_rendering`, and the two must not be the same test. The loose
|
||
one decides what to LOOK AT (a token that shares most of a known rendering's prefix); the strict
|
||
one decides what to COUNT AS ONE. Using the strict test for both made the instrument blind to
|
||
exactly what it exists to find: «Ли Чанфына» stopped being a variant of «Ли Чанфэн» and the
|
||
second shape vanished from the tally instead of being counted as a second shape."""
|
||
short = min(len(cap), len(known))
|
||
if short < stem:
|
||
return False
|
||
return lcp(cap.lower(), known.lower()) >= max(stem, (short * 3 + 4) // 5) and abs(len(cap) - len(known)) <= 5
|
||
|
||
|
||
def variants_in(target, renderings, stem, tail):
|
||
"""Which renderings of this term the target shows — known ones and near-variants of them."""
|
||
found = set()
|
||
for r in renderings:
|
||
if r and r in target:
|
||
found.add(r)
|
||
for cap in CAP.findall(target):
|
||
for r in renderings:
|
||
if r and cap != r and near(cap, r, stem):
|
||
found.add(cap)
|
||
# fold the found set: inflections of one rendering are one rendering
|
||
folded = []
|
||
for f in sorted(found, key=len):
|
||
if not any(same_rendering(f, g, stem, tail) for g in folded):
|
||
folded.append(f)
|
||
return folded
|
||
|
||
|
||
def run(pairs, renderings, stem, tail):
|
||
"""pairs: [{chapter, unit, source, target}]. Returns rows sorted by spread."""
|
||
rows = []
|
||
for src, known in sorted(renderings.items()):
|
||
seen = {}
|
||
units = 0
|
||
for p in pairs:
|
||
if src not in (p.get("source") or ""):
|
||
continue
|
||
units += 1
|
||
for v in variants_in(p.get("target") or "", known, stem, tail):
|
||
# ⚠ FOLDED ACROSS UNITS, not only inside one. Folding per unit and keying the tally
|
||
# on the raw string counted «Чжао Сяомань» in chapter 1 and «Чжао Сяоманя» in
|
||
# chapter 2 as two shapes — the instrument's own self-test caught it.
|
||
key = next((k for k in seen if same_rendering(k, v, stem, tail)), v)
|
||
seen.setdefault(key, set()).add(p.get("chapter"))
|
||
if units == 0:
|
||
continue
|
||
rows.append({"src": src, "units_with_source": units, "renderings": {k: sorted(v) for k, v in seen.items()},
|
||
"spread": len(seen)})
|
||
rows.sort(key=lambda r: (-r["spread"], r["src"]))
|
||
return rows
|
||
|
||
|
||
def selftest():
|
||
"""A planted text: one term shipped in two shapes, one term shipped consistently (inflected),
|
||
one term never rendered at all. The instrument must say 2, 1 and 0 — the third is what tells a
|
||
real zero from an instrument that found nothing because it looked nowhere."""
|
||
renderings = {"李长风": {"Ли Чанфэн"}, "赵小满": {"Чжао Сяомань"}, "青锋剑": {"Цинфэн"}}
|
||
pairs = [
|
||
{"chapter": 1, "source": "李长风走在山路上。赵小满跟着。", "target": "Ли Чанфэн шёл по горной тропе. Чжао Сяомань шёл следом."},
|
||
{"chapter": 2, "source": "李长风看着赵小满。", "target": "Ли Чанфына видел Чжао Сяоманя."},
|
||
{"chapter": 3, "source": "青锋剑在手。", "target": "Меч был в руке."},
|
||
# a third unit for 赵小满, inflected further: the length allowance must not split it off
|
||
{"chapter": 3, "source": "赵小满笑了。", "target": "С Чжао Сяоманем всё было ясно."},
|
||
]
|
||
rows = run(pairs, renderings, stem=4, tail=1)
|
||
got = {r["src"]: r["spread"] for r in rows}
|
||
want = {"李长风": 2, "赵小满": 1, "青锋剑": 0}
|
||
ok = got == want
|
||
print("self-test rows:")
|
||
for r in rows:
|
||
print(" ", r)
|
||
print(f"self-test: got {got} want {want} → {'PASS' if ok else 'FAIL'}")
|
||
print(" (control: the instrument was given 3 terms and 3 units; a silent instrument would print"
|
||
" spread 0 for all three, which is why one term is planted with two shapes and one with none)")
|
||
return 0 if ok else 1
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--pairs")
|
||
ap.add_argument("--bank")
|
||
ap.add_argument("--signature")
|
||
ap.add_argument("--stem", type=int, default=4)
|
||
ap.add_argument("--tail", type=int, default=1)
|
||
ap.add_argument("--json", dest="out")
|
||
ap.add_argument("--selftest", action="store_true")
|
||
a = ap.parse_args()
|
||
if a.selftest:
|
||
sys.exit(selftest())
|
||
if not a.pairs:
|
||
ap.error("--pairs is required (tmctl export --json --pairs)")
|
||
doc = json.load(open(a.pairs, encoding="utf-8"))
|
||
pairs = normalise_pairs(doc)
|
||
renderings = known_renderings(a.bank, a.signature)
|
||
rows = run(pairs, renderings, a.stem, a.tail)
|
||
multi = [r for r in rows if r["spread"] >= 2]
|
||
withr = [r for r in rows if r["spread"] >= 1]
|
||
print(f"units read: {len(pairs)} canonical terms known: {len(renderings)}")
|
||
print(f"terms whose source appears in at least one unit: {len(rows)}")
|
||
print(f"terms rendered at all: {len(withr)} (control: terms considered {len(rows)})")
|
||
print(f"TERMS SHIPPED IN MORE THAN ONE SHAPE: {len(multi)} (control: terms rendered at all {len(withr)})")
|
||
for r in multi:
|
||
print(f" {r['src']}: {r['spread']} shapes over {r['units_with_source']} units")
|
||
for shape, chapters in r["renderings"].items():
|
||
print(f" {shape!r} in chapters {chapters}")
|
||
if a.out:
|
||
json.dump(rows, open(a.out, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
||
print(f"rows written to {a.out}")
|
||
|
||
|
||
def normalise_pairs(doc):
|
||
"""`tmctl export --json --pairs` shape, kept tolerant: the fields this needs are the chapter
|
||
number and the two texts, wherever the envelope puts them."""
|
||
# `tm-export-v1` puts them under `chunks`, with `final_text` for the shipped side and `source`
|
||
# for the other. Named first and explicitly, so a change of envelope is a loud KeyError rather
|
||
# than an empty list that reads as «this book has no terms out of line».
|
||
if isinstance(doc, dict):
|
||
for key in ("chunks", "pairs", "units", "items", "rows"):
|
||
if isinstance(doc.get(key), list):
|
||
doc = doc[key]
|
||
break
|
||
out = []
|
||
for p in doc if isinstance(doc, list) else []:
|
||
out.append({
|
||
"chapter": p.get("chapter", p.get("chapter_number", p.get("ch"))),
|
||
"unit": p.get("chunk_idx", p.get("unit", p.get("unit_id", p.get("id")))),
|
||
"source": p.get("source", p.get("src", "")) or "",
|
||
"target": p.get("final_text", p.get("target", p.get("translation", p.get("dst", "")))) or "",
|
||
})
|
||
return out
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|