textmachine/eval/exp15/structural_profile.py

#!/usr/bin/env python3
"""exp15 — deterministic structural profiler ($0), §B1.3 StructureProfile mirror.

Characterizes ANY text slice so the report can print a per-slice structural profile BEFORE the
paid run and verify the slice actually exercises the lever it is meant to (anti-null, polygon-sync
lesson). This is an EVAL-side profiler (Python); the production profiler is Go (chunker StructureProfile,
backend zone) — at divergence trust Go. Pure function of input bytes; no model calls.

Metrics (per §A.3 step 0 / §B1.3):
  - lines, nonblank_lines, blank_run_density (fraction of blank-line runs >=2 among all line gaps)
  - median_chars_per_nonblank_line  (line-per-sentence proxy: guzhenren 一句一段 => short lines)
  - dialogue_opener_share            (lines opening with 「『“«»"— - etc.)
  - terminator_density               (。！？!?… per 100 chars — sentence-boundary signal)
  - cjk_share                        (Han+Kana / all non-space chars)
  - ornament_lines                   (lines that are runs of separator glyphs ＊◇◆■○※*#—═… or 分割线)
  - separator_inventory              (which ornament glyphs actually occur as separator lines)
  - section_markers                  (第…节 / 第…卷 / 第…話 / Chapter … counts)
Verdict heuristic per lever:
  flat_line_per_sentence  = median_chars_per_nonblank_line small AND dialogue lines line-per-utterance
  scene_marked            = ornament_lines > 0 (real separators present)

Run: eval/.venv/bin/python eval/exp15/structural_profile.py <file> [<file> ...]
     eval/.venv/bin/python eval/exp15/structural_profile.py --json <file>   # machine-readable
"""
from __future__ import annotations

import json
import statistics
import sys
import unicodedata
from pathlib import Path

DIALOGUE_OPENERS = tuple("「『“«»\"”—")  # plus ascii "- " handled separately
ORNAMENT_CHARS = set("＊◇◆■○※*#—═―─•·∗☆★＝=~〜─━＿_＄§")  # separator glyphs (per §B1.3 + common webnovel)
TERMINATORS = set("。！？!?…‼⁇．")


def is_cjk(ch: str) -> bool:
    o = ord(ch)
    return (0x4E00 <= o <= 0x9FFF or 0x3040 <= o <= 0x30FF or 0x3400 <= o <= 0x4DBF
            or 0xF900 <= o <= 0xFAFF or 0x20000 <= o <= 0x2A6DF)


def is_ornament_line(s: str) -> bool:
    t = s.strip()
    if "分割线" in t or "分隔线" in t:
        return True
    if len(t) < 2:
        return False
    core = [c for c in t if not c.isspace()]
    if len(core) < 2:
        return False
    return all(c in ORNAMENT_CHARS for c in core)


def profile(text: str) -> dict:
    lines = text.split("\n")
    nonblank = [ln for ln in lines if ln.strip()]
    # blank-run density: count maximal runs of >=2 consecutive blank lines
    blank_runs2 = 0
    run = 0
    for ln in lines:
        if ln.strip():
            if run >= 2:
                blank_runs2 += 1
            run = 0
        else:
            run += 1
    if run >= 2:
        blank_runs2 += 1
    total_gaps = max(1, len(nonblank))

    lens = [len(ln.strip()) for ln in nonblank]
    dlg = 0
    for ln in nonblank:
        s = ln.lstrip()
        if s.startswith(DIALOGUE_OPENERS) or s.startswith("- ") or s.startswith("― "):
            dlg += 1
    ornament = [ln.strip() for ln in nonblank if is_ornament_line(ln)]
    orn_inv = sorted({c for o in ornament for c in o if not c.isspace()})

    allchars = [c for c in text if not c.isspace()]
    cjk = sum(1 for c in allchars if is_cjk(c))
    terms = sum(1 for c in text if c in TERMINATORS)

    import re
    sec_jie = len(re.findall(r"第[0-9一二三四五六七八九十百千零两]{1,10}[节節]", text))
    sec_juan = len(re.findall(r"第[0-9一二三四五六七八九十百千零两]{1,10}[卷]", text))
    sec_hua = len(re.findall(r"第[０-９0-9一二三四五六七八九十百]+[話话]", text))
    sec_ch = len(re.findall(r"(?mi)^\s*chapter\s+[a-z0-9]", text))

    median_len = statistics.median(lens) if lens else 0
    return {
        "chars": len(text),
        "lines": len(lines),
        "nonblank_lines": len(nonblank),
        "blank_run2_density": round(blank_runs2 / total_gaps, 4),
        "median_chars_per_nonblank_line": median_len,
        "mean_chars_per_nonblank_line": round(statistics.mean(lens), 1) if lens else 0,
        "dialogue_opener_share": round(dlg / max(1, len(nonblank)), 4),
        "terminator_per_100char": round(100 * terms / max(1, len(allchars)), 3),
        "cjk_share": round(cjk / max(1, len(allchars)), 4),
        "ornament_lines": len(ornament),
        "separator_inventory": orn_inv,
        "section_markers": {"jie": sec_jie, "juan": sec_juan, "hua": sec_hua, "chapter": sec_ch},
        "verdict": {
            "flat_line_per_sentence": bool(median_len and median_len <= 40 and cjk > 0),
            "scene_marked": len(ornament) > 0,
        },
    }


def main():
    args = [a for a in sys.argv[1:] if a != "--json"]
    as_json = "--json" in sys.argv
    out = {}
    for path in args:
        p = Path(path)
        if not p.exists():
            print(f"[MISSING] {path}", file=sys.stderr)
            continue
        raw = p.read_bytes()
        try:
            text = raw.decode("utf-8")
        except UnicodeDecodeError:
            text = raw.decode("gb18030", errors="replace")
        pr = profile(text)
        out[p.name] = pr
        if not as_json:
            v = pr["verdict"]
            print(f"\n### {p.name}  ({pr['chars']:,} chars, {pr['nonblank_lines']:,} nonblank lines)")
            print(f"  median chars/line = {pr['median_chars_per_nonblank_line']} (mean {pr['mean_chars_per_nonblank_line']})  "
                  f"cjk={pr['cjk_share']}  term/100c={pr['terminator_per_100char']}")
            print(f"  dialogue-opener share = {pr['dialogue_opener_share']}   blank-run>=2 density = {pr['blank_run2_density']}")
            print(f"  ornament/separator lines = {pr['ornament_lines']}  inventory={pr['separator_inventory']}")
            print(f"  section markers = {pr['section_markers']}")
            print(f"  VERDICT: flat_line_per_sentence={v['flat_line_per_sentence']}  scene_marked={v['scene_marked']}")
    if as_json:
        print(json.dumps(out, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()