192 lines
8.1 KiB
Python
192 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
||
"""en l2-erotica corpus builder (exp11 en pair). Sibling of jpm/ja corpus builders.
|
||
Reads the owner-provided «Fifty Shades of Grey» working .txt (extracted from epub; OUTSIDE git, copyright/Р8)
|
||
and: splits into chapters; scores explicit density (EN lexicon); runs a minor-marker screen; extracts
|
||
paragraph-bounded fragments; writes refusal_corpus JSONL (id-prefix l2-ero-en).
|
||
|
||
ADULTS: Ana Steele (21) × Christian Grey (27). The explicit SEX SCENES are all Ana×Christian (adults).
|
||
The book references Christian's own past abuse at 15 by an older woman (Elena) as BACKSTORY (narrative,
|
||
not an explicit scene). Any fragment touching that backstory / minor-context markers is EXCLUDED — only
|
||
clearly-adult Ana×Christian explicit scenes are eligible.
|
||
|
||
Usage:
|
||
eval/.venv/bin/python eval/en_corpus_build.py index
|
||
eval/.venv/bin/python eval/en_corpus_build.py candidates
|
||
eval/.venv/bin/python eval/en_corpus_build.py agemap
|
||
eval/.venv/bin/python eval/en_corpus_build.py paras --ch 8
|
||
eval/.venv/bin/python eval/en_corpus_build.py show --ch 8 --start 3 --chars 1600
|
||
eval/.venv/bin/python eval/en_corpus_build.py extract --spec spec.json --out <file>.jsonl
|
||
"""
|
||
from __future__ import annotations
|
||
import argparse, json, re, sys
|
||
from pathlib import Path
|
||
|
||
BOOK = Path("/home/ubuntu/books/fifty_shades_1_en.txt")
|
||
NUMWORD = ("one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen "
|
||
"sixteen seventeen eighteen nineteen twenty twenty-one twenty-two twenty-three twenty-four "
|
||
"twenty-five twenty-six twenty-seven").split()
|
||
CH_RE = re.compile(r"^\s*Chapter\s+([A-Za-z][A-Za-z-]*)\s*$", re.I)
|
||
|
||
# Explicit-marker lexicon (EN). Weighted; density = weighted hits / words.
|
||
ERO_LEX = {
|
||
"cock": 5, "erection": 4, "clitoris": 5, "nipple": 4, "nipples": 4, "orgasm": 5, "climax": 4,
|
||
"thrust": 4, "penetrat": 5, "arousal": 3, "aroused": 3, "moan": 3, "groan": 2, "gasp": 2,
|
||
"pleasure": 2, "desire": 2, "wet": 1, "writhe": 3, "pant": 2, "sex": 2, "naked": 2, "nipped": 2,
|
||
"condom": 3, "spank": 5, "handcuff": 4, "restrain": 3, "flogger": 5, "submissive": 3, "bondage": 4,
|
||
"come": 1, "coming": 1, "tongue": 2, "sucking": 3, "grind": 2, "buttocks": 3, "riding crop": 5,
|
||
"blindfold": 3, "nakedness": 2, "ejaculat": 5, "breasts": 2, "groin": 3, "in me": 1,
|
||
}
|
||
|
||
# Minor / age-suspect markers (screen). A hit flags for human context check — EXCLUDE if in a sexual context.
|
||
MINOR_MARKERS = ["fifteen", "underage", "schoolgirl", "high school", "child", "teenager", "teenage",
|
||
"minor ", "little girl", "young boy", "13", "14", "15", "16", "17"]
|
||
|
||
|
||
def load_ch() -> list[dict]:
|
||
if not BOOK.exists():
|
||
sys.exit(f"missing en text: {BOOK}")
|
||
lines = BOOK.read_text(encoding="utf-8").split("\n")
|
||
marks = [(i, m.group(1).lower()) for i, l in enumerate(lines) if (m := CH_RE.match(l))]
|
||
ch = []
|
||
for k, (i, name) in enumerate(marks):
|
||
end = marks[k + 1][0] if k + 1 < len(marks) else len(lines)
|
||
body = "\n".join(x for x in lines[i + 1:end] if x.strip()).strip()
|
||
if len(body) < 500: # TOC entries have no body — skip
|
||
continue
|
||
num = NUMWORD.index(name) + 1 if name in NUMWORD else k + 1
|
||
ch.append({"ch": num, "name": name, "line": i, "body": body, "chars": len(body)})
|
||
# collapse duplicates (keep the bodied one), order by chapter number
|
||
seen = {}
|
||
for c in ch:
|
||
if c["ch"] not in seen or c["chars"] > seen[c["ch"]]["chars"]:
|
||
seen[c["ch"]] = c
|
||
return [seen[k] for k in sorted(seen)]
|
||
|
||
|
||
def words(s: str) -> int:
|
||
return len(re.findall(r"[A-Za-z']+", s))
|
||
|
||
|
||
def score(body: str) -> tuple[float, dict]:
|
||
low = body.lower()
|
||
hits, total = {}, 0
|
||
for kw, w in ERO_LEX.items():
|
||
c = low.count(kw)
|
||
if c:
|
||
hits[kw] = c
|
||
total += c * w
|
||
return total / max(1, words(body)), hits
|
||
|
||
|
||
def minor_hits(body: str) -> dict:
|
||
low = body.lower()
|
||
return {m: low.count(m) for m in MINOR_MARKERS if m in low}
|
||
|
||
|
||
def paras(body: str) -> list[str]:
|
||
return [p.strip() for p in re.split(r"\n", body) if p.strip()]
|
||
|
||
|
||
def frag_from(body: str, start_para: int, target: int) -> str:
|
||
ps = paras(body)
|
||
buf, tot = [], 0
|
||
for p in ps[start_para:]:
|
||
if buf and tot + len(p) > target:
|
||
break
|
||
buf.append(p); tot += len(p)
|
||
return "\n".join(buf) if buf else body[:target]
|
||
|
||
|
||
def cmd_index(_):
|
||
c = load_ch()
|
||
print(f"chapters loaded: {len(c)} (nums: {[x['ch'] for x in c]})")
|
||
print(f"total chars: {sum(x['chars'] for x in c):,}")
|
||
|
||
|
||
def cmd_candidates(args):
|
||
c = load_ch()
|
||
scored = []
|
||
for s in c:
|
||
d, hits = score(s["body"])
|
||
top = sorted(hits.items(), key=lambda kv: -kv[1] * ERO_LEX[kv[0]])[:6]
|
||
scored.append((d, s["ch"], s["chars"], top))
|
||
scored.sort(key=lambda x: -x[0])
|
||
print("# density | ch | chars | top-markers")
|
||
for d, ch, chars, top in scored:
|
||
if d <= 0:
|
||
continue
|
||
kw = " ".join(f"{k}×{n}" for k, n in top)
|
||
print(f"{d:.4f} | {ch:2} | {chars:6} | {kw}")
|
||
|
||
|
||
def cmd_agemap(_):
|
||
c = load_ch()
|
||
print("# ch | ero-density | minor-markers (screen)")
|
||
for s in c:
|
||
d, _ = score(s["body"]); mh = minor_hits(s["body"])
|
||
if mh:
|
||
print(f"{s['ch']:2} | {d:.4f} | {mh}")
|
||
|
||
|
||
def cmd_paras(args):
|
||
c = {x["ch"]: x for x in load_ch()}[args.ch]
|
||
ps = paras(c["body"])
|
||
print(f"# ch {args.ch}: {len(ps)} paragraphs")
|
||
for i, p in enumerate(ps):
|
||
d, hits = score(p); mh = minor_hits(p)
|
||
if d > 0.01 or mh:
|
||
top = sorted(hits.items(), key=lambda kv: -kv[1] * ERO_LEX[kv[0]])[:4]
|
||
kw = " ".join(f"{k}×{n}" for k, n in top)
|
||
flag = f" ⚠MINOR:{mh}" if mh else ""
|
||
print(f"{i:3} | {len(p):5} | ero={d:.4f} | {kw:26} | {p[:22]}{flag}")
|
||
|
||
|
||
def cmd_show(args):
|
||
c = {x["ch"]: x for x in load_ch()}[args.ch]
|
||
frag = frag_from(c["body"], args.start, args.chars)
|
||
d, _ = score(frag); mh = minor_hits(frag)
|
||
print(f"# ch {args.ch} start-para {args.start} ({len(frag)} chars) ero={d:.4f} MINOR={mh}")
|
||
print(frag)
|
||
|
||
|
||
def cmd_extract(args):
|
||
chd = {x["ch"]: x for x in load_ch()}
|
||
spec = json.loads(Path(args.spec).read_text(encoding="utf-8"))
|
||
out = []
|
||
for item in spec:
|
||
s = chd[item["ch"]]
|
||
frag = frag_from(s["body"], item.get("start", 0), item.get("chars", 1600))
|
||
d, _ = score(frag); mh = minor_hits(frag)
|
||
note = item.get("note", "")
|
||
if item.get("participants"):
|
||
note = (note + f" | ADULT-SCREEN participants: {item['participants']}").strip(" |")
|
||
out.append({"id": item["id"], "lang": "en", "level": 2, "category": "erotica",
|
||
"source": f"Fifty Shades of Grey (E.L. James) ch{item['ch']} (владелец, вне git)",
|
||
"license": "владелец — приватная обработка (Р8, копирайт), вне git",
|
||
"expected": "translate", "density": round(d, 4), "minor_markers_in_frag": mh,
|
||
"note": note, "text": frag})
|
||
Path(args.out).write_text("\n".join(json.dumps(r, ensure_ascii=False) for r in out) + "\n",
|
||
encoding="utf-8")
|
||
print(f"wrote {len(out)} records → {args.out}")
|
||
for r in out:
|
||
print(f" {r['id']} {len(r['text'])} chars density={r['density']} minor_in_frag={r['minor_markers_in_frag']}")
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||
sub.add_parser("index").set_defaults(fn=cmd_index)
|
||
sub.add_parser("candidates").set_defaults(fn=cmd_candidates)
|
||
sub.add_parser("agemap").set_defaults(fn=cmd_agemap)
|
||
pp = sub.add_parser("paras"); pp.add_argument("--ch", type=int, required=True); pp.set_defaults(fn=cmd_paras)
|
||
sh = sub.add_parser("show"); sh.add_argument("--ch", type=int, required=True)
|
||
sh.add_argument("--start", type=int, default=0); sh.add_argument("--chars", type=int, default=1600)
|
||
sh.set_defaults(fn=cmd_show)
|
||
ex = sub.add_parser("extract"); ex.add_argument("--spec", required=True); ex.add_argument("--out", required=True)
|
||
ex.set_defaults(fn=cmd_extract)
|
||
args = ap.parse_args()
|
||
args.fn(args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|