#!/usr/bin/env python3
"""FIRST-OCCURRENCE test of the KWIC consolidator (backlog 73 candidate). $0 — no model calls.

Positive test of the hypothesis "the terminologist is a KWIC/first-occurrence consolidator, not a
frequency-majority picker" (vs the conclusion-BY-EXCLUSION of D39.65 §2). Reads only durable coldrun-a
artifacts read-only (D39.58).

DATA (grounded by author!=reviewer, 01.08):
  - first-occurrence-in-BANKNOTE per source term: retrieval_state.banknote_detail (JSON {k,s,d,t}),
    ordered by (chapter, chunk_idx), first-wins. NOTE the honest scope: this is first-seen-in-BANKNOTE
    (when the drafter first DECLARED the term), NOT first-appearance-in-source-text.
  - freq-majority top + FINAL terminologist dst: parse_bankstop() (drafts: are score/freq-ordered;
    r['dst'] is the terminologist FINAL, == terminologist checkpoint output).
  - join is EXACT on the normalized source key (banknote k is already NormalizeSourceKey'd; bank-stop
    src is raw Han → for Han these coincide; we also try a light-normalized fallback and REPORT coverage).

DISCIPLINE (eval-aggregation lesson + prompt): UNITS ARE PRINTED FIRST. The discriminating subpopulation
(first-occ != freq-top) is isolated and read unit-by-unit BEFORE any aggregate; ties (first==freq),
coined FINALs (== neither draft), and catastrophes are screened against the winner too. No aggregate is
emitted before the units.
"""
import json, sys
from parse_common import con, parse_bankstop

def freq_top(variants):          # inlined (do not import m22: its module-level analysis would re-run)
    return sorted(variants, key=lambda x: (-x[1], x[0]))[0][0]

def norm(s):
    return " ".join(s.strip().lower().replace("«", "").replace("»", "").replace('"', "").split())

# --- first-occurrence-in-banknote per normalized source key ---
c = con()
firstocc = {}          # k -> (chapter, chunk_idx, dst)
nchunks = 0
for chapter, chunk_idx, detail in c.execute(
        "SELECT chapter, chunk_idx, banknote_detail FROM retrieval_state "
        "WHERE banknote_detail != '' ORDER BY chapter, chunk_idx"):
    nchunks += 1
    try:
        arr = json.loads(detail)
    except Exception:
        continue
    for e in arr:
        k = e.get("k", "")
        if k and k not in firstocc:
            firstocc[k] = (chapter, chunk_idx, e.get("d", ""))
c.close()
print(f"# banknote chunks with detail = {nchunks}; distinct first-occ keys = {len(firstocc)}")

bs = parse_bankstop()
# n>=2 population: terms with >=2 DISTINCT draft variants (the only ones where a consolidation CHOICE exists)
n2 = {s: r for s, r in bs.items() if len({d for d, _ in r["drafts"]}) >= 2}

# join bank-stop term -> its first-occ dst; try raw then light-normalized key
def firstocc_for(src):
    if src in firstocc:
        return firstocc[src]
    nk = norm(src)
    for k, v in firstocc.items():
        if norm(k) == nk:
            return v
    return None

reached = {s: r for s, r in n2.items() if firstocc_for(s) is not None}
print(f"# n>=2 (>=2 distinct drafts) = {len(n2)}; reached via banknote first-occ = {len(reached)}\n")

# ============ UNITS FIRST — the discriminating subpopulation (first-occ != freq-top) ============
rows = []
for s, r in reached.items():
    ch, ci, first_d = firstocc_for(s)
    drafts = r["drafts"]
    ft = freq_top(drafts)
    final = r["dst"]
    draft_set = {norm(d) for d, _ in drafts}
    label_first = norm(final) == norm(first_d)
    label_freq  = norm(final) == norm(ft)
    coined      = norm(final) not in draft_set
    rows.append(dict(src=s, typ=r.get("type", ""), ch=ch, ci=ci, first_d=first_d, ft=ft,
                     final=final, drafts=[d for d, _ in drafts],
                     first_eq_freq=(norm(first_d) == norm(ft)),
                     label_first=label_first, label_freq=label_freq, coined=coined))

discriminating = [x for x in rows if not x["first_eq_freq"]]
print(f"===== DISCRIMINATING SUBPOPULATION: first-occ != freq-top  (N={len(discriminating)} of {len(rows)} reached) =====")
print("(units read individually BEFORE any aggregate)\n")
for x in sorted(discriminating, key=lambda z: (z["ch"], z["ci"])):
    tag = "FINAL=first" if x["label_first"] and not x["label_freq"] else \
          "FINAL=freq"  if x["label_freq"] and not x["label_first"] else \
          "FINAL=both"  if x["label_first"] and x["label_freq"] else \
          "FINAL=COINED" if x["coined"] else "FINAL=other-draft"
    print(f"  {x['src']} [{x['typ']}] ch{x['ch']}.{x['ci']}  first='{x['first_d']}'  freq='{x['ft']}'  "
          f"FINAL='{x['final']}'  -> {tag}")
    print(f"       drafts={x['drafts']}")

# ties (first==freq: NON-discriminating) and coined — screened separately
ties = [x for x in rows if x["first_eq_freq"]]
coined_all = [x for x in rows if x["coined"]]
print(f"\n===== NON-DISCRIMINATING (first-occ == freq-top): N={len(ties)} (cannot separate the two hypotheses) =====")
print(f"===== COINED FINALs (== neither any draft variant): N={len(coined_all)} (role invented, screened separately) =====")
for x in coined_all[:20]:
    print(f"  {x['src']} [{x['typ']}]  FINAL='{x['final']}'  drafts={x['drafts']}  first='{x['first_d']}'")

# ============ AGGREGATE (only over the DISCRIMINATING, non-coined units) ============
disc_nc = [x for x in discriminating if not x["coined"]]
first_only = sum(1 for x in disc_nc if x["label_first"] and not x["label_freq"])
freq_only  = sum(1 for x in disc_nc if x["label_freq"] and not x["label_first"])
other_draft = sum(1 for x in disc_nc if not x["label_first"] and not x["label_freq"])
disc_coined = sum(1 for x in discriminating if x["coined"])
print(f"\n===== AGGREGATE over DISCRIMINATING non-coined units (N={len(disc_nc)}) =====")
print(f"  FINAL == first-occ ONLY : {first_only}")
print(f"  FINAL == freq-top  ONLY : {freq_only}")
print(f"  FINAL == some other draft (neither first nor freq): {other_draft}")
print(f"  (+ {disc_coined} discriminating units where FINAL was COINED — excluded from the ratio)")
print("\nNOTE (honest scope): 'first-occ' = first-seen-in-BANKNOTE, not first-in-source-text; 8/75 multi-")
print("variant terms unreachable via retrieval_state (superseded generations) are OUT of this first cut.")