textmachine/backend/internal/pipeline/flagseverity_test.go

package pipeline

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"go/types"
	"os"
	"strconv"
	"strings"
	"testing"

	"textmachine/backend/internal/llm"
)

// flagseverity_test.go pins the chapter passport's "worst flag" ranking.
//
// ⚠ WHY THIS FILE EXISTS AT ALL. flagReasonSeverity had no test of any kind, and the shape of the bug it
// carried is the one this project keeps meeting: a hand-written list that stopped tracking the constants it
// lists. `off_target_lang` — the flag an entire pack was built to raise — was never added to the switch, so
// it fell to the default and became the MOST BENIGN reason in the passport, below `length`. A chapter that
// came back in the wrong language reported «length» as its worst problem. Nothing was wrong with the code;
// the list was simply not tied to anything.

// TestEveryFlagReasonIsRanked reads the FlagReason constants OUT OF THE SOURCE rather than retyping them
// here, because a hand-written list in the test would rot exactly like the hand-written list in the code —
// and would then certify the rot. This is the guarantee that the next flag someone adds cannot land in the
// unknown bucket silently: it fails here on the day it is declared, not on the day it is misreported.
func TestEveryFlagReasonIsRanked(t *testing.T) {
	// ⚠ EVERY FILE OF THE PACKAGE, not disposition.go alone — the first version of this test named one
	// file, and a planting that declared a FlagReason anywhere else walked straight past it while the test
	// still reported «every declared reason is ranked». The guarantee was one file narrower than its own
	// sentence, which is the defect class this file exists to prevent, committed by the file itself.
	entries, err := os.ReadDir(".")
	if err != nil {
		t.Fatalf("read the package directory: %v", err)
	}
	var sources []string
	for _, e := range entries {
		n := e.Name()
		if e.IsDir() || !strings.HasSuffix(n, ".go") || strings.HasSuffix(n, "_test.go") {
			continue
		}
		sources = append(sources, n)
	}
	if len(sources) < 10 {
		t.Fatalf("only %d source files found — the walk stopped seeing the package: %v", len(sources), sources)
	}
	fset := token.NewFileSet()
	var decls []ast.Decl
	for _, name := range sources {
		f, perr := parser.ParseFile(fset, name, nil, 0)
		if perr != nil {
			t.Fatalf("parse %s: %v", name, perr)
		}
		decls = append(decls, f.Decls...)
	}
	var found int
	for _, d := range decls {
		gd, ok := d.(*ast.GenDecl)
		if !ok || gd.Tok != token.CONST {
			continue
		}
		for _, sp := range gd.Specs {
			vs, ok := sp.(*ast.ValueSpec)
			if !ok {
				continue
			}
			// ⚠ THE TYPE MUST BE WRITTEN ON THIS SPEC. An earlier version carried the previous spec's type
			// forward, which is not how Go types a const block: a spec that has a VALUE and no type is its
			// own UNTYPED constant and inherits nothing. That version raised a FALSE ALARM — an ordinary
			// string constant placed after a FlagReason spec in the same block was accused of being an
			// unranked flag, and a doc-URL constant would have failed this test with a message about the
			// chapter passport. A test that accuses the innocent is not enforcing its own sentence, and this
			// one exists to say what IS a declared reason.
			//
			// ⚠ AND BOTH GO SPELLINGS COUNT. `X FlagReason = "…"` writes the type in the spec; the equally
			// ordinary `X = FlagReason("…")` writes it as a CONVERSION, leaving vs.Type nil — and a walk
			// that looked only at vs.Type was blind to it, which an adversarial pass exhibited as a pair:
			// the typed spelling of one declaration RED, the conversion spelling of the same declaration
			// SURVIVED. The sentence that used to stand here — «a missed declaration would trip the found
			// floor rather than pass silently» — was simply false: one missing constant leaves the count at
			// 16 against a floor of 15, and nothing trips.
			if len(vs.Values) == 0 {
				continue
			}
			// ⚠ AND A THIRD SPELLING, which this walk used to drop on the floor: the value can be a
			// conversion of a constant from ANOTHER package — `X FlagReason = FlagReason(llm.Y)` — used so
			// that a vocabulary the wire, the checkpoint, the flag and the operator all read has ONE
			// carrier. There is no literal to unquote there, the old code hit `lit == nil` and `continue`d,
			// and both flags declared that way landed in the unknown bucket while THIS TEST STAYED GREEN.
			// Measured when it happened: 18 ranks against 16 constants the walk could see.
			//
			// So an unreadable value is now a FAILURE, not a skip. A silent `continue` in an
			// exhaustiveness test is the same defect the test exists to catch, one level up.
			decl, isDecl := flagReasonValue(vs)
			if !isDecl {
				continue
			}
			val, err := decl.resolve()
			if err != nil {
				t.Errorf("%s: %v", vs.Names[0].Name, err)
				continue
			}
			if val == "" {
				continue // reasonOK — the sentinel for "not a flag", deliberately unranked
			}
			found++
			if _, ranked := flagSeverity[FlagReason(val)]; !ranked {
				t.Errorf("%s (%q) has no entry in flagSeverity: it would rank %d — BELOW every real diagnosis, "+
					"so it can never be reported as a chapter's worst flag", vs.Names[0].Name, val, severityUnknown)
			}
		}
	}
	// The reader of this test must know it actually read something: a parse that silently matched nothing
	// would pass forever and guard nothing — the same failure class one layer up.
	if found < 15 {
		t.Fatalf("only %d FlagReason constants were read out of the package's %d source files — the walk "+
			"stopped matching the source, and this test is no longer checking anything", found, len(sources))
	}
	if len(flagSeverity) != found {
		t.Errorf("flagSeverity has %d entries for %d declared reasons — a rank exists for a reason that no "+
			"longer does", len(flagSeverity), found)
	}
}

// TestOffTargetLangIsRankedWithTheContentFailures is the specific harm, asserted as a number and not as a
// membership: the flag must out-rank the budget symptoms, because those are what it was losing to.
func TestOffTargetLangIsRankedWithTheContentFailures(t *testing.T) {
	off := flagReasonSeverity(string(FlagOffTargetLang))
	if off == severityUnknown {
		t.Fatal("off_target_lang fell to the unknown bucket — the exact defect this file was written for")
	}
	for _, milder := range []FlagReason{FlagLength, FlagEmpty, FlagGlossaryMiss, FlagSanitizerStripped} {
		if off >= flagReasonSeverity(string(milder)) {
			t.Errorf("off_target_lang (%d) must be WORSE than %s (%d): a chapter in the wrong language is not "+
				"a budget symptom", off, milder, flagReasonSeverity(string(milder)))
		}
	}
	// It is a content failure, ranked with the echo it sits beside in the classifier.
	if off != flagReasonSeverity(string(FlagCJKArtifact)) {
		t.Errorf("off_target_lang (%d) and cjk_artifact (%d) are the same class — «the output is not a "+
			"translation of this text into the asked-for language» — and must rank together",
			off, flagReasonSeverity(string(FlagCJKArtifact)))
	}
	// A refusal still outranks both: the provider said no, which is a different and worse fact.
	if off <= flagReasonSeverity(string(FlagHardRefusal)) {
		t.Error("a hard refusal must still outrank an off-target completion")
	}
}

// isFlagReasonIdent reports whether an expression names the FlagReason type — as a spec's type or as the
// function of a conversion. One helper for both spellings, so the walk cannot learn one and forget the other.
func isFlagReasonIdent(e ast.Expr) bool {
	id, ok := e.(*ast.Ident)
	return ok && id.Name == "FlagReason"
}

// flagValueDecl is a FlagReason constant declaration the walk has recognised, in whichever of the three
// spellings it was written.
type flagValueDecl struct {
	lit  *ast.BasicLit // `X FlagReason = "y"` or `X = FlagReason("y")`
	conv string        // `X FlagReason = FlagReason(pkg.Y)` — the rendered source expression
}

// convertedFlagValues resolves the conversion spelling. The keys are SOURCE EXPRESSIONS and the values
// are compile-time references to the very constants those expressions name, so the map cannot drift in
// VALUE — only in membership, and a member it lacks fails loud below instead of being skipped.
var convertedFlagValues = map[string]FlagReason{
	"llm.CutBySelfDeadline": FlagReason(llm.CutBySelfDeadline),
	"llm.CutByParent":       FlagReason(llm.CutByParent),
	"llm.CutByConnection":   FlagReason(llm.CutByConnection),
}

func (d flagValueDecl) resolve() (string, error) {
	if d.lit != nil {
		if d.lit.Kind != token.STRING {
			return "", fmt.Errorf("declared with a non-string value %s — a FlagReason is a string", d.lit.Value)
		}
		return strconv.Unquote(d.lit.Value)
	}
	v, ok := convertedFlagValues[d.conv]
	if !ok {
		return "", fmt.Errorf("declared as a conversion of %s, whose value this test cannot read from the "+
			"source; add it to convertedFlagValues (a compile-time reference, not a retyped string) so the "+
			"exhaustiveness check can see it instead of skipping it", d.conv)
	}
	return string(v), nil
}

// flagReasonValue recognises a FlagReason declaration in any of its three spellings and reports how its
// value can be read. It returns false for a spec that is not a FlagReason at all — an ordinary string
// constant sitting in the same block is not this test's business.
func flagReasonValue(vs *ast.ValueSpec) (flagValueDecl, bool) {
	typed := isFlagReasonIdent(vs.Type)
	if lit, ok := vs.Values[0].(*ast.BasicLit); ok {
		return flagValueDecl{lit: lit}, typed
	}
	call, isCall := vs.Values[0].(*ast.CallExpr)
	if !isCall || !isFlagReasonIdent(call.Fun) || len(call.Args) != 1 {
		return flagValueDecl{}, false
	}
	if lit, ok := call.Args[0].(*ast.BasicLit); ok {
		return flagValueDecl{lit: lit}, true
	}
	return flagValueDecl{conv: types.ExprString(call.Args[0])}, true
}

// TestTheSeverityTableMeansWhatItsCommentsSay pins the ORDER, not the membership. The exhaustiveness
// test beside it proves every reason has a rank; it says nothing about what the ranks are, so the whole
// table could be re-ordered — `cancelled` declared a chapter's worst problem, a lost chunk declared its
// mildest — and the battery would stay green.
//
// Every assertion below is a sentence the table already writes about itself, turned into a check. That
// is the point: a rank is a claim, and a claim with no carrier is a comment.
func TestTheSeverityTableMeansWhatItsCommentsSay(t *testing.T) {
	rank := func(r FlagReason) int {
		s, ok := flagSeverity[r]
		if !ok {
			t.Fatalf("premise broken: %s carries no rank, so the comparisons below compare nothing", r)
		}
		return s
	}
	// The control first: the table is not empty and not all one value, or every «is milder than» below
	// would hold trivially.
	seen := map[int]bool{}
	for _, s := range flagSeverity {
		seen[s] = true
	}
	if len(flagSeverity) < 5 || len(seen) < 3 {
		t.Fatalf("premise broken: %d reasons across %d distinct ranks — an ordering test needs an order",
			len(flagSeverity), len(seen))
	}

	// «The mildest mark there is» — a passport that reported `cancelled` as a chapter's worst problem
	// would hide a durable finding behind a state the next run erases.
	for r, s := range flagSeverity {
		if r == FlagCancelled {
			continue
		}
		if rank(FlagCancelled) <= s {
			t.Fatalf("`cancelled` must be milder than every other mark — it is the only one the engine "+
				"removes by itself — but it ranks %d against %s at %d", rank(FlagCancelled), r, s)
		}
	}
	// «They rank together because they are the same thing to a reader — the chunk is lost and the money
	// is spent.» A lost connection is not a member: it reaches no disposition, so it wears no rank.
	//
	// ⚠ THE THIRD MEMBER JOINED WITH A CLAIM OF ITS OWN (row 291): `retry_unaffordable` is a position that
	// paid for an attempt and whose re-attack a ceiling refused, so to a reader it is the same sentence —
	// the chunk is lost and the money is spent — and the table says so. Asserted here because a rank
	// written in a comment and nowhere else is a claim with no carrier, and a third member added to the
	// comment alone would have left the trio's equality measured on two of three.
	for _, r := range []FlagReason{FlagAttemptTimeout, FlagRetryUnaffordable} {
		if rank(FlagDecodeError) != rank(r) {
			t.Fatalf("the paid-and-nothing-came-back reasons must share one rank: decode=%d %s=%d",
				rank(FlagDecodeError), r, rank(r))
		}
	}
	// «It IS one of those two, plus the fact that the remedy could not be bought» — a chapter whose unit
	// died for lack of money must not report `length` as its worst problem and send a person to fix a
	// budget formula instead of topping up.
	if rank(FlagRetryUnaffordable) >= rank(FlagLength) || rank(FlagRetryUnaffordable) >= rank(FlagEmpty) {
		t.Fatalf("a re-attack nobody could afford must out-rank the budget symptom it supersedes: "+
			"unaffordable=%d length=%d empty=%d", rank(FlagRetryUnaffordable), rank(FlagLength), rank(FlagEmpty))
	}
	// «An unrecognised string must not out-rank a diagnosis the engine actually made.»
	for r, s := range flagSeverity {
		if severityUnknown <= s {
			t.Fatalf("severityUnknown (%d) must be milder than every diagnosis the engine makes, but %s "+
				"ranks %d", severityUnknown, r, s)
		}
	}
	// «Ranked below a budget symptom (the chunk is not lost)» — a stripped chunk SHIPPED.
	if rank(FlagSanitizerStripped) <= rank(FlagLength) || rank(FlagSanitizerStripped) <= rank(FlagEmpty) {
		t.Fatalf("an auto-cleaned chunk that SHIPPED must be milder than a budget symptom that lost one: "+
			"stripped=%d length=%d empty=%d", rank(FlagSanitizerStripped), rank(FlagLength), rank(FlagEmpty))
	}
	// «Ranked with the deterministic content failures, above a mere budget symptom» — a DROPPED
	// contaminated output is unreadable as shipped.
	if rank(FlagSanitizerDefect) >= rank(FlagLength) {
		t.Fatalf("a contaminated output that was DROPPED must be more severe than a budget symptom: "+
			"defect=%d length=%d", rank(FlagSanitizerDefect), rank(FlagLength))
	}
}