package pipeline import ( "fmt" "go/ast" "go/parser" "go/token" "go/types" "os" "strconv" "strings" "testing" "textmachine/backend/internal/llm" ) // flagseverity_test.go pins the chapter passport's "worst flag" ranking. // // ⚠ WHY THIS FILE EXISTS AT ALL. flagReasonSeverity had no test of any kind, and the shape of the bug it // carried is the one this project keeps meeting: a hand-written list that stopped tracking the constants it // lists. `off_target_lang` — the flag an entire pack was built to raise — was never added to the switch, so // it fell to the default and became the MOST BENIGN reason in the passport, below `length`. A chapter that // came back in the wrong language reported «length» as its worst problem. Nothing was wrong with the code; // the list was simply not tied to anything. // TestEveryFlagReasonIsRanked reads the FlagReason constants OUT OF THE SOURCE rather than retyping them // here, because a hand-written list in the test would rot exactly like the hand-written list in the code — // and would then certify the rot. This is the guarantee that the next flag someone adds cannot land in the // unknown bucket silently: it fails here on the day it is declared, not on the day it is misreported. func TestEveryFlagReasonIsRanked(t *testing.T) { // ⚠ EVERY FILE OF THE PACKAGE, not disposition.go alone — the first version of this test named one // file, and a planting that declared a FlagReason anywhere else walked straight past it while the test // still reported «every declared reason is ranked». The guarantee was one file narrower than its own // sentence, which is the defect class this file exists to prevent, committed by the file itself. entries, err := os.ReadDir(".") if err != nil { t.Fatalf("read the package directory: %v", err) } var sources []string for _, e := range entries { n := e.Name() if e.IsDir() || !strings.HasSuffix(n, ".go") || strings.HasSuffix(n, "_test.go") { continue } sources = append(sources, n) } if len(sources) < 10 { t.Fatalf("only %d source files found — the walk stopped seeing the package: %v", len(sources), sources) } fset := token.NewFileSet() var decls []ast.Decl for _, name := range sources { f, perr := parser.ParseFile(fset, name, nil, 0) if perr != nil { t.Fatalf("parse %s: %v", name, perr) } decls = append(decls, f.Decls...) } var found int for _, d := range decls { gd, ok := d.(*ast.GenDecl) if !ok || gd.Tok != token.CONST { continue } for _, sp := range gd.Specs { vs, ok := sp.(*ast.ValueSpec) if !ok { continue } // ⚠ THE TYPE MUST BE WRITTEN ON THIS SPEC. An earlier version carried the previous spec's type // forward, which is not how Go types a const block: a spec that has a VALUE and no type is its // own UNTYPED constant and inherits nothing. That version raised a FALSE ALARM — an ordinary // string constant placed after a FlagReason spec in the same block was accused of being an // unranked flag, and a doc-URL constant would have failed this test with a message about the // chapter passport. A test that accuses the innocent is not enforcing its own sentence, and this // one exists to say what IS a declared reason. // // ⚠ AND BOTH GO SPELLINGS COUNT. `X FlagReason = "…"` writes the type in the spec; the equally // ordinary `X = FlagReason("…")` writes it as a CONVERSION, leaving vs.Type nil — and a walk // that looked only at vs.Type was blind to it, which an adversarial pass exhibited as a pair: // the typed spelling of one declaration RED, the conversion spelling of the same declaration // SURVIVED. The sentence that used to stand here — «a missed declaration would trip the found // floor rather than pass silently» — was simply false: one missing constant leaves the count at // 16 against a floor of 15, and nothing trips. if len(vs.Values) == 0 { continue } // ⚠ AND A THIRD SPELLING, which this walk used to drop on the floor: the value can be a // conversion of a constant from ANOTHER package — `X FlagReason = FlagReason(llm.Y)` — used so // that a vocabulary the wire, the checkpoint, the flag and the operator all read has ONE // carrier. There is no literal to unquote there, the old code hit `lit == nil` and `continue`d, // and both flags declared that way landed in the unknown bucket while THIS TEST STAYED GREEN. // Measured when it happened: 18 ranks against 16 constants the walk could see. // // So an unreadable value is now a FAILURE, not a skip. A silent `continue` in an // exhaustiveness test is the same defect the test exists to catch, one level up. decl, isDecl := flagReasonValue(vs) if !isDecl { continue } val, err := decl.resolve() if err != nil { t.Errorf("%s: %v", vs.Names[0].Name, err) continue } if val == "" { continue // reasonOK — the sentinel for "not a flag", deliberately unranked } found++ if _, ranked := flagSeverity[FlagReason(val)]; !ranked { t.Errorf("%s (%q) has no entry in flagSeverity: it would rank %d — BELOW every real diagnosis, "+ "so it can never be reported as a chapter's worst flag", vs.Names[0].Name, val, severityUnknown) } } } // The reader of this test must know it actually read something: a parse that silently matched nothing // would pass forever and guard nothing — the same failure class one layer up. if found < 15 { t.Fatalf("only %d FlagReason constants were read out of the package's %d source files — the walk "+ "stopped matching the source, and this test is no longer checking anything", found, len(sources)) } if len(flagSeverity) != found { t.Errorf("flagSeverity has %d entries for %d declared reasons — a rank exists for a reason that no "+ "longer does", len(flagSeverity), found) } } // TestOffTargetLangIsRankedWithTheContentFailures is the specific harm, asserted as a number and not as a // membership: the flag must out-rank the budget symptoms, because those are what it was losing to. func TestOffTargetLangIsRankedWithTheContentFailures(t *testing.T) { off := flagReasonSeverity(string(FlagOffTargetLang)) if off == severityUnknown { t.Fatal("off_target_lang fell to the unknown bucket — the exact defect this file was written for") } for _, milder := range []FlagReason{FlagLength, FlagEmpty, FlagGlossaryMiss, FlagSanitizerStripped} { if off >= flagReasonSeverity(string(milder)) { t.Errorf("off_target_lang (%d) must be WORSE than %s (%d): a chapter in the wrong language is not "+ "a budget symptom", off, milder, flagReasonSeverity(string(milder))) } } // It is a content failure, ranked with the echo it sits beside in the classifier. if off != flagReasonSeverity(string(FlagCJKArtifact)) { t.Errorf("off_target_lang (%d) and cjk_artifact (%d) are the same class — «the output is not a "+ "translation of this text into the asked-for language» — and must rank together", off, flagReasonSeverity(string(FlagCJKArtifact))) } // A refusal still outranks both: the provider said no, which is a different and worse fact. if off <= flagReasonSeverity(string(FlagHardRefusal)) { t.Error("a hard refusal must still outrank an off-target completion") } } // isFlagReasonIdent reports whether an expression names the FlagReason type — as a spec's type or as the // function of a conversion. One helper for both spellings, so the walk cannot learn one and forget the other. func isFlagReasonIdent(e ast.Expr) bool { id, ok := e.(*ast.Ident) return ok && id.Name == "FlagReason" } // flagValueDecl is a FlagReason constant declaration the walk has recognised, in whichever of the three // spellings it was written. type flagValueDecl struct { lit *ast.BasicLit // `X FlagReason = "y"` or `X = FlagReason("y")` conv string // `X FlagReason = FlagReason(pkg.Y)` — the rendered source expression } // convertedFlagValues resolves the conversion spelling. The keys are SOURCE EXPRESSIONS and the values // are compile-time references to the very constants those expressions name, so the map cannot drift in // VALUE — only in membership, and a member it lacks fails loud below instead of being skipped. var convertedFlagValues = map[string]FlagReason{ "llm.CutBySelfDeadline": FlagReason(llm.CutBySelfDeadline), "llm.CutByParent": FlagReason(llm.CutByParent), "llm.CutByConnection": FlagReason(llm.CutByConnection), } func (d flagValueDecl) resolve() (string, error) { if d.lit != nil { if d.lit.Kind != token.STRING { return "", fmt.Errorf("declared with a non-string value %s — a FlagReason is a string", d.lit.Value) } return strconv.Unquote(d.lit.Value) } v, ok := convertedFlagValues[d.conv] if !ok { return "", fmt.Errorf("declared as a conversion of %s, whose value this test cannot read from the "+ "source; add it to convertedFlagValues (a compile-time reference, not a retyped string) so the "+ "exhaustiveness check can see it instead of skipping it", d.conv) } return string(v), nil } // flagReasonValue recognises a FlagReason declaration in any of its three spellings and reports how its // value can be read. It returns false for a spec that is not a FlagReason at all — an ordinary string // constant sitting in the same block is not this test's business. func flagReasonValue(vs *ast.ValueSpec) (flagValueDecl, bool) { typed := isFlagReasonIdent(vs.Type) if lit, ok := vs.Values[0].(*ast.BasicLit); ok { return flagValueDecl{lit: lit}, typed } call, isCall := vs.Values[0].(*ast.CallExpr) if !isCall || !isFlagReasonIdent(call.Fun) || len(call.Args) != 1 { return flagValueDecl{}, false } if lit, ok := call.Args[0].(*ast.BasicLit); ok { return flagValueDecl{lit: lit}, true } return flagValueDecl{conv: types.ExprString(call.Args[0])}, true } // TestTheSeverityTableMeansWhatItsCommentsSay pins the ORDER, not the membership. The exhaustiveness // test beside it proves every reason has a rank; it says nothing about what the ranks are, so the whole // table could be re-ordered — `cancelled` declared a chapter's worst problem, a lost chunk declared its // mildest — and the battery would stay green. // // Every assertion below is a sentence the table already writes about itself, turned into a check. That // is the point: a rank is a claim, and a claim with no carrier is a comment. func TestTheSeverityTableMeansWhatItsCommentsSay(t *testing.T) { rank := func(r FlagReason) int { s, ok := flagSeverity[r] if !ok { t.Fatalf("premise broken: %s carries no rank, so the comparisons below compare nothing", r) } return s } // The control first: the table is not empty and not all one value, or every «is milder than» below // would hold trivially. seen := map[int]bool{} for _, s := range flagSeverity { seen[s] = true } if len(flagSeverity) < 5 || len(seen) < 3 { t.Fatalf("premise broken: %d reasons across %d distinct ranks — an ordering test needs an order", len(flagSeverity), len(seen)) } // «The mildest mark there is» — a passport that reported `cancelled` as a chapter's worst problem // would hide a durable finding behind a state the next run erases. for r, s := range flagSeverity { if r == FlagCancelled { continue } if rank(FlagCancelled) <= s { t.Fatalf("`cancelled` must be milder than every other mark — it is the only one the engine "+ "removes by itself — but it ranks %d against %s at %d", rank(FlagCancelled), r, s) } } // «They rank together because they are the same thing to a reader — the chunk is lost and the money // is spent.» A lost connection is not a member: it reaches no disposition, so it wears no rank. // // ⚠ THE THIRD MEMBER JOINED WITH A CLAIM OF ITS OWN (row 291): `retry_unaffordable` is a position that // paid for an attempt and whose re-attack a ceiling refused, so to a reader it is the same sentence — // the chunk is lost and the money is spent — and the table says so. Asserted here because a rank // written in a comment and nowhere else is a claim with no carrier, and a third member added to the // comment alone would have left the trio's equality measured on two of three. for _, r := range []FlagReason{FlagAttemptTimeout, FlagRetryUnaffordable} { if rank(FlagDecodeError) != rank(r) { t.Fatalf("the paid-and-nothing-came-back reasons must share one rank: decode=%d %s=%d", rank(FlagDecodeError), r, rank(r)) } } // «It IS one of those two, plus the fact that the remedy could not be bought» — a chapter whose unit // died for lack of money must not report `length` as its worst problem and send a person to fix a // budget formula instead of topping up. if rank(FlagRetryUnaffordable) >= rank(FlagLength) || rank(FlagRetryUnaffordable) >= rank(FlagEmpty) { t.Fatalf("a re-attack nobody could afford must out-rank the budget symptom it supersedes: "+ "unaffordable=%d length=%d empty=%d", rank(FlagRetryUnaffordable), rank(FlagLength), rank(FlagEmpty)) } // «An unrecognised string must not out-rank a diagnosis the engine actually made.» for r, s := range flagSeverity { if severityUnknown <= s { t.Fatalf("severityUnknown (%d) must be milder than every diagnosis the engine makes, but %s "+ "ranks %d", severityUnknown, r, s) } } // «Ranked below a budget symptom (the chunk is not lost)» — a stripped chunk SHIPPED. if rank(FlagSanitizerStripped) <= rank(FlagLength) || rank(FlagSanitizerStripped) <= rank(FlagEmpty) { t.Fatalf("an auto-cleaned chunk that SHIPPED must be milder than a budget symptom that lost one: "+ "stripped=%d length=%d empty=%d", rank(FlagSanitizerStripped), rank(FlagLength), rank(FlagEmpty)) } // «Ranked with the deterministic content failures, above a mere budget symptom» — a DROPPED // contaminated output is unreadable as shipped. if rank(FlagSanitizerDefect) >= rank(FlagLength) { t.Fatalf("a contaminated output that was DROPPED must be more severe than a budget symptom: "+ "defect=%d length=%d", rank(FlagSanitizerDefect), rank(FlagLength)) } }