441 lines
16 KiB
Go
441 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"go/ast"
|
|
"go/parser"
|
|
"go/token"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"textmachine/backend/internal/pipeline"
|
|
"textmachine/backend/internal/runevents"
|
|
"textmachine/backend/internal/store"
|
|
)
|
|
|
|
// exitcontract_test.go drives the REAL binary, because the shell contract is only a contract at the
|
|
// shell: a supervisor learns how a run ended from the number the process exits with, and the acceptance
|
|
// of the platform's runner found that out the hard way — its fake probe died FROM the signal, so it
|
|
// never saw that the engine catches SIGTERM and left with 1 (PD-152).
|
|
|
|
var (
|
|
tmctlOnce sync.Once
|
|
tmctlPath string
|
|
tmctlErr error
|
|
)
|
|
|
|
// buildTmctl compiles the CLI once for the whole package.
|
|
func buildTmctl(t *testing.T) string {
|
|
t.Helper()
|
|
tmctlOnce.Do(func() {
|
|
dir, err := os.MkdirTemp("", "tmctl-bin")
|
|
if err != nil {
|
|
tmctlErr = err
|
|
return
|
|
}
|
|
tmctlPath = filepath.Join(dir, "tmctl")
|
|
out, err := exec.Command("go", "build", "-o", tmctlPath, ".").CombinedOutput()
|
|
if err != nil {
|
|
tmctlErr = fmt.Errorf("build tmctl: %w\n%s", err, out)
|
|
}
|
|
})
|
|
if tmctlErr != nil {
|
|
t.Fatal(tmctlErr)
|
|
}
|
|
return tmctlPath
|
|
}
|
|
|
|
func exitCodeOf(t *testing.T, err error) int {
|
|
t.Helper()
|
|
if err == nil {
|
|
return 0
|
|
}
|
|
var ee *exec.ExitError
|
|
if !errors.As(err, &ee) {
|
|
t.Fatalf("not an exit status: %v", err)
|
|
}
|
|
return ee.ExitCode()
|
|
}
|
|
|
|
// slowProvider answers after `delay`, so a signal can arrive while a call is in flight.
|
|
func slowProvider(t *testing.T, delay time.Duration) *httptest.Server {
|
|
t.Helper()
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
time.Sleep(delay)
|
|
fmt.Fprint(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":"ПЕРЕВОД"},"finish_reason":"stop"}],
|
|
"usage":{"prompt_tokens":100,"completion_tokens":50}}`)
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func journalOf(t *testing.T, bookPath string) []runevents.Envelope {
|
|
t.Helper()
|
|
body, err := os.ReadFile(filepath.Join(filepath.Dir(bookPath), runevents.JournalFile))
|
|
if err != nil {
|
|
t.Fatalf("read journal: %v", err)
|
|
}
|
|
var out []runevents.Envelope
|
|
for _, l := range strings.Split(strings.TrimSpace(string(body)), "\n") {
|
|
if l == "" {
|
|
continue
|
|
}
|
|
var ev runevents.Envelope
|
|
if err := json.Unmarshal([]byte(l), &ev); err != nil {
|
|
t.Fatalf("malformed journal line %q: %v", l, err)
|
|
}
|
|
out = append(out, ev)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func lastOutcome(t *testing.T, envs []runevents.Envelope) string {
|
|
t.Helper()
|
|
if len(envs) == 0 {
|
|
t.Fatal("empty journal")
|
|
}
|
|
last := envs[len(envs)-1]
|
|
if last.Type != runevents.TypeFinished {
|
|
t.Fatalf("the stream's last line is %s, not the terminal event", last.Type)
|
|
}
|
|
var f runevents.Finished
|
|
if err := json.Unmarshal(last.Data, &f); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return f.Outcome
|
|
}
|
|
|
|
func TestACeilingHaltLeavesItsOwnExitCode(t *testing.T) {
|
|
// PD-113: the platform's contract calls this stop `paused` and forbids `failed`, and exit 1 was the
|
|
// only thing it had to go on. Since the ceiling argument is set flush against the hold (PD-158), this
|
|
// is the number that separates "the money ran out" from "the machine broke".
|
|
if testing.Short() {
|
|
t.Skip("builds and runs the binary")
|
|
}
|
|
bin := buildTmctl(t)
|
|
srv := slowProvider(t, 0)
|
|
bookPath := setupCLIProject(t, srv.URL)
|
|
raiseCeiling(t, bookPath, "ceilings: { book_usd: 0.0000001, day_usd: 2.0 }")
|
|
|
|
err := exec.Command(bin, "translate", "--config", bookPath).Run()
|
|
if got := exitCodeOf(t, err); got != 4 {
|
|
t.Fatalf("a ceiling halt exited %d, want 4", got)
|
|
}
|
|
envs := journalOf(t, bookPath)
|
|
if got := lastOutcome(t, envs); got != runevents.OutcomeCeiling {
|
|
t.Fatalf("terminal outcome %q, want %q", got, runevents.OutcomeCeiling)
|
|
}
|
|
found := false
|
|
for _, ev := range envs {
|
|
if ev.Type == runevents.TypeCeiling {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("the stream must carry the ceiling event too: the exit code and the event fail independently")
|
|
}
|
|
}
|
|
|
|
func TestAGracefulStopLeavesItsOwnExitCode(t *testing.T) {
|
|
// The other half of row 165 and of PD-152: tmctl CATCHES SIGTERM, so it does not die from the signal —
|
|
// it winds down and exits. While that exit was 1, a host reboot closed every live run as `failed`.
|
|
if testing.Short() {
|
|
t.Skip("builds and runs the binary")
|
|
}
|
|
bin := buildTmctl(t)
|
|
srv := slowProvider(t, 3*time.Second)
|
|
bookPath := setupCLIProject(t, srv.URL)
|
|
|
|
cmd := exec.Command(bin, "translate", "--config", bookPath)
|
|
cmd.Stderr = os.Stderr
|
|
if err := cmd.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitForJournal(t, bookPath)
|
|
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := exitCodeOf(t, cmd.Wait()); got != 5 {
|
|
t.Fatalf("a caught SIGTERM exited %d, want 5", got)
|
|
}
|
|
if got := lastOutcome(t, journalOf(t, bookPath)); got != runevents.OutcomeStopped {
|
|
t.Fatalf("terminal outcome %q, want %q", got, runevents.OutcomeStopped)
|
|
}
|
|
}
|
|
|
|
// TestTheRefusalTableIsTotal: a class with no entry in refusalExit does not fail — it falls to
|
|
// exitRefusedOther, silently, and the distinction the class was introduced for is gone. The table is the
|
|
// only place that mapping exists, so its completeness is asserted rather than assumed.
|
|
// refusalClassesFromSource reads the refusal vocabulary out of the SOURCE that declares it.
|
|
//
|
|
// A hand-kept list here would be the very thing the gate is against: a class added to
|
|
// pipeline.RefusalClass and to nobody's list reads as the catch-all 19 at the shell, and the test that
|
|
// was supposed to notice would be one of the places that had not been updated. The acceptance planted
|
|
// exactly that and the battery stayed green.
|
|
//
|
|
// ⚠ It walks GenDecl/ValueSpec, NOT case-clause literals. The sibling gate over the dispatch switch is
|
|
// the precedent for the TECHNIQUE and a trap as an implementation: it keys on *ast.BasicLit inside a
|
|
// `case`, so a command declared as a CONSTANT is invisible to it (its own ping). These values are
|
|
// constants, so copying that shape would have reproduced the blindness one file over.
|
|
func refusalClassesFromSource(t *testing.T) map[string]bool {
|
|
t.Helper()
|
|
// The whole PACKAGE, not one file: a class declared in a sibling file of package pipeline would be
|
|
// invisible to a single-file read, and invisible is exactly the state this gate exists to end.
|
|
// (Walked by hand rather than with parser.ParseDir, which the linter refuses as deprecated.)
|
|
const dir = "../../internal/pipeline"
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
t.Fatalf("the refusal vocabulary must be readable under %s: %v", dir, err)
|
|
}
|
|
fset := token.NewFileSet()
|
|
var decls []ast.Decl
|
|
read := 0
|
|
for _, e := range entries {
|
|
name := e.Name()
|
|
if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
|
|
continue
|
|
}
|
|
f, perr := parser.ParseFile(fset, filepath.Join(dir, name), nil, 0)
|
|
if perr != nil {
|
|
t.Fatalf("parse %s: %v", name, perr)
|
|
}
|
|
decls = append(decls, f.Decls...)
|
|
read++
|
|
}
|
|
if read == 0 {
|
|
t.Fatalf("no source files read under %s — the gate would pass on anything", dir)
|
|
}
|
|
out := map[string]bool{}
|
|
for _, d := range decls {
|
|
gd, ok := d.(*ast.GenDecl)
|
|
if !ok || gd.Tok != token.CONST {
|
|
continue
|
|
}
|
|
for _, sp := range gd.Specs {
|
|
vs, ok := sp.(*ast.ValueSpec)
|
|
if !ok {
|
|
continue
|
|
}
|
|
typed := false
|
|
if id, ok := vs.Type.(*ast.Ident); ok && id.Name == "RefusalClass" {
|
|
typed = true
|
|
}
|
|
for _, v := range vs.Values {
|
|
// Two spellings reach the same vocabulary: `X RefusalClass = "s"` and the CONVERSION
|
|
// `X = RefusalClass("s")`, which carries no Type on the spec at all. Reading only the
|
|
// first would leave the second invisible — the same class of blindness as reading one file.
|
|
lit, ok := v.(*ast.BasicLit)
|
|
if !ok {
|
|
call, isCall := v.(*ast.CallExpr)
|
|
if !isCall || len(call.Args) != 1 {
|
|
continue
|
|
}
|
|
if id, isID := call.Fun.(*ast.Ident); !isID || id.Name != "RefusalClass" {
|
|
continue
|
|
}
|
|
lit, ok = call.Args[0].(*ast.BasicLit)
|
|
if !ok {
|
|
t.Fatalf("a RefusalClass conversion of something that is not a string literal: %#v", v)
|
|
}
|
|
} else if !typed {
|
|
continue
|
|
}
|
|
if lit.Kind != token.STRING {
|
|
t.Fatalf("a RefusalClass constant whose value is not a string literal: %#v", v)
|
|
}
|
|
value, uerr := strconv.Unquote(lit.Value)
|
|
if uerr != nil {
|
|
t.Fatal(uerr)
|
|
}
|
|
out[value] = true
|
|
}
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
t.Fatalf("no RefusalClass constants found under %s — the gate is reading the wrong shape and would pass on anything", dir)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestTheRefusalTableIsTotal(t *testing.T) {
|
|
declared := refusalClassesFromSource(t)
|
|
var classes []pipeline.RefusalClass
|
|
for c := range declared {
|
|
classes = append(classes, pipeline.RefusalClass(c))
|
|
}
|
|
sort.Slice(classes, func(i, j int) bool { return classes[i] < classes[j] })
|
|
if len(refusalExit) != len(classes) {
|
|
t.Fatalf("refusalExit maps %d classes, pipeline declares %d (%v) — a class added to pipeline.RefusalClass "+
|
|
"without a number here reads as the catch-all %d", len(refusalExit), len(classes), classes, exitRefusedOther)
|
|
}
|
|
for c := range refusalExit {
|
|
if !declared[string(c)] {
|
|
t.Errorf("refusalExit numbers %q, which pipeline no longer declares", c)
|
|
}
|
|
}
|
|
seen := map[int]pipeline.RefusalClass{}
|
|
for _, c := range classes {
|
|
code, ok := refusalExit[c]
|
|
if !ok {
|
|
t.Errorf("class %q has no number and would arrive as the catch-all %d", c, exitRefusedOther)
|
|
continue
|
|
}
|
|
if code < refusalFirst || code > refusalLast {
|
|
t.Errorf("class %q maps to %d, outside the band [%d,%d]", c, code, refusalFirst, refusalLast)
|
|
}
|
|
if prior, dup := seen[code]; dup {
|
|
t.Errorf("classes %q and %q share the number %d — a caller cannot tell them apart", prior, c, code)
|
|
}
|
|
seen[code] = c
|
|
}
|
|
}
|
|
|
|
func TestEveryRefusalClassHasItsOwnNumberInsideTheBand(t *testing.T) {
|
|
// PD-196: the platform's intake rejects a book as `source_unreadable` after five failures and deletes
|
|
// the upload. While every refusal was exit 1, an operator's typo in a hand-written book.yaml looked
|
|
// exactly like an unusable source.
|
|
if testing.Short() {
|
|
t.Skip("builds and runs the binary")
|
|
}
|
|
bin := buildTmctl(t)
|
|
srv := slowProvider(t, 0)
|
|
|
|
t.Run("a broken config", func(t *testing.T) {
|
|
bookPath := setupCLIProject(t, srv.URL)
|
|
writeCLIFile(t, bookPath, "book_id: cli-book\n bad indentation: [\n")
|
|
if got := exitCodeOf(t, exec.Command(bin, "translate", "--config", bookPath).Run()); got != exitConfigInvalid {
|
|
t.Fatalf("exit %d, want %d", got, exitConfigInvalid)
|
|
}
|
|
})
|
|
|
|
t.Run("a source the config names but the host does not have", func(t *testing.T) {
|
|
// The CONFIG class, not the text class, and the difference is a user's file. A missing path says
|
|
// something about the deployment — a template naming a filename the upload route did not use, a
|
|
// mount that went away — and exit 11 is what an automated intake acts on by deleting the upload.
|
|
bookPath := setupCLIProject(t, srv.URL)
|
|
if err := os.Remove(filepath.Join(filepath.Dir(bookPath), "source.txt")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := exitCodeOf(t, exec.Command(bin, "translate", "--config", bookPath).Run()); got != exitConfigInvalid {
|
|
t.Fatalf("exit %d, want %d — a missing path must not be reported as the user's text being unusable", got, exitConfigInvalid)
|
|
}
|
|
})
|
|
|
|
t.Run("a source that exists but cannot be read", func(t *testing.T) {
|
|
// Passes the config's existence check and fails at the READ — a permission, an I/O error, a path
|
|
// that is not a file. Still the config/host class: nothing here says the user's text is bad, and
|
|
// exit 11 is what deletes it.
|
|
bookPath := setupCLIProject(t, srv.URL)
|
|
src := filepath.Join(filepath.Dir(bookPath), "source.txt")
|
|
if err := os.Remove(src); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Mkdir(src, 0o755); err != nil { // stat succeeds, every read fails
|
|
t.Fatal(err)
|
|
}
|
|
if got := exitCodeOf(t, exec.Command(bin, "translate", "--config", bookPath).Run()); got != exitConfigInvalid {
|
|
t.Fatalf("exit %d, want %d — an unreadable path must not be reported as the user's text being unusable", got, exitConfigInvalid)
|
|
}
|
|
})
|
|
|
|
t.Run("a source that reads but holds no book", func(t *testing.T) {
|
|
// The ONE thing the engine can assert about the TEXT with no configuration in the way: the bytes
|
|
// were read and cut, and there is nothing to translate. No `encoding`, `source_lang` or path
|
|
// setting explains an empty result from a successful read.
|
|
bookPath := setupCLIProject(t, srv.URL)
|
|
writeCLIFile(t, filepath.Join(filepath.Dir(bookPath), "source.txt"), " \n\n")
|
|
if got := exitCodeOf(t, exec.Command(bin, "translate", "--config", bookPath).Run()); got != exitSourceUnreadable {
|
|
t.Fatalf("exit %d, want %d", got, exitSourceUnreadable)
|
|
}
|
|
})
|
|
|
|
t.Run("a project another process holds", func(t *testing.T) {
|
|
bookPath := setupCLIProject(t, srv.URL)
|
|
held, err := store.Open(filepath.Join(filepath.Dir(bookPath), "cli-book.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer held.Close()
|
|
if got := exitCodeOf(t, exec.Command(bin, "translate", "--config", bookPath).Run()); got != exitProjectLocked {
|
|
t.Fatalf("exit %d, want %d", got, exitProjectLocked)
|
|
}
|
|
})
|
|
|
|
// The band is the contract, not the individual numbers: a caller that treats [10,19] as "refused"
|
|
// stays safe when a class it has never heard of appears.
|
|
for _, code := range []int{exitConfigInvalid, exitSourceUnreadable, exitProjectLocked, exitSchemaMismatch, exitDecisionsRejected, exitRefusedOther} {
|
|
if code < refusalFirst || code > refusalLast {
|
|
t.Fatalf("refusal code %d is outside the band [%d,%d] every consumer keys on", code, refusalFirst, refusalLast)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheCallerMayNameTheRun(t *testing.T) {
|
|
// Row 102: a run the platform spawned should be ONE trace, and the id is also the `engine_run_id`
|
|
// half of the seam's idempotency key — so letting the caller supply it makes that namespace theirs
|
|
// by construction instead of something they have to learn from the handshake.
|
|
if testing.Short() {
|
|
t.Skip("builds and runs the binary")
|
|
}
|
|
bin := buildTmctl(t)
|
|
srv := slowProvider(t, 0)
|
|
bookPath := setupCLIProject(t, srv.URL)
|
|
|
|
cmd := exec.Command(bin, "translate", "--config", bookPath)
|
|
cmd.Env = append(os.Environ(), "TM_TRACE_ID=platform-run-42")
|
|
cmd.Stderr = os.Stderr
|
|
if err := cmd.Run(); err != nil {
|
|
t.Fatalf("run: %v", err)
|
|
}
|
|
envs := journalOf(t, bookPath)
|
|
var h runevents.Hello
|
|
if err := json.Unmarshal(envs[0].Data, &h); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if h.EngineRunID != "platform-run-42" {
|
|
t.Fatalf("engine_run_id = %q, want the id the caller passed", h.EngineRunID)
|
|
}
|
|
}
|
|
|
|
// raiseCeiling rewrites the fixture's ceilings line.
|
|
func raiseCeiling(t *testing.T, bookPath, line string) {
|
|
t.Helper()
|
|
body, err := os.ReadFile(bookPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
patched := strings.Replace(string(body), "ceilings: { book_usd: 1.0, day_usd: 2.0 }", line, 1)
|
|
if patched == string(body) {
|
|
t.Fatalf("the ceilings line moved; fixture needs updating:\n%s", body)
|
|
}
|
|
writeCLIFile(t, bookPath, patched)
|
|
}
|
|
|
|
// waitForJournal blocks until the run has written its handshake, so a signal cannot arrive before the
|
|
// process has started working.
|
|
func waitForJournal(t *testing.T, bookPath string) {
|
|
t.Helper()
|
|
path := filepath.Join(filepath.Dir(bookPath), runevents.JournalFile)
|
|
deadline := time.Now().Add(30 * time.Second)
|
|
for {
|
|
if body, err := os.ReadFile(path); err == nil && strings.Contains(string(body), `"hello"`) {
|
|
return
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("the run wrote no handshake in 30s")
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
}
|