398 lines
16 KiB
Go
398 lines
16 KiB
Go
package main
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"database/sql"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"strings"
|
||
"syscall"
|
||
"testing"
|
||
"time"
|
||
|
||
"textmachine/backend/internal/pipeline"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// migrate_cli_test.go is the CLI half of backlog row 174: the deploy deadlock is a SHELL fact — the
|
||
// platform learns what happened from an exit code and a line of stderr — so the contract is exercised
|
||
// where it is a contract. The migration mechanics themselves are pinned generically in
|
||
// internal/store/migrate_test.go.
|
||
|
||
// staleTheProjectSchema rolls the project database's recorded version back one step, which is what an
|
||
// older engine binary left behind.
|
||
//
|
||
// ⚠ This fixture re-applies the newest migration on the next write open, which is fine while that step
|
||
// is `CREATE … IF NOT EXISTS` and is NOT guaranteed for a future ALTER step (backlog row 49а). It is
|
||
// used here because these tests are about the exit codes and the output, and the deep fixture — the
|
||
// migration chain truncated to an older vintage — is only reachable from inside the store package,
|
||
// where those properties are tested. If a future migration makes this fail, move the setup rather than
|
||
// weakening the assertions.
|
||
func staleTheProjectSchema(t *testing.T, dbPath string) int {
|
||
t.Helper()
|
||
db, err := sql.Open("sqlite", "file:"+dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer db.Close()
|
||
if _, err := db.Exec(`DELETE FROM schema_version WHERE version = (SELECT MAX(version) FROM schema_version)`); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return schemaVersionOf(t, dbPath)
|
||
}
|
||
|
||
// schemaVersionOf reads the recorded schema version through its own connection — the state under test
|
||
// must be asserted independently of the product code that reports it.
|
||
func schemaVersionOf(t *testing.T, dbPath string) int {
|
||
t.Helper()
|
||
db, err := sql.Open("sqlite", "file:"+dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer db.Close()
|
||
var v int
|
||
if err := db.QueryRow(`SELECT COALESCE(MAX(version), 0) FROM schema_version`).Scan(&v); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return v
|
||
}
|
||
|
||
// holdTheProject takes the project's exclusive flock the way another tmctl process would, WITHOUT
|
||
// opening the store — because a write open would migrate the database, and the state under test is a
|
||
// project that is both stale and held.
|
||
func holdTheProject(t *testing.T, dbPath string) {
|
||
t.Helper()
|
||
f, err := os.OpenFile(dbPath+".lock", os.O_CREATE|os.O_RDWR, 0o644)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() {
|
||
_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||
f.Close()
|
||
})
|
||
}
|
||
|
||
func projectDBOf(bookPath string) string {
|
||
return filepath.Join(filepath.Dir(bookPath), "cli-book.db")
|
||
}
|
||
|
||
func backupsOf(t *testing.T, dbPath string) []string {
|
||
t.Helper()
|
||
entries, err := os.ReadDir(backupDirFor(dbPath))
|
||
if os.IsNotExist(err) {
|
||
return nil
|
||
}
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var names []string
|
||
for _, e := range entries {
|
||
names = append(names, e.Name())
|
||
}
|
||
return names
|
||
}
|
||
|
||
// mockProvider answers every call with a usable draft, so a `translate` really settles money.
|
||
func mockProvider(t *testing.T) *httptest.Server {
|
||
t.Helper()
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
_, _ = io.ReadAll(r.Body)
|
||
fmt.Fprint(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":"ЧЕРНОВИК ПЕРЕВОДА"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500}}`)
|
||
}))
|
||
t.Cleanup(srv.Close)
|
||
return srv
|
||
}
|
||
|
||
// TestTheDeployDeadlockIsBrokenAtTheShell drives the real binary through the exact sequence the
|
||
// platform meets on an engine upgrade: `status --json` before a spawn refuses the older schema, and the
|
||
// contract must make that refusal actionable — its own number inside the refusal band, and the two
|
||
// versions machine-readable — so the caller can migrate and start the run instead of stopping the world.
|
||
func TestTheDeployDeadlockIsBrokenAtTheShell(t *testing.T) {
|
||
if testing.Short() {
|
||
t.Skip("builds and runs the binary")
|
||
}
|
||
bin := buildTmctl(t)
|
||
bookPath := setupCLIProject(t, mockProvider(t).URL)
|
||
dbPath := projectDBOf(bookPath)
|
||
|
||
// A book an older binary left behind.
|
||
st, err := store.Open(dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
st.Close()
|
||
from := staleTheProjectSchema(t, dbPath)
|
||
|
||
var stderr bytes.Buffer
|
||
status := exec.Command(bin, "status", "--config", bookPath, "--json")
|
||
status.Stderr = &stderr
|
||
if got := exitCodeOf(t, status.Run()); got != exitSchemaMismatch {
|
||
t.Fatalf("`status --json` on an older schema exited %d, want %d", got, exitSchemaMismatch)
|
||
}
|
||
// The number says WHAT; these two say which way and how far — without them a caller cannot tell the
|
||
// direction a migration repairs from the one where it would loop forever.
|
||
want := fmt.Sprintf("schema_mismatch found=%d expected=%d", from, store.SchemaHead())
|
||
if !strings.Contains(stderr.String(), want) {
|
||
t.Fatalf("the refusal must carry %q machine-readably; got:\n%s", want, stderr.String())
|
||
}
|
||
if exitSchemaMismatch < refusalFirst || exitSchemaMismatch > refusalLast {
|
||
t.Fatalf("%d is outside the refusal band [%d,%d] every consumer keys on", exitSchemaMismatch, refusalFirst, refusalLast)
|
||
}
|
||
|
||
out, err := exec.Command(bin, "migrate", "--config", bookPath).Output()
|
||
if got := exitCodeOf(t, err); got != 0 {
|
||
t.Fatalf("`migrate` exited %d: %s", got, out)
|
||
}
|
||
if want := fmt.Sprintf("schema v%d -> v%d", from, store.SchemaHead()); !strings.Contains(string(out), want) {
|
||
t.Fatalf("migrate must report the transition %q; got:\n%s", want, out)
|
||
}
|
||
if got := exitCodeOf(t, exec.Command(bin, "status", "--config", bookPath, "--json").Run()); got != 0 {
|
||
t.Fatalf("`status --json` after the migration exited %d, want 0 — the deadlock is not broken", got)
|
||
}
|
||
// …and the run the migration unblocked actually starts. It follows within milliseconds, which is
|
||
// what put migrate's restore point and the paid path's pre-flight guard in the same second.
|
||
if run := exec.Command(bin, "translate", "--config", bookPath); exitCodeOf(t, run.Run()) != 0 {
|
||
t.Fatal("the run spawned right after the migration must start, not die in its pre-flight guard")
|
||
}
|
||
}
|
||
|
||
// TestTheMigrationRestorePointNeverCollidesWithThePaidPath is the deterministic half of the same fact:
|
||
// both producers are handed the SAME stamp, because in the deploy sequence they really do land in the
|
||
// same second. `store.BackupSQLite` refuses to overwrite a restore point (backlog row 173), so sharing
|
||
// the name would make the run refuse to start with exit 1 — outside the refusal band, and after a
|
||
// migration that already succeeded.
|
||
func TestTheMigrationRestorePointNeverCollidesWithThePaidPath(t *testing.T) {
|
||
bookPath := setupCLIProject(t, "http://127.0.0.1:1")
|
||
dbPath := projectDBOf(bookPath)
|
||
st, err := store.Open(dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
st.Close()
|
||
staleTheProjectSchema(t, dbPath)
|
||
|
||
// PRODUCTION takes the restore point — the point of the test. Supplying the name here instead would
|
||
// only pin that the constant is non-empty, and the regression to catch is the call site dropping it.
|
||
if err := migrateCmd(bookPath, io.Discard); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
names := backupsOf(t, dbPath)
|
||
if len(names) != 1 {
|
||
t.Fatalf("restore points after the migration: %v, want exactly one", names)
|
||
}
|
||
|
||
// Strip the tag production chose and the bare second-stamp must still be free, so translate's
|
||
// pre-flight guard can take its own backup in the SAME second. Untagged, the two names are equal and
|
||
// BackupSQLite refuses to overwrite — the run the migration just unblocked would die in its
|
||
// pre-flight guard with exit 1. No wall clock is involved: the collision is decided by the name.
|
||
tagged := strings.TrimSuffix(names[0], ".db")
|
||
bare := strings.TrimSuffix(tagged, migrateSuffix)
|
||
if bare == tagged {
|
||
t.Fatalf("the migration's restore point %q carries no tag of its own — it is inside the paid path's namespace", names[0])
|
||
}
|
||
if _, err := store.BackupSQLite(dbPath, backupDirFor(dbPath), bare); err != nil {
|
||
t.Fatalf("the paid path's pre-flight backup in the SAME second must still succeed: %v", err)
|
||
}
|
||
}
|
||
|
||
// TestMigrateNeedsNeitherPricesNorModels: the $0 path must not carry the paid path's gates. A
|
||
// models.yaml whose prices are older than 120 days refuses to load (backlog row 146), and if the
|
||
// migration were built on the full config stack a stand with stale prices would meet the same deadlock
|
||
// this command exists to remove — the read command refuses, and the one thing that could migrate
|
||
// refuses too.
|
||
func TestMigrateNeedsNeitherPricesNorModels(t *testing.T) {
|
||
bookPath := setupCLIProject(t, mockProvider(t).URL)
|
||
modelsPath := filepath.Join(filepath.Dir(bookPath), "models.yaml")
|
||
raw, err := os.ReadFile(modelsPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
stale := time.Now().UTC().AddDate(0, 0, -200).Format("2006-01-02")
|
||
writeCLIFile(t, modelsPath, strings.Replace(string(raw), time.Now().UTC().Format("2006-01-02"), stale, 1))
|
||
|
||
// The contrast: the read-only projection is refused by the price gate before it reads a schema.
|
||
if got := exitCode(status(context.Background(), bookPath, true)); got != exitConfigInvalid {
|
||
t.Fatalf("setup: `status` on stale prices exited %d, want %d", got, exitConfigInvalid)
|
||
}
|
||
var out bytes.Buffer
|
||
if err := migrateCmd(bookPath, &out); err != nil {
|
||
t.Fatalf("migrate must not consult prices: %v", err)
|
||
}
|
||
if !strings.Contains(out.String(), fmt.Sprintf("v%d", store.SchemaHead())) {
|
||
t.Fatalf("migrate must report the head it brought the project to; got %q", out.String())
|
||
}
|
||
|
||
// The structural version of the same claim, which also covers the provider keys the models file
|
||
// carries: with no models.yaml at all the migration still runs, so it cannot be consulting either.
|
||
if err := os.Remove(modelsPath); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if got := exitCode(status(context.Background(), bookPath, true)); got != exitConfigInvalid {
|
||
t.Fatalf("setup: `status` without models.yaml exited %d, want %d", got, exitConfigInvalid)
|
||
}
|
||
if err := migrateCmd(bookPath, io.Discard); err != nil {
|
||
t.Fatalf("migrate must not load the model stack at all: %v", err)
|
||
}
|
||
}
|
||
|
||
// TestMigrateKeepsTheMoneyAndRepeatsAsANoOp is the money half at the command level: a real run's
|
||
// settled spend, a reservation its process never settled, and a migration between them. `committed` is
|
||
// what the platform's ceiling argument is computed from (PD-158) and what a settle reconciles against,
|
||
// so it must survive the deploy step untouched; `reserved` is advisory and the write open's recovery
|
||
// pass is entitled to zero it.
|
||
func TestMigrateKeepsTheMoneyAndRepeatsAsANoOp(t *testing.T) {
|
||
bookPath := setupCLIProject(t, mockProvider(t).URL)
|
||
dbPath := projectDBOf(bookPath)
|
||
if err := translate(context.Background(), bookPath, false, pipeline.RebillConsent{}, false, 0); err != nil {
|
||
t.Fatalf("setup run: %v", err)
|
||
}
|
||
|
||
st, err := store.Open(dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
committed, _, err := st.SpentUSD("cli-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if committed <= 0 {
|
||
t.Fatalf("setup: the run must have settled money, committed=%v", committed)
|
||
}
|
||
// The reservation a killed run leaves behind.
|
||
if _, verdict, err := st.Reserve("cli-book", 0.25, store.Ceilings{BookUSD: 10, DayUSD: 10}); err != nil || verdict != store.ReserveOK {
|
||
t.Fatalf("stale reserve: %v %v", verdict, err)
|
||
}
|
||
st.Close()
|
||
from := staleTheProjectSchema(t, dbPath)
|
||
|
||
for i := range 2 {
|
||
var out bytes.Buffer
|
||
if err := migrateCmd(bookPath, &out); err != nil {
|
||
t.Fatalf("call %d: %v", i, err)
|
||
}
|
||
ro, err := store.OpenReadOnly(dbPath)
|
||
if err != nil {
|
||
t.Fatalf("call %d: the read path must work after migrating: %v", i, err)
|
||
}
|
||
gotCommitted, gotReserved, err := ro.SpentUSD("cli-book")
|
||
ro.Close()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if gotCommitted != committed {
|
||
t.Fatalf("call %d: committed %v -> %v; a migration must not move settled money by a cent", i, committed, gotCommitted)
|
||
}
|
||
if gotReserved != 0 {
|
||
t.Fatalf("call %d: reserved=%v, want the recovery pass to have zeroed it", i, gotReserved)
|
||
}
|
||
// The restore point is taken for the call that MIGRATES and for no other, and it holds the
|
||
// database as it was BEFORE the step — a copy taken afterwards is not a restore point at all.
|
||
names := backupsOf(t, dbPath)
|
||
if len(names) != 1 {
|
||
t.Fatalf("call %d: restore points %v, want exactly the one the migrating call took", i, names)
|
||
}
|
||
if v := schemaVersionOf(t, filepath.Join(backupDirFor(dbPath), names[0])); v != from {
|
||
t.Fatalf("call %d: the restore point is at schema v%d, want the pre-migration v%d", i, v, from)
|
||
}
|
||
if i == 1 && !strings.Contains(out.String(), "already at schema") {
|
||
t.Fatalf("the repeat call must report a no-op; got %q", out.String())
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestMigrateRefusesWithTheSameVocabularyAsEveryOtherCommand(t *testing.T) {
|
||
t.Run("a broken config", func(t *testing.T) {
|
||
bookPath := setupCLIProject(t, "http://127.0.0.1:1")
|
||
writeCLIFile(t, bookPath, "book_id: cli-book\n bad indentation: [\n")
|
||
if got := exitCode(migrateCmd(bookPath, io.Discard)); got != exitConfigInvalid {
|
||
t.Fatalf("exit %d, want %d", got, exitConfigInvalid)
|
||
}
|
||
})
|
||
|
||
t.Run("a project another process holds", func(t *testing.T) {
|
||
// The deploy step runs while the operator believes the runs are stopped. If one is not, the
|
||
// answer is "come back later" — never a migration under a live writer's feet, whose reservations
|
||
// the recovery pass would zero. And nothing may be written for that refusal: a restore point
|
||
// taken before the lock would copy the whole database once per retry.
|
||
bookPath := setupCLIProject(t, "http://127.0.0.1:1")
|
||
dbPath := projectDBOf(bookPath)
|
||
st, err := store.Open(dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
st.Close()
|
||
staleTheProjectSchema(t, dbPath)
|
||
holdTheProject(t, dbPath)
|
||
|
||
if got := exitCode(migrateCmd(bookPath, io.Discard)); got != exitProjectLocked {
|
||
t.Fatalf("exit %d, want %d", got, exitProjectLocked)
|
||
}
|
||
if got := backupsOf(t, dbPath); len(got) != 0 {
|
||
t.Fatalf("a refused migration wrote %v; nothing may be written for a project it does not own", got)
|
||
}
|
||
// The retry the refusal invites reaches the lock too, instead of dying on the leftovers of the
|
||
// previous attempt.
|
||
if got := exitCode(migrateCmd(bookPath, io.Discard)); got != exitProjectLocked {
|
||
t.Fatalf("retry exited %d, want %d again", got, exitProjectLocked)
|
||
}
|
||
})
|
||
|
||
t.Run("a project newer than this binary", func(t *testing.T) {
|
||
// The direction a migration cannot repair. It must not read as success, or a caller looping
|
||
// "status refused → migrate → retry" would loop forever on a rolled-back binary.
|
||
bookPath := setupCLIProject(t, "http://127.0.0.1:1")
|
||
dbPath := projectDBOf(bookPath)
|
||
st, err := store.Open(dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
st.Close()
|
||
db, err := sql.Open("sqlite", "file:"+dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := db.Exec(`INSERT INTO schema_version (version) VALUES (?)`, store.SchemaHead()+7); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
db.Close()
|
||
err = migrateCmd(bookPath, io.Discard)
|
||
if got := exitCode(err); got != exitSchemaMismatch {
|
||
t.Fatalf("exit %d, want %d", got, exitSchemaMismatch)
|
||
}
|
||
if !strings.Contains(err.Error(), fmt.Sprintf("schema_mismatch found=%d expected=%d", store.SchemaHead()+7, store.SchemaHead())) {
|
||
t.Fatalf("the refusal must name both versions machine-readably: %v", err)
|
||
}
|
||
if got := backupsOf(t, dbPath); len(got) != 0 {
|
||
t.Fatalf("a refused migration wrote %v", got)
|
||
}
|
||
})
|
||
|
||
t.Run("a book that has never been run", func(t *testing.T) {
|
||
// The deploy sweep does not have to know which books have a database yet. Nothing is backed up:
|
||
// a database this call creates has no history to lose.
|
||
bookPath := setupCLIProject(t, "http://127.0.0.1:1")
|
||
var out bytes.Buffer
|
||
if err := migrateCmd(bookPath, &out); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !strings.Contains(out.String(), "created") {
|
||
t.Fatalf("migrate must say it created the project; got %q", out.String())
|
||
}
|
||
if v := schemaVersionOf(t, projectDBOf(bookPath)); v != store.SchemaHead() {
|
||
t.Fatalf("created at v%d, want v%d", v, store.SchemaHead())
|
||
}
|
||
if got := backupsOf(t, projectDBOf(bookPath)); len(got) != 0 {
|
||
t.Fatalf("a project being created was backed up: %v", got)
|
||
}
|
||
})
|
||
}
|