package main import ( "context" "errors" "flag" "fmt" "io" "os" "path/filepath" "strings" "text/tabwriter" "time" "unicode" "unicode/utf8" "textmachine/platform/internal/money" "textmachine/platform/internal/pgstore" "textmachine/platform/internal/runner" "textmachine/platform/internal/runs" ) // exitMarker records how a run's unit ended. systemd runs it as ExecStopPost=, so it must work with // nothing at all: no database, no credentials, no network. Everything it writes comes from the // environment systemd sets ($SERVICE_RESULT, $EXIT_CODE, $EXIT_STATUS) and from its two arguments. // // It lives in this CLI rather than in a shell one-liner inside the unit for three reasons that all // bit somebody once: a shell command inside a systemd property has its own quoting rules and a state // directory with a space in it silently splits into two arguments; the write has to be ATOMIC, // because the reader polls and half a marker reads as a finished run; and a Go function can be // tested, whereas a quoted string in a property cannot. func exitMarker(args []string) error { fs := flag.NewFlagSet("exit-marker", flag.ContinueOnError) if err := fs.Parse(args); err != nil { return err } if fs.NArg() != 2 { return errors.New("exit-marker takes ") } return runner.WriteMarker(fs.Arg(0), runner.MarkerFromEnv(fs.Arg(1), os.Getenv)) } // listBooks answers the question an engine upgrade asks: which books may have their project file // migrated right now (unified backlog row 174). // // It is a COMMAND and not a paragraph in the deploy note because the deploy note's step has to be // executable: `--migratable` prints one directory per line and nothing else, so the upgrade reads // // for d in $(tmplatformctl books --migratable); do \ // /opt/textmachine/engine//tmctl migrate --config "$d/book.yaml"; done // // ⚠ The VERSIONED path and never a bare `tmctl` from PATH, which is what this example used to show: // the migration has to run under the NEW binary, and the old one migrates the file into its own // schema — that is, does nothing — leaving the deadlock in place and the operator with a command // that printed success (deploy/README.md, "Апгрейд ДВИЖКА"). // // A book somebody can still resume is simply not in that list. Written the other way round — an // operator eyeballing a table — the book that gets migrated by mistake is the one whose owner comes // back to a run that no longer starts, or to a hold nothing can close. func listBooks(ctx context.Context, store *pgstore.Store, args []string, out io.Writer) error { fs := flag.NewFlagSet("books", flag.ContinueOnError) migratable := fs.Bool("migratable", false, "print only the project directories that are safe to migrate") abandoned := fs.Bool("abandoned", false, "print only the books whose reading surface was given up on") if err := fs.Parse(args); err != nil { return err } if *abandoned { return listAbandoned(ctx, store, out) } books, err := store.BooksForMigration(ctx) if err != nil { return err } if *migratable { for _, b := range books { if b.Migratable() { if _, err := fmt.Fprintln(out, b.Workdir); err != nil { return err } } } return nil } w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) _, _ = fmt.Fprintln(w, "BOOK\tMIGRATE\tWHY NOT\tWORKDIR\tTITLE") for _, b := range books { verdict, why := "yes", "" if !b.Migratable() { var blockers []string for _, c := range []struct { blocked bool name string }{{b.Live, "live run"}, {b.Resumable, "resumable run"}, {b.Unsettled, "unsettled hold"}} { if c.blocked { blockers = append(blockers, c.name) } } verdict, why = "no", strings.Join(blockers, ", ") } _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", b.ID, verdict, why, b.Workdir, oneLine(b.Title)) } return w.Flush() } // addBook registers a book that already exists on disk. // // A DEV tool, and named as one: the contract's own intake (POST /books, a multipart upload) is not // built, so the library would otherwise have nothing to list. What matters for the pack is the rule // it enforces — the library reads the read-model, so a book put there by ANY route is a book the API // serves, and the upload handle is one more writer of the same row rather than a second world. func addBook(ctx context.Context, store *pgstore.Store, args []string, out io.Writer) error { in, err := parseBookIntake(args) if err != nil { return err } id, err := store.AddBook(ctx, in) if err != nil { return err } _, err = fmt.Fprintf(out, "added %s\n", id) return err } // parseBookIntake validates the arguments and nothing else. Separate from addBook because what a // book needs to be usable — an engine project directory, a chapter count the ceiling scale can be // built from — is the part worth asserting, and it is not worth a database to assert it. func parseBookIntake(args []string) (pgstore.NewBook, error) { fs := flag.NewFlagSet("book add", flag.ContinueOnError) user := fs.String("user", "", "account that owns the book") workdir := fs.String("workdir", "", "the engine's project directory (holds book.yaml)") title := fs.String("title", "", "title as the reader sees it") src := fs.String("source-lang", "", "source language CODE, never a name") dst := fs.String("target-lang", "", "target language CODE, never a name") // Chapters are given rather than counted: counting them means chunking the source, which is the // engine's job and costs seconds of CPU per call until it persists a manifest (unified backlog // row 100). The number matters because it clamps the ceiling scale. chapters := fs.Int("chapters", 0, "how many chapters the book has") characters := fs.Int64("characters", 0, "size in characters") if err := fs.Parse(args); err != nil { return pgstore.NewBook{}, err } switch { case *user == "" || *workdir == "" || *title == "": return pgstore.NewBook{}, errors.New("book add needs --user, --workdir and --title") case *src == "" || *dst == "": return pgstore.NewBook{}, errors.New("book add needs --source-lang and --target-lang") case *chapters <= 0: return pgstore.NewBook{}, errors.New("book add needs --chapters: it is what bounds the run ceiling scale") } abs, err := filepath.Abs(*workdir) if err != nil { return pgstore.NewBook{}, err } // Checked here because the alternative is a run that dies at argument parsing inside a transient // unit, where the only trace is a marker saying "exit-code 1". if _, err := os.Stat(filepath.Join(abs, runner.ConfigFile)); err != nil { return pgstore.NewBook{}, fmt.Errorf("%s does not look like an engine project directory: %w", abs, err) } return pgstore.NewBook{ OwnerID: *user, Title: *title, SourceLang: *src, TargetLang: *dst, ChapterCount: *chapters, CharacterCount: *characters, Workdir: abs, Now: time.Now().UTC(), }, nil } // listRuns is the diagnosis half of the operator's handle on a wedged control plane. // // It exists because the only signal there was is a counter with no names in it: // `tm_platform_sweep_unfinished_total` says a pass ran out of time and never says on WHAT, so an // operator could watch it climb for days with nothing to look at and nothing to do (register row // PD-169, re-opened by the P7 acceptance). What they need is the four facts below — which run, how // many times running, what it said, and what it is holding — because those decide whether to wait, // to fix the host, or to end the run. // // `--stalled` is the same list narrowed to the ones that have crossed the threshold: on a healthy // deployment it prints nothing, which is what makes it usable from a cron line. // // ⚠ IT COVERS BOTH PHASES since PD-385. A run whose RECONCILIATION is stuck and one whose // SETTLEMENT is stuck are one question to an operator — what is wedged and what is it holding — and // they were not one list: the store's query was the live phase's alone, so the ERROR logged at the // threshold named this command and this command answered "no run is failing to reconcile" over a // frozen hold. The PHASE column is what tells them apart, because the remedy differs. func listRuns(ctx context.Context, store *pgstore.Store, args []string, out io.Writer) error { fs := flag.NewFlagSet("runs", flag.ContinueOnError) stalled := fs.Bool("stalled", false, "print only what the reconciler keeps failing on: live runs it cannot finish AND finished runs whose settlement will not close") if err := fs.Parse(args); err != nil { return err } // ⚠ ZERO, not one. The usage line promises "live runs and what the reconciler cannot finish", and // a floor of one silently drops the first half of that: on a healthy deployment the command // printed "no run is failing to reconcile" and never showed the runs that were going perfectly // well — which is also the answer an operator gets while a run IS wedged and the deferral has not // counted it yet. from := 0 if *stalled { from = runs.StalledAfter } list, err := store.StalledRuns(ctx, from) if err != nil { return err } if len(list) == 0 { // Said out loud rather than printing an empty table: "no output" and "the query found nothing" // look the same to a person and one of them means the command did not run. what := "no run is live and no settlement is stuck" if *stalled { what = "no run is failing to reconcile and no settlement is failing to close" } _, err := fmt.Fprintln(out, what) return err } w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) _, _ = fmt.Fprintln(w, "RUN\tPHASE\tATTEMPT\tSTATUS\tFAILS\tNEXT TRY\tHELD\tSPENT\tHELD FOR\tUNIT\tBOOK\tLAST ERROR") for _, r := range list { next := "now" if r.NextTry != nil { next = r.NextTry.UTC().Format(time.RFC3339) } held := "-" if r.HeldMicroUSD > 0 { held = money.MicroUSD(r.HeldMicroUSD).USD() } unit := r.UnitName if unit == "" { // The difference decides what the operator may do: `run abandon` refuses a run whose // attempt still names a unit, because closing one over a live engine strands it spending. unit = "(none)" } // "?" and not "$0.00": an attempt with no baseline is the one the settlement refuses to price, // and a zero here would be a figure an operator could write a hold off against. spent := "?" if r.SpentMicroUSD != nil { spent = money.MicroUSD(*r.SpentMicroUSD).USD() } // The two phases need different things done to them, so the row says which it is rather than // leaving an operator to infer it from a run status that reads the same either way: a // `settling` row's engine is already gone, and stopping a unit is not its remedy. // ⚠ A `settling` row is NOT an invitation to `run abandon` — that command writes the hold off // whole and refuses anything below the stalled threshold, while this table shows the row from // its FIRST failure so the trouble is visible while it is only trouble. Seeing early and // acting late are the point; the FAILS column is what says which of the two this row is. phase := "live" if r.Settling { phase = "settling" } _, _ = fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", r.RunID, phase, r.AttemptNo, r.Status, r.Failures, next, held, spent, time.Duration(r.HeldSeconds)*time.Second, unit, oneLine(r.Title), oneLine(r.LastError)) } return w.Flush() } // oneLine keeps a table a table: a driver's error string carries newlines, and one of them turns a // row into several that no longer line up with the header. // // Two kinds of text reach it and neither is trustworthy: the engine's stderr, verbatim and unbounded // (runner.readEngine), and a book's TITLE, which the intake strips control characters from only when // it came from a FILENAME — so a hand-typed one arrives with its tabs and newlines able to forge // columns and rows here. // // ⚠ CUT ON A RUNE BOUNDARY: the product translates zh/ja, so a fixed byte offset lands inside a // three-byte rune about two times in three. func oneLine(s string) string { s = strings.Map(func(r rune) rune { if unicode.IsControl(r) { return ' ' // a tab forges a column; a newline forges a row } return r }, s) s = strings.Join(strings.Fields(s), " ") if len(s) <= 120 { return s } cut := 117 for cut > 0 && !utf8.RuneStart(s[cut]) { cut-- } return s[:cut] + "..." } // abandonRun is the acting half, and the whole reason the diagnosis above is worth printing. // // What it does NOT do is decide anything about money by itself, and it does not have to: abandoning // is refused for any attempt that reached the engine, so the hold of one that did not comes back // WHOLE through the ordinary settlement. `--release-hold` only does it now rather than on the next // sweep — which matters when the daemon is stopped, the state this CLI exists for. // // ⚠ THE PARAGRAPH ABOVE IS THE LIVE HALF ONLY, since PD-385. On a run whose PHASE is `settling` — // finished, with a settlement that has failed and will not close — there is no sweep left to hand // the money to, so the command closes it inline whatever the flag says, and the refusal above does // not apply because the attempt has already ended. What DOES apply there is a different refusal: a // settlement that has not failed even once is turned away, because this command returns the hold // WHOLE and that one is about to be settled correctly for what it really spent. // // It is not the escrow design (unified backlog row 136, zone backlog П-18) and does not pretend to // be one: that is a mechanism for deciding this automatically, and this is a person deciding one // case with their reason on the record. func abandonRun(ctx context.Context, store *pgstore.Store, args []string, out io.Writer) error { fs := flag.NewFlagSet("run abandon", flag.ContinueOnError) runID := fs.String("run", "", "the run to end") reason := fs.String("reason", "", "why — it is written to the attempt and to the ledger") release := fs.Bool("release-hold", false, "give the hold back now instead of on the next sweep (use when the daemon is stopped; ignored for a PHASE=settling run, whose money is always closed inline because no sweep will ever come for it)") if err := fs.Parse(args); err != nil { return err } if *runID == "" || *reason == "" { // The reason is required and not defaulted: this row is the only account anyone will ever have // of why a paid-for run was declared over, and "" is not one. return errors.New("run abandon needs --run and --reason") } verdict, err := store.AbandonRun(ctx, *runID, *reason, *release, time.Now().UTC()) switch { case errors.Is(err, pgstore.ErrNoRun): return fmt.Errorf("there is no run %s", *runID) case errors.Is(err, pgstore.ErrSettlementNotStuck): // Refused, and the refusal is a MONEY refusal: this command gives the hold back whole, so on a // settlement that has not failed it would be handing back whatever the run actually spent. return fmt.Errorf("run %s has finished and its settlement is not stalled yet: it has failed "+ "fewer than %d times running, so the sweep is still retrying it — abandoning it here "+ "would return the WHOLE hold and charge nothing for what the run actually spent. Wait "+ "for it to reach the threshold, or watch it in `tmplatformctl runs`", *runID, runs.StalledAfter) case errors.Is(err, pgstore.ErrMoneyAlreadyClosed): // A different sentence from "no such run" on purpose: this one means the operator was right // about the run and late about the state, and the answer is "nothing to do" rather than "look // somewhere else". The old code answered both with "is not a live run", which is how a frozen // hold read as a typo (PD-385). return fmt.Errorf("run %s has finished and its money is already closed; nothing to abandon", *runID) case errors.Is(err, pgstore.ErrRunMayHaveAProcess): // Refused rather than forced, and the message is the operator's next step: systemd owns the // process, so ending the run here would leave an engine spending against a run nothing looks // at any more. Stopping the unit makes the ordinary reconciler close the run properly, which // is better than this command in every way. The error names the unit, including the case where // the column is empty and only the spend baseline says a process may exist. return fmt.Errorf("%w — ask systemd first (systemctl --user status ), stop it if it is there and let the reconciler close the run, or wait for its exit marker", err) case err != nil: return err } what := "its hold comes back whole on the next sweep" if *release { what = "the hold was returned to the account whole" } if verdict == pgstore.AbandonedSettlement { // The settlement branch has no sweep to hand the money to — that IS the state — so it closes // the hold inside its own transaction whether or not the flag was given, and says so rather // than repeating a promise about a pass that will never come for this run. what = "its settlement was given up on and the hold was returned to the account whole" } _, err = fmt.Fprintf(out, "run %s abandoned; %s\n", *runID, what) return err } // refreshBook re-arms a reading surface the platform gave up materializing. // // The give-up is deliberately not final (see pgstore.AbandonReadModelDebt): the next boundary of // real work stamps a fresh debt by itself. This is for the case where there is no next boundary // coming soon — an operator who has just fixed the workdir, or moved the project back — and who // should not have to start a paid run to make the reader's screen fill in. func refreshBook(ctx context.Context, store *pgstore.Store, args []string, out io.Writer) error { fs := flag.NewFlagSet("book refresh", flag.ContinueOnError) book := fs.String("book", "", "the book whose reading surface to ask for again") if err := fs.Parse(args); err != nil { return err } if *book == "" { return errors.New("book refresh needs --book") } armed, err := store.RearmReadModelDebt(ctx, *book, time.Now().UTC()) if err != nil { return err } if !armed { // Two facts, one message, because the remedy for both is to wait: either the book already owes // a surface (the sweep will pay it) or there is no such book. _, err := fmt.Fprintf(out, "%s already owes a reading surface, or there is no such book; nothing to do\n", *book) return err } _, err = fmt.Fprintf(out, "%s will be materialized by the next sweep\n", *book) return err } // listAbandoned names the books behind the `reading_surfaces_abandoned` gauge. // // A book here is not broken to its owner: it still answers, with whatever text was materialized // before. What it stopped doing is getting FRESHER — so the operator's question is whether the cause // is theirs (a workdir that moved, an engine build that cannot read the project) and, once it is // fixed, `book refresh` asks again without waiting for the book's next run. func listAbandoned(ctx context.Context, store *pgstore.Store, out io.Writer) error { list, err := store.AbandonedSurfaces(ctx) if err != nil { return err } if len(list) == 0 { _, err := fmt.Fprintln(out, "no book has been given up on") return err } w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) _, _ = fmt.Fprintln(w, "BOOK\tTRIES\tGIVEN UP\tWORKDIR\tTITLE\tLAST ERROR") for _, b := range list { _, _ = fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\n", b.ID, b.Attempts, b.AbandonedAt.UTC().Format(time.RFC3339), b.Workdir, oneLine(b.Title), oneLine(b.LastError)) } return w.Flush() }