430 lines
17 KiB
Go
430 lines
17 KiB
Go
// tmctl is the TextMachine CLI (Фаза 0: translate один чанк + report).
|
||
package main
|
||
|
||
import (
|
||
"bufio"
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"flag"
|
||
"fmt"
|
||
"os"
|
||
"os/signal"
|
||
"path/filepath"
|
||
"strings"
|
||
"syscall"
|
||
|
||
"textmachine/backend/internal/obs"
|
||
"textmachine/backend/internal/pipeline"
|
||
)
|
||
|
||
// Exit codes (Веха 2): 0 clean · 2 completed-with-flags (приёмка допускает N
|
||
// флагов) · 1 infra failure. The exit-2 case is carried up as a typed sentinel
|
||
// (*pipeline.CompletedWithFlags) so a "clean run, attention needed" never
|
||
// masquerades as an infra crash and vice-versa.
|
||
func main() {
|
||
err := run()
|
||
var flagged *pipeline.CompletedWithFlags
|
||
switch {
|
||
case err == nil:
|
||
return
|
||
case errors.As(err, &flagged):
|
||
fmt.Fprintln(os.Stderr, "tmctl:", err)
|
||
os.Exit(2)
|
||
default:
|
||
fmt.Fprintln(os.Stderr, "tmctl:", err)
|
||
os.Exit(1)
|
||
}
|
||
}
|
||
|
||
func run() error {
|
||
if len(os.Args) < 2 {
|
||
return fmt.Errorf("usage: tmctl <translate|report|status|redrive> --config book.yaml")
|
||
}
|
||
cmd, args := os.Args[1], os.Args[2:]
|
||
|
||
// ContinueOnError (not ExitOnError): the stdlib's ExitOnError calls os.Exit(2)
|
||
// on a parse failure, which would COLLIDE with exit code 2 (completed-with-
|
||
// flags). Returning the error routes a bad flag to main()'s default branch →
|
||
// exit 1, keeping 2 exclusive to the flagged-chunks case (self-review Веха 2).
|
||
fs := flag.NewFlagSet(cmd, flag.ContinueOnError)
|
||
cfgPath := fs.String("config", "", "path to book.yaml")
|
||
resnapshot := fs.Bool("resnapshot", false, "re-pin existing jobs to the current config snapshot (пере-перевод оплаченных чанков — явное согласие)")
|
||
asJSON := fs.Bool("json", false, "status: emit the projection as JSON (stable disposition/flag_reason enums) for CI/IDE")
|
||
chapter := fs.Int("chapter", -1, "redrive: restrict to this chapter (default: any)")
|
||
chunk := fs.Int("chunk", -1, "redrive: restrict to this chunk index within the chapter (default: any)")
|
||
reason := fs.String("reason", "", "redrive: restrict to this flag_reason (default: any)")
|
||
dryRun := fs.Bool("dry-run", false, "redrive: report what would be re-attacked without touching anything")
|
||
if err := fs.Parse(args); err != nil {
|
||
return err
|
||
}
|
||
if *cfgPath == "" {
|
||
return fmt.Errorf("--config book.yaml is required")
|
||
}
|
||
|
||
loadDotEnv(filepath.Join(filepath.Dir(*cfgPath), ".env"))
|
||
loadDotEnv(".env")
|
||
|
||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||
defer stop()
|
||
// LogBodies решается один раз на допуске (privacy by default): содержимое
|
||
// LLM-обменов попадает в логи только при LOG_LLM_BODIES=1 И LOG_LEVEL=debug.
|
||
ctx = obs.WithReqInfo(ctx, obs.ReqInfo{
|
||
TraceID: obs.NewTraceID(),
|
||
LogBodies: os.Getenv("LOG_LLM_BODIES") == "1",
|
||
})
|
||
|
||
switch cmd {
|
||
case "translate":
|
||
return translate(ctx, *cfgPath, *resnapshot)
|
||
case "report":
|
||
return report(*cfgPath)
|
||
case "status":
|
||
return status(ctx, *cfgPath, *asJSON)
|
||
case "redrive":
|
||
return redrive(ctx, *cfgPath, *resnapshot, pipeline.RedriveSelector{
|
||
Chapter: *chapter, ChunkIdx: *chunk, Reason: *reason, DryRun: *dryRun,
|
||
})
|
||
default:
|
||
return fmt.Errorf("unknown command %q (want translate|report|status|redrive)", cmd)
|
||
}
|
||
}
|
||
|
||
func translate(ctx context.Context, cfgPath string, resnapshot bool) error {
|
||
r, err := pipeline.NewRunner(cfgPath, obs.NewLogger())
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer r.Close()
|
||
r.Resnapshot = resnapshot
|
||
|
||
res, err := r.TranslateBook(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
for _, ch := range res.Chunks {
|
||
fmt.Printf("=== ГЛАВА %d ЧАНК %d — %s%s ===\n", ch.Chapter, ch.ChunkIdx, ch.Disposition, flagSuffix(ch.FlagReason))
|
||
if ch.Disposition == pipeline.DispOK {
|
||
fmt.Println(ch.FinalText)
|
||
} else {
|
||
fmt.Printf("[ФЛАГ %s] чанк не переведён — черновик/редактура непригодны, помечен для человека\n", ch.FlagReason)
|
||
}
|
||
for _, st := range ch.Stages {
|
||
how := "call"
|
||
switch {
|
||
case st.Disposition == pipeline.DispSkipped:
|
||
how = "skipped"
|
||
case st.FromResume:
|
||
how = "resume"
|
||
}
|
||
fmt.Printf(" %-8s %-22s %-8s %-8s%s $%.6f (cum $%.6f) in=%d (cached=%d) out=%d+%d att=%d %dms finish=%s\n",
|
||
st.Stage, st.Model, how, st.Disposition, flagSuffix(st.FlagReason),
|
||
st.CostUSD, st.CumCostUSD,
|
||
st.Usage.PromptTokens, st.Usage.CachedTokens,
|
||
st.Usage.CompletionTokens, st.Usage.ReasoningTokens, st.Attempts, st.LatencyMS, st.FinishReason)
|
||
}
|
||
fmt.Println()
|
||
}
|
||
|
||
fmt.Printf("ИТОГО (этот запуск): $%.6f — чанков %d, флагов %d\n", res.TotalUSD, len(res.Chunks), res.Flagged)
|
||
committed, reserved, err := r.Store.SpentUSD(res.BookID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
fmt.Printf("Ledger книги: committed=$%.6f reserved=$%.6f\n", committed, reserved)
|
||
|
||
// Exit code 2 is a typed sentinel, not an infra error: the report above is
|
||
// already on stdout; main() maps this to a non-zero exit for the operator.
|
||
if res.Flagged > 0 {
|
||
return &pipeline.CompletedWithFlags{Flagged: res.Flagged, Total: len(res.Chunks)}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// flagSuffix renders a non-empty flag reason as "(reason)".
|
||
func flagSuffix(r pipeline.FlagReason) string {
|
||
if r == "" {
|
||
return ""
|
||
}
|
||
return "(" + string(r) + ")"
|
||
}
|
||
|
||
func report(cfgPath string) error {
|
||
// Read-only ($0): no API keys required (D20.4 — a store audit must not demand provider keys).
|
||
r, err := pipeline.NewReadOnlyRunner(cfgPath, obs.NewLogger())
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer r.Close()
|
||
|
||
rows, err := r.Store.RequestLogRows(r.Book.BookID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
fmt.Printf("%-20s %-8s %-12s %-22s %8s %8s %8s %8s %8s %10s %8s %-8s %-5s %-3s\n",
|
||
"ts", "stage", "role", "model", "prompt", "cached", "cwrite", "compl", "reason", "cost_usd", "ms", "finish", "tmhit", "ok")
|
||
for _, row := range rows {
|
||
fmt.Printf("%-20s %-8s %-12s %-22s %8d %8d %8d %8d %8d %10.6f %8d %-8s %-5d %-3d\n",
|
||
row.TS, row.Stage, row.Role, row.ModelActual, row.PromptTokens, row.CachedTokens,
|
||
row.CacheCreationTokens, row.CompletionTokens, row.ReasoningTokens, row.CostUSD,
|
||
row.LatencyMS, row.FinishReason, row.TMHit, row.OK)
|
||
}
|
||
|
||
// Flag section (Веха 2): every chunk×stage whose disposition ≠ ok — the
|
||
// "флаг редактору" the plan requires (02-mvp Фаза-1 приёмка допускает N).
|
||
flags, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
headerPrinted := false
|
||
for _, f := range flags {
|
||
if f.Disposition == "ok" {
|
||
continue
|
||
}
|
||
if !headerPrinted {
|
||
fmt.Printf("\n=== ФЛАГИ (disposition ≠ ok) ===\n")
|
||
fmt.Printf("%-4s %-6s %-8s %-9s %-16s %5s %10s %s\n",
|
||
"ch", "chunk", "stage", "disp", "reason", "att", "cost_usd", "detail")
|
||
headerPrinted = true
|
||
}
|
||
fmt.Printf("%-4d %-6d %-8s %-9s %-16s %5d %10.6f %s\n",
|
||
f.Chapter, f.ChunkIdx, f.Stage, f.Disposition, f.FlagReason, f.Attempts, f.CostUSD, f.Detail)
|
||
}
|
||
|
||
// Memory section (шаг 4): the per-chunk retrieval-state, surfaced so silent glossary
|
||
// degradation is LOUD. The aggregate line always prints when a glossary is in use;
|
||
// each chunk with a post-check miss is listed (the flagger-mode signal — the model
|
||
// ignored an approved term, or we injected the wrong dst) with the offending src→dst.
|
||
states, err := r.Store.RetrievalStatesForBook(r.Book.BookID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if len(states) > 0 {
|
||
var injected, sticky, ambiguous, spoiler, evicted, misses, style int
|
||
for _, rs := range states {
|
||
injected += rs.NExactHits
|
||
sticky += rs.NSticky
|
||
ambiguous += rs.NAmbiguousFlagged
|
||
spoiler += rs.NSpoilerBlocked
|
||
evicted += rs.NEvicted
|
||
misses += rs.NPostcheckMiss
|
||
style += rs.NStyleFlags
|
||
}
|
||
fmt.Printf("\n=== ПАМЯТЬ (retrieval-state) ===\n")
|
||
fmt.Printf("инъекций(точных)=%d sticky=%d ambiguous=%d спойлер-блок=%d вытеснено=%d post-check-промахов=%d\n",
|
||
injected, sticky, ambiguous, spoiler, evicted, misses)
|
||
printed := false
|
||
for _, rs := range states {
|
||
if rs.NPostcheckMiss == 0 {
|
||
continue
|
||
}
|
||
if !printed {
|
||
fmt.Printf("%-4s %-6s %6s %s\n", "ch", "chunk", "misses", "detail (src→dst не найден в выводе)")
|
||
printed = true
|
||
}
|
||
fmt.Printf("%-4d %-6d %6d %s\n", rs.Chapter, rs.ChunkIdx, rs.NPostcheckMiss, rs.PostcheckDetail)
|
||
}
|
||
// Cheap style/number flaggers (observability, not gates): total + per-chunk detail.
|
||
fmt.Printf("\n=== СТИЛЬ-ГЕЙТЫ (наблюдаемость: тире-диалоги, ё, транслит-междометия, разряды 万/億) — всего %d ===\n", style)
|
||
stylePrinted := false
|
||
for _, rs := range states {
|
||
if rs.NStyleFlags == 0 {
|
||
continue
|
||
}
|
||
if !stylePrinted {
|
||
fmt.Printf("%-4s %-6s %6s %s\n", "ch", "chunk", "flags", "detail")
|
||
stylePrinted = true
|
||
}
|
||
fmt.Printf("%-4d %-6d %6d %s\n", rs.Chapter, rs.ChunkIdx, rs.NStyleFlags, rs.StyleDetail)
|
||
}
|
||
}
|
||
|
||
committed, reserved, err := r.Store.SpentUSD(r.Book.BookID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
fmt.Printf("\nLedger книги: committed=$%.6f reserved=$%.6f\n", committed, reserved)
|
||
return nil
|
||
}
|
||
|
||
// status prints the READ-ONLY progress projection (D15.3): 0 LLM, 0 replay. --json emits the
|
||
// StatusReport with stable disposition/flag_reason enums for CI/IDE; the default is a human
|
||
// dashboard (unit counts, per-chapter quality passports, money, secondary ETA). BOTH modes exit 2
|
||
// (completed-with-flags) when flagged>0 (minor 1d — --json used to always exit 0), so a consumer
|
||
// never reads a clean 0 over a book that still needs attention.
|
||
func status(ctx context.Context, cfgPath string, asJSON bool) error {
|
||
// Read-only ($0): no API keys required (D20.4 — a progress projection must not demand keys).
|
||
r, err := pipeline.NewReadOnlyRunner(cfgPath, obs.NewLogger())
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer r.Close()
|
||
|
||
rep, err := r.Status(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
if asJSON {
|
||
enc := json.NewEncoder(os.Stdout)
|
||
enc.SetIndent("", " ")
|
||
if err := enc.Encode(rep); err != nil {
|
||
return err
|
||
}
|
||
// Symmetric with the human mode (minor 1d): flagged>0 is exit 2 (completed-with-flags), not
|
||
// exit 0. The full JSON is already on stdout; the sentinel only sets the shell disposition
|
||
// (its message goes to stderr in main), so a CI/IDE consumer both parses the projection AND
|
||
// sees "attention needed" in the exit code instead of silently reading 0.
|
||
if rep.Flagged > 0 {
|
||
return &pipeline.CompletedWithFlags{Flagged: rep.Flagged, Total: rep.TotalChunks}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
snap := rep.Snapshot
|
||
if snap == "" {
|
||
snap = "—"
|
||
} else if len(snap) > 12 {
|
||
snap = snap[:12]
|
||
}
|
||
drift := ""
|
||
if rep.SnapshotDrift {
|
||
drift += " ⚠ SNAPSHOT-DRIFT (несколько снапшотов в chunk_status — конфиг менялся; --resnapshot)"
|
||
}
|
||
if rep.ConfigDrift {
|
||
drift += " ⚠ CONFIG-DRIFT (текущий конфиг рендерит другой снапшот — правка после прогона; translate потребует --resnapshot = переоплата книги)"
|
||
}
|
||
fmt.Printf("=== СТАТУС: %s (snapshot %s)%s ===\n", rep.BookID, snap, drift)
|
||
fmt.Printf("Прогресс: %d/%d чанков (%.1f%%) — done=%d in_progress=%d flagged=%d pending=%d\n",
|
||
rep.Done, rep.TotalChunks, rep.PercentDone, rep.Done, rep.InProgress, rep.Flagged, rep.Pending)
|
||
fmt.Printf("Эскалаций: %d · post-check промахов (confirmed): %d · стиль-флагов (наблюдаемость): %d\n",
|
||
rep.Escalations, rep.PostcheckMisses, rep.StyleFlags)
|
||
ceil := ""
|
||
if rep.BookCeilingUSD > 0 {
|
||
ceil = fmt.Sprintf(" (потолок $%.2f — %.1f%%)", rep.BookCeilingUSD, rep.CeilingPct)
|
||
}
|
||
fmt.Printf("Деньги: committed=$%.6f reserved=$%.6f%s · прогноз книги ~$%.6f\n",
|
||
rep.CommittedUSD, rep.ReservedUSD, ceil, rep.ProjectedBookUSD)
|
||
if rep.ETASeconds > 0 {
|
||
fmt.Printf("ETA: ~%.0fс (средний throughput fresh-вызовов, НЕ EWMA — D12-отступление; вторично)\n", rep.ETASeconds)
|
||
}
|
||
|
||
if len(rep.Chapters) > 0 {
|
||
// style = cheap deterministic style/number flags (observability, not a disposition) — a per-
|
||
// chapter glance count so a human sees where the linters fired without opening `report` (D20.4).
|
||
fmt.Printf("\n%-5s %-9s %-10s %-6s %-4s %-6s %-16s %10s\n", "ch", "done/all", "verdict", "flag", "esc", "style", "worst_flag", "cost_usd")
|
||
for _, p := range rep.Chapters {
|
||
fmt.Printf("%-5d %d/%-7d %-10s %-6d %-4d %-6d %-16s %10.6f\n",
|
||
p.Chapter, p.ChunksDone, p.ChunksTotal, p.Verdict, p.ChunksFlagged, p.Escalations, p.StyleFlags,
|
||
dashIfEmpty(p.WorstFlagReason), p.CostUSD)
|
||
}
|
||
}
|
||
// flagged≠failed (jobs.status.failed = infra-only, D12): flagged chunks are a "clean run,
|
||
// attention needed" signal. Two kinds need DIFFERENT actions (minor 1d): a chunk_status flag
|
||
// (soft_refusal, length, …) is re-drivable with `tmctl redrive`; a gate-promoted glossary_miss
|
||
// is NOT (redrive is a no-op — the miss re-derives from the unchanged glossary), so it needs a
|
||
// seed fix + `translate --resnapshot`. Advise each set separately instead of one blanket hint.
|
||
if rep.Flagged > 0 {
|
||
if redrivable := rep.Flagged - rep.GlossaryMissFlagged; redrivable > 0 {
|
||
fmt.Printf("\n%d флагнутых чанк(ов) ждут внимания — переатаковать: tmctl redrive --config %s [--chapter N --chunk M --reason R]\n",
|
||
redrivable, cfgPath)
|
||
}
|
||
if rep.GlossaryMissFlagged > 0 {
|
||
fmt.Printf("\n%d чанк(ов) флагнуты post-check-гейтом (glossary_miss) — redrive для них no-op: исправьте сид-глоссарий (термин/dst) и перезапустите `tmctl translate --config %s --resnapshot` (переоплата затронутых чанков)\n",
|
||
rep.GlossaryMissFlagged, cfgPath)
|
||
}
|
||
return &pipeline.CompletedWithFlags{Flagged: rep.Flagged, Total: rep.TotalChunks}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// dashIfEmpty renders "" as "—" for a table cell.
|
||
func dashIfEmpty(s string) string {
|
||
if s == "" {
|
||
return "—"
|
||
}
|
||
return s
|
||
}
|
||
|
||
// redrive re-attacks the FLAGGED chunks matching the selector (D15.3): it resets their terminal
|
||
// flag (chunk_status + checkpoints of the flagged/skipped stages) and re-runs the durable loop
|
||
// with a FRESH retry/escalation budget, never touching DispOK work. On --dry-run it only reports
|
||
// the plan. Redrive-calls bill normally; money already spent on the discarded attempts stays
|
||
// committed (honest). By default it requires the current config to render the SAME snapshot the
|
||
// flagged rows carry (else it fails loud); passing --resnapshot wires r.Resnapshot so Redrive skips
|
||
// that drift guard and accepts the re-pin/re-pay explicitly (D20.4: the flag used to be parsed but
|
||
// silently ignored — Redrive already honoured r.Resnapshot, only the CLI never set it).
|
||
func redrive(ctx context.Context, cfgPath string, resnapshot bool, sel pipeline.RedriveSelector) error {
|
||
r, err := pipeline.NewRunner(cfgPath, obs.NewLogger())
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer r.Close()
|
||
r.Resnapshot = resnapshot
|
||
|
||
summary, res, err := r.Redrive(ctx, sel)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
fmt.Printf("=== REDRIVE: %s ===\n", r.Book.BookID)
|
||
if len(summary.Targets) == 0 {
|
||
fmt.Println("Флагнутых чанков под селектор не найдено — нечего переатаковать.")
|
||
return nil
|
||
}
|
||
fmt.Printf("Целей: %d флагнутых чанк(ов) — сброс терминального флага + свежий retry/escalation-бюджет (DispOK не тронут):\n", len(summary.Targets))
|
||
for _, t := range summary.Targets {
|
||
fmt.Printf(" ch%d/chunk%d (%s) → стадии: %s\n", t.Chapter, t.ChunkIdx, t.FlagReason, strings.Join(t.Stages, ", "))
|
||
}
|
||
if summary.DryRun {
|
||
fmt.Println("[dry-run: ничего не сброшено, вызовов не сделано]")
|
||
return nil
|
||
}
|
||
fmt.Println("[сброшено; перезапуск durable-цикла — DispOK чанки резюмируются за $0, сброшенные переатакуются]")
|
||
fmt.Println()
|
||
|
||
if res != nil {
|
||
fmt.Printf("ИТОГО re-run (этот запуск): $%.6f — чанков %d, флагов %d\n", res.TotalUSD, len(res.Chunks), res.Flagged)
|
||
committed, reserved, serr := r.Store.SpentUSD(res.BookID)
|
||
if serr == nil {
|
||
fmt.Printf("Ledger книги: committed=$%.6f reserved=$%.6f\n", committed, reserved)
|
||
}
|
||
if res.Flagged > 0 {
|
||
return &pipeline.CompletedWithFlags{Flagged: res.Flagged, Total: len(res.Chunks)}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// loadDotEnv reads KEY=VALUE lines into the environment without overriding
|
||
// already-set variables. Ключи — из backend/.env (gitignored); секреты никогда
|
||
// не попадают в конфиги/репо.
|
||
func loadDotEnv(path string) {
|
||
f, err := os.Open(path)
|
||
if err != nil {
|
||
return
|
||
}
|
||
defer f.Close()
|
||
sc := bufio.NewScanner(f)
|
||
for sc.Scan() {
|
||
line := strings.TrimSpace(sc.Text())
|
||
if line == "" || strings.HasPrefix(line, "#") {
|
||
continue
|
||
}
|
||
k, v, ok := strings.Cut(line, "=")
|
||
if !ok {
|
||
continue
|
||
}
|
||
k, v = strings.TrimSpace(k), strings.TrimSpace(v)
|
||
// Снимаем только ПАРНЫЕ обрамляющие кавычки: Trim по набору символов
|
||
// откусил бы легитимную кавычку в конце ключа (находка ревью).
|
||
if n := len(v); n >= 2 && (v[0] == '"' || v[0] == '\'') && v[n-1] == v[0] {
|
||
v = v[1 : n-1]
|
||
}
|
||
if os.Getenv(k) == "" {
|
||
os.Setenv(k, v)
|
||
}
|
||
}
|
||
}
|