package books import ( "context" "errors" "fmt" "sync/atomic" "time" "golang.org/x/sync/semaphore" "textmachine/platform/internal/jobs" ) // DefaultMaxCuts is how many books this platform lets the engine cut AT ONCE — counted across every // way of starting one: the upload that cuts its own book, the queue's workers and the backstop sweep. // // The number is small on purpose, and the reason is what a cut IS on this host. `tmctl manifest` is // spawned as a plain child process (runner.readEngine) — no transient unit, no cgroup, no MemoryMax, // unlike a translation run, which is wrapped in all three. // // ⚠ What this cap bounds, said exactly, because the loose version of it is false: it bounds the CUTS, // on every path that starts one — the upload that cuts its own book, the queue's workers and the // backstop sweep. It is NOT a bound on every engine process the host may hold: the materializer reads // a manifest and an export of its own through the same uncapped `runner.readEngine` // (readmodel.refresh), and the run reconciler reads `status` the same way. Those are bounded by the // queue's worker count and by their sweeps being sequential, which is a different bound and a looser // one. The path that had NO bound at all was the synchronous cut — N uploads were N engine processes // — and that is the one this closes. // // The figure is the queue's own default worker count and is taken from there rather than repeated // here (jobs.DefaultWorkers): the host was already sized for that many engine processes, and this is // the same number said once for every way of starting one. const DefaultMaxCuts = jobs.DefaultWorkers // ErrHostAtCutCapacity is a cut that never started because the host is already running as many as it // will run at once. // // Not a fact about the book and not a fault of the deployment, and no path treats it as either: at // intake it leaves the upload to the queue, and on the queue's own path it gives the claim back // without spending an attempt of a budget that exists for a BROKEN host. A host at its cap is a // working host. var ErrHostAtCutCapacity = errors.New("books: the host is already cutting as many books as it will cut at once") // ErrNoTimeToCut is a cut that was not started because what is left of the caller's budget is less // than a cut needs. // // The SAME class as the cap and treated identically — the engine was never asked, so this pass knows // nothing about the book and records nothing — and it exists because waiting for a slot spends the // caller's budget. Without it a pass could win a slot with seconds left, hand the engine those // seconds, and have the killed process read back as `parser_unavailable`: a verdict about the // DEPLOYMENT, which spends an attempt of a budget that five times over deletes the user's file. var ErrNoTimeToCut = errors.New("books: what is left of this pass is shorter than a cut") // cutSlots is the cap itself, plus what an operator has to be able to see of it: a cap nobody can // watch is indistinguishable from latency somebody has to guess at. type cutSlots struct { sem *semaphore.Weighted limit int64 // inFlight and waiting are the state RIGHT NOW — how saturated the host is, and how deep the line // for it is. Gauges, because the question they answer stops being true the moment it changes. inFlight atomic.Int64 waiting atomic.Int64 // waited and gaveUp are cumulative, because their question is the opposite one: how often has this // cap been reached at all, and how often did reaching it cost a cut. A gauge would answer it only // for whoever happened to be looking. waited atomic.Uint64 gaveUp atomic.Uint64 } // cuts builds the cap on first use. // // Lazily, because this service is assembled as a struct literal by its deployment (cmd/tmplatformd) // and by every test that exercises intake, and a cap that only exists when a constructor was called // is a cap absent from exactly the paths nobody remembered to route through one. func (s *Service) cuts() *cutSlots { s.cutsOnce.Do(func() { limit := int64(s.Cfg.MaxCuts) if limit <= 0 { limit = DefaultMaxCuts } s.cutSlots = &cutSlots{sem: semaphore.NewWeighted(limit), limit: limit} }) return s.cutSlots } // takeCutSlot holds one of the host's cut slots for the caller and returns what gives it back. // // It WAITS rather than refusing, and the caller's own context is what bounds the wait — the upload's // cut budget, the queue job's timeout, the sweep's per-book slice. Waiting is right here because // every one of those already has a milder answer than a refusal for running out: the upload is // accepted `parsing` and the queue finishes it; a queued pass gives the book straight back. A // refusal at the door would turn a host that is merely busy into an upload the user has to do again. // // It cannot lengthen the walk it is called inside, either — that is not a promise about this code // but a property of the context it takes: the cut runs on a step of the upload's walk (books.step), // so time spent here is time NOT spent on the engine, never time added to the tail. // // ⛔ `reserve` is what a WON slot must still be worth. Waiting spends the caller's budget, so a slot // won at the very end of it buys a cut the engine has no time to finish — and a killed engine reads // back as a fault of the DEPLOYMENT, which spends an attempt of the budget that deletes a user's // file after five. Zero means the caller has nothing at stake in losing (the intake spends no // attempts), and then waiting to the very end is free. func (s *Service) takeCutSlot(ctx context.Context, reserve time.Duration) (func(), error) { c := s.cuts() release := func() { c.inFlight.Add(-1) c.sem.Release(1) } if c.sem.TryAcquire(1) { c.inFlight.Add(1) return release, nil } if err := worthStarting(ctx, reserve); err != nil { // Every slot is taken and there is not enough left to make winning one worth it. Counted as a // give-up, because from the operator's side it is the cap that cost this cut. c.gaveUp.Add(1) return nil, err } c.waited.Add(1) c.waiting.Add(1) started := time.Now() wait, stopWaiting := waitCtx(ctx, reserve) err := c.sem.Acquire(wait, 1) stopWaiting() c.waiting.Add(-1) if err != nil { c.gaveUp.Add(1) return nil, fmt.Errorf("%w: waited %s for one of %d slots: %w", ErrHostAtCutCapacity, time.Since(started).Round(time.Millisecond), c.limit, err) } // Won — but the wait spent time, so the question of whether it is still worth cutting is asked // AGAIN. A slot handed back unused is a slot the next caller gets. if err := worthStarting(ctx, reserve); err != nil { c.sem.Release(1) c.gaveUp.Add(1) return nil, err } c.inFlight.Add(1) // The fact and the wait, at INFO: this is the host doing what it was configured to do, and an // operator reading it learns the cap is the thing shaping their latency. What it must NOT do is // carry the book — a cut waits because of the HOST, and the book it happens to be for is no more // at fault than any other (ENGINEERING_STANDARDS §Наблюдаемость). s.log().InfoContext(ctx, "a cut waited for one of the host's cut slots", "waited_seconds", time.Since(started).Seconds(), "slots", c.limit) return release, nil } // CutCapacity is one reading of the cap, for the telemetry pass that publishes it. // // Read from the service rather than collected on scrape, which is the same rule the rest of this // deployment's gauges follow: a scrape must not be able to set the load on anything. type CutCapacity struct { Limit int InFlight int Waiting int // Waited and GaveUp are cumulative counts of cuts that had to wait at all, and of cuts whose // caller ran out of budget while waiting. Waited uint64 GaveUp uint64 } // CutCapacity reports where the host's cut capacity stands. func (s *Service) CutCapacity() CutCapacity { c := s.cuts() return CutCapacity{ Limit: int(c.limit), InFlight: int(c.inFlight.Load()), Waiting: int(c.waiting.Load()), Waited: c.waited.Load(), GaveUp: c.gaveUp.Load(), } } // waitCtx bounds a wait so that what is left when it ends is still worth a cut. With no reserve, or // with no deadline to take it out of, the caller's own context is the bound. func waitCtx(ctx context.Context, reserve time.Duration) (context.Context, context.CancelFunc) { deadline, ok := ctx.Deadline() if !ok || reserve <= 0 { return ctx, func() {} } // The parent stays the parent, so its cancellation still ends the wait; only the deadline is // pulled in by the reserve. return context.WithDeadline(ctx, deadline.Add(-reserve)) } // worthStarting reports whether a cut started now would have the time a cut needs. func worthStarting(ctx context.Context, reserve time.Duration) error { if reserve <= 0 { return nil } deadline, ok := ctx.Deadline() if !ok { return nil } if left := time.Until(deadline); left < reserve { return fmt.Errorf("%w: %s left, a cut is given %s", ErrNoTimeToCut, left.Round(time.Millisecond), reserve) } return nil } // engineNotAsked reports whether an error means the engine was never asked at all — the host was at // its cap, or what was left of the pass was shorter than a cut. Neither says anything about the book. func engineNotAsked(err error) bool { return errors.Is(err, ErrHostAtCutCapacity) || errors.Is(err, ErrNoTimeToCut) }