textmachine/platform/internal/books/limit.go

package books

import (
	"context"
	"errors"
	"fmt"
	"sync/atomic"
	"time"

	"golang.org/x/sync/semaphore"

	"textmachine/platform/internal/jobs"
)

// DefaultMaxCuts is how many books this platform lets the engine cut AT ONCE — counted across every
// way of starting one: the upload that cuts its own book, the queue's workers and the backstop sweep.
//
// The number is small on purpose, and the reason is what a cut IS on this host. `tmctl manifest` is
// spawned as a plain child process (runner.readEngine) — no transient unit, no cgroup, no MemoryMax,
// unlike a translation run, which is wrapped in all three.
//
// ⚠ What this cap bounds, said exactly, because the loose version of it is false: it bounds the CUTS,
// on every path that starts one — the upload that cuts its own book, the queue's workers and the
// backstop sweep. It is NOT a bound on every engine process the host may hold: the materializer reads
// a manifest and an export of its own through the same uncapped `runner.readEngine`
// (readmodel.refresh), and the run reconciler reads `status` the same way. Those are bounded by the
// queue's worker count and by their sweeps being sequential, which is a different bound and a looser
// one. The path that had NO bound at all was the synchronous cut — N uploads were N engine processes
// — and that is the one this closes.
//
// The figure is the queue's own default worker count and is taken from there rather than repeated
// here (jobs.DefaultWorkers): the host was already sized for that many engine processes, and this is
// the same number said once for every way of starting one.
const DefaultMaxCuts = jobs.DefaultWorkers

// ErrHostAtCutCapacity is a cut that never started because the host is already running as many as it
// will run at once.
//
// Not a fact about the book and not a fault of the deployment, and no path treats it as either: at
// intake it leaves the upload to the queue, and on the queue's own path it gives the claim back
// without spending an attempt of a budget that exists for a BROKEN host. A host at its cap is a
// working host.
var ErrHostAtCutCapacity = errors.New("books: the host is already cutting as many books as it will cut at once")

// ErrNoTimeToCut is a cut that was not started because what is left of the caller's budget is less
// than a cut needs.
//
// The SAME class as the cap and treated identically — the engine was never asked, so this pass knows
// nothing about the book and records nothing — and it exists because waiting for a slot spends the
// caller's budget. Without it a pass could win a slot with seconds left, hand the engine those
// seconds, and have the killed process read back as `parser_unavailable`: a verdict about the
// DEPLOYMENT, which spends an attempt of a budget that five times over deletes the user's file.
var ErrNoTimeToCut = errors.New("books: what is left of this pass is shorter than a cut")

// cutSlots is the cap itself, plus what an operator has to be able to see of it: a cap nobody can
// watch is indistinguishable from latency somebody has to guess at.
type cutSlots struct {
	sem   *semaphore.Weighted
	limit int64
	// inFlight and waiting are the state RIGHT NOW — how saturated the host is, and how deep the line
	// for it is. Gauges, because the question they answer stops being true the moment it changes.
	inFlight atomic.Int64
	waiting  atomic.Int64
	// waited and gaveUp are cumulative, because their question is the opposite one: how often has this
	// cap been reached at all, and how often did reaching it cost a cut. A gauge would answer it only
	// for whoever happened to be looking.
	waited atomic.Uint64
	gaveUp atomic.Uint64
}

// cuts builds the cap on first use.
//
// Lazily, because this service is assembled as a struct literal by its deployment (cmd/tmplatformd)
// and by every test that exercises intake, and a cap that only exists when a constructor was called
// is a cap absent from exactly the paths nobody remembered to route through one.
func (s *Service) cuts() *cutSlots {
	s.cutsOnce.Do(func() {
		limit := int64(s.Cfg.MaxCuts)
		if limit <= 0 {
			limit = DefaultMaxCuts
		}
		s.cutSlots = &cutSlots{sem: semaphore.NewWeighted(limit), limit: limit}
	})
	return s.cutSlots
}

// takeCutSlot holds one of the host's cut slots for the caller and returns what gives it back.
//
// It WAITS rather than refusing, and the caller's own context is what bounds the wait — the upload's
// cut budget, the queue job's timeout, the sweep's per-book slice. Waiting is right here because
// every one of those already has a milder answer than a refusal for running out: the upload is
// accepted `parsing` and the queue finishes it; a queued pass gives the book straight back. A
// refusal at the door would turn a host that is merely busy into an upload the user has to do again.
//
// It cannot lengthen the walk it is called inside, either — that is not a promise about this code
// but a property of the context it takes: the cut runs on a step of the upload's walk (books.step),
// so time spent here is time NOT spent on the engine, never time added to the tail.
//
// ⛔ `reserve` is what a WON slot must still be worth. Waiting spends the caller's budget, so a slot
// won at the very end of it buys a cut the engine has no time to finish — and a killed engine reads
// back as a fault of the DEPLOYMENT, which spends an attempt of the budget that deletes a user's
// file after five. Zero means the caller has nothing at stake in losing (the intake spends no
// attempts), and then waiting to the very end is free.
func (s *Service) takeCutSlot(ctx context.Context, reserve time.Duration) (func(), error) {
	c := s.cuts()
	release := func() {
		c.inFlight.Add(-1)
		c.sem.Release(1)
	}
	if c.sem.TryAcquire(1) {
		c.inFlight.Add(1)
		return release, nil
	}
	if err := worthStarting(ctx, reserve); err != nil {
		// Every slot is taken and there is not enough left to make winning one worth it. Counted as a
		// give-up, because from the operator's side it is the cap that cost this cut.
		c.gaveUp.Add(1)
		return nil, err
	}
	c.waited.Add(1)
	c.waiting.Add(1)
	started := time.Now()
	wait, stopWaiting := waitCtx(ctx, reserve)
	err := c.sem.Acquire(wait, 1)
	stopWaiting()
	c.waiting.Add(-1)
	if err != nil {
		c.gaveUp.Add(1)
		return nil, fmt.Errorf("%w: waited %s for one of %d slots: %w",
			ErrHostAtCutCapacity, time.Since(started).Round(time.Millisecond), c.limit, err)
	}
	// Won — but the wait spent time, so the question of whether it is still worth cutting is asked
	// AGAIN. A slot handed back unused is a slot the next caller gets.
	if err := worthStarting(ctx, reserve); err != nil {
		c.sem.Release(1)
		c.gaveUp.Add(1)
		return nil, err
	}
	c.inFlight.Add(1)
	// The fact and the wait, at INFO: this is the host doing what it was configured to do, and an
	// operator reading it learns the cap is the thing shaping their latency. What it must NOT do is
	// carry the book — a cut waits because of the HOST, and the book it happens to be for is no more
	// at fault than any other (ENGINEERING_STANDARDS §Наблюдаемость).
	s.log().InfoContext(ctx, "a cut waited for one of the host's cut slots",
		"waited_seconds", time.Since(started).Seconds(), "slots", c.limit)
	return release, nil
}

// CutCapacity is one reading of the cap, for the telemetry pass that publishes it.
//
// Read from the service rather than collected on scrape, which is the same rule the rest of this
// deployment's gauges follow: a scrape must not be able to set the load on anything.
type CutCapacity struct {
	Limit    int
	InFlight int
	Waiting  int
	// Waited and GaveUp are cumulative counts of cuts that had to wait at all, and of cuts whose
	// caller ran out of budget while waiting.
	Waited uint64
	GaveUp uint64
}

// CutCapacity reports where the host's cut capacity stands.
func (s *Service) CutCapacity() CutCapacity {
	c := s.cuts()
	return CutCapacity{
		Limit:    int(c.limit),
		InFlight: int(c.inFlight.Load()),
		Waiting:  int(c.waiting.Load()),
		Waited:   c.waited.Load(),
		GaveUp:   c.gaveUp.Load(),
	}
}

// waitCtx bounds a wait so that what is left when it ends is still worth a cut. With no reserve, or
// with no deadline to take it out of, the caller's own context is the bound.
func waitCtx(ctx context.Context, reserve time.Duration) (context.Context, context.CancelFunc) {
	deadline, ok := ctx.Deadline()
	if !ok || reserve <= 0 {
		return ctx, func() {}
	}
	// The parent stays the parent, so its cancellation still ends the wait; only the deadline is
	// pulled in by the reserve.
	return context.WithDeadline(ctx, deadline.Add(-reserve))
}

// worthStarting reports whether a cut started now would have the time a cut needs.
func worthStarting(ctx context.Context, reserve time.Duration) error {
	if reserve <= 0 {
		return nil
	}
	deadline, ok := ctx.Deadline()
	if !ok {
		return nil
	}
	if left := time.Until(deadline); left < reserve {
		return fmt.Errorf("%w: %s left, a cut is given %s",
			ErrNoTimeToCut, left.Round(time.Millisecond), reserve)
	}
	return nil
}

// engineNotAsked reports whether an error means the engine was never asked at all — the host was at
// its cap, or what was left of the pass was shorter than a cut. Neither says anything about the book.
func engineNotAsked(err error) bool {
	return errors.Is(err, ErrHostAtCutCapacity) || errors.Is(err, ErrNoTimeToCut)
}