Take in the platform zone's intake work as the zone built it: a cut waits for one of the host's slots, the tail's budget comes from the path, and the zone's own head tells the truth
This commit is contained in:
parent
c3039be9b9
commit
ddcbf0c03d
26 changed files with 2181 additions and 139 deletions
|
|
@ -225,6 +225,21 @@ func startBackup(cfg config.Config, db *pgstore.Store, engine backup.Engine, log
|
|||
}
|
||||
}
|
||||
|
||||
// intakeConfig is what an operator chose about intake, in the intake's own terms.
|
||||
//
|
||||
// A function of its own, and for the reason the runner's knobs got one: a mapping written inline in
|
||||
// the wiring is a mapping nothing can witness, and a knob dropped from it fails at nothing — the
|
||||
// service simply runs on its default while the boot line prints the operator's number.
|
||||
func intakeConfig(cfg config.Config) books.Config {
|
||||
return books.Config{
|
||||
BooksDir: cfg.Intake.BooksDir,
|
||||
EngineBinary: cfg.Runner.EngineBinary,
|
||||
BookTemplate: cfg.Intake.BookTemplate,
|
||||
MaxCuts: cfg.Intake.MaxCuts,
|
||||
Pairs: intakePairs(cfg),
|
||||
}
|
||||
}
|
||||
|
||||
// startIntake wires the book upload, or explains at boot why this deployment takes none.
|
||||
//
|
||||
// A nil service leaves POST /books unmounted, which is the same shape every unbuilt contract route
|
||||
|
|
@ -245,13 +260,8 @@ func startIntake(cfg config.Config, db *pgstore.Store, engine books.Manifester,
|
|||
Store: db,
|
||||
Engine: engine,
|
||||
Reader: reader,
|
||||
Cfg: books.Config{
|
||||
BooksDir: cfg.Intake.BooksDir,
|
||||
EngineBinary: cfg.Runner.EngineBinary,
|
||||
BookTemplate: cfg.Intake.BookTemplate,
|
||||
Pairs: intakePairs(cfg),
|
||||
},
|
||||
Log: log,
|
||||
Cfg: intakeConfig(cfg),
|
||||
Log: log,
|
||||
}
|
||||
deps.Intake = svc
|
||||
deps.Upload = httpapi.UploadLimits{
|
||||
|
|
@ -554,6 +564,15 @@ func sweep(ctx context.Context, s sweeps, every, sweepBudget time.Duration, log
|
|||
// observe publishes the state of the control plane. A failure to measure never fails the sweep: it
|
||||
// is one WARN and the next tick tries again.
|
||||
func observe(ctx context.Context, s sweeps, log *slog.Logger) {
|
||||
// First, and from memory rather than from the database: an instance whose database is unreachable
|
||||
// is exactly when an operator wants to know whether its intake is saturated, and every reading
|
||||
// below returns early on that error.
|
||||
if s.books != nil {
|
||||
c := s.books.CutCapacity()
|
||||
s.metrics.ObserveCuts(metrics.Cuts{
|
||||
Limit: c.Limit, InFlight: c.InFlight, Waiting: c.Waiting, Waited: c.Waited, GaveUp: c.GaveUp,
|
||||
})
|
||||
}
|
||||
o, err := s.db.Observe(ctx, runs.StalledAfter)
|
||||
if err != nil {
|
||||
log.Warn("the control plane's own state could not be read", "err", err)
|
||||
|
|
|
|||
|
|
@ -161,3 +161,33 @@ func TestTheOperatorsExportKnobsReachTheDoor(t *testing.T) {
|
|||
"still running and answer their polls with a failure", got.StaleAfter, jobs.JobTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// The operator's intake knobs reach the intake, and the cap among them.
|
||||
//
|
||||
// Same class as the runner's knobs above and the same reason for existing: a knob dropped from the
|
||||
// wiring changes nothing visible — the service runs on its package default while the boot line
|
||||
// prints the number the operator set, so the configuration LOOKS applied. The cap is the one that
|
||||
// costs memory when it is silently four instead of the operator's figure.
|
||||
func TestTheOperatorsIntakeKnobsReachTheIntake(t *testing.T) {
|
||||
cfg := config.Config{}
|
||||
cfg.Intake.BooksDir = "/srv/tm/books"
|
||||
cfg.Intake.BookTemplate = "/etc/tm/book.yaml"
|
||||
// Distinct from the package default (books.DefaultMaxCuts) on purpose: equal to it, this
|
||||
// assertion would pass on a wiring that dropped the field entirely.
|
||||
cfg.Intake.MaxCuts = books.DefaultMaxCuts + 3
|
||||
cfg.Runner.EngineBinary = "/opt/tm/tmctl"
|
||||
cfg.LanguagePairs = []config.LanguagePair{{Source: "zh", Target: "ru", Available: true}}
|
||||
|
||||
got := intakeConfig(cfg)
|
||||
switch {
|
||||
case got.MaxCuts != cfg.Intake.MaxCuts:
|
||||
t.Errorf("MaxCuts is %d, want the operator's %d: TM_PLATFORM_MAX_CUTS does nothing and the host cuts on the package default",
|
||||
got.MaxCuts, cfg.Intake.MaxCuts)
|
||||
case got.BooksDir != cfg.Intake.BooksDir || got.BookTemplate != cfg.Intake.BookTemplate:
|
||||
t.Errorf("the intake's paths did not arrive: %+v", got)
|
||||
case got.EngineBinary != cfg.Runner.EngineBinary:
|
||||
t.Errorf("EngineBinary is %q, want %q", got.EngineBinary, cfg.Runner.EngineBinary)
|
||||
case len(got.Pairs) != 1:
|
||||
t.Errorf("the declared pairs did not arrive: %+v", got.Pairs)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -698,6 +698,36 @@ install -d -m0750 -o tmplatform -g tmplatform /srv/textmachine/books
|
|||
(10 минут). Прокси перед сервисом обязан разрешать столько же: у него свои `client_max_body_size` и
|
||||
свои таймауты чтения, и молчаливо режет их он, а не мы.
|
||||
|
||||
⛔ **И ПОСЛЕ последнего байта маршрут МОЛЧИТ ещё до 3 мин 40 с — это отдельный таймаут прокси, не тот,
|
||||
что выше.** Приняв тело, сервис доводит книгу до ответа: переводит строку в `parsing`, синхронно режет
|
||||
книгу движком, записывает исход, перечитывает строку и пишет квитанцию идемпотентности. Верхняя граница
|
||||
всего этого — `books.UploadSettle` = **220 с** (`internal/books/parse.go`), и это ГРАНИЦА, а не оценка:
|
||||
хвост целиком идёт под одним дедлайном (`walk`), и шаг, добавленный в него завтра, её не сдвинет.
|
||||
⇒ **у прокси таймаут ОТВЕТА (`proxy_read_timeout` у nginx, `timeout server` у HAProxy) обязан быть не
|
||||
меньше `TM_PLATFORM_UPLOAD_DEADLINE + 220 с`** — на дефолтах это 10 мин + 3 мин 40 с = **13 мин 40 с**,
|
||||
и брать с запасом. Дефолт у nginx — 60 с, то есть НИЖЕ этой границы более чем вдвое.
|
||||
⚠ Цена ошибки несимметрична и потому её стоит назвать: прокси, оборвавший ожидание, вернёт человеку
|
||||
ошибку на книге, которая на самом деле **ПРИНЯТА** — она уже в библиотеке и её дорежет очередь, — а
|
||||
человек, повторив «неудавшуюся» загрузку, получит вторую копию книги. Поднимаете
|
||||
`TM_PLATFORM_UPLOAD_DEADLINE` — поднимайте и этот таймаут на ту же величину; сам сервис за прокси не
|
||||
отвечает и молча его не подвинет.
|
||||
⚠ Число берётся ИЗ КОДА, не отсюда: `grep -n 'UploadSettle =' internal/books/parse.go` и сумма его
|
||||
слагаемых там же. Бут отказывается стартовать, если `TM_PLATFORM_UPLOAD_DEADLINE` плюс этот хвост не
|
||||
влезает в самое узкое из трёх окон приёма (`internal/config/config.go`), так что расходиться с кодом
|
||||
эта величина может только в одну сторону — в бо́льшую осторожность прокси.
|
||||
|
||||
⚠ **`TM_PLATFORM_MAX_CUTS` (дефолт 4) — сколько книг хост режет ОДНОВРЕМЕННО**, считая все три пути
|
||||
сразу: загрузку, которая режет свою книгу в запросе, воркеров очереди и страховочный свип. Это
|
||||
**сайзинг, а не отказ**: упёршаяся загрузка ждёт слот, а не получает ошибку, и если не дождалась —
|
||||
книга принимается `parsing`, её дорезает очередь. Поднимать имеет смысл ровно настолько, насколько
|
||||
хост тянет одновременных `tmctl manifest`: разбор идёт обычным дочерним процессом БЕЗ cgroup и без
|
||||
`MemoryMax` (в отличие от прогонов), то есть этот потолок — единственная граница памяти, какая у
|
||||
приёма есть. Насыщение видно метриками `tm_platform_cuts_in_flight` / `tm_platform_cuts_waiting`
|
||||
против `tm_platform_cut_slots` и счётчиками `tm_platform_cut_waits_total`,
|
||||
`tm_platform_cut_slot_timeouts_total`; растущий второй счётчик означает, что латентность приёма
|
||||
делает ПОТОЛОК, а не движок. Ноль бут отвергает: «без лимита» и «не режем ничего» — противоположные
|
||||
прочтения одного значения.
|
||||
|
||||
⚠ **`TM_PLATFORM_STATE_DIR` меняется только когда живых прогонов нет.** Exit-маркер пишется по пути,
|
||||
вычисленному при СПАВНЕ, а читается по пути из текущей конфигурации: после смены каталога конец
|
||||
идущего прогона становится невидим, и реконсилятор перезапускает его как потерянный (PD-155). Путь
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -522,7 +522,7 @@ system_messages not found in type config.CapabilitiesConfig`. Диагноз с
|
|||
состояние `cgroup.subtree_control` среза `tm-runs.slice` в момент прогона — назван КАНДИДАТОМ и
|
||||
только: диагноз `PD-423` не установлен, и вносить его в рецепт как проверку нельзя.
|
||||
|
||||
Ожидание при всех четырёх: 19 пакетов, exit 0, **скипов 0**, линтер «0 issues». Замерено 29.08: с гейтами — 0 скипов на обоих деревьях; без них — exit 0 и **287 скипов на HEAD
|
||||
Ожидание при всех четырёх: 20 пакетов (`go list ./...`), exit 0, **скипов 0**, линтер «0 issues». Замерено 29.08: с гейтами — 0 скипов на обоих деревьях; без них — exit 0 и **287 скипов на HEAD
|
||||
`fbe6cf3`**, **304 на дереве пака P11** (пак добавил 17 пинов, гейченных тем же DSN). Число зависит
|
||||
от дерева, и переносить его между ними нельзя.
|
||||
|
||||
|
|
|
|||
|
|
@ -3,17 +3,445 @@
|
|||
> **Что это.** Состояние зоны и её живые остатки. Обратно-хронологический: свежее выше.
|
||||
> Отработавшие эры вынесены срезами в [`archive/`](archive/) — читать только по конкретной ссылке.
|
||||
|
||||
## Состояние зоны на 07.09.2026
|
||||
## Состояние зоны на 08.09.2026
|
||||
|
||||
| Вопрос | Ответ |
|
||||
|---|---|
|
||||
| последний заленджённый пак | «форма заказа перевода» (05–06.09, `628cc56`+`36ea8b8`, акты **D39.208**, **D39.211**–**D39.214**), канон контракта `0.12.0` |
|
||||
| пак в дереве, не закоммиченный | «деньги и правда на экране» — отчёт ниже |
|
||||
| последний заленджённый пак | «деньги и правда на экране» (06–07.09, `fda0679`, акт **D39.221**), канон контракта `0.13.0`. ⚠ Обе величины берутся ПРИБОРОМ, а не отсюда: `git log --oneline -1 -- platform/` и `grep '^ version:' ../../docs/architecture/14-api-contract/openapi.yaml` — эта строка стареет, они нет |
|
||||
| пак в дереве, не закоммиченный | «разрез приёма до готовности и правда о себе» (08.09) — отчёт ниже |
|
||||
| открытые дефекты | `DEFECT_REGISTER.md` (счёт — `python3 docs/scripts/counts.py` от корня) |
|
||||
| нормы и приёмка | `ENGINEERING_STANDARDS.md` · направление — `PLATFORM_DIRECTION.md` · стек и стенд — `STACK_DECISIONS.md` |
|
||||
| незакрытые куски работы | `../BACKLOG.md` (`П-N`) |
|
||||
| как разворачивается | `../deploy/README.md` |
|
||||
|
||||
## ПАК «РАЗРЕЗ ПРИЁМА ДО ГОТОВНОСТИ И ПРАВДА О СЕБЕ» — ОТЧЁТ (08.09, `textmachine-fa`)
|
||||
|
||||
> Промт `docs/PLATFORM_INTAKE_TRUTH_SESSION_PROMPT.md`, вход HEAD `3f4680c`, дерево на входе чисто.
|
||||
> Зона НЕ коммитит — дерево передано оркестратору №23 (`textmachine-a8`). Пак $0, платных вызовов **0**.
|
||||
> **Работа завершена, править не планирую.** Сказано ПОСЛЕ адверсариального круга, а не до него: первая
|
||||
> редакция этого отчёта несла ту же фразу при шести неисправленных дефектах, введённых этим паком.
|
||||
> Дерево — 26 файлов, все в `platform/` (`git status --porcelain -- platform/`), 23 правленых и 3 новых.
|
||||
|
||||
### Исход по каждому пункту §4
|
||||
|
||||
| Пункт | Исход |
|
||||
|---|---|
|
||||
| §4.1 ограничитель параллелизма | **сделано**; форма — `x/sync/semaphore` на общей точке порождения, ожидание с деградацией в очередь. Предъявлено НАГРУЗКОЙ |
|
||||
| §4.2 бюджет хвоста из кода | **сделано, форма Б (структурная)**; попутно вскрыт и закрыт ЧЕТВЁРТЫЙ промах суммы — квитанция не входила в неё |
|
||||
| §4.3 рантбук | **сделано**; правок рантбука ДВЕ, вторая объявлена ниже с доводом |
|
||||
| §4.4 три места неразличимого сбоя | **сделано все три**, каждое своим лечением; свип вылечен БЕЗ миграции |
|
||||
| §4.5 правда о себе | **сделано**: шапка, два ряда флипнуты, маркер третьего починен, черты заэкранированы |
|
||||
| §4.6 честная причина человеку | **закрыто по построению на моей стороне** — и ПРЕМИСА пака при этом опровергнута замером (ниже) |
|
||||
| §4.7 живой гейт | **рецепт исполнен и РАБОТАЕТ**; довод зоны опровергнут, разрез впервые встретился с настоящим движком |
|
||||
| §4.8 п.2 (комментарий `cutNow`) | **взят вместе с §4.4**, как и предписано |
|
||||
| §4.8 п.4 (мёртвый `enqueue`) | **взят вместе с §4.1**: предикат сведён в одно названное место |
|
||||
| §4.8 п.6 («ВСЕГДА» контракта) | **не беру — пинг оркестратору** с моим выбором из двух (ниже) |
|
||||
| §4.8 остальное | **не делаю**, как объявлено паком |
|
||||
| ⚠ сверх пака | константа контракта `0.12.0` → `0.13.0` — красное на входе, взято по явному указанию оркестратора (ниже) |
|
||||
|
||||
### Что стало с деревом — находка → что сделано → чем предъявлено
|
||||
|
||||
| Находка | Что стало с деревом | Чем предъявлено |
|
||||
|---|---|---|
|
||||
| §4.1 у синхронного входа нет ограничителя параллелизма | `internal/books/limit.go`: потолок на `semaphore.Weighted`, взводится в `s.manifest` — ОДНОЙ строке, через которую к движку идут все три входа. Дефолт — `DefaultMaxCuts = jobs.DefaultWorkers` (4), то есть число, подо что хост уже рассчитан, и носитель у него ОДИН. Конфиг `TM_PLATFORM_MAX_CUTS`; ноль отвергает читатель чисел (`loader.number`), а не отдельная проверка интейка. Наблюдаемость: 3 гейджа + 2 счётчика, публикует существующий телеметрический проход | `TestTheHostRunsNoMoreCutsAtOnceThanItsCapAllows` — 6 загрузок при потолке 2, пик **2**, и это НЕ вакуум: тест сперва дожидается контрольной величины «4 из 6 стоят в очереди» из счётчиков самого потолка · парный `TestWithRoomForEveryCutTheHostRunsThemAllAtOnce` — та же нагрузка при потолке 6 даёт пик **6** (иначе первый тест проходил бы и на фикстуре, где ничего не совпало по времени) · посадка M4 |
|
||||
| упор в потолок не должен стоить пользователю загрузки | ожидание, а не отказ: не дождался ⇒ `errNotConclusive` ⇒ `201 parsing`, дорезает очередь. На очередном пути — claim обратно, НОЛЬ потраченных попыток и `river.JobSnooze`, то есть задание возвращается, не тратя единственную попытку (`giveBack` + `jobs.ErrTryAgainLater`) | `TestAnUploadThatRunsOutOfBudgetWaitingForASlotIsAcceptedRatherThanRefused` · `TestAnUploadTheHostCouldNotCutStillLeavesSomebodyToFinishTheBook` (claim ОТДАН и задание ПОСТАВЛЕНО — то, чего первая редакция не утверждала) · `TestAQueuedParseThatCannotCutSpendsNothingAndGivesTheBookBack` (оба исхода: упор в потолок и нехватка бюджета) · `TestAPassThatEstablishedNothingGetsItsJobBackInsteadOfSpendingIt` · посадки M9, N7, N8 |
|
||||
| §4.2 граница хвоста выводилась руками и трижды была неверна | ОДИН отсоединённый дедлайн на весь хвост (`walk`), каждый шаг берёт `min(свой бюджет, остаток)` (`step`). Шаг, добавленный завтра, границу не двигает ПО ПОСТРОЕНИЮ | `TestNoStepOfAnUploadsTailOutlivesTheWalk` — в т.ч. **50** вложенных шагов, каждый просит час · `TestTheCutOfAnUploadIsBoundedByTheWalkAndNotByItsOwnBudget` (по дедлайну, который движок РЕАЛЬНО получил, а не по секундомеру) · `TestAStepOutsideAWalkKeepsItsOwnBudgetAndSurvivesItsCaller` · посадки M1, M2 |
|
||||
| ⚠ ЧЕТВЁРТЫЙ промах той же суммы, найден моей же посадкой | квитанция идемпотентности (10 с) писалась ПОСЛЕ `Accept` и в сумму не входила ⇒ хвост был длиннее объявленного ровно на неё. Квитанция стала ТЕРМИНОМ: `UploadSettle = CutBudget + 4*writeBudget + ReceiptBudget` = **220 с** (ровно замер 07.09), носитель величины ОДИН — `books.ReceiptBudget`, тратит её `httpapi.settleCtx` | `TestTheWalkLeavesTheReceiptItsShareOfTheSettleBudget` · `TestTheReceiptSpendsTheShareTheIntakeSetAsideForIt` (httpapi) · посадки M3, M7'' |
|
||||
| §4.3 рантбук молчал про таймаут ОТВЕТА прокси | `deploy/README.md`: молчание названо числом (220 с), требование к прокси — `TM_PLATFORM_UPLOAD_DEADLINE + 220 с` = 13 мин 40 с на дефолтах, цена ошибки названа (человек получает ошибку на ПРИНЯТОЙ книге и повтором делает вторую) | директивы сверены с вендор-доками ЭТОЙ сессией: nginx `proxy_read_timeout`, дефолт **60 с** (nginx.org, ngx_http_proxy_module) · HAProxy `timeout server` (docs.haproxy.org 3.0, индекс ключевых слов) |
|
||||
| §4.4 `ClaimParse` в `cutNow` молчал о сбое БД | гонка и «хранилище не спросили» разведены: первая — INFO, вторая — ERROR, и текст называет следствие (книга `parsing` без задания до свипа) | `TestTheIntakeTellsALostRaceApartFromAStoreItCouldNotAsk` — обе фикстуры, и каждое сообщение проверено ОТСУТСТВУЮЩИМ в чужой |
|
||||
| §4.4 ветвь `RowsAffected()==0 ⇒ задание НЕ ставить` не запинена | пин на ОБЕ стороны: чужой claim ⇒ задания нет и чужой claim цел; свой ⇒ задание есть и claim снят | `TestOnlyAClaimThatWasReallyGivenBackQueuesTheJobThatFinishesTheBook` · посадка M6' красная ТЕКСТОМ про задание |
|
||||
| §4.4 свип `StuckIntake` считает от `added_at` (штамп ДО тела) | ⭐ вылечено БЕЗ миграции и без колонки: бутовый гейт сверял дедлайн с `min(UploadGrace, ClaimStale)`; добавлено ТРЕТЬЕ окно `books.ClaimGrace` (20 мин — самое узкое). Условие «свип забирает claim у идущей загрузки» стало недостижимо настройкой | посадка M5 красная текстом «does not fit the parse claim's grace» + `TestEveryWindowAnUploadMustFitInsideIsActuallyConsulted`. ⚠ **Исправление к первой редакции этой строки:** я написала, что у каждого из трёх окон свой случай, недостижимый двум другим — это БЫЛО НЕВЕРНО и найдено адверсариальным проходом. `ClaimGrace` сегодня самое узкое, поэтому все пять случаев ловит один терм, и два других можно было удалить из гейта при зелёной батарее. Вылечено выносом выбора окна в `intakeWindow(...)`, который тест кормит значениями, делающими каждое окно самым узким по очереди |
|
||||
| §4.8 п.2 комментарий `cutNow` лгал о том, кто ставит задание | снят тем же движением, что и правка ветви | `grep -rn "the queue job is already enqueued" platform/ --include=*.go` → **0** при 200 осмотренных `.go` (единственный хит по дереву — цитата находки в этом журнале, и она историческая) |
|
||||
| §4.8 п.4 предикат «кто ставит задание» размазан по двум местам | `cutsItsOwnUploads()` — одно названное место, читают оба; doc параметра `StartParsing` больше не выдаёт его за штатный путь | `grep "s.Engine != nil" internal/books/*.go` (без тестов, 4 файла) → **1 хит, и он внутри самого предиката**. ⚠ Рядом остаётся `s.Engine == nil` в `s.manifest` — это НЕ тот предикат, а nil-гард самого вызова, и он не решает, кто ставит задание |
|
||||
| §4.5 шапка журнала лгала о том, где зона | пере-снята прибором: последний пак «деньги и правда» `fda0679`, коммитов в `platform/` после него 0, канон `0.13.0`; в шапку вписаны КОМАНДЫ, которыми числа берутся | `git log --oneline fda0679..HEAD -- platform/ \| wc -l` → **0** · `grep '^ version:' openapi.yaml` → `0.13.0` |
|
||||
| §4.5 три ряда `open` при легшем лечении | `PD-424` и `PD-438` → `fixed`; у `PD-441` починен МАРКЕР, статус оставлен `open` и в ячейке названо, что держит его движковая половина (строка **331**). Фраза «в дереве» снята во всех трёх | `grep -c 'статус флипает лендинг'` → **0** при 465 рядах |
|
||||
| §4.5 незаэкранированные `\|` | заэкранированы в трёх рядах (`PD-375`, `PD-422`, `PD-197`) | эскейп-аware счёт колонок: рядов с числом колонок ≠ 7 — **0** из 465 |
|
||||
| `PD-464` (строка регистра, моя зона) | закрыт ОБЕИМИ половинами и переведён в `fixed` с диспозицией | см. ячейку ряда |
|
||||
|
||||
### §6 ось 4: существующее ПРЕЖДЕ велосипеда — что рассмотрено и чем отвергнуто
|
||||
|
||||
| Кандидат | Исход | Довод |
|
||||
|---|---|---|
|
||||
| **`golang.org/x/sync/semaphore`** | **ВЗЯТ** | `Acquire(ctx, 1)` — ровно нужная семантика: ждёт до слота или до конца контекста, очередь **FIFO** (в отличие от буферизованного канала, где поздний может обогнать раннего и часть загрузок ждала бы весь бюджет). `TryAcquire` у него не «барджит»: `success := s.size-s.cur >= n && s.waiters.Len() == 0` — быстрый путь отказывает, пока список ожидающих непуст, и слот у стоящих в очереди не ворует. ⚠ Прочитано в ИСХОДНИКЕ пинованной версии (`$(go env GOMODCACHE)/golang.org/x/sync@v0.22.0/semaphore/semaphore.go`, `TryAcquire`), а не по памяти: на этом свойстве держится довод про FIFO. Уже был в `go.sum` **косвенной** зависимостью той же версии `v0.22.0`; правка `go.mod` — перевод в прямые, БЕЗ смены версии (`git diff platform/go.mod`: одна строка вверх, одна вниз; `go.sum` −1 строка) |
|
||||
| `errgroup.SetLimit` | отвергнут | ограничивает горутины, которые запускает САМА группа. Здесь группы нет и быть не может: вызывающие независимы и приходят из разных мест (HTTP-обработчик, воркер River, свип). Форма не подходит по существу, а не по вкусу |
|
||||
| `netutil.LimitListener` | отвергнут | ограничивает СОЕДИНЕНИЯ на слушателе — то есть весь API разом, включая чтения, листинги и логин, из-за нагрузки на приём. И не накрывает ни воркера, ни свип: это ровно «потолок в маршруте», который пак запрещает |
|
||||
| `MaxWorkers` очереди (уже стоит) | отвергнут как ЕДИНСТВЕННОЕ средство, но учтён как число | ограничить синхронный вход им нельзя: этот путь намеренно НЕ ставит задание, чтобы воркер не гонялся с разрезом за claim. Зато он назвал дефолт: 4 — то, подо что хост уже рассчитан, и потолок сказан один раз для всех способов запустить движок, а не только для того, что идёт через очередь |
|
||||
| самописный счётчик / буферизованный канал | отвергнут | норма зоны «stdlib или устоявшаяся библиотека прежде своего», и здесь у своего есть конкретная цена — отсутствие FIFO |
|
||||
|
||||
### Посадки мутаций — вердикт по ТЕКСТУ падения, а не по цвету
|
||||
|
||||
Копия дерева с каноном (`cp -a --parents platform docs/architecture/14-api-contract`), базовая линия копии зелёная на всех четырёх пакетах.
|
||||
|
||||
| Посадка | Вердикт | Текст, по которому он вынесен |
|
||||
|---|---|---|
|
||||
| M1 разрез снова отсоединён от хвоста | **RED** | `the cut was granted 1m29.99s inside a 700ms walk` |
|
||||
| M2 `step` перестаёт капать по хвосту | **RED** | `the walk is not capping it` + `adding a step moves the boundary` |
|
||||
| M3 хвост съедает долю квитанции | **RED** | `want UploadSettle (3m30s) less the receipt's share (10s)` |
|
||||
| M4 потолок не применяется вовсе | **RED** | контрольная величина легла в ноль: `{Limit:2 InFlight:0 Waiting:0 Waited:0 GaveUp:0}` |
|
||||
| M5 гейт забывает окно claim'а | **RED** | `an upload deadline of 21m0s was accepted, though ... does not fit the parse claim's grace` |
|
||||
| M6′ release ставит задание, ничего не вернув | **RED** | `a release that gave back nothing still queued a job (2 in total)` |
|
||||
| M7 значение `ReceiptBudget` изменено | ⚠ **ВЫЖИЛА, и это ВЕРНЫЙ исход** | пере-сайзинг остаётся согласованным по обе стороны шва; ловить надо не значение, а ДРЕЙФ — см. M7'' |
|
||||
| M7'' квитанция возвращается к своему литералу | **RED** (после того, как по находке M7 заведён пин) | `the receipt was given 30.0s, want the share the intake declared for it (10s)` |
|
||||
| M9 занятый хост тратит попытку книги | **RED** | `a queued parse that found no slot answered <nil>, want the cap` |
|
||||
| M10 два способа не получить claim свёрнуты обратно в один молчаливый `return` | **RED** | `losing the race said nothing` + `a claim that could not be asked for said nothing, so the book sits `parsing` with no job and nobody knows` + `was not logged at ERROR` |
|
||||
|
||||
**ТРЕТИЙ круг посадок — по коду, ПЕРЕПИСАННОМУ после адверсариального прохода.** Первые две редакции
|
||||
двух пинов оказались тавтологичны, и посадка это показала, а не рассуждение.
|
||||
|
||||
| Посадка | Вердикт | Текст |
|
||||
|---|---|---|
|
||||
| N1 у ожидания удалена строка INFO | **RED** | `a cut that queued for a slot and got one said nothing` |
|
||||
| N2 переименована причина `host_at_cut_capacity` | ⚠ ВЫЖИЛА → **N2′ RED** | первая редакция пина сверяла КОНСТАНТУ с самой собой и проходила при любом значении; переписана на литерал: `the reason is "MUTATED", want the stable "host_at_cut_capacity"` |
|
||||
| N4 два гейджа потолка поменяны местами | **RED** | `the exposition is missing "tm_platform_cuts_in_flight 4"` |
|
||||
| N5 из проводки выброшен `MaxCuts` оператора | **RED** | `MaxCuts is 0, want the operator's 7: TM_PLATFORM_MAX_CUTS does nothing` |
|
||||
| N6 `cutsItsOwnUploads` всегда истинен | ⚠ ВЫЖИЛА → **N6′ RED** | счёт заданий РАЗЛИЧИТЬ НЕ МОЖЕТ (сломанный предикат ставит то же одно задание через релиз); переписано на лог с положительным контролем: `a deployment with no engine attempted a cut anyway` |
|
||||
| N7 разрез перестаёт оставлять хвосту резерв | **RED** | `the intake's parse claim was NOT given back` + `the queue was handed 0 jobs` + `has spent 1 attempts` |
|
||||
| N8 `giveBack` перестаёт возвращать задание очереди | **RED** | `the error does not tell the queue to bring the job back, so the single attempt is spent` |
|
||||
| N9 разрез запускается даже когда места на него нет | **RED** | `a cut was started with no room for it (1 calls)` + `the upload is "not_started", want ` + `the pass does not say WHY it did not cut ("no_time_to_cut")` |
|
||||
|
||||
⚠ **Единственная выжившая, которую я НЕ чиню и объявляю:** смена значения `DefaultMaxCuts`. Это
|
||||
сайзинг, а не свойство: изменённый потолок остаётся согласованным по всей системе, и «поймать» его
|
||||
можно было бы только пином на литерал, то есть запретом менять число. Что запинено — ПРОВОДКА
|
||||
(оператор получает своё число) и ЕДИНСТВЕННОСТЬ носителя (`DefaultMaxCuts = jobs.DefaultWorkers`).
|
||||
|
||||
⚠ **Первая редакция M6 и M7 НЕ КОМПИЛИРОВАЛАСЬ** (`declared and not used: tag`, `imported and not used`). Это не вердикт, а его отсутствие: посадка, которая не собралась, красит батарею по причине, не имеющей отношения к предмету. Обе пере-посажены компилирующимися.
|
||||
|
||||
### Классы и знаменатели — «закрыт в N из M», M посчитан командой
|
||||
|
||||
| Класс | Знаменатель | Как посчитан |
|
||||
|---|---|---|
|
||||
| входы, порождающие процесс движка на разрезе | **3 из 3** (интейк · `parseWorker` · свип `Sweep`) | `s.manifest` имеет РОВНО ОДНОГО вызывающего (`grep -rn 's\.manifest(' --include=*.go internal/ \| grep -v _test` → 1: `parse.go:178`), у `parseClaimed` их два (`Parse`, `cutNow`), у `Parse` — воркер `jobs.go:117` и `Sweep`. Все три сходятся в одну строку |
|
||||
| порождения движка ВНЕ потолка | **1** — `internal/readmodel/readmodel.go:156` | `grep -rn '\.Manifest(' --include=*.go \| grep -v _test` → 2 вызывающих, один из них мой. Оставлен снаружи сознательно, довод — ниже |
|
||||
| шаги хвоста под общим дедлайном | **все** (в `Accept` их 5 + квитанция) | построением, а не перечнем: `writeCtx` идёт через `step`, `step` капает по хвосту. Проверено на 50 шагах, которых в коде нет |
|
||||
| ряды регистра с диспозицией «статус флипает лендинг» | **3 из 3** | `grep -c 'статус флипает лендинг'` → было 3, стало **0** |
|
||||
| ряды с числом колонок ≠ 7 | **3 из 3** | эскейп-аware счёт: было 3, стало **0** при 465 осмотренных |
|
||||
| открытые ряды регистра по МОИМ файлам | 10 путей осмотрено, совпадений — 20 рядов, из них МОЙ предмет **1** (`PD-464`, закрыт); остальные 19 — соседние классы, не тронутые этим паком | `grep` по ПОЛНЫМ путям (норма §3 п.8), контроль: открытых рядов всего **109** |
|
||||
|
||||
### Числа и команды — сняты ПОСЛЕ последней правки
|
||||
|
||||
```
|
||||
$ python3 docs/scripts/counts.py --check → EXIT=1, и это ОЖИДАЕМО: ровно два расхождения,
|
||||
✗ docs/PROGRESS.md: «открытых рядов регистра платформы — 112» против пере-счёта 110
|
||||
✗ docs/PROGRESS.md: «... (major 3)» против пере-счёта 1
|
||||
⚠ оба литерала — в ЧУЖОЙ зоне (`docs/PROGRESS.md`), туда не лезу. После лендинга и закрытия
|
||||
PD-464 верные числа: **открытых 109, major 1, minor 36, info 72** (`counts.py` по дереву).
|
||||
|
||||
$ git log --oneline fda0679..HEAD -- platform/ | wc -l → 0
|
||||
$ grep '^ version:' docs/architecture/14-api-contract/openapi.yaml → 0.13.0
|
||||
$ go list ./... | wc -l → 20 (носитель скипов чинен первым движением)
|
||||
|
||||
$ TM_PLATFORM_TEST_DSN=… TM_PLATFORM_TEST_ENGINE_BIN=… TM_PLATFORM_TEST_BOOK_TEMPLATE=… \
|
||||
TM_PLATFORM_TEST_PGDUMP=… TM_PLATFORM_TEST_PGRESTORE=… make check
|
||||
MAKE-EXIT=0 · пакетов `ok` **20** · строк FAIL **0** · линтер «0 issues» · скипов **5**
|
||||
⚠ Прогон ПОСЛЕДНИЙ — после адверсариального круга и после последнего добавленного пина. Прежние
|
||||
редакции этих чисел (до круга) в отчёте не оставлены: они были верны и уже не про этот код.
|
||||
```
|
||||
|
||||
⚠ **Скипы 5, и условие у всех одно, названное** (§10 ниже): нет деплой-артефакта
|
||||
`backend/configs/mining-contrast.zh.txt`. Из четырёх гейтов батареи на этом хосте закрыты все:
|
||||
Postgres · движковый бинарь + шаблон (собран из `git archive HEAD backend`, §4.7) · достижимый
|
||||
пользовательский `systemd` · `MemoryMax` — судится самим `TestARunIsBoundedByItsOwnCgroup` (`STACK_DECISIONS` §«Гейты батареи»: прямой пробы у этого условия нет), и он ОТРАБОТАЛ: в полном перечне скипов, который печатает `check`, его нет, а `FAIL` в прогоне нет вовсе.
|
||||
⚠ Числа Go-батареи сняты ПОСЛЕ последней правки кода; доковые правки после них Go-батарею не касаются.
|
||||
|
||||
### Живой гейт (§4.7) — довод зоны опровергнут ИСПОЛНЕНИЕМ
|
||||
|
||||
Зона писала, что второй гейт батареи требует `$0`-пайплайна рядом с `backend/prompts/`, то есть записи в
|
||||
чужую зону или полной копии дерева. **Проверено исполнением — неверно.** Рецепт:
|
||||
|
||||
```
|
||||
$ git archive HEAD backend | tar -x -C $W # снапшот чужой зоны, ни байта записи в неё
|
||||
$ cd $W/backend && go build -o $W/tmctl ./cmd/tmctl
|
||||
$ sed -e 's|pipeline: ../configs/...|pipeline: $W/backend/configs/pipeline-c1.yaml|' \
|
||||
-e 's|models: ../configs/...|models: $W/backend/configs/models.yaml|' \
|
||||
$W/backend/example/book.yaml > $W/template.yaml # пути абсолютные
|
||||
$ TM_PLATFORM_TEST_ENGINE_BIN=$W/tmctl TM_PLATFORM_TEST_BOOK_TEMPLATE=$W/template.yaml go test ./internal/books/
|
||||
```
|
||||
`TestTheRenderedConfigurationIsOneTheEngineActuallyLoads` — **PASS**.
|
||||
|
||||
⭐ **И где именно рассуждение зоны свернуло не туда:** «нужен `$0`-пайплайн» верно для теста
|
||||
`internal/runner`, который гоняет `translate` и падает на `missing API keys`. Оно было ОБОБЩЕНО на гейт
|
||||
целиком — а `manifest` есть `$0`-глагол и ключей не требует по `D20.4`, поэтому боевой `pipeline-c1.yaml`
|
||||
(«платный») загружается и режет без единого ключа. То есть довод был верен про один тест и ложен про гейт,
|
||||
и разница видна только исполнением.
|
||||
|
||||
### ⚠ Правки, вызванные заказанной сменой поведения (объявляю по `D39.183`)
|
||||
|
||||
1. **`TestAnUploadDeadlineIsRefusedUnlessTheWholeUploadFitsTheTighterWindow` → `...TightestWindow`.** Гейт
|
||||
получил третье окно (§4.4, п.7 десятки) — прежний тест утверждал, что дедлайн `26m29s` ПРИНИМАЕТСЯ, и
|
||||
после лечения это неверно. Тест не «починен под зелень»: он пере-написан строже — у каждого из трёх окон
|
||||
свой случай, недостижимый двум другим, и посадка M5 краснит его именем нового окна.
|
||||
2. **`ClaimGrace` экспортирована** (была `claimGrace`) — переименование затронуло 3 тестовых файла зоны
|
||||
механически; утверждений не тронуто. Основание — то же, по которому экспортирована `UploadGrace`: бут
|
||||
обязан отказывать конфигурации, которая её нарушает.
|
||||
3. **Вторая правка рантбука сверх §4.3** — абзац про `TM_PLATFORM_MAX_CUTS`. Довод: §4.1 требует, чтобы
|
||||
потолок был «конфигурируемым и наблюдаемым», а ручка, о которой рантбук молчит, оператору не доступна;
|
||||
документировать ручку, которую этот же пак и завёл, — часть §4.1, а не «остальное про выкат».
|
||||
|
||||
### ⚠ Константа контракта `0.12.0` → `0.13.0` — что было сломано и кем
|
||||
|
||||
Батарея была КРАСНОЙ на входе, до единой моей правки: `internal/gates`
|
||||
`TestTheAnnouncedContractVersionIsTheOneTheCanonRatified` — «this build announces contract 0.12.0 and the
|
||||
ratified canon is 0.13.0». Улика: файл-носитель (`internal/httpapi/capabilities.go:36`) в моём диффе
|
||||
отсутствовал (`git diff --name-only HEAD | grep -i contract` → пусто). Канон увёл на `0.13.0` коммит
|
||||
оркестратора `3d90943`, константу за собой не потянув; последняя правка константы — `6ceb133`, до него.
|
||||
То есть гейт красен с момента ратификации, **сутки**, и заметила это входная сверка следующей сессии зоны.
|
||||
Взято мной по ЯВНОМУ указанию оркестратора с названным основанием: ратифицированный порядок `D39.208` п.1
|
||||
— код первым с честно красным гейтом, канон вторым; здесь порядок был обратный, и правка возвращает мир к
|
||||
гейту, а не гейт к миру (`D39.183`, обслуживание). **Авторство ошибки — оркестратор, не прошлый пак:** на
|
||||
`fda0679` канон и константа обе были `0.12.0`, гейт был зелёным, и число батареи в акте `D39.221` честное.
|
||||
|
||||
### §4.6 — ответ (а): закрыто по построению НА МОЕЙ СТОРОНЕ, но премиса пака опровергнута
|
||||
|
||||
Пункт 10 десятки предполагал, что причина отказа не доезжает. **Опровергнуто:** перечислены ВСЕ семь
|
||||
пользовательских отказов приёма — `payload_too_large` и `request_timeout` (корневые коды), `malformed` ×2,
|
||||
`unsupported_pair` ×2, `no_book`, `no_chapter_structure`, `too_long`, `missing_or_late`. Причины, не
|
||||
выразимой перечислимым кодом, я не нашла; расширять `errors[]`/`cause.code` нечем, и минор не нужен.
|
||||
⚠ **Собственную первую находку снимаю:** я решила, что слишком длинное поле формы уходит с ПУСТЫМ
|
||||
`errors[]` — неверно, оно названо на месте чтения (`v0.go:843`, `ItemTooLong`), а ветвь `Invalid(w, r)` без
|
||||
элемента до него не доходит.
|
||||
|
||||
⛔ **А вот премиса пака про клиента ЗАМЕРОМ НЕ ПОДТВЕРЖДАЕТСЯ, и следующая смена не должна её унаследовать.**
|
||||
Пак пишет: «Таблица „код → русская фраза“ у клиента уже есть (`14-api-contract/README.md`, и фронт её
|
||||
рисует)». Замер: `no_chapter_structure` в живых доках — **11 хитов при 1104 осмотренных `.md`**, и НИ ОДИН
|
||||
не таблица фраз; в `14-api-contract/README.md` нет ни `no_book`, ни `no_chapter_structure`. В зоне фронта
|
||||
`no_book`/`unsupported_pair` — **0 хитов при 8114 осмотренных `.ts`/`.tsx`**. Что там есть на самом деле —
|
||||
правило «клиент диспетчеризует по СТАТУСУ и показывает одну нейтральную фразу» (README §вход) и решение
|
||||
владельца 16.08 о машинном коде. ⇒ вывод пака (моя работа тут закончена, клиентская половина — фронт, а он
|
||||
заморожен) остаётся ВЕРНЫМ, но не потому, что таблица есть, а потому, что платформа дала клиенту всё, по
|
||||
чему её можно нарисовать. Разница существенна: с премисой пака работа выглядит сделанной у обеих сторон.
|
||||
|
||||
### Пинги оркестратору
|
||||
|
||||
1. ⛔ **Литералы в `docs/PROGRESS.md` под гардом `counts.py --check` протухли моим флипом — двигать их
|
||||
тебе.** После лендинга верно: **открытых 109, major 1** (было «112 (major 3)»). Разница в три ряда:
|
||||
`PD-424` и `PD-438` переведены в `fixed` по твоему же маркеру, `PD-464` закрыт этим паком. Гейт красен
|
||||
ОЖИДАЕМО и ровно на этих двух строках — других расхождений он не даёт.
|
||||
2. **§4.8 п.6, «ВСЕГДА» в дельте контракта — моё мнение, как просил пак: нужна ОГОВОРКА В КАНОНЕ, а не
|
||||
структурная гарантия.** Довод из кода, а не из вкуса. Между `FinishParse` и `ReadBook` свип
|
||||
материализатора может взять долг (он записан ИМЕННО `FinishParse`) и дописать строке `source_chars` и
|
||||
`structure` (`readmodel.refresh` → `SaveStructure`). Но он НЕ МОЖЕТ ни снять вердикт разреза, ни
|
||||
изменить `status`/`chapter_count`: единственный писатель `reject_reason` — `RejectBook`
|
||||
(`pgstore/books.go:394`, один хит по всему коду), а статус двигают только `FinishParse`/`reject`.
|
||||
⇒ расхождение возможно только В СТОРОНУ БОЛЬШЕГО: ответ либо уже несёт поверхностные поля, либо ещё
|
||||
нет. Структурная гарантия потребовала бы держать что-то поперёк `FinishParse`→`ReadBook` на горячем
|
||||
пути ради полей, которые клиент всё равно перечитывает карточкой. ⚠ И отдельно: **саму фразу «ВСЕГДА» я
|
||||
в каноне не нашла** — `grep 'ВСЕГДА' 14-api-contract/README.md` даёт 0, `grep -i always` в
|
||||
`openapi.yaml` — 12 хитов, все про другое (SSE-кадры, `about:blank`). Назови предложение адресом, и
|
||||
если оно живёт не там, где я искала, мой довод надо перепроверить против него.
|
||||
3. ⚠ **Премиса пака в §4.6 неверна — см. секцию выше.** «Таблица код → русская фраза у клиента уже есть, и
|
||||
фронт её рисует» замером не подтверждается (0 хитов `no_book`/`unsupported_pair` при 8114 осмотренных
|
||||
`.ts`/`.tsx`; в `14-api-contract/README.md` ни `no_book`, ни `no_chapter_structure`). Вывод пака устоял,
|
||||
основание — нет. Стоит поправить, иначе следующая смена унаследует «у клиента всё готово».
|
||||
4. **Мелкая неточность адреса в §4.7:** «а 35 строками ниже в том же журнале лежит рецепт снапшота» —
|
||||
реально 261 строкой ниже. Адреса в дереве, которое я сдаю: довод — `platform-PROGRESS.md:452`, рецепт —
|
||||
`:713` (на входном `HEAD` это были `:173` и `:434`; расстояние то же). На существо не влияет: рецепт там
|
||||
и есть, и он работает.
|
||||
5. **Твой вопрос «есть ли дешёвый способ закрыть сутки красноты между ратификацией и следующей сессией
|
||||
зоны» — есть, и он в ТВОЕЙ зоне.** Пара «канон ↔ объявленная константа» сегодня судится только Go-тестом,
|
||||
который гоняет зона. А `docs/scripts/counts.py --check` уже читает оба дерева, уже висит на зонном
|
||||
pre-commit и уже срабатывает **именно на коммитах с D-логом или PROGRESS** — то есть ровно на
|
||||
ратификационных. Добавить туда одну проверку — `grep '^ version:' openapi.yaml` против
|
||||
`const ContractVersion` в `platform/internal/httpapi/capabilities.go` — стоит десятка строк и ловит
|
||||
ровно тот класс, который стоил суток: он предупреждает того, КТО ДВИГАЕТ КАНОН, в момент движения.
|
||||
Заказом не делаю (файл в `docs/`), рекомендацию записываю.
|
||||
|
||||
### Адверсариальный проход по СВОЕЙ готовой работе — восемь находок, и они были настоящие
|
||||
|
||||
Проход заказан §5.4 и выполнен субагентом (author ≠ reviewer) по готовому диффу, с направлением на
|
||||
классы, которые уже стоили зоне денег. **Круги НЕ сошлись с первого раза: он нашёл восемь, и шесть из
|
||||
них — дефекты, которые ввёл ЭТОТ пак.** Каждую я пере-проверила по коду прежде, чем чинить.
|
||||
|
||||
| # | Находка | Чем оказалась | Что сделано |
|
||||
|---|---|---|---|
|
||||
| **F1** | комментарий `giveBack` обещал повтор задания с бэкоффом | ⛔ ЛОЖЬ: `ParseArgs.InsertOpts` — `MaxAttempts: 1`, повтора нет вовсе; книга на занятом хосте ждала свип **20 минут** | заведён `jobs.ErrTryAgainLater`; воркер переводит его в `river.JobSnooze(RetryDelay)`, который НЕ тратит единственную попытку. Комментарий приведён к правде. Пин — `TestAPassThatEstablishedNothingGetsItsJobBackInsteadOfSpendingIt` |
|
||||
| **F2** | у выигравшего слот не проверялось, осталось ли время на разбор | ⛔ настоящий: слот, выигранный в конце бюджета, отдавал движку миллисекунды; убитый процесс читается как `parser_unavailable`, а он ТРАТИТ попытку — пять таких удаляют файл пользователя | `takeCutSlot(ctx, reserve)`: `worthStarting` до и ПОСЛЕ ожидания, `waitCtx` обрывает ожидание на резерв раньше. Резерв — `CutBudget` на очередном пути, `0` на интейке (там попытка не тратится) |
|
||||
| **F3** | отказ бута при `MaxCuts < 1` | ⛔ МЁРТВЫЙ КОД: `l.number` уже отвергает всё непозитивное, и мой тест пинил чужой охранник, а не мой | ветвь удалена; в тесте названо, ГДЕ живёт отказ |
|
||||
| **F4** | тест трёх окон | ⛔ ВАКУУМЕН для двух окон из трёх, и его комментарий утверждал обратное — ровно тот класс, который он якобы чинил | выбор окна вынесен в `intakeWindow(...)`; новый тест делает каждое окно самым узким по очереди. Ревьюер пере-мутировал независимо: теперь красный |
|
||||
| **F5** | терминальная запись могла родиться истёкшей | ⛔ настоящий и злой: при спетом хвосте claim НЕ отдавался и задание НЕ ставилось — книга «принята», а доделать её некому 20 минут. Мой тест этого не утверждал | `stepLeaving` + `cutTailReserve`: слабину забирает РАЗРЕЗ, а не записи. Пин — `TestAnUploadTheHostCouldNotCutStillLeavesSomebodyToFinishTheBook` |
|
||||
| **F6** | комментарий потолка обещал больше, чем потолок делает | верно: `readmodel` порождает те же процессы мимо него; плюс `DefaultMaxCuts` был вторым литералом числа воркеров | комментарий сужен до правды и называет, что осталось снаружи; `DefaultMaxCuts = jobs.DefaultWorkers` — один носитель, и `config.Runner.Workers` берёт его же |
|
||||
| **F7** | шесть поверхностей пережили мутацию | верно все шесть | закрыты пинами (ниже), кроме значения `DefaultMaxCuts` — это САЙЗИНГ, и его смена не дефект; названо в §10 |
|
||||
| **F8** | баннер `capabilities.go` противоречил себе | верно, и сломала его Я этой же сменой | баннер разводит два порядка: код первым (канон отстаёт) — ратифицированный, канон первым (код отстаёт) — тот, что стоил суток |
|
||||
|
||||
### ⛔ НАХОДКА №9 — класс, которого не ловит НИ батарея, НИ мутация
|
||||
|
||||
Поймана мной при починке F5: **моя починка была дефектной, и её дефект не имел цвета.**
|
||||
|
||||
`cutTailReserve` был КОНСТАНТОЙ `3 * writeBudget`, а бюджет записи в тестах — полем сервиса
|
||||
(`s.writeBudget`, который фикстуры укорачивают, чтобы достать случаи, недостижимые за 30 секунд). В
|
||||
фикстуре с хвостом 300 мс резерв оставался 90 с — больше всего хвоста ⇒ шаг разреза рождался истёкшим,
|
||||
движок не звался НИКОГДА, и пакет `books` **зависал навсегда** на `<-first`.
|
||||
|
||||
⭐ **Почему это отдельный класс.** Батарея его не ловит, потому что зелёного вердикта просто не
|
||||
наступает — но и красного тоже: прогон висит до таймаута `go test`, и в CI это читается как «долго», а
|
||||
не как «сломано». Мутация его не ловит по той же причине: у посадки нет вердикта, есть тайм-аут.
|
||||
Единственное, что его назвало — прогон с УКОРОЧЕННЫМ `-timeout` и чтение стека упавшего по нему
|
||||
процесса (`limit_test.go:162`, `<-first`); по цвету он неотличим от медленной машины. Две вещи из этого:
|
||||
- резерв сделан производным от бюджета В СИЛЕ (`s.cutTailReserve()` = `3 * s.write()`), иначе фикстура
|
||||
молча моделирует не то;
|
||||
- «нет места для разреза» больше не притворяется отказом хранилища: claim берётся на СВОЁМ бюджете
|
||||
записи, разрез — на своём, и пустой разрез отвечает «вердикта нет» (`ReasonNoTimeToCut`), а не падает
|
||||
внутри `ClaimParse`. Это тот же класс, что F1: диагноз, который называет не то, что случилось.
|
||||
Запинено `TestAnUploadWithNoRoomLeftForACutSaysThatAndHandsTheBookOver` + посадка N9.
|
||||
|
||||
⚠ **И правило, которое стоит пережить пак:** число, которое фикстура умеет укорачивать, и число,
|
||||
выведенное из него, обязаны быть выведены ОДИНАКОВО. Константа рядом с полем — это две величины,
|
||||
которые совпадают в бою и расходятся в тесте, то есть ровно то, что фикстура сделать не может увидеть.
|
||||
|
||||
⚠ **Урок, который стоит пережить этот пак:** шесть из восьми находок — в коде, который я СДАВАЛА как
|
||||
готовый, с зелёной батареей, десятью посадками и отчётом, где написано «круги сошлись». Батарея была
|
||||
зелёной на всех восьми. Ловит их не цвет, а второй читатель, которому названо, ГДЕ у этого пака мягко.
|
||||
|
||||
### Якоря, убитые моим переездом — норма §3 п.8
|
||||
|
||||
Мои правки сдвинули строки в `internal/config/config.go`, `internal/pgstore/books.go`,
|
||||
`internal/metrics/metrics.go` и `cmd/tmplatformd/runner.go` (везде вставки, сдвиг +6 в первых двух).
|
||||
|
||||
**Замер дифференциальный, а не «посмотрела»:** линтер на входном `HEAD` даёт **8** проблемных якорей,
|
||||
моё дерево давало **26**. Чтобы отделить своё от унаследованного и от WIP чужой сессии, собрала
|
||||
`git archive HEAD` в /tmp, подменила в копии ТОЛЬКО `platform/` своим и сравнила списки `comm`-ом.
|
||||
⚠ Кап вывода линтера — 25 строк; на 26 проблемах обрезка читается как отсутствие, поэтому в копии
|
||||
скрипта кап поднят до 500. Появившихся из-за меня — **18**.
|
||||
|
||||
| Где | Сколько | Что сделано |
|
||||
|---|---|---|
|
||||
| `platform/docs/DEFECT_REGISTER.md` | **14 из 14** | пере-наведены механически: токен найден в цели, адрес заменён; ни одного «руками» |
|
||||
| `docs/PROGRESS.md`, `docs/architecture/05-decisions-log.md` | **4** | ЧУЖАЯ зона — ушли пингом с готовыми адресами и токенами |
|
||||
|
||||
⭐ **Находка из этого же хода:** экранирование `\|` в ячейке регистра (§4.5) **ломает якорь**, если черта
|
||||
попала в его токен. `PD-422` держал `internal/runs/runs.go:408`=`resnapshot := book.BankMoved || book.HasPriorRun`;
|
||||
после экранирования токен перестал совпадать с кодом. Вылечено укорочением токена до
|
||||
`resnapshot := book.BankMoved` (единственный хит в файле). Счёт колонок и сверка токена тянут ячейку в
|
||||
разные стороны — следующий, кто пойдёт экранировать черты, наступит на то же. Ушло пингом.
|
||||
|
||||
Итог: мой лес **12** проблемных якорей против **8** на `HEAD`; остаток — ровно те 4 чужой зоны.
|
||||
|
||||
### §10 — что НЕ удалось и что НЕ проверено (это разные исходы)
|
||||
|
||||
- **Скипов 5 (было 6), и условие у всех ОДНО и названное:** `backend/configs/mining-contrast.zh.txt` нет
|
||||
на этом хосте, и это деплой-артефакт, которого нет в репозитории (снапшот `git archive HEAD backend`
|
||||
его не несёт — `ls backend/configs` даёт `langpacks pairs models.yaml pipeline-*.yaml`, и всё).
|
||||
Скипающиеся: `TestTheRealEngineNamesItsRestorePointInTheLineThisPlatformParses`,
|
||||
`TestALivePreviewWritesNothingAndALiveApplyWrites`,
|
||||
`TestALiveBuildOfAHollowBookWritesTheMarkedCopyInsteadOfRefusing`,
|
||||
`TestWithoutPartialTheSameBookIsRefusedWithTheBuildsOwnNumber`,
|
||||
`TestTheSnapshotGuardIsLoudWithoutTheFlagsAndPassesWithThem`. Шестой (`TestARestorePointCanActuallyBeRestored`)
|
||||
закрыт: `pg_dump`/`pg_restore` есть в `~/.local/pgsql/bin`, переменные выставлены.
|
||||
⚠ Это НЕ «не проверено» про мой предмет: ни один из пяти не касается разреза приёма — они про банк,
|
||||
выдачу и снапшот-гард. Живой гейт МОЕГО предмета закрыт и зелёный.
|
||||
- ⛔ **ЭТО МЕСТО БЫЛО «НЕ ПРОВЕРЕНО» И ОКАЗАЛОСЬ ДЕФЕКТОМ — оставляю как след.** Первая редакция отчёта
|
||||
писала: «полагаюсь на то, что River повторит задание с бэкоффом; сколько попыток он даёт, я не
|
||||
измеряла». Замер (адверсариальный проход, подтверждён мной по коду): `ParseArgs.InsertOpts` —
|
||||
`MaxAttempts: 1`, повтора НЕТ ВООБЩЕ, задание просто списывается, и книга ждала свип 20 минут. То
|
||||
есть моё «рассуждение, а не замер» было не осторожностью, а неверным утверждением в комментарии кода.
|
||||
Вылечено `river.JobSnooze`, который возвращает задание не тратя единственную попытку. ⭐ Урок ровно
|
||||
тот, что записан в каноне зоны: строка «не проверено» — это не смягчение, это место, где ещё не
|
||||
посмотрели, и смотреть надо ДО сдачи.
|
||||
- **НЕ ЗАПИНЕНО ИМЕНЕМ, но покрыто исполнением** (проверено посадками, не грепом по именам):
|
||||
`cutTailReserve` — посадкой N7, `worthStarting` и `waitCtx` — обоими исходами
|
||||
`TestAQueuedParseThatCannotCutSpendsNothingAndGivesTheBookBack` (ожидание обрывается на резерв раньше,
|
||||
поэтому случай «упор в потолок» кончается за ~1,5 с, а не за весь бюджет), `engineNotAsked` — обоими
|
||||
ветвями там же, `jobs.RetryDelay` — новым тестом очереди. Собственного теста по имени у них нет.
|
||||
- **НЕ ПРОВЕРЕНО экспериментально: сколько памяти реально держит один `tmctl manifest`.** Дефолт 4 выбран
|
||||
как число, под которое хост уже был рассчитан (`MaxWorkers` очереди), а не измерен на большой книге.
|
||||
Ручка конфигурируема именно поэтому.
|
||||
- **Опровержение премисы §4.6 сделано ГРЕПОМ ПО КОДАМ**, а не чтением рендера фронта: я искала строки
|
||||
`no_book`/`unsupported_pair` в 8114 `.ts`/`.tsx`. Таблица, ключуемая иначе (например, по корневому
|
||||
`code`), таким грепом не нашлась бы. Утверждаю ровно замеренное.
|
||||
- **Одно новое условное сообщение НЕ запинено:** `«the parse claim could not be given back; the backstop
|
||||
sweep takes the book»` в `giveBack` — ветвь, где отказ `ReleaseParseClaim` накладывается на упор в
|
||||
потолок. Четыре остальных новых сообщения запинены ОБЕИМИ фикстурами (где обязано прозвучать и где
|
||||
обязано молчать), это пятое — нет: чтобы его достать, нужен отказ хранилища ВНУТРИ уже насыщенного
|
||||
потолка, и фикстуру такой конъюнкции я не построила. Называю прямо, а не выдаю шесть из семи за семь.
|
||||
Остальные шесть — включая обе новые («движок не спрошен» и «не осталось места на разрез») — запинены
|
||||
фикстурой, где сообщение обязано прозвучать, И фикстурой, где обязано молчать.
|
||||
- **Ограничитель НЕ накрывает `readmodel`** (`internal/readmodel/readmodel.go:156` — второй и последний
|
||||
вызывающий `Engine.Manifest`) и глаголы `export`/`status`. Это осознанная граница, а не пропуск:
|
||||
материализатор и `status` идут внутри очереди, которая уже ограничена одним `MaxWorkers` на все три
|
||||
типа заданий (`internal/jobs/jobs.go:170`), свипы последовательны, а `readEngine` накрыл бы ещё
|
||||
`Status` на пути СТАРТА платного прогона (`internal/runs/spawn.go:242`) и связал бы запуск прогонов с
|
||||
нагрузкой приёма. Единственным неограниченным источником процессов был синхронный интейк — он и закрыт.
|
||||
⚠ Если приёмка считает, что хосту нужен потолок на ВСЕ порождения, это отдельная работа со своим
|
||||
дизайном (развязка денежного пути), а не райдер к этому паку.
|
||||
|
||||
### Попутно: совместимость с новой секцией `bank.json` (пришло пингом от движковой зоны, проверено моим кодом)
|
||||
|
||||
Движковый пак добавил в `bank.json` секцию `consolidation` (полнота банка) и поле `never_asked`, версия
|
||||
`tm-bank-v1` НЕ бампнута. **Пере-проверено на моей стороне, не принято на слово:**
|
||||
- `bank.json` разбирает `internal/ingest/bank.go:78` `DecodeBank` — простой `json.Unmarshal`, неизвестный
|
||||
член игнорируется. ⚠ Строгий декодер в зоне ЕСТЬ ровно один (`internal/httpapi/bank.go:137`,
|
||||
`grep -rn DisallowUnknownFields --include=*.go` → **1 хит при 200 `.go`**), но он на ДРУГОМ пути — тело
|
||||
запроса на правки ОТ КЛИЕНТА, где строгость требует сам канон. Пути не пересекаются ⇒ лендинг движка
|
||||
приём банка не ломает.
|
||||
- Читателей секции у зоны нет. ⚠ По подстроке их **2 при 186 `.go` в `internal/`**
|
||||
(`internal/ingest/manifest.go:97`, `internal/pricing/pricing.go:112`) — и оба английская ПРОЗА про
|
||||
«terminology consolidation» в денежных комментариях, а не чтение поля. Счёт по подстроке и счёт по
|
||||
владению здесь расходятся на два: следующему, кто будет снимать этот ноль, читать хиты, а не число.
|
||||
- ⛔ **Закон на будущее:** бит `complete` брать ГОТОВЫМ из артефакта, не выводить у себя (п.6 закона
|
||||
входной двери шва). У движка он считается от среза рендер-паса, а срез классификатора полноты банка не
|
||||
означает — самостоятельный вывод разошёлся бы с движковым молча.
|
||||
|
||||
Строку под читателя НЕ завожу: это следующий пак зоны, и решение оркестратора — не торопить.
|
||||
|
||||
### Вопросы оркестратору
|
||||
|
||||
- **Нужен ли ряд регистра на остаток §4.1** (порождения вне потолка: `readmodel` + `export`/`status`)?
|
||||
Я его НЕ завела: это не дефект сегодняшнего поведения, а названная граница механизма, и заводить ряд
|
||||
«мы решили иначе» — засорять регистр. Скажи, если хочешь ряд.
|
||||
- **`ReasonHostAtCapacity` — константа, которая НИКОГДА не пишется в БД** (`reject` — единственный писатель
|
||||
причины, а класс потолка возвращает claim до него). Я оставила её строкой рядом с пятью
|
||||
`ReasonX`-константами, потому что читатель приходит за ними туда же, и написала это в комментарии.
|
||||
Если считаешь, что не-хранимой причине там не место — скажу, куда унести.
|
||||
|
||||
## ПАК «РАЗРЕЗ ПРИЁМА ДО ГОТОВНОСТИ И ПРАВДА О СЕБЕ» — ЗАПИСКА-ПЛАН (08.09, `textmachine-fa`)
|
||||
|
||||
> Промт `docs/PLATFORM_INTAKE_TRUTH_SESSION_PROMPT.md`, вход HEAD `3f4680c`, дерево на входе чисто
|
||||
> (`git status --porcelain` — пусто). Зона НЕ коммитит. Пак $0.
|
||||
> Baseline снят сам: `python3 docs/scripts/counts.py --check` → «Литералы сходятся с пере-счётом
|
||||
> (8 проверок)», регистр 465 рядов / open 112 / major 3.
|
||||
|
||||
**Что беру и в каком порядке.** Сначала то, что стоит $0 и является предусловием остального (шапка
|
||||
этого журнала, носитель числа пакетов, ряды регистра), потом код в порядке связности: бюджет хвоста —
|
||||
ограничитель — три неразличимых сбоя, потому что первые два связаны структурно и чинить их по
|
||||
отдельности значит ломать один другим. Живой гейт и рантбук — последними, они судят уже построенное.
|
||||
|
||||
**Разметка решений, принятых ДО кода — чтобы их можно было опровергнуть по этой записке.**
|
||||
|
||||
1. **Бюджет (§4.2) — форма Б, структурная.** Перечень шагов подвёл трижды, и четвёртый перечень был бы
|
||||
той же заплатой (`D39.216`). Беру ОДИН отсоединённый контекст хвоста с дедлайном `UploadSettle`,
|
||||
от которого наследуются все шаги: `context.WithTimeout` на потомке с более ранним дедлайном сам даёт
|
||||
`min(шаг, остаток)`, поэтому добавленный шаг границу не двигает ПО ПОСТРОЕНИЮ, а не по внимательности
|
||||
следующего автора. Пин утверждает САМО свойство (§5.3), а не сумму слагаемых.
|
||||
2. **Ограничитель (§4.1) — на `books.Service.manifest`.** Замер входов, а не память: `Engine.Manifest`
|
||||
зовут ДВА места (`internal/books/parse.go:440`, `internal/readmodel/readmodel.go:156`), а `s.manifest` —
|
||||
ровно те три, что названы заказом (интейк · `parseWorker` · свип `Sweep`). Шире (`runner.readEngine`,
|
||||
общий на `manifest`/`export`/`status`) НЕ ставлю, и довод замером: очередь у платформы ОДНА и уже
|
||||
ограничена (`internal/jobs/jobs.go:170`, `MaxWorkers` дефолт 4) на все три типа заданий сразу, свипы
|
||||
последовательны — то есть единственный неограниченный источник процессов на хосте это и есть
|
||||
синхронный интейк; а `readEngine` накрыл бы ещё `Status`, который стоит на пути СТАРТА платного
|
||||
прогона (`internal/runs/spawn.go:242`, `bookMeter`), и связал бы запуск прогонов с нагрузкой приёма.
|
||||
Что осталось снаружи — называю в отчёте числом, а не умолчанием.
|
||||
3. **Форма — ожидание, а не немедленный отказ.** Ожидание внутри уже стоящего `CutBudget` к хвосту
|
||||
ничего не добавляет (приор оркестратора, проверяю кодом), а упор даёт штатную деградацию: не уложился
|
||||
⇒ `errNotConclusive` ⇒ `201 parsing` ⇒ книгу доделывает очередь. Это строго лучше `503`: пользователь
|
||||
получает книгу. Существующее прежде своего (§6): `x/sync/semaphore`, `errgroup.SetLimit`,
|
||||
`netutil.LimitListener`, `MaxWorkers` — рассматриваю и отвергнутое называю с доводом.
|
||||
4. **Свип `StuckIntake` (§4.4) — гипотеза лечения БЕЗ миграции.** `StartParsing` не штампует
|
||||
`parse_started_at` (`internal/pgstore/books.go:213`), поэтому `coalesce(parse_started_at, added_at)`
|
||||
в предикате свипа — это `added_at`, поставленный ДО прихода тела; бутовый гейт
|
||||
(`internal/config/config.go:729`) сверяет дедлайн только с `min(UploadGrace, ClaimStale)` = 30 мин и
|
||||
пропускает дедлайн до 26m29s, а `claimGrace` = 20 мин ⇒ условие достижимо настройкой. Кандидат —
|
||||
добавить `claimGrace` третьим окном в тот же `min()`: колонки не нужно, форма гейта уже ровно эта.
|
||||
Не выйдет — пинг, а не полумера молча (§4.8).
|
||||
|
||||
**Что считаю рискованным.** (а) Форма Б трогает контексты на ВСЁМ пути приёма — класс ошибок здесь
|
||||
«тихо-зелёный»: путь продолжает работать, а гарантия исчезает, поэтому пин обязан быть структурным
|
||||
(дедлайны шагов), а не «уложились по часам». (б) Ограничитель на общей точке касается и очередного
|
||||
входа — нагрузочное предъявление обязано считать ОДНОВРЕМЕННЫЕ процессы, а не суммарные. (в) Фикстуры
|
||||
зоны уже делали два разных числа одним (`D39.208` п.5): везде, где в фикстуре встречаются `writeBudget`,
|
||||
`CutBudget` и `UploadSettle`, беру ТРИ РАЗНЫХ значения.
|
||||
|
||||
**Чего не делаю:** п.6 десятки (текст контракта) — пинг оркестратору; всё из §4.8.
|
||||
|
||||
## ПАК «ДЕНЬГИ И ПРАВДА НА ЭКРАНЕ» — ОТЧЁТ (06–07.09, `textmachine-bf`)
|
||||
|
||||
> Промт `docs/PLATFORM_MONEY_TRUTH_SESSION_PROMPT.md`, вход HEAD `e4097cb`, дерево на входе чисто.
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ require (
|
|||
github.com/riverqueue/river v0.42.0
|
||||
github.com/riverqueue/river/riverdriver/riverpgxv5 v0.42.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/time v0.15.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
|
@ -49,7 +50,6 @@ require (
|
|||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
go.uber.org/goleak v1.3.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
|
|
|
|||
|
|
@ -93,7 +93,6 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
|||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import (
|
|||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
|
@ -65,6 +66,10 @@ type Config struct {
|
|||
// which is what every deployment did before the form was ratified: an unprovisioned book then
|
||||
// WAITS rather than being rejected.
|
||||
BookTemplate string
|
||||
// MaxCuts caps how many books this deployment lets the engine cut at once, across the upload that
|
||||
// cuts its own book, the queue's workers and the backstop sweep together. Zero takes
|
||||
// DefaultMaxCuts. See limit.go for why the number is the only memory bound this path has.
|
||||
MaxCuts int
|
||||
// Pairs is what this deployment declares it can translate, from its configuration — the AVAILABLE
|
||||
// half of it. EMPTY refuses every upload: "declares nothing" is not "declares this pair", and the
|
||||
// boot refuses to mount an intake with an empty list at all, so this is the second half of one
|
||||
|
|
@ -86,6 +91,13 @@ type Service struct {
|
|||
// writeBudget shortens what a terminal write gets. Unexported and zero by default: only this
|
||||
// package's own tests set it, to reach the case where the cut outlives that budget.
|
||||
writeBudget time.Duration
|
||||
// uploadSettle shortens the whole tail of an upload (walk). Unexported and zero by default, for
|
||||
// the same reason as writeBudget: reaching the end of a three-and-a-half-minute budget is a test
|
||||
// nobody would run otherwise, and the boot compares the CONSTANT, never this.
|
||||
uploadSettle time.Duration
|
||||
// The host's cap on concurrent engine cuts, built on first use — see limit.go.
|
||||
cutsOnce sync.Once
|
||||
cutSlots *cutSlots
|
||||
}
|
||||
|
||||
func (s *Service) now() time.Time {
|
||||
|
|
@ -216,12 +228,14 @@ func (s *Service) Accept(ctx context.Context, in Intake) (pgstore.Book, error) {
|
|||
s.abandon(c, id, dir)
|
||||
return pgstore.Book{}, err
|
||||
}
|
||||
// On a context that outlives the request: every byte is in, and a client that hung up while
|
||||
// waiting for the 201 must not cost the upload it already finished. The ROW is what makes the
|
||||
// book findable, so it is written even when nobody is left to read the answer. Bounded on its own
|
||||
// (writeCtx): detached is not the same as unlimited, and a hung statement here would hold the
|
||||
// goroutine of a request that is already over.
|
||||
start, cancelStart := s.writeCtx(ctx)
|
||||
// Every byte is in, so the upload's TAIL starts here and everything past this line is spent from
|
||||
// one budget. It outlives the request — the ROW is what makes the book findable, and a client
|
||||
// that hung up while waiting for the 201 must not cost the upload it already finished — and it is
|
||||
// bounded as a whole rather than step by step, which is what keeps its end where the boot was
|
||||
// promised no matter how many steps it grows (walk, step, UploadSettle).
|
||||
walk, cancelWalk := s.walk(ctx)
|
||||
defer cancelWalk()
|
||||
start, cancelStart := s.writeCtx(walk)
|
||||
defer cancelStart()
|
||||
// The job is enqueued here ONLY where no cut runs in this request. Where one does, a job that
|
||||
// exists while the cut is going races it for the parse claim — and both outcomes cost the book:
|
||||
|
|
@ -230,17 +244,17 @@ func (s *Service) Accept(ctx context.Context, in Intake) (pgstore.Book, error) {
|
|||
// needs it (ReleaseParseClaim).
|
||||
//
|
||||
// The price, named: a process that dies between this row and the cut leaves the book `parsing`
|
||||
// with no job, and the backstop sweep takes it after claimGrace rather than at once.
|
||||
enqueue := s.enqueue
|
||||
if s.Engine != nil {
|
||||
enqueue = nil
|
||||
// with no job, and the backstop sweep takes it after ClaimGrace rather than at once.
|
||||
var enqueue func(context.Context, pgstore.Tx, string) error
|
||||
if !s.cutsItsOwnUploads() {
|
||||
enqueue = s.enqueue
|
||||
}
|
||||
book, err := s.Store.StartParsing(start, id, streamRunes, enqueue)
|
||||
if err != nil {
|
||||
// The row is gone or unreachable, and the directory holds a file nothing points at — which is
|
||||
// the one thing the row-first order exists to prevent, so it is undone here too. The ordinary
|
||||
// cause is the sweep having abandoned this upload while it was still arriving.
|
||||
c, cancel := s.writeCtx(ctx)
|
||||
c, cancel := s.writeCtx(walk)
|
||||
defer cancel()
|
||||
s.abandon(c, id, dir)
|
||||
return pgstore.Book{}, err
|
||||
|
|
@ -249,8 +263,8 @@ func (s *Service) Accept(ctx context.Context, in Intake) (pgstore.Book, error) {
|
|||
// book in this file" and the book's size are both knowable before the upload is over (backlog row
|
||||
// 285). Only the engine's own verdict about the SOURCE refuses; a deployment fault falls through
|
||||
// to the asynchronous path this route always had — see cutNow.
|
||||
if cut := s.cutNow(ctx, book); cut.err != nil {
|
||||
c, cancel := s.writeCtx(ctx)
|
||||
if cut := s.cutNow(walk, book); cut.err != nil {
|
||||
c, cancel := s.writeCtx(walk)
|
||||
defer cancel()
|
||||
// Nothing was accepted, so nothing is left behind: no row, no file (row 285 closes row 254).
|
||||
s.discard(c, id, dir, cut.claimedAt)
|
||||
|
|
@ -261,7 +275,7 @@ func (s *Service) Accept(ctx context.Context, in Intake) (pgstore.Book, error) {
|
|||
// On its OWN budget: `start` was opened before the bytes were received and the cut runs inside
|
||||
// this call, so by now that context can be spent — and a re-read on a dead context answers with
|
||||
// the pre-cut row, which is the one thing this line exists to avoid.
|
||||
read, cancelRead := s.writeCtx(ctx)
|
||||
read, cancelRead := s.writeCtx(walk)
|
||||
defer cancelRead()
|
||||
fresh, err := s.Store.ReadBook(read, id)
|
||||
if err == nil {
|
||||
|
|
@ -292,25 +306,58 @@ type cutResult struct {
|
|||
// a user as a verdict about their file.
|
||||
//
|
||||
// Bounded by its own budget, which keeps a large book from holding the request open: past it the
|
||||
// upload is accepted `parsing`. The claim is taken here so this and the queue cannot run the engine
|
||||
// over one project directory at once (see Parse).
|
||||
// upload is accepted `parsing`. That budget is a STEP of the upload's walk, so a cut cannot spend
|
||||
// what the rest of the tail still needs (step). The claim is taken here so this and the queue cannot
|
||||
// run the engine over one project directory at once (see Parse).
|
||||
func (s *Service) cutNow(ctx context.Context, book pgstore.Book) cutResult {
|
||||
if s.Engine == nil {
|
||||
if !s.cutsItsOwnUploads() {
|
||||
return cutResult{}
|
||||
}
|
||||
c, cancel := context.WithTimeout(context.WithoutCancel(ctx), CutBudget)
|
||||
defer cancel()
|
||||
// The claim is a WRITE and gets a write's budget, not the cut's: it is what lets this pass end the
|
||||
// book at all, and tying it to the budget of the work it guards was what made «no room for a cut»
|
||||
// arrive as «the store could not be asked».
|
||||
cl, cancelClaim := s.writeCtx(ctx)
|
||||
defer cancelClaim()
|
||||
now := s.now()
|
||||
claim, err := s.Store.ClaimParse(c, book.ID, now, now.Add(-claimGrace))
|
||||
if err != nil {
|
||||
return cutResult{} // already claimed, or unreachable: not an answer about the file
|
||||
claim, err := s.Store.ClaimParse(cl, book.ID, now, now.Add(-ClaimGrace))
|
||||
// The two ways this does not produce a claim are told apart, because one of them needs a human
|
||||
// and the other is how the walk is supposed to go. Neither is an answer about the FILE, so both
|
||||
// leave the upload to be accepted `parsing`.
|
||||
switch {
|
||||
case errors.Is(err, pgstore.ErrParseClaimed):
|
||||
// Somebody else holds this book: on this route that is the backstop sweep, since no job was
|
||||
// enqueued for a book the intake cuts itself. Whoever holds it finishes the walk.
|
||||
s.log().InfoContext(ctx, "the intake did not get the parse claim; the pass that holds it finishes the book")
|
||||
return cutResult{}
|
||||
case err != nil:
|
||||
// NOT the race above: the claim could not be ASKED for. Said out loud, because the book is now
|
||||
// `parsing` with no claim and no job — this route enqueues none — and nothing comes back for it
|
||||
// until the backstop sweep does, a claim grace later. Silent, this is indistinguishable from
|
||||
// the ordinary line above, which needs no operator at all.
|
||||
s.log().ErrorContext(ctx, "the parse claim could not be taken, so this upload is not cut here; the backstop sweep finishes the book after its grace", "err", err)
|
||||
return cutResult{}
|
||||
}
|
||||
switch err := s.parseClaimed(c, claim, true); {
|
||||
// The cut itself, on what the walk can spare after the writes that must follow it (stepLeaving).
|
||||
// Below zero there is no cut to run, and it is not ATTEMPTED: a pass started on a spent context
|
||||
// fails somewhere inside itself and is diagnosed as whatever failed first, which is never the
|
||||
// truth. Answered as «no verdict», which is what the queue finishing the book already means.
|
||||
c, cancel := s.stepLeaving(ctx, CutBudget, s.cutTailReserve())
|
||||
defer cancel()
|
||||
cut := c.Err()
|
||||
if cut == nil {
|
||||
cut = s.parseClaimed(c, claim, true)
|
||||
} else {
|
||||
s.log().InfoContext(ctx, "what is left of this upload is shorter than a cut plus the writes that follow it; the queue takes the book",
|
||||
"reason", ReasonNoTimeToCut)
|
||||
cut = notConclusive(ReasonNoTimeToCut)
|
||||
}
|
||||
switch err := cut; {
|
||||
case errors.Is(err, ErrBadIntake):
|
||||
return cutResult{err: err, claimedAt: claim.At}
|
||||
case err != nil: // errNotConclusive, or a context this pass could not finish inside
|
||||
// Nothing was established, so the claim goes back at once: the queue job is already enqueued
|
||||
// and a standing claim would make it do nothing (ReleaseParseClaim).
|
||||
// Nothing was established, so the claim goes back at once — and the job that finishes the book
|
||||
// goes in WITH it, in the one transaction: this route enqueued none at StartParsing, precisely
|
||||
// so that no worker could race this cut for the claim (Accept above, and ReleaseParseClaim).
|
||||
s.log().WarnContext(ctx, "the intake's own cut was not conclusive; the queue takes the book", "err", err)
|
||||
w, wcancel := s.writeCtx(ctx)
|
||||
defer wcancel()
|
||||
|
|
@ -405,6 +452,20 @@ func (c *counter) Write(p []byte) (int, error) {
|
|||
return n, err
|
||||
}
|
||||
|
||||
// cutsItsOwnUploads reports whether this deployment answers an upload with a verdict about the file,
|
||||
// or hands the book to the queue and answers `parsing`.
|
||||
//
|
||||
// ONE function because it decides TWO things that must never disagree: whether the row's transaction
|
||||
// carries a job (Accept), and whether a cut runs at all (cutNow). Written twice, the two readings of
|
||||
// one predicate drift, and each way of drifting costs the book — a job enqueued for a cut that then
|
||||
// runs races it for the claim, and a cut skipped where no job was enqueued leaves the book waiting
|
||||
// out the backstop sweep's whole grace.
|
||||
//
|
||||
// In a deployment it is always true: the boot refuses to mount an intake without an engine binary
|
||||
// (config.IntakeEnabled), so the false branch belongs to the development path and to this package's
|
||||
// own tests, where it stands for a deployment that has a queue and no engine.
|
||||
func (s *Service) cutsItsOwnUploads() bool { return s.Engine != nil }
|
||||
|
||||
func (s *Service) enqueue(ctx context.Context, tx pgstore.Tx, bookID string) error {
|
||||
if s.Queue == nil {
|
||||
return nil // no queue configured: the sweep picks the book up on its next pass
|
||||
|
|
|
|||
|
|
@ -500,7 +500,7 @@ func TestASourceTheEngineRefusesIsRejectedAndItsFileRemoved(t *testing.T) {
|
|||
t.Fatalf("the source was deleted on the first refusal: %v", err)
|
||||
}
|
||||
for range parseAttempts {
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -554,7 +554,7 @@ func TestOnlyTheOneRefusalClassAboutTheTextEverCostsTheUpload(t *testing.T) {
|
|||
dir := f.provision(t, book)
|
||||
f.engine.set(ingest.Manifest{}, refusal(t, tc.code))
|
||||
for range parseAttempts + 1 {
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -618,7 +618,7 @@ func TestAnEngineThatCannotBeRunIsRetriedAndThenGivenUpOn(t *testing.T) {
|
|||
if f.engine.called() != before {
|
||||
t.Fatalf("attempt %d was retried before the grace passed", i)
|
||||
}
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
}
|
||||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -650,7 +650,7 @@ func TestABookWithNoEngineConfigurationWaitsRatherThanDies(t *testing.T) {
|
|||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.now = f.now.Add(claimGrace + time.Minute) // the claim spaces the retries
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute) // the claim spaces the retries
|
||||
}
|
||||
if f.engine.called() != 0 {
|
||||
t.Fatalf("the engine was asked %d times about a book it has no configuration for", f.engine.called())
|
||||
|
|
@ -668,7 +668,7 @@ func TestABookWithNoEngineConfigurationWaitsRatherThanDies(t *testing.T) {
|
|||
}
|
||||
// And the moment somebody provisions it, the very next pass parses it.
|
||||
f.provision(t, pgstore.Book{ID: book.ID})
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -953,9 +953,9 @@ func TestTheSweepNeverRemovesTheSourceOfABookItCouldNotDelete(t *testing.T) {
|
|||
// loser dies on the engine's exclusive lock with exit 1 — and exit 1 is how the engine says "I cannot
|
||||
// cut this source".
|
||||
func TestTheClaimGraceOutlivesTheQueuesJobTimeout(t *testing.T) {
|
||||
if claimGrace <= jobs.JobTimeout {
|
||||
t.Fatalf("claimGrace %s does not outlive the queue's job timeout %s: the sweep would steal a claim from a running parse",
|
||||
claimGrace, jobs.JobTimeout)
|
||||
if ClaimGrace <= jobs.JobTimeout {
|
||||
t.Fatalf("ClaimGrace %s does not outlive the queue's job timeout %s: the sweep would steal a claim from a running parse",
|
||||
ClaimGrace, jobs.JobTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1043,7 +1043,7 @@ func TestWaitingForAConfigurationDoesNotBringDeletionCloser(t *testing.T) {
|
|||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
}
|
||||
var attempts int
|
||||
if err := f.store.Pool().QueryRow(f.ctx,
|
||||
|
|
@ -1144,7 +1144,7 @@ func TestAVanishedStorageRootIsNotEveryBooksFault(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
}
|
||||
if f.engine.called() != 0 {
|
||||
t.Fatalf("the engine was asked %d times while the storage root was gone", f.engine.called())
|
||||
|
|
@ -1245,7 +1245,7 @@ func TestAnUnmountedVolumeLooksLikeAnEmptyRootAndStillIsNotTheBooksFault(t *test
|
|||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
}
|
||||
if f.engine.called() != 0 {
|
||||
t.Fatalf("the engine was asked %d times about storage that is not mounted", f.engine.called())
|
||||
|
|
@ -1344,7 +1344,7 @@ func TestTheIntakeRefusesTheDocumentItsOwnMaterialiserWouldReject(t *testing.T)
|
|||
// NON-DESTRUCTIVE, through the whole budget: a document this build cannot read says nothing about
|
||||
// the user's text.
|
||||
for range parseAttempts + 1 {
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1381,7 +1381,7 @@ func TestAManifestShapeThisBuildDoesNotKnowNeverDeletesTheUpload(t *testing.T) {
|
|||
// reader that does not ask which shape it is holding.
|
||||
f.engine.set(ingest.Manifest{Version: "tm-manifest-v3"}, nil)
|
||||
for range parseAttempts + 1 {
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1398,7 +1398,7 @@ func TestAManifestShapeThisBuildDoesNotKnowNeverDeletesTheUpload(t *testing.T) {
|
|||
emptyDir := f.provision(t, empty)
|
||||
f.engine.set(ingest.Manifest{Version: ingest.KnownManifestVersion}, nil)
|
||||
for range parseAttempts + 1 {
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, empty.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1429,7 +1429,7 @@ func TestAManifestThatContradictsItselfNeverCostsTheUpload(t *testing.T) {
|
|||
// No chapters, yet units and chunks were cut: no book produces this and no engine reports it.
|
||||
f.engine.set(ingest.Manifest{Version: "tm-manifest-v9", UnitsTotal: 4402}, nil)
|
||||
for range parseAttempts + 1 {
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1445,7 +1445,7 @@ func TestAManifestThatContradictsItselfNeverCostsTheUpload(t *testing.T) {
|
|||
emptyDir := f.provision(t, empty)
|
||||
f.engine.set(ingest.Manifest{Version: "tm-manifest-v2"}, nil)
|
||||
for range parseAttempts + 1 {
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, empty.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -348,3 +348,58 @@ func remove(t *testing.T, path string) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// The job goes in ONLY where the claim was really given back, and the half that decides is
|
||||
// `RowsAffected` on the release itself (pgstore.ReleaseParseClaim).
|
||||
//
|
||||
// Both halves are one transaction because either alone is a state nobody finishes. The guard is the
|
||||
// side nothing covered: a pass whose claim has been taken over by another has nothing to hand the
|
||||
// queue, and a job enqueued for a book somebody else is already parsing is a worker that wakes up,
|
||||
// finds the claim held and does nothing — while the pass that DOES hold it still owes the book its
|
||||
// finish. Both directions are asserted here, because a case that only ever proves the job is ABSENT
|
||||
// passes just as well when the release stopped enqueueing altogether.
|
||||
func TestOnlyAClaimThatWasReallyGivenBackQueuesTheJobThatFinishesTheBook(t *testing.T) {
|
||||
f := newFixture(t) // no template: the intake's own pass reaches no verdict and releases once
|
||||
queue := &countingQueue{}
|
||||
f.svc.Queue = queue
|
||||
book := f.accept(t, "蛊真人.txt", "первая глава")
|
||||
if queue.n != 1 {
|
||||
t.Fatalf("the intake handed the queue %d jobs, want the one that finishes this book", queue.n)
|
||||
}
|
||||
// Somebody takes the book: from here a release carrying any OTHER stamp is a pass talking about
|
||||
// work it is no longer doing.
|
||||
claim, err := f.store.ClaimParse(f.ctx, book.ID, f.now, f.now.Add(-ClaimGrace))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.store.ReleaseParseClaim(f.ctx, book.ID, claim.At.Add(-time.Second), f.svc.enqueue); err != nil {
|
||||
t.Fatalf("a release carrying a stale stamp failed instead of doing nothing: %v", err)
|
||||
}
|
||||
if queue.n != 1 {
|
||||
t.Errorf("a release that gave back nothing still queued a job (%d in total): the worker it wakes will find the claim held and do nothing, and the pass that holds it is still the one that owes the book",
|
||||
queue.n)
|
||||
}
|
||||
var stamp *time.Time
|
||||
if err := f.store.Pool().QueryRow(f.ctx,
|
||||
`select parse_started_at from books where id = $1`, book.ID).Scan(&stamp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stamp == nil {
|
||||
t.Fatal("a release carrying a stale stamp cleared somebody else's claim")
|
||||
}
|
||||
// And the holder's OWN release does both halves: without this the assertion above would hold on a
|
||||
// release that has stopped queueing anything at all.
|
||||
if err := f.store.ReleaseParseClaim(f.ctx, book.ID, claim.At, f.svc.enqueue); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queue.n != 2 {
|
||||
t.Errorf("the holder gave the claim back and the queue has %d jobs, want a second one: the book has nobody to finish it", queue.n)
|
||||
}
|
||||
if err := f.store.Pool().QueryRow(f.ctx,
|
||||
`select parse_started_at from books where id = $1`, book.ID).Scan(&stamp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stamp != nil {
|
||||
t.Errorf("the holder's own release left the claim standing (%v): the job it queued will do nothing", stamp)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
207
platform/internal/books/limit.go
Normal file
207
platform/internal/books/limit.go
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
package books
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"textmachine/platform/internal/jobs"
|
||||
)
|
||||
|
||||
// DefaultMaxCuts is how many books this platform lets the engine cut AT ONCE — counted across every
|
||||
// way of starting one: the upload that cuts its own book, the queue's workers and the backstop sweep.
|
||||
//
|
||||
// The number is small on purpose, and the reason is what a cut IS on this host. `tmctl manifest` is
|
||||
// spawned as a plain child process (runner.readEngine) — no transient unit, no cgroup, no MemoryMax,
|
||||
// unlike a translation run, which is wrapped in all three.
|
||||
//
|
||||
// ⚠ What this cap bounds, said exactly, because the loose version of it is false: it bounds the CUTS,
|
||||
// on every path that starts one — the upload that cuts its own book, the queue's workers and the
|
||||
// backstop sweep. It is NOT a bound on every engine process the host may hold: the materializer reads
|
||||
// a manifest and an export of its own through the same uncapped `runner.readEngine`
|
||||
// (readmodel.refresh), and the run reconciler reads `status` the same way. Those are bounded by the
|
||||
// queue's worker count and by their sweeps being sequential, which is a different bound and a looser
|
||||
// one. The path that had NO bound at all was the synchronous cut — N uploads were N engine processes
|
||||
// — and that is the one this closes.
|
||||
//
|
||||
// The figure is the queue's own default worker count and is taken from there rather than repeated
|
||||
// here (jobs.DefaultWorkers): the host was already sized for that many engine processes, and this is
|
||||
// the same number said once for every way of starting one.
|
||||
const DefaultMaxCuts = jobs.DefaultWorkers
|
||||
|
||||
// ErrHostAtCutCapacity is a cut that never started because the host is already running as many as it
|
||||
// will run at once.
|
||||
//
|
||||
// Not a fact about the book and not a fault of the deployment, and no path treats it as either: at
|
||||
// intake it leaves the upload to the queue, and on the queue's own path it gives the claim back
|
||||
// without spending an attempt of a budget that exists for a BROKEN host. A host at its cap is a
|
||||
// working host.
|
||||
var ErrHostAtCutCapacity = errors.New("books: the host is already cutting as many books as it will cut at once")
|
||||
|
||||
// ErrNoTimeToCut is a cut that was not started because what is left of the caller's budget is less
|
||||
// than a cut needs.
|
||||
//
|
||||
// The SAME class as the cap and treated identically — the engine was never asked, so this pass knows
|
||||
// nothing about the book and records nothing — and it exists because waiting for a slot spends the
|
||||
// caller's budget. Without it a pass could win a slot with seconds left, hand the engine those
|
||||
// seconds, and have the killed process read back as `parser_unavailable`: a verdict about the
|
||||
// DEPLOYMENT, which spends an attempt of a budget that five times over deletes the user's file.
|
||||
var ErrNoTimeToCut = errors.New("books: what is left of this pass is shorter than a cut")
|
||||
|
||||
// cutSlots is the cap itself, plus what an operator has to be able to see of it: a cap nobody can
|
||||
// watch is indistinguishable from latency somebody has to guess at.
|
||||
type cutSlots struct {
|
||||
sem *semaphore.Weighted
|
||||
limit int64
|
||||
// inFlight and waiting are the state RIGHT NOW — how saturated the host is, and how deep the line
|
||||
// for it is. Gauges, because the question they answer stops being true the moment it changes.
|
||||
inFlight atomic.Int64
|
||||
waiting atomic.Int64
|
||||
// waited and gaveUp are cumulative, because their question is the opposite one: how often has this
|
||||
// cap been reached at all, and how often did reaching it cost a cut. A gauge would answer it only
|
||||
// for whoever happened to be looking.
|
||||
waited atomic.Uint64
|
||||
gaveUp atomic.Uint64
|
||||
}
|
||||
|
||||
// cuts builds the cap on first use.
|
||||
//
|
||||
// Lazily, because this service is assembled as a struct literal by its deployment (cmd/tmplatformd)
|
||||
// and by every test that exercises intake, and a cap that only exists when a constructor was called
|
||||
// is a cap absent from exactly the paths nobody remembered to route through one.
|
||||
func (s *Service) cuts() *cutSlots {
|
||||
s.cutsOnce.Do(func() {
|
||||
limit := int64(s.Cfg.MaxCuts)
|
||||
if limit <= 0 {
|
||||
limit = DefaultMaxCuts
|
||||
}
|
||||
s.cutSlots = &cutSlots{sem: semaphore.NewWeighted(limit), limit: limit}
|
||||
})
|
||||
return s.cutSlots
|
||||
}
|
||||
|
||||
// takeCutSlot holds one of the host's cut slots for the caller and returns what gives it back.
|
||||
//
|
||||
// It WAITS rather than refusing, and the caller's own context is what bounds the wait — the upload's
|
||||
// cut budget, the queue job's timeout, the sweep's per-book slice. Waiting is right here because
|
||||
// every one of those already has a milder answer than a refusal for running out: the upload is
|
||||
// accepted `parsing` and the queue finishes it; a queued pass gives the book straight back. A
|
||||
// refusal at the door would turn a host that is merely busy into an upload the user has to do again.
|
||||
//
|
||||
// It cannot lengthen the walk it is called inside, either — that is not a promise about this code
|
||||
// but a property of the context it takes: the cut runs on a step of the upload's walk (books.step),
|
||||
// so time spent here is time NOT spent on the engine, never time added to the tail.
|
||||
//
|
||||
// ⛔ `reserve` is what a WON slot must still be worth. Waiting spends the caller's budget, so a slot
|
||||
// won at the very end of it buys a cut the engine has no time to finish — and a killed engine reads
|
||||
// back as a fault of the DEPLOYMENT, which spends an attempt of the budget that deletes a user's
|
||||
// file after five. Zero means the caller has nothing at stake in losing (the intake spends no
|
||||
// attempts), and then waiting to the very end is free.
|
||||
func (s *Service) takeCutSlot(ctx context.Context, reserve time.Duration) (func(), error) {
|
||||
c := s.cuts()
|
||||
release := func() {
|
||||
c.inFlight.Add(-1)
|
||||
c.sem.Release(1)
|
||||
}
|
||||
if c.sem.TryAcquire(1) {
|
||||
c.inFlight.Add(1)
|
||||
return release, nil
|
||||
}
|
||||
if err := worthStarting(ctx, reserve); err != nil {
|
||||
// Every slot is taken and there is not enough left to make winning one worth it. Counted as a
|
||||
// give-up, because from the operator's side it is the cap that cost this cut.
|
||||
c.gaveUp.Add(1)
|
||||
return nil, err
|
||||
}
|
||||
c.waited.Add(1)
|
||||
c.waiting.Add(1)
|
||||
started := time.Now()
|
||||
wait, stopWaiting := waitCtx(ctx, reserve)
|
||||
err := c.sem.Acquire(wait, 1)
|
||||
stopWaiting()
|
||||
c.waiting.Add(-1)
|
||||
if err != nil {
|
||||
c.gaveUp.Add(1)
|
||||
return nil, fmt.Errorf("%w: waited %s for one of %d slots: %w",
|
||||
ErrHostAtCutCapacity, time.Since(started).Round(time.Millisecond), c.limit, err)
|
||||
}
|
||||
// Won — but the wait spent time, so the question of whether it is still worth cutting is asked
|
||||
// AGAIN. A slot handed back unused is a slot the next caller gets.
|
||||
if err := worthStarting(ctx, reserve); err != nil {
|
||||
c.sem.Release(1)
|
||||
c.gaveUp.Add(1)
|
||||
return nil, err
|
||||
}
|
||||
c.inFlight.Add(1)
|
||||
// The fact and the wait, at INFO: this is the host doing what it was configured to do, and an
|
||||
// operator reading it learns the cap is the thing shaping their latency. What it must NOT do is
|
||||
// carry the book — a cut waits because of the HOST, and the book it happens to be for is no more
|
||||
// at fault than any other (ENGINEERING_STANDARDS §Наблюдаемость).
|
||||
s.log().InfoContext(ctx, "a cut waited for one of the host's cut slots",
|
||||
"waited_seconds", time.Since(started).Seconds(), "slots", c.limit)
|
||||
return release, nil
|
||||
}
|
||||
|
||||
// CutCapacity is one reading of the cap, for the telemetry pass that publishes it.
|
||||
//
|
||||
// Read from the service rather than collected on scrape, which is the same rule the rest of this
|
||||
// deployment's gauges follow: a scrape must not be able to set the load on anything.
|
||||
type CutCapacity struct {
|
||||
Limit int
|
||||
InFlight int
|
||||
Waiting int
|
||||
// Waited and GaveUp are cumulative counts of cuts that had to wait at all, and of cuts whose
|
||||
// caller ran out of budget while waiting.
|
||||
Waited uint64
|
||||
GaveUp uint64
|
||||
}
|
||||
|
||||
// CutCapacity reports where the host's cut capacity stands.
|
||||
func (s *Service) CutCapacity() CutCapacity {
|
||||
c := s.cuts()
|
||||
return CutCapacity{
|
||||
Limit: int(c.limit),
|
||||
InFlight: int(c.inFlight.Load()),
|
||||
Waiting: int(c.waiting.Load()),
|
||||
Waited: c.waited.Load(),
|
||||
GaveUp: c.gaveUp.Load(),
|
||||
}
|
||||
}
|
||||
|
||||
// waitCtx bounds a wait so that what is left when it ends is still worth a cut. With no reserve, or
|
||||
// with no deadline to take it out of, the caller's own context is the bound.
|
||||
func waitCtx(ctx context.Context, reserve time.Duration) (context.Context, context.CancelFunc) {
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || reserve <= 0 {
|
||||
return ctx, func() {}
|
||||
}
|
||||
// The parent stays the parent, so its cancellation still ends the wait; only the deadline is
|
||||
// pulled in by the reserve.
|
||||
return context.WithDeadline(ctx, deadline.Add(-reserve))
|
||||
}
|
||||
|
||||
// worthStarting reports whether a cut started now would have the time a cut needs.
|
||||
func worthStarting(ctx context.Context, reserve time.Duration) error {
|
||||
if reserve <= 0 {
|
||||
return nil
|
||||
}
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if left := time.Until(deadline); left < reserve {
|
||||
return fmt.Errorf("%w: %s left, a cut is given %s",
|
||||
ErrNoTimeToCut, left.Round(time.Millisecond), reserve)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// engineNotAsked reports whether an error means the engine was never asked at all — the host was at
|
||||
// its cap, or what was left of the pass was shorter than a cut. Neither says anything about the book.
|
||||
func engineNotAsked(err error) bool {
|
||||
return errors.Is(err, ErrHostAtCutCapacity) || errors.Is(err, ErrNoTimeToCut)
|
||||
}
|
||||
562
platform/internal/books/limit_test.go
Normal file
562
platform/internal/books/limit_test.go
Normal file
|
|
@ -0,0 +1,562 @@
|
|||
package books
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"textmachine/platform/internal/jobs"
|
||||
)
|
||||
|
||||
// waitUntil polls a condition and fails with what it last saw, so a test that never contended reads
|
||||
// as a test that never contended rather than as a passing one.
|
||||
func waitUntil(t *testing.T, what string, saw func() string, ok func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatalf("%s did not happen within 10s; last reading: %s", what, saw())
|
||||
}
|
||||
|
||||
// upload runs one intake and reports what it answered.
|
||||
func (f *fixture) uploadAsync(name string) <-chan error {
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||||
Filename: name, File: strings.NewReader("первая глава\fвторая глава")})
|
||||
done <- err
|
||||
}()
|
||||
return done
|
||||
}
|
||||
|
||||
// The host never runs more cuts at once than its cap allows, and this is asserted under LOAD rather
|
||||
// than read off the code: six uploads arrive together at a cap of two.
|
||||
//
|
||||
// ⚠ The control value is what makes the assertion mean anything. "The peak never exceeded two" is
|
||||
// also true of a fixture where two uploads never managed to overlap at all, so the test first waits
|
||||
// until FOUR cuts are provably queued for a slot — that reading comes from the cap's own counters,
|
||||
// which is the number an operator reads too.
|
||||
func TestTheHostRunsNoMoreCutsAtOnceThanItsCapAllows(t *testing.T) {
|
||||
const slots, uploads = 2, 6
|
||||
f := newFixture(t)
|
||||
templated(t, f)
|
||||
f.svc.Cfg.MaxCuts = slots
|
||||
|
||||
var mu sync.Mutex
|
||||
inFlight, peak := 0, 0
|
||||
hold := make(chan struct{})
|
||||
f.engine.onManifest = func(context.Context) {
|
||||
mu.Lock()
|
||||
inFlight++
|
||||
if inFlight > peak {
|
||||
peak = inFlight
|
||||
}
|
||||
mu.Unlock()
|
||||
<-hold
|
||||
mu.Lock()
|
||||
inFlight--
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
done := make([]<-chan error, uploads)
|
||||
for i := range done {
|
||||
done[i] = f.uploadAsync("book.txt")
|
||||
}
|
||||
waitUntil(t, "four of six uploads queued for one of two cut slots",
|
||||
func() string { return fmt.Sprintf("%+v", f.svc.CutCapacity()) },
|
||||
func() bool { return f.svc.CutCapacity().Waiting == uploads-slots })
|
||||
// Only now: with the line proven, whatever the peak turns out to be is a fact about the cap.
|
||||
close(hold)
|
||||
for _, ch := range done {
|
||||
if err := <-ch; err != nil {
|
||||
t.Errorf("an upload under the cap failed: %v", err)
|
||||
}
|
||||
}
|
||||
if peak != slots {
|
||||
t.Errorf("%d cuts ran at once at a cap of %d", peak, slots)
|
||||
}
|
||||
if c := f.svc.CutCapacity(); c.Waited < uploads-slots {
|
||||
t.Errorf("the cap counted %d waits for %d uploads over %d slots: an operator would not see the saturation",
|
||||
c.Waited, uploads, slots)
|
||||
}
|
||||
if c := f.svc.CutCapacity(); c.InFlight != 0 || c.Waiting != 0 {
|
||||
t.Errorf("after every upload finished the cap still reads in-flight %d waiting %d: a slot was not given back",
|
||||
c.InFlight, c.Waiting)
|
||||
}
|
||||
}
|
||||
|
||||
// The same load with room for all of it reaches all of it at once.
|
||||
//
|
||||
// Without this the test above would also pass on a fixture that never got two cuts to overlap — the
|
||||
// vacuous shape that made a whole class of this zone's assertions invisible (D39.208 §5). Here the
|
||||
// cap is the ONLY thing changed between the two, so the difference in the peak is the cap's doing.
|
||||
func TestWithRoomForEveryCutTheHostRunsThemAllAtOnce(t *testing.T) {
|
||||
const uploads = 6
|
||||
f := newFixture(t)
|
||||
templated(t, f)
|
||||
f.svc.Cfg.MaxCuts = uploads
|
||||
|
||||
var mu sync.Mutex
|
||||
inFlight, peak := 0, 0
|
||||
hold := make(chan struct{})
|
||||
f.engine.onManifest = func(context.Context) {
|
||||
mu.Lock()
|
||||
inFlight++
|
||||
if inFlight > peak {
|
||||
peak = inFlight
|
||||
}
|
||||
mu.Unlock()
|
||||
<-hold
|
||||
mu.Lock()
|
||||
inFlight--
|
||||
mu.Unlock()
|
||||
}
|
||||
done := make([]<-chan error, uploads)
|
||||
for i := range done {
|
||||
done[i] = f.uploadAsync("book.txt")
|
||||
}
|
||||
waitUntil(t, "all six uploads cutting at once",
|
||||
func() string { return fmt.Sprintf("%+v", f.svc.CutCapacity()) },
|
||||
func() bool { return f.svc.CutCapacity().InFlight == uploads })
|
||||
close(hold)
|
||||
for _, ch := range done {
|
||||
<-ch
|
||||
}
|
||||
if peak != uploads {
|
||||
t.Errorf("only %d of %d cuts ran at once with a slot for each: the load never contended and the cap's test proves nothing",
|
||||
peak, uploads)
|
||||
}
|
||||
}
|
||||
|
||||
// An upload that runs out of budget waiting for a slot is ACCEPTED `parsing` and finished by the
|
||||
// queue — never refused. A busy host must not cost a user the upload they already made.
|
||||
//
|
||||
// This is also the fixture where the cap's log line MUST sound: a message pinned only by its silence
|
||||
// says nothing about the case where it is wrong.
|
||||
func TestAnUploadThatRunsOutOfBudgetWaitingForASlotIsAcceptedRatherThanRefused(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
templated(t, f)
|
||||
f.svc.Cfg.MaxCuts = 1
|
||||
// Short enough that the second upload's whole walk expires while the first holds the only slot,
|
||||
// and different from every other budget in this fixture so the two cannot be confused.
|
||||
// THREE distinct numbers, and the third one is load-bearing rather than tidy: the reserve a cut
|
||||
// must leave for the writes after it is three write budgets, so a fixture that shortens only the
|
||||
// walk leaves no room for a cut at all and models nothing (see stepLeaving).
|
||||
f.svc.uploadSettle = 300 * time.Millisecond
|
||||
f.svc.writeBudget = 20 * time.Millisecond
|
||||
var buf bytes.Buffer
|
||||
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
|
||||
hold := make(chan struct{})
|
||||
first := make(chan struct{})
|
||||
var once sync.Once
|
||||
f.engine.onManifest = func(context.Context) {
|
||||
once.Do(func() { close(first) })
|
||||
<-hold
|
||||
}
|
||||
held := f.uploadAsync("held.txt")
|
||||
<-first
|
||||
waitUntil(t, "the only slot taken",
|
||||
func() string { return fmt.Sprintf("%+v", f.svc.CutCapacity()) },
|
||||
func() bool { return f.svc.CutCapacity().InFlight == 1 })
|
||||
|
||||
book, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||||
Filename: "waited.txt", File: strings.NewReader("первая глава\fвторая глава")})
|
||||
if err != nil {
|
||||
t.Fatalf("an upload that met a busy host was refused: %v", err)
|
||||
}
|
||||
if book.Status != "parsing" {
|
||||
t.Errorf("an upload that met a busy host is %q, want `parsing` with the queue to finish it", book.Status)
|
||||
}
|
||||
if c := f.svc.CutCapacity(); c.GaveUp != 1 {
|
||||
t.Errorf("the cap counted %d cuts that ran out of budget waiting, want 1", c.GaveUp)
|
||||
}
|
||||
if got := buf.String(); !strings.Contains(got, "the engine was not asked") || !strings.Contains(got, ReasonHostAtCapacity) {
|
||||
t.Errorf("nothing in the log says the engine was not asked and why (%q), so an operator sees latency and no cause; log: %s",
|
||||
ReasonHostAtCapacity, got)
|
||||
}
|
||||
close(hold)
|
||||
<-held
|
||||
|
||||
// And the same message is SILENT on a host with room: a line that is always there is a line that
|
||||
// tells an operator nothing.
|
||||
f.engine.onManifest = nil
|
||||
buf.Reset()
|
||||
f.svc.uploadSettle = 0
|
||||
if _, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||||
Filename: "free.txt", File: strings.NewReader("первая глава\fвторая глава")}); err != nil {
|
||||
t.Fatalf("an upload on an idle host failed: %v", err)
|
||||
}
|
||||
if got := buf.String(); strings.Contains(got, ReasonHostAtCapacity) || strings.Contains(got, "waited for one of the host's cut slots") {
|
||||
t.Errorf("an idle host still logged about its cap: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A queued pass that cannot cut gives the book back and spends NOTHING, and the two reasons it can
|
||||
// have are told apart: the host is at its cap, or what is left of the pass is shorter than a cut.
|
||||
//
|
||||
// Both matter, and the second is what makes the first safe. Waiting for a slot spends the caller's
|
||||
// budget, so without a reserve a pass could win a slot with seconds left, hand the engine those
|
||||
// seconds, and have the killed process read back as a fault of the DEPLOYMENT — which DOES spend an
|
||||
// attempt, and five of those delete the user's file.
|
||||
func TestAQueuedParseThatCannotCutSpendsNothingAndGivesTheBookBack(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.svc.Cfg.MaxCuts = 1
|
||||
// Two books that reached `parsing` without any engine call: with no template configured the
|
||||
// intake's own pass stops before the engine is asked, which is the state the queue picks up.
|
||||
held := f.accept(t, "held.txt", "первая глава\fвторая глава")
|
||||
waiting := f.accept(t, "waiting.txt", "первая глава\fвторая глава")
|
||||
f.provision(t, held)
|
||||
f.provision(t, waiting)
|
||||
|
||||
hold, running := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
f.engine.onManifest = func(context.Context) {
|
||||
once.Do(func() { close(running) })
|
||||
<-hold
|
||||
}
|
||||
parsed := make(chan error, 1)
|
||||
go func() { parsed <- f.svc.Parse(f.ctx, held.ID) }()
|
||||
<-running
|
||||
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
budget time.Duration
|
||||
want error
|
||||
}{
|
||||
// Enough left that a cut would be worth starting, so this pass WAITS — and the wait is bounded
|
||||
// by what it must leave the engine, which is why it ends in about a second rather than in the
|
||||
// whole budget.
|
||||
{"the host is at its cap", CutBudget + 1500*time.Millisecond, ErrHostAtCutCapacity},
|
||||
// Less left than a cut needs, so no slot is even waited for.
|
||||
{"there is no time left for a cut", 300 * time.Millisecond, ErrNoTimeToCut},
|
||||
} {
|
||||
before, _ := f.parseState(t, waiting.ID)
|
||||
ctx, cancel := context.WithTimeout(f.ctx, c.budget)
|
||||
err := f.svc.Parse(ctx, waiting.ID)
|
||||
cancel()
|
||||
if !errors.Is(err, c.want) {
|
||||
t.Fatalf("%s: the pass answered %v, want %v", c.what, err, c.want)
|
||||
}
|
||||
if !errors.Is(err, jobs.ErrTryAgainLater) {
|
||||
t.Errorf("%s: the error does not tell the queue to bring the job back, so the single attempt is spent and the book waits out the sweep's grace", c.what)
|
||||
}
|
||||
after, claimed := f.parseState(t, waiting.ID)
|
||||
if after != before {
|
||||
t.Errorf("%s: a pass that never asked the engine spent the book's budget: attempts %d → %d", c.what, before, after)
|
||||
}
|
||||
if claimed {
|
||||
t.Errorf("%s: a pass that established nothing kept the claim, so the next pass waits out the whole grace for it", c.what)
|
||||
}
|
||||
}
|
||||
close(hold)
|
||||
if err := <-parsed; err != nil {
|
||||
t.Errorf("the parse that held the slot failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// parseState reads how much of a book's attempt budget is spent and whether a claim stands on it.
|
||||
func (f *fixture) parseState(t *testing.T, id string) (attempts int, claimed bool) {
|
||||
t.Helper()
|
||||
var stamp *time.Time
|
||||
if err := f.store.Pool().QueryRow(f.ctx,
|
||||
`select parse_attempts, parse_started_at from books where id = $1`, id).Scan(&attempts, &stamp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return attempts, stamp != nil
|
||||
}
|
||||
|
||||
// The two ways an intake gets no parse claim are told apart in the log, because one of them needs a
|
||||
// human and the other is how the walk is meant to go.
|
||||
//
|
||||
// Both messages are asserted in a fixture where each MUST sound, and each is asserted ABSENT from the
|
||||
// other's: a line pinned only by its silence says nothing about the case where it is wrong, and these
|
||||
// two are one `if` apart — the shape that made this branch silent about a database it could not reach
|
||||
// for a whole claim grace.
|
||||
func TestTheIntakeTellsALostRaceApartFromAStoreItCouldNotAsk(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
var buf bytes.Buffer
|
||||
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
// No template, so the intake's own pass reaches no verdict: the book is left `parsing` with its
|
||||
// claim given back, which is the state another pass can take it in.
|
||||
book := f.accept(t, "蛊真人.txt", "первая глава")
|
||||
|
||||
// (1) Somebody else holds the claim. The ordinary race, and the only other claimant an upload can
|
||||
// meet on this route: no job was enqueued for a book the intake cuts itself.
|
||||
if _, err := f.store.ClaimParse(f.ctx, book.ID, f.now, f.now.Add(-ClaimGrace)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf.Reset()
|
||||
f.svc.Cfg.BookTemplate = "" // still no verdict to reach; what is asserted is which line is written
|
||||
walk, cancel := f.svc.walk(f.ctx)
|
||||
defer cancel()
|
||||
if got := f.svc.cutNow(walk, book); got.err != nil {
|
||||
t.Fatalf("losing the race was answered as a verdict about the file: %v", got.err)
|
||||
}
|
||||
lost := buf.String()
|
||||
if !strings.Contains(lost, "did not get the parse claim") {
|
||||
t.Errorf("losing the race said nothing: %s", lost)
|
||||
}
|
||||
if strings.Contains(lost, "could not be taken") {
|
||||
t.Errorf("an ordinary race was reported as a store that could not be asked: %s", lost)
|
||||
}
|
||||
if strings.Contains(lost, `"level":"ERROR"`) {
|
||||
t.Errorf("an ordinary race was logged at ERROR, which is a page for an operator who has nothing to do: %s", lost)
|
||||
}
|
||||
|
||||
// (2) The store could not be ASKED at all — here because the walk this cut belongs to is already
|
||||
// over, which is what a request whose whole tail is spent looks like from inside cutNow.
|
||||
buf.Reset()
|
||||
f.svc.uploadSettle = time.Nanosecond
|
||||
spent, cancelSpent := f.svc.walk(f.ctx)
|
||||
defer cancelSpent()
|
||||
<-spent.Done()
|
||||
if got := f.svc.cutNow(spent, book); got.err != nil {
|
||||
t.Fatalf("a store that could not be asked was answered as a verdict about the file: %v", got.err)
|
||||
}
|
||||
unreachable := buf.String()
|
||||
if !strings.Contains(unreachable, "could not be taken") {
|
||||
t.Errorf("a claim that could not be asked for said nothing, so the book sits `parsing` with no job and nobody knows: %s", unreachable)
|
||||
}
|
||||
if !strings.Contains(unreachable, `"level":"ERROR"`) {
|
||||
t.Errorf("a store that could not be asked was not logged at ERROR: %s", unreachable)
|
||||
}
|
||||
if strings.Contains(unreachable, "did not get the parse claim") {
|
||||
t.Errorf("a store that could not be asked was reported as an ordinary race, which needs no operator: %s", unreachable)
|
||||
}
|
||||
}
|
||||
|
||||
// A cut that WAITS and then gets a slot says so, and a cut that never waited does not.
|
||||
//
|
||||
// The line was asserted only by its absence before, which is the vacuous half: a message pinned by
|
||||
// silence alone is a message nothing defends where it matters. Here the waiter actually wins its
|
||||
// slot — the holder lets go — so the line is one the fixture FORCES.
|
||||
func TestACutThatWaitedAndThenGotASlotSaysSo(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
templated(t, f)
|
||||
f.svc.Cfg.MaxCuts = 1
|
||||
var buf bytes.Buffer
|
||||
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
|
||||
hold, running := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
f.engine.onManifest = func(context.Context) {
|
||||
once.Do(func() { close(running) })
|
||||
<-hold
|
||||
}
|
||||
first := f.uploadAsync("held.txt")
|
||||
<-running
|
||||
second := f.uploadAsync("waited.txt")
|
||||
waitUntil(t, "the second upload queued for the only slot",
|
||||
func() string { return fmt.Sprintf("%+v", f.svc.CutCapacity()) },
|
||||
func() bool { return f.svc.CutCapacity().Waiting == 1 })
|
||||
// The holder lets go, so the waiter WINS rather than times out — the case the earlier fixture
|
||||
// never reached, because there the waiter always ran out of budget.
|
||||
close(hold)
|
||||
if err := <-first; err != nil {
|
||||
t.Fatalf("the upload that held the slot failed: %v", err)
|
||||
}
|
||||
if err := <-second; err != nil {
|
||||
t.Fatalf("the upload that waited for a slot failed: %v", err)
|
||||
}
|
||||
if got := buf.String(); !strings.Contains(got, "a cut waited for one of the host's cut slots") {
|
||||
t.Errorf("a cut that queued for a slot and got one said nothing, so an operator reading latency has no cause to read: %s", got)
|
||||
}
|
||||
|
||||
// And an idle host does not say it: a line that is always there tells nobody anything.
|
||||
buf.Reset()
|
||||
f.engine.onManifest = nil
|
||||
if _, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||||
Filename: "free.txt", File: strings.NewReader("первая глава\fвторая глава")}); err != nil {
|
||||
t.Fatalf("an upload on an idle host failed: %v", err)
|
||||
}
|
||||
if got := buf.String(); strings.Contains(got, "waited for one of the host's cut slots") {
|
||||
t.Errorf("an upload that never queued still reported a wait: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A deployment with no engine hands the book to the queue in the row's OWN transaction, and runs no
|
||||
// cut. This is the false branch of cutsItsOwnUploads, which nothing exercised: with every fixture
|
||||
// carrying an engine, the predicate could be replaced by `true` and the whole battery stayed green.
|
||||
func TestADeploymentWithNoEngineQueuesTheBookWithTheRowAndCutsNothing(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
templated(t, f)
|
||||
queue := &countingQueue{}
|
||||
f.svc.Queue = queue
|
||||
f.svc.Engine = nil
|
||||
var buf bytes.Buffer
|
||||
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
|
||||
book := f.accept(t, "蛊真人.txt", "первая глава\fвторая глава")
|
||||
if book.Status != "parsing" {
|
||||
t.Errorf("a deployment that cannot cut answered %q, want `parsing` for the queue to finish", book.Status)
|
||||
}
|
||||
if queue.n != 1 {
|
||||
t.Errorf("the queue was handed %d jobs, want the one that goes in with the row: nothing else enqueues on this path", queue.n)
|
||||
}
|
||||
// ⚠ Counting jobs CANNOT tell the two apart, and that is why the log is read instead. With the
|
||||
// predicate broken to «this deployment always cuts», the cut is attempted, stops at «no engine is
|
||||
// configured», reaches no verdict — and its release enqueues the very same one job. The observable
|
||||
// difference is that a cut was ATTEMPTED at all.
|
||||
if got := buf.String(); strings.Contains(got, "the intake's own cut was not conclusive") {
|
||||
t.Errorf("a deployment with no engine attempted a cut anyway: %s", got)
|
||||
}
|
||||
if _, claimed := f.parseState(t, book.ID); claimed {
|
||||
t.Error("a book nobody cut carries a parse claim")
|
||||
}
|
||||
|
||||
// The positive control, in the same fixture: an engine that reaches no verdict DOES take the path
|
||||
// above and DOES say so. Without it the assertion is satisfied by a log that never says anything.
|
||||
buf.Reset()
|
||||
f.svc.Engine = f.engine
|
||||
f.svc.Cfg.BookTemplate = "" // no configuration to cut against: the pass reaches no verdict
|
||||
second := f.accept(t, "второй.txt", "первая глава\fвторая глава")
|
||||
if got := buf.String(); !strings.Contains(got, "the intake's own cut was not conclusive") {
|
||||
t.Errorf("a deployment WITH an engine did not report its inconclusive cut, so the assertion above proves nothing: %s", got)
|
||||
}
|
||||
if queue.n != 2 {
|
||||
t.Errorf("the second book got %d jobs in total, want one of its own", queue.n)
|
||||
}
|
||||
_ = second
|
||||
}
|
||||
|
||||
// The reason an unfinished intake carries is the one an operator greps for. Its VALUE is asserted,
|
||||
// not just its presence: the constant could be renamed to anything and every other test stayed green.
|
||||
func TestAnIntakeStoppedByTheCapNamesTheReasonInWordsAnOperatorCanFind(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
templated(t, f)
|
||||
f.svc.Cfg.MaxCuts = 1
|
||||
f.svc.uploadSettle = 300 * time.Millisecond
|
||||
f.svc.writeBudget = 20 * time.Millisecond // the reserve follows it — see the fixture above
|
||||
var buf bytes.Buffer
|
||||
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
|
||||
hold, running := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
f.engine.onManifest = func(context.Context) {
|
||||
once.Do(func() { close(running) })
|
||||
<-hold
|
||||
}
|
||||
held := f.uploadAsync("held.txt")
|
||||
<-running
|
||||
if _, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||||
Filename: "waited.txt", File: strings.NewReader("первая глава\fвторая глава")}); err != nil {
|
||||
t.Fatalf("an upload that met a busy host was refused: %v", err)
|
||||
}
|
||||
// The LITERAL and not the constant. Asserting `strings.Contains(log, ReasonHostAtCapacity)` renames
|
||||
// both sides at once and passes at any value — the tautology this assertion replaced, and the
|
||||
// mutation that survived it. What is being defended is a word an operator's runbook and grep can
|
||||
// hold still, so the word is written out here.
|
||||
const word = "host_at_cut_capacity"
|
||||
if ReasonHostAtCapacity != word {
|
||||
t.Errorf("the reason is %q, want the stable %q that operators and the runbook grep for", ReasonHostAtCapacity, word)
|
||||
}
|
||||
if got := buf.String(); !strings.Contains(got, word) {
|
||||
t.Errorf("the pass that did not cut carries no %q anywhere an operator would grep: %s", word, got)
|
||||
}
|
||||
close(hold)
|
||||
<-held
|
||||
}
|
||||
|
||||
// An upload the host had no room to cut still leaves the book with somebody to finish it: the claim
|
||||
// goes back and the job goes in, in the one transaction.
|
||||
//
|
||||
// ⛔ This is what the reserve buys, and it is the assertion the first edition of these tests left out
|
||||
// — it checked only that the upload was ACCEPTED, which is true either way. Without the reserve the
|
||||
// release is the step the spent walk truncates, and then the book sits `parsing`, claimed, with no
|
||||
// job: nothing comes back for it until the backstop sweep's grace runs out, twenty minutes later,
|
||||
// and every surface says the upload went fine.
|
||||
func TestAnUploadTheHostCouldNotCutStillLeavesSomebodyToFinishTheBook(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
templated(t, f)
|
||||
queue := &countingQueue{}
|
||||
f.svc.Queue = queue
|
||||
f.svc.Cfg.MaxCuts = 1
|
||||
f.svc.uploadSettle = 300 * time.Millisecond
|
||||
f.svc.writeBudget = 20 * time.Millisecond
|
||||
|
||||
hold, running := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
f.engine.onManifest = func(context.Context) {
|
||||
once.Do(func() { close(running) })
|
||||
<-hold
|
||||
}
|
||||
held := f.uploadAsync("held.txt")
|
||||
<-running
|
||||
|
||||
book, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||||
Filename: "waited.txt", File: strings.NewReader("первая глава\fвторая глава")})
|
||||
if err != nil {
|
||||
t.Fatalf("an upload that met a busy host was refused: %v", err)
|
||||
}
|
||||
if book.Status != "parsing" {
|
||||
t.Fatalf("the upload is %q, want `parsing`", book.Status)
|
||||
}
|
||||
attempts, claimed := f.parseState(t, book.ID)
|
||||
if claimed {
|
||||
t.Error("the intake's parse claim was NOT given back: the book sits `parsing`, claimed, with nobody to finish it until the claim grace runs out")
|
||||
}
|
||||
if queue.n != 1 {
|
||||
t.Errorf("the queue was handed %d jobs for a book the intake could not cut, want the one that finishes it", queue.n)
|
||||
}
|
||||
if attempts != 0 {
|
||||
t.Errorf("a book the engine was never asked about has spent %d attempts of its budget", attempts)
|
||||
}
|
||||
close(hold)
|
||||
<-held
|
||||
}
|
||||
|
||||
// An upload whose walk cannot hold a cut AT ALL says so in those words, and still leaves the book to
|
||||
// the queue.
|
||||
//
|
||||
// The branch is reachable only when what remains of the walk is shorter than the writes that must
|
||||
// follow a cut, which no other fixture produces — and it is the branch that used to arrive as «the
|
||||
// parse claim could not be taken», a diagnosis pointing at the store for a walk that had simply run
|
||||
// out. A message that names the wrong thing is worse than none: it sends whoever reads it to the
|
||||
// database.
|
||||
func TestAnUploadWithNoRoomLeftForACutSaysThatAndHandsTheBookOver(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
templated(t, f)
|
||||
queue := &countingQueue{}
|
||||
f.svc.Queue = queue
|
||||
// The walk is SHORTER than the reserve three writes need, so no cut can be started at all.
|
||||
f.svc.writeBudget = 25 * time.Millisecond
|
||||
f.svc.uploadSettle = 60 * time.Millisecond
|
||||
var buf bytes.Buffer
|
||||
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
|
||||
book, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||||
Filename: "late.txt", File: strings.NewReader("первая глава\fвторая глава")})
|
||||
if err != nil {
|
||||
t.Fatalf("an upload with no room left for a cut was refused: %v", err)
|
||||
}
|
||||
if book.Status != "parsing" {
|
||||
t.Errorf("the upload is %q, want `parsing` for the queue to finish", book.Status)
|
||||
}
|
||||
if f.engine.called() != 0 {
|
||||
t.Errorf("a cut was started with no room for it (%d calls)", f.engine.called())
|
||||
}
|
||||
got := buf.String()
|
||||
if !strings.Contains(got, ReasonNoTimeToCut) {
|
||||
t.Errorf("the pass does not say WHY it did not cut (%q): %s", ReasonNoTimeToCut, got)
|
||||
}
|
||||
if strings.Contains(got, "the parse claim could not be taken") {
|
||||
t.Errorf("a walk that ran out was reported as a store that could not be asked, which sends an operator to the database: %s", got)
|
||||
}
|
||||
if queue.n != 1 {
|
||||
t.Errorf("the queue was handed %d jobs, want the one that finishes a book nobody cut", queue.n)
|
||||
}
|
||||
if _, claimed := f.parseState(t, book.ID); claimed {
|
||||
t.Error("the claim was not given back, so nothing comes for the book until the grace runs out")
|
||||
}
|
||||
}
|
||||
|
|
@ -18,17 +18,23 @@ import (
|
|||
// walk itself, and every one of them is a number an operator would have to reason about the
|
||||
// reconciler to choose.
|
||||
const (
|
||||
// claimGrace is how long a parse may be somebody's business before another pass may take it. It
|
||||
// ClaimGrace is how long a parse may be somebody's business before another pass may take it. It
|
||||
// covers the ordinary case — the queue job is claimed in milliseconds — and the failure it exists
|
||||
// for: the process holding the claim was restarted mid-parse.
|
||||
//
|
||||
// Exported for the same reason UploadGrace below is, and against the same kind of configuration:
|
||||
// the sweep's staleness for a `parsing` book falls back to `added_at` when no claim is stamped
|
||||
// (pgstore.StuckIntake), and `added_at` is stamped when the row is created — BEFORE the body has
|
||||
// arrived. So an upload allowed to take longer than this grace is one the sweep may claim while
|
||||
// its own request is still walking, and the boot refuses that configuration outright.
|
||||
//
|
||||
// ⚠ It MUST outlive the queue's own job timeout, and it is written as that constant plus a margin
|
||||
// so the two cannot drift apart. Shorter, the sweep steals the claim from a parse that is still
|
||||
// legitimately running: thief and holder meet on one project directory and the loser dies on the
|
||||
// engine's exclusive lock. That is no longer the data-loss it was — the lock has its own exit code
|
||||
// now (`project_locked`, ingest.ExitProjectLocked) and reads as the host's state rather than as a
|
||||
// verdict about the book — but it still spends an attempt of the budget on nothing.
|
||||
claimGrace = jobs.JobTimeout + 5*time.Minute
|
||||
ClaimGrace = jobs.JobTimeout + 5*time.Minute
|
||||
// UploadGrace is how long a book may stay `uploading`. A request that is still arriving holds the
|
||||
// row, so anything older than this is an upload whose request is gone — and that reasoning holds
|
||||
// only while the route's own read deadline is SHORTER. Exported so the boot can refuse a
|
||||
|
|
@ -74,6 +80,17 @@ const (
|
|||
// would otherwise reject every book it holds. It is also the state the deploy note's own step
|
||||
// exists to prevent (`tmplatformctl books --migratable`).
|
||||
ReasonSchemaMismatch = ingest.RejectSchemaMismatch
|
||||
// ReasonHostAtCapacity is why a pass did nothing, and it is the one name here that is NOT a
|
||||
// rejection reason: it never reaches a book's row. `reject` is the only writer of a reason, and
|
||||
// the class that carries this one gives the claim back before any of that (parseClaimed,
|
||||
// atCutCapacity) — so it exists for the operator's log and for the message an unfinished intake
|
||||
// carries, and nothing stores it.
|
||||
ReasonHostAtCapacity = "host_at_cut_capacity"
|
||||
// ReasonNoTimeToCut is the other half of the same class and is stored no more than the one above:
|
||||
// what remained of the upload's walk was less than a cut plus the writes that have to follow it,
|
||||
// so no cut was started. Told apart from the cap because the remedies are opposite — one is a
|
||||
// busier host than usual, the other a walk that had already spent itself.
|
||||
ReasonNoTimeToCut = "no_time_to_cut"
|
||||
)
|
||||
|
||||
// ErrNotProvisioned is a book with no usable engine configuration and no way for this platform to
|
||||
|
|
@ -117,7 +134,7 @@ var ErrStorageGone = errors.New("books: the book storage root is gone")
|
|||
// the engine holds exclusively.
|
||||
func (s *Service) Parse(ctx context.Context, bookID string) error {
|
||||
now := s.now()
|
||||
claim, err := s.Store.ClaimParse(ctx, bookID, now, now.Add(-claimGrace))
|
||||
claim, err := s.Store.ClaimParse(ctx, bookID, now, now.Add(-ClaimGrace))
|
||||
if errors.Is(err, pgstore.ErrParseClaimed) {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -163,7 +180,16 @@ func notConclusive(reason string) error {
|
|||
// (parseAttempts). At intake the uploader still holds the file, so being wrong costs a retry — which
|
||||
// is why the budget's reasoning does not reach here.
|
||||
func (s *Service) parseClaimed(ctx context.Context, claim pgstore.ParseClaim, atIntake bool) error {
|
||||
m, err := s.manifest(ctx, claim)
|
||||
// The reserve a cut must leave itself, and it is a property of WHO is asking. At intake nothing is
|
||||
// at stake in giving up — no attempt is spent and the queue finishes the book — so waiting to the
|
||||
// very end of the step is free. On the queue's own path an attempt IS at stake, so a cut is not
|
||||
// started, and a slot is not waited for, unless what remains is what this platform calls a cut's
|
||||
// worth of time.
|
||||
reserve := CutBudget
|
||||
if atIntake {
|
||||
reserve = 0
|
||||
}
|
||||
m, err := s.manifest(ctx, claim, reserve)
|
||||
if err == nil {
|
||||
// THE FLOOR, and it is FIRST for a reason that is the whole of it: below this line a document
|
||||
// that could not be read correctly is indistinguishable from a book with nothing in it, and
|
||||
|
|
@ -265,6 +291,17 @@ func (s *Service) parseClaimed(ctx context.Context, claim pgstore.ParseClaim, at
|
|||
}
|
||||
return nil
|
||||
}
|
||||
if engineNotAsked(err) {
|
||||
// The engine was never asked, so this pass knows NOTHING about the book and records nothing:
|
||||
// no attempt spent, no reason stored, and the claim handed straight back. That is the opposite
|
||||
// of the deployment faults below, where waiting is the point — the cap is the host's state at
|
||||
// this instant, and the next pass may find a slot a second later.
|
||||
s.log().InfoContext(ctx, "the engine was not asked, so this pass does not cut", "err", err)
|
||||
if atIntake {
|
||||
return notConclusive(ReasonHostAtCapacity)
|
||||
}
|
||||
return s.giveBack(ctx, claim, err)
|
||||
}
|
||||
if errors.Is(err, ErrStorageGone) {
|
||||
// The host cannot see its own storage. Nothing about this book is known yet, so it waits with
|
||||
// the budget untouched — exactly like a book waiting for a configuration.
|
||||
|
|
@ -305,6 +342,27 @@ func (s *Service) parseClaimed(ctx context.Context, claim pgstore.ParseClaim, at
|
|||
return s.defer_(ctx, claim, reason)
|
||||
}
|
||||
|
||||
// giveBack ends a pass that established NOTHING and hands the claim straight back, so the next pass
|
||||
// can take the book at once instead of after the grace that exists for a process which DIED holding
|
||||
// it. The attempt goes back with the claim (ReleaseParseClaim), because nothing was tried.
|
||||
//
|
||||
// No job is enqueued with it, unlike the intake's own release, and the reason is that the job this
|
||||
// pass is running must COME BACK rather than be replaced: the error is wrapped in jobs.ErrTryAgainLater,
|
||||
// and the worker turns that into a snooze, which does not spend the single attempt this queue's
|
||||
// policy allows (jobs.InsertOpts, jobs.RetryDelay). Enqueueing a second job here would be one more
|
||||
// job per busy moment at the same book.
|
||||
//
|
||||
// ⚠ The sweep is what catches the case where nothing comes back at all — a pass that is not a queue
|
||||
// job (books.Sweep) simply logs and moves on, and the book is offered again once its claim goes stale.
|
||||
func (s *Service) giveBack(ctx context.Context, claim pgstore.ParseClaim, cause error) error {
|
||||
w, cancel := s.writeCtx(ctx)
|
||||
defer cancel()
|
||||
if err := s.Store.ReleaseParseClaim(w, claim.BookID, claim.At, nil); err != nil {
|
||||
s.log().ErrorContext(ctx, "the parse claim could not be given back; the backstop sweep takes the book", "err", err)
|
||||
}
|
||||
return fmt.Errorf("%w: %w", jobs.ErrTryAgainLater, cause)
|
||||
}
|
||||
|
||||
// defer_ spends one attempt of the budget and, when the budget is gone, ends the intake.
|
||||
//
|
||||
// ONE path for every way a parse can fail, and the reason has outlived the defect that produced it.
|
||||
|
|
@ -362,45 +420,135 @@ func waitsForTheDeployment(reason string) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// writeCtx is the context a TERMINAL write uses: detached from the caller's deadline and bounded on
|
||||
// its own.
|
||||
// walkKey carries the deadline of an upload's tail to every step taken under it.
|
||||
//
|
||||
// The engine call this follows can legitimately consume the whole budget of the pass — and then the
|
||||
// write that records what happened would run on an already-expired context and be lost, leaving the
|
||||
// book to be retried and the attempt to be spent again, forever. What must survive is the record.
|
||||
func (s *Service) writeCtx(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
budget := writeBudget
|
||||
if s.writeBudget > 0 {
|
||||
budget = s.writeBudget // tests shorten it; nothing outside this package can set it
|
||||
// A context value rather than a parameter because the steps are not all in this function or in this
|
||||
// file: the cut is shared with the queue's worker, and a bound only the callers who remember to pass
|
||||
// it are subject to is the hand-written list this replaces.
|
||||
type walkKey struct{}
|
||||
|
||||
// walk opens the ONE budget an upload's tail is spent from, and marks it so every step taken under
|
||||
// it — including steps nobody has written yet — is bounded by the same deadline.
|
||||
//
|
||||
// Detached from the request, because the tail must outlive it: every byte is in, and a client that
|
||||
// hung up while waiting for the 201 must not cost the upload it already finished. Bounded, because
|
||||
// detached is not unlimited and this is the number the boot compared against the windows an upload
|
||||
// has to finish inside (internal/config).
|
||||
func (s *Service) walk(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
// UploadSettle is the WHOLE tail and part of it happens after this package is done: the receipt
|
||||
// is written by the HTTP surface once Accept has returned (ReceiptBudget). Taking the whole of it
|
||||
// here would put the walk's end exactly one receipt past the number the boot was promised, which
|
||||
// is the same off-by-a-step this walk exists to make impossible.
|
||||
budget := UploadSettle - ReceiptBudget
|
||||
if s.uploadSettle > 0 {
|
||||
budget = s.uploadSettle // tests shorten it; nothing outside this package can set it
|
||||
}
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), budget)
|
||||
// The wall clock and not s.Now: this deadline is compared by the context package against its own
|
||||
// clock, and a test that freezes the injected one would otherwise open a walk already over.
|
||||
deadline := time.Now().Add(budget)
|
||||
c, cancel := context.WithDeadline(context.WithoutCancel(ctx), deadline)
|
||||
return context.WithValue(c, walkKey{}, deadline), cancel
|
||||
}
|
||||
|
||||
// step is the context ONE action of an intake runs on: its own budget, and never past the deadline
|
||||
// of the walk it belongs to.
|
||||
//
|
||||
// It does two things and both are load-bearing. It DETACHES from the caller's cancellation, because
|
||||
// the engine call a terminal write follows can legitimately consume everything the step before it
|
||||
// had — and a write that then runs on an already-expired context is a record that happened and was
|
||||
// not written down, leaving the book to be retried and the attempt spent again. And it CAPS at the
|
||||
// walk's own deadline, so what a step escapes is the budget of the step before it and never the
|
||||
// budget of the whole tail.
|
||||
//
|
||||
// That second half is what makes UploadSettle true for any number of steps: a step added below this
|
||||
// line costs latency inside the tail and cannot move its end.
|
||||
//
|
||||
// Outside a walk — the queue's worker and the backstop sweep, whose budgets are their own — there is
|
||||
// no deadline to cap against, and this is the detached, self-bounded write it has always been.
|
||||
func (s *Service) step(ctx context.Context, budget time.Duration) (context.Context, context.CancelFunc) {
|
||||
return s.stepLeaving(ctx, budget, 0)
|
||||
}
|
||||
|
||||
// stepLeaving is step for an action that must not spend what the rest of the walk still needs.
|
||||
//
|
||||
// ⛔ The reserve exists because the two ways a short walk can end are NOT equal, and without it the
|
||||
// wrong one happens. A cut that is given less time ends as «no verdict», and the queue finishes the
|
||||
// book — a degradation this intake is built around. A terminal WRITE that is given less time leaves
|
||||
// the book claimed with no job, and nothing comes back for it until the backstop sweep's grace runs
|
||||
// out: twenty minutes, for a book whose only misfortune was arriving late in a walk. So the slack is
|
||||
// taken out of the cut and never out of the writes that record what the cut found.
|
||||
//
|
||||
// It is the same rule the intake sweep and the materializer already follow — do not START work the
|
||||
// remaining budget cannot hold (Sweep, readmodel.Drain) — applied one level down, to the steps of
|
||||
// one upload rather than to the books of one pass.
|
||||
func (s *Service) stepLeaving(ctx context.Context, budget, reserve time.Duration) (context.Context, context.CancelFunc) {
|
||||
deadline := time.Now().Add(budget)
|
||||
if walk, ok := ctx.Value(walkKey{}).(time.Time); ok {
|
||||
if keep := walk.Add(-reserve); keep.Before(deadline) {
|
||||
deadline = keep
|
||||
}
|
||||
}
|
||||
return context.WithDeadline(context.WithoutCancel(ctx), deadline)
|
||||
}
|
||||
|
||||
// cutTailReserve is what the walk must still hold for the steps that FOLLOW a cut: the write that
|
||||
// records what it found, the release that follows a write which failed, and the re-read the response
|
||||
// is built from. See stepLeaving for why the cut is the step that gives way.
|
||||
//
|
||||
// Derived from the write budget IN FORCE rather than from the constant, and that is not a nicety: a
|
||||
// test shortens the write budget to reach cases a thirty-second one cannot, and a reserve pinned to
|
||||
// the constant would then be three real writes' worth of a walk measured in milliseconds — that is,
|
||||
// larger than the whole walk, so no cut would ever run and the fixture would silently model nothing.
|
||||
func (s *Service) cutTailReserve() time.Duration { return 3 * s.write() }
|
||||
|
||||
// write is the budget one terminal write gets: the constant, or what a test shortened it to.
|
||||
func (s *Service) write() time.Duration {
|
||||
if s.writeBudget > 0 {
|
||||
return s.writeBudget // tests shorten it; nothing outside this package can set it
|
||||
}
|
||||
return writeBudget
|
||||
}
|
||||
|
||||
// writeCtx is the context a TERMINAL write uses.
|
||||
func (s *Service) writeCtx(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
return s.step(ctx, s.write())
|
||||
}
|
||||
|
||||
// writeBudget is what a terminal write gets. Short: it is one statement against a database this
|
||||
// process is already connected to.
|
||||
const writeBudget = 30 * time.Second
|
||||
|
||||
// UploadSettle is the worst case of everything an upload still does once its body has arrived, and
|
||||
// the boot leaves room for it (internal/config): the upload's deadline bounds the BODY, and the
|
||||
// windows an upload must finish inside bound all of it.
|
||||
// UploadSettle is what an upload's tail gets once its body has arrived: the ONE deadline every step
|
||||
// after the last byte runs under (walk), and the room the boot leaves for it (internal/config).
|
||||
//
|
||||
// The steps, in order and by the budget each actually gets, because the sum is only as right as this
|
||||
// list — and an earlier edition of it dropped the first one:
|
||||
// It is an ALLOWANCE and not a sum, and that difference is the whole of this constant's history.
|
||||
// Three editions of it were a hand-written list of the steps it covers, and all three were short —
|
||||
// one dropped a step, one called two steps alternatives where the code runs both, and each was found
|
||||
// by the overrun rather than by the list. A list has to be re-derived by whoever adds a step, and
|
||||
// three times running nobody did. The walk caps every step at what is left of this deadline instead:
|
||||
// the tail cannot outlive the number regardless of how many steps it grows, so what an added step
|
||||
// costs is latency inside the tail and never the boundary the boot was promised.
|
||||
//
|
||||
// 1. StartParsing, which moves the book out of `uploading` — writeBudget;
|
||||
// 2. the synchronous cut — CutBudget;
|
||||
// 3. whichever end the cut reaches (FinishParse, ReleaseParseClaim or DeleteRefusedIntake) — writeBudget;
|
||||
// 4. the re-read that gives the response the row as it stands — writeBudget;
|
||||
// 5. the idempotency receipt the HTTP surface writes (httpapi.settleCtx, 10s), which is shorter
|
||||
// than a writeBudget and is covered by the fourth one here.
|
||||
// The SIZE is still chosen and not arbitrary: it is every step's own budget summed, so that in the
|
||||
// worst ordinary case none of them is truncated and the cap never bites. The terms are the cut, the
|
||||
// four terminal writes of the walk's longest chain — StartParsing, the end the cut reaches, the
|
||||
// release that follows an end which FAILED, and the re-read — and the receipt. Running out of it
|
||||
// anyway is not a failed upload: a step that finds nothing left leaves the book `parsing`, which the
|
||||
// backstop sweep finishes, the same degradation an overrun cut already has.
|
||||
//
|
||||
// None of them is bounded BY another: every one takes a detached context of its own, so the worst
|
||||
// case is the sum.
|
||||
// ⚠ The sum is written out and it is NOT what makes this constant true; the walk is, and that
|
||||
// difference is the whole lesson of PD-464. What the SUM buys is that no step is cut short in the
|
||||
// ordinary case. What the WALK buys is that the tail ends here even when the sum is wrong again.
|
||||
//
|
||||
// The reading surface is deliberately NOT in this list: materializing it runs the engine twice more
|
||||
// It covers the tail END TO END, including the part that is not this package's: the receipt is a
|
||||
// TERM, and the walk takes this number minus it, because the receipt is written after Accept has
|
||||
// returned. The edition before this one stopped at the fourth write, and was short by exactly one
|
||||
// receipt — the third miss in a row, and the last one the sum can make on its own.
|
||||
//
|
||||
// The reading surface is deliberately not in it: materializing runs the engine twice more
|
||||
// (readmodel.MaterializeBudget), and the intake leaves that debt to the materializer's sweep rather
|
||||
// than making an uploader wait for it (parseClaimed, the `!atIntake` guard).
|
||||
const UploadSettle = CutBudget + 4*writeBudget
|
||||
const UploadSettle = CutBudget + 4*writeBudget + ReceiptBudget
|
||||
|
||||
// CutBudget bounds the cut an upload waits for. Past it the book is accepted `parsing` and the queue
|
||||
// finishes the job, so overrunning costs a less informative response and never a failed upload.
|
||||
|
|
@ -410,9 +558,18 @@ const UploadSettle = CutBudget + 4*writeBudget
|
|||
// past the window in which its key can still be replayed.
|
||||
const CutBudget = 90 * time.Second
|
||||
|
||||
// ReceiptBudget is the share of UploadSettle that belongs to the HTTP surface rather than to this
|
||||
// package: the idempotency receipt, written once Accept has returned (httpapi.settleCtx).
|
||||
//
|
||||
// Declared HERE and used there, rather than written twice. It is a term of the sum the boot compares
|
||||
// against the windows an upload must finish inside, and a second copy of it in the package that
|
||||
// actually spends it is a copy that can drift — which is how every earlier edition of UploadSettle
|
||||
// came to be short.
|
||||
const ReceiptBudget = 10 * time.Second
|
||||
|
||||
// manifest asks the engine to cut the book, once its configuration is there to cut it against —
|
||||
// rendering that configuration first, if this deployment carries a template (form Б, D39.130).
|
||||
func (s *Service) manifest(ctx context.Context, claim pgstore.ParseClaim) (ingest.Manifest, error) {
|
||||
func (s *Service) manifest(ctx context.Context, claim pgstore.ParseClaim, reserve time.Duration) (ingest.Manifest, error) {
|
||||
if s.Engine == nil {
|
||||
return ingest.Manifest{}, errors.New("books: no engine is configured")
|
||||
}
|
||||
|
|
@ -437,6 +594,14 @@ func (s *Service) manifest(ctx context.Context, claim pgstore.ParseClaim) (inges
|
|||
}); err != nil {
|
||||
return ingest.Manifest{}, err
|
||||
}
|
||||
// Under the host's cap, and taken HERE rather than in any of the three callers: this is the one
|
||||
// line all of them reach the engine through, and a cap on a route leaves the other routes outside
|
||||
// it (limit.go).
|
||||
release, err := s.takeCutSlot(ctx, reserve)
|
||||
if err != nil {
|
||||
return ingest.Manifest{}, err
|
||||
}
|
||||
defer release()
|
||||
return s.Engine.Manifest(ctx, s.Cfg.EngineBinary, workdir)
|
||||
}
|
||||
|
||||
|
|
@ -523,7 +688,7 @@ func (s *Service) reject(ctx context.Context, claim pgstore.ParseClaim, reason s
|
|||
// One book's failure never stops the sweep: these are independent books of independent accounts.
|
||||
func (s *Service) Sweep(ctx context.Context) error {
|
||||
now := s.now()
|
||||
stuck, err := s.Store.StuckIntake(ctx, now.Add(-UploadGrace), now.Add(-claimGrace))
|
||||
stuck, err := s.Store.StuckIntake(ctx, now.Add(-UploadGrace), now.Add(-ClaimGrace))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ func TestARenderedConfigurationIsNeverRewritten(t *testing.T) {
|
|||
}
|
||||
f.engine.set(wholeManifest(3, 1), nil)
|
||||
// Another walk of the same book — a re-parse after the claim went stale.
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -292,7 +292,7 @@ func TestABrokenTemplateStopsNoIntakeAndSpendsNoBudget(t *testing.T) {
|
|||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
}
|
||||
if got := f.card(t, book.ID); got.Status != "parsing" {
|
||||
t.Fatalf("a book on a host with a broken template is %q, want parsing", got.Status)
|
||||
|
|
@ -314,7 +314,7 @@ func TestABrokenTemplateStopsNoIntakeAndSpendsNoBudget(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
f.engine.set(wholeManifest(3, 1), nil)
|
||||
f.now = f.now.Add(claimGrace + time.Minute)
|
||||
f.now = f.now.Add(ClaimGrace + time.Minute)
|
||||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
|||
154
platform/internal/books/walk_test.go
Normal file
154
platform/internal/books/walk_test.go
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
package books
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The tail of an upload cannot outlive UploadSettle, whatever it does on the way there.
|
||||
//
|
||||
// The PROPERTY and not a sum of the steps, and that is the whole of the form. The sum was derived by
|
||||
// hand three times and was short all three, each time for a different reason (register row PD-464); a
|
||||
// test that re-listed the steps would be the fourth hand-derivation and would hold for exactly as
|
||||
// long as somebody remembered to update it. What is asserted here survives steps nobody has written
|
||||
// yet, which is why the fourth case takes fifty of them.
|
||||
func TestNoStepOfAnUploadsTailOutlivesTheWalk(t *testing.T) {
|
||||
// THREE different numbers on purpose: in a fixture where the walk, a step's own budget and what a
|
||||
// step asks for coincide, "capped by the walk" and "given what it asked for" are the same
|
||||
// observation, and the whole class of differences between them goes invisible (D39.208 §5).
|
||||
const walkBudget, writeShort, asksForever = 400 * time.Millisecond, 90 * time.Millisecond, time.Hour
|
||||
s := &Service{writeBudget: writeShort, uploadSettle: walkBudget}
|
||||
walk, cancel := s.walk(context.Background())
|
||||
defer cancel()
|
||||
end, ok := walk.Deadline()
|
||||
if !ok {
|
||||
t.Fatal("the walk carries no deadline, so nothing taken under it is bounded by anything")
|
||||
}
|
||||
|
||||
// A step that asks for more than the walk has left gets the walk's end and not what it asked for.
|
||||
long, cancelLong := s.step(walk, asksForever)
|
||||
defer cancelLong()
|
||||
if d, _ := long.Deadline(); !d.Equal(end) {
|
||||
t.Errorf("a step that asked for %s ends at %s, want the walk's own end %s: the walk is not capping it",
|
||||
asksForever, d.Format(time.StampMilli), end.Format(time.StampMilli))
|
||||
}
|
||||
|
||||
// A step that asks for less keeps its own budget: the cap is a ceiling, not a replacement.
|
||||
short, cancelShort := s.step(walk, writeShort)
|
||||
defer cancelShort()
|
||||
if d, _ := short.Deadline(); !d.Before(end) {
|
||||
t.Errorf("a step that asked for %s ends at %s, at or past the walk's %s: it was given the ceiling instead of its budget",
|
||||
writeShort, d.Format(time.StampMilli), end.Format(time.StampMilli))
|
||||
}
|
||||
|
||||
// FIFTY steps, each asking for an hour, each taken under the one before it. This is the property
|
||||
// the form exists for: the number of steps in the tail is not a term of where the tail ends.
|
||||
under := walk
|
||||
for i := range 50 {
|
||||
next, cancelNext := s.step(under, asksForever)
|
||||
defer cancelNext()
|
||||
if d, _ := next.Deadline(); d.After(end) {
|
||||
t.Fatalf("step %d of the tail ends at %s, past the walk's %s: adding a step moves the boundary",
|
||||
i, d.Format(time.StampMilli), end.Format(time.StampMilli))
|
||||
}
|
||||
under = next
|
||||
}
|
||||
|
||||
// A terminal write taken after the step before it is SPENT is still usable — that is the whole
|
||||
// reason a step detaches at all: the engine call may legitimately consume everything, and the
|
||||
// record of what happened must still be written. It is capped by the walk all the same.
|
||||
spent, cancelSpent := s.step(walk, time.Nanosecond)
|
||||
defer cancelSpent()
|
||||
<-spent.Done()
|
||||
after, cancelAfter := s.step(spent, writeShort)
|
||||
defer cancelAfter()
|
||||
if err := after.Err(); err != nil {
|
||||
t.Errorf("a write taken after an exhausted step was born cancelled (%v): what happened would go unrecorded", err)
|
||||
}
|
||||
if d, _ := after.Deadline(); d.After(end) {
|
||||
t.Errorf("a write taken after an exhausted step ends at %s, past the walk's %s",
|
||||
d.Format(time.StampMilli), end.Format(time.StampMilli))
|
||||
}
|
||||
}
|
||||
|
||||
// Outside a walk — the queue's worker and the backstop sweep — a step is the detached, self-bounded
|
||||
// write it has always been: their budgets are their own, and there is no upload waiting on them.
|
||||
func TestAStepOutsideAWalkKeepsItsOwnBudgetAndSurvivesItsCaller(t *testing.T) {
|
||||
const writeShort = 90 * time.Millisecond
|
||||
s := &Service{writeBudget: writeShort}
|
||||
parent, cancelParent := context.WithCancel(context.Background())
|
||||
free, cancelFree := s.step(parent, writeShort)
|
||||
defer cancelFree()
|
||||
cancelParent()
|
||||
if err := free.Err(); err != nil {
|
||||
t.Errorf("a write outside a walk died with its caller (%v): a job whose deadline is spent would lose its record", err)
|
||||
}
|
||||
d, ok := free.Deadline()
|
||||
if !ok {
|
||||
t.Fatal("a write outside a walk got no deadline: detached is not the same as unlimited")
|
||||
}
|
||||
if left := time.Until(d); left > writeShort {
|
||||
t.Errorf("a write outside a walk got %s, want no more than its own %s", left, writeShort)
|
||||
}
|
||||
}
|
||||
|
||||
// UploadSettle covers the tail END TO END, and part of it is spent after this package is done: the
|
||||
// idempotency receipt is written once Accept has returned. So the walk takes the settle budget MINUS
|
||||
// the receipt's share — otherwise the tail ends exactly one receipt past the number the boot compared
|
||||
// against the windows an upload has to finish inside, which is the shape of every earlier miss.
|
||||
func TestTheWalkLeavesTheReceiptItsShareOfTheSettleBudget(t *testing.T) {
|
||||
s := &Service{}
|
||||
before := time.Now()
|
||||
walk, cancel := s.walk(context.Background())
|
||||
defer cancel()
|
||||
end, ok := walk.Deadline()
|
||||
if !ok {
|
||||
t.Fatal("the walk carries no deadline")
|
||||
}
|
||||
// `before` is read a hair earlier than the walk stamps its deadline, so the reading is the budget
|
||||
// plus scheduling slack and never less than it.
|
||||
got, want := end.Sub(before), UploadSettle-ReceiptBudget
|
||||
if got < want || got > want+time.Second {
|
||||
t.Errorf("the walk runs for about %s, want UploadSettle (%s) less the receipt's share (%s) = %s",
|
||||
got, UploadSettle, ReceiptBudget, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The cut an upload waits for runs on the WALK's deadline, not on a budget of its own.
|
||||
//
|
||||
// Asserted at the real call site and by the deadline the engine is HANDED, rather than by a stopwatch
|
||||
// around the request: the question is which budget bounds the cut, and a clock answers it only on a
|
||||
// machine that happened to be slow enough. A cut detached from the walk — the shape this replaced —
|
||||
// hands the engine CutBudget here, three orders of magnitude past the walk it is inside.
|
||||
func TestTheCutOfAnUploadIsBoundedByTheWalkAndNotByItsOwnBudget(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
templated(t, f)
|
||||
const tail, write = 700 * time.Millisecond, 20 * time.Millisecond
|
||||
f.svc.uploadSettle = tail
|
||||
// Shortened with the walk, because the reserve a cut leaves for the writes after it is three of
|
||||
// these: against the real thirty seconds a 700 ms walk holds no cut at all, and the fixture would
|
||||
// then be asserting about a cut that never ran (stepLeaving).
|
||||
f.svc.writeBudget = write
|
||||
var granted time.Duration
|
||||
var bounded bool
|
||||
f.engine.onManifest = func(ctx context.Context) {
|
||||
d, ok := ctx.Deadline()
|
||||
bounded = ok
|
||||
granted = time.Until(d)
|
||||
}
|
||||
f.accept(t, "蛊真人.txt", "первая глава\fвторая глава")
|
||||
if f.engine.called() != 1 {
|
||||
t.Fatalf("the engine was asked %d times, so there is no granted deadline to judge", f.engine.called())
|
||||
}
|
||||
if !bounded {
|
||||
t.Fatal("the cut ran with no deadline at all")
|
||||
}
|
||||
if granted <= 0 || granted > tail {
|
||||
t.Errorf("the cut was granted %s inside a %s walk, want a slice of the walk", granted, tail)
|
||||
}
|
||||
if granted >= CutBudget {
|
||||
t.Errorf("the cut was granted %s, its own CutBudget (%s): it is detached from the walk it runs inside",
|
||||
granted, CutBudget)
|
||||
}
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@ import (
|
|||
|
||||
"textmachine/platform/internal/books"
|
||||
"textmachine/platform/internal/exports"
|
||||
"textmachine/platform/internal/jobs"
|
||||
"textmachine/platform/internal/login"
|
||||
"textmachine/platform/internal/money"
|
||||
"textmachine/platform/internal/pgstore"
|
||||
|
|
@ -257,6 +258,12 @@ type IntakeConfig struct {
|
|||
// against the template's. Empty means this deployment provisions books by hand, which is what
|
||||
// every deployment did before the form was ratified.
|
||||
BookTemplate string
|
||||
// MaxCuts caps how many books this host lets the engine cut at once, counting every way of
|
||||
// starting one. A SIZING and not a refusal, unlike MaxUploadBytes: an upload that meets the cap
|
||||
// waits for a slot and is accepted `parsing` if none frees up, so raising this buys latency at
|
||||
// the cost of memory. Zero takes the default (books.DefaultMaxCuts), which is the queue's own
|
||||
// worker count.
|
||||
MaxCuts int
|
||||
}
|
||||
|
||||
// LanguagePair is one translation direction this deployment knows about.
|
||||
|
|
@ -552,7 +559,7 @@ func (c *Config) loadRunner(l *loader) error {
|
|||
RunBudget: 60 * time.Second,
|
||||
ResyncEvery: 5 * time.Minute,
|
||||
HoldFactorPercent: pricing.DefaultHoldFactorPercent,
|
||||
Workers: 4,
|
||||
Workers: jobs.DefaultWorkers,
|
||||
}
|
||||
// An ABSOLUTE state directory or none. The exit marker is written by a systemd unit whose
|
||||
// WorkingDirectory is the BOOK's directory and read by this daemon from its own working directory,
|
||||
|
|
@ -698,6 +705,7 @@ func (c *Config) loadIntake(l *loader) error {
|
|||
// promises — the numbers an operator raises when their users hit them.
|
||||
MaxUploadBytes: 64 << 20,
|
||||
UploadDeadline: 10 * time.Minute,
|
||||
MaxCuts: books.DefaultMaxCuts,
|
||||
}
|
||||
if in.BooksDir != "" && !filepath.IsAbs(in.BooksDir) {
|
||||
return fmt.Errorf("config: TM_PLATFORM_BOOKS_DIR must be an absolute path, got %q", in.BooksDir)
|
||||
|
|
@ -716,24 +724,51 @@ func (c *Config) loadIntake(l *loader) error {
|
|||
if in.UploadDeadline, err = l.duration("TM_PLATFORM_UPLOAD_DEADLINE", in.UploadDeadline); err != nil {
|
||||
return err
|
||||
}
|
||||
// Zero and negatives are refused by the READER (l.number), which is where every count of this
|
||||
// configuration is refused — and the refusal matters here rather than being a formality: the two
|
||||
// readings of zero are opposite, and taking it as "no limit" would give a deployment that meant
|
||||
// "cut nothing" an unbounded one. A second check here would be unreachable, and an unreachable
|
||||
// guard is a guard no test can defend.
|
||||
if in.MaxCuts, err = l.number("TM_PLATFORM_MAX_CUTS", in.MaxCuts); err != nil {
|
||||
return err
|
||||
}
|
||||
// An upload has to FINISH inside two windows, and the boot is where a deployment that breaks
|
||||
// either is cheap to find. The intake sweep treats a book that has been `uploading` longer than
|
||||
// its grace as an upload whose request is gone, and deletes the row and the directory under a
|
||||
// request still writing into them; past the claim window a retry takes the idempotency claim from
|
||||
// an upload that is still running, and the user gets the second book the key exists to prevent.
|
||||
//
|
||||
// Against the TIGHTER of the two rather than against each: written as two checks, whichever is
|
||||
// Against the TIGHTEST of them rather than against each: written as separate checks, whichever is
|
||||
// looser could never fire, and the pin that was supposed to cover it was passing on the other one.
|
||||
// And the deadline is only the BODY — what an upload still has to do afterwards is added here,
|
||||
// or a 29m59s deadline settled its key a moment after the claim became stealable.
|
||||
if window := min(books.UploadGrace, pgstore.ClaimStale); in.UploadDeadline+books.UploadSettle >= window {
|
||||
return fmt.Errorf("config: TM_PLATFORM_UPLOAD_DEADLINE (%s) plus what follows an upload (%s) must fit inside the tighter of the intake sweep's grace (%s) and the idempotency claim window (%s)",
|
||||
in.UploadDeadline, books.UploadSettle, books.UploadGrace, pgstore.ClaimStale)
|
||||
//
|
||||
// THREE windows and not two, and the third is the parse claim's. A `parsing` book with no claim
|
||||
// stamped on it is stale to the sweep by `added_at` (pgstore.StuckIntake), which is written when
|
||||
// the row is created and therefore BEFORE the body has arrived — so an upload allowed to outlast
|
||||
// books.ClaimGrace is one the sweep may claim while the request is still walking, and the two of
|
||||
// them then race for the same book. It is the loosest-looking of the three and the tightest in
|
||||
// fact: 20 minutes against 30 and 60.
|
||||
if window := intakeWindow(books.UploadGrace, books.ClaimGrace, pgstore.ClaimStale); in.UploadDeadline+books.UploadSettle >= window {
|
||||
return fmt.Errorf("config: TM_PLATFORM_UPLOAD_DEADLINE (%s) plus what follows an upload (%s) must fit inside the tightest of the intake sweep's grace (%s), the parse claim's grace (%s) and the idempotency claim window (%s)",
|
||||
in.UploadDeadline, books.UploadSettle, books.UploadGrace, books.ClaimGrace, pgstore.ClaimStale)
|
||||
}
|
||||
c.Intake = in
|
||||
return nil
|
||||
}
|
||||
|
||||
// intakeWindow is the tightest of the windows an upload's whole walk has to finish inside.
|
||||
//
|
||||
// A named function and not an inline `min`, and the reason is not readability — it is that the risk
|
||||
// here is a TERM GOING MISSING, and today's constants cannot expose one. Whichever window is tightest
|
||||
// hides the other two: with the parse claim's grace at 20 minutes against 30 and 60, a gate written
|
||||
// against that one alone refuses exactly the same deployments, and every case built from the real
|
||||
// constants passes either way. Choosing here lets a test hand it values that make each window the
|
||||
// tightest in turn — which is the only shape that catches a term nobody consulted.
|
||||
func intakeWindow(sweepGrace, parseClaimGrace, idempotencyClaim time.Duration) time.Duration {
|
||||
return min(sweepGrace, parseClaimGrace, idempotencyClaim)
|
||||
}
|
||||
|
||||
// loader reads the environment and REMEMBERS what it read, so that the boot line can say where each
|
||||
// value came from. Every setting of this service goes through it: a print that covers most of the
|
||||
// configuration is worse than none, because the variable an operator is hunting is exactly the one
|
||||
|
|
|
|||
|
|
@ -281,32 +281,44 @@ func TestARelativeBooksDirectoryIsRefusedAtBoot(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// The upload deadline is bounded by two windows, and a deployment that breaks either loses data:
|
||||
// past the intake sweep's grace the sweep deletes the row and the directory out from under a request
|
||||
// still writing into them; past the idempotency claim window a retry takes the claim from an upload
|
||||
// that is still running and the user gets a second book.
|
||||
// The upload deadline is bounded by THREE windows, and a deployment that breaks any of them loses
|
||||
// something: past the intake sweep's grace the sweep deletes the row and the directory out from under
|
||||
// a request still writing into them; past the parse claim's grace the sweep claims the parse of a
|
||||
// book whose own upload is still walking, and the two race for it; past the idempotency claim window
|
||||
// a retry takes the claim from an upload that is still running and the user gets a second book.
|
||||
//
|
||||
// ⚠ The boundary is pinned to the EXACT value, and that is the whole of this test. The window that
|
||||
// binds today is the claim's, so any deadline over the sweep's grace is over it too — the earlier
|
||||
// form asserted two over-long values and both were caught by the same check, so dropping the other
|
||||
// one changed nothing. And "under the window" is not enough: what an upload does AFTER its body has
|
||||
// to fit as well.
|
||||
func TestAnUploadDeadlineIsRefusedUnlessTheWholeUploadFitsTheTighterWindow(t *testing.T) {
|
||||
// ⚠ The boundary is pinned to the EXACT value. What this test does NOT do — said here because an
|
||||
// earlier edition of this very comment claimed it did — is tell the three windows apart: the parse
|
||||
// claim's grace is the tightest of them today (20 minutes against 30 and 60), so every case below is
|
||||
// refused by that one term, and deleting either of the other two from the gate leaves this test
|
||||
// green. Their coverage lives in TestEveryWindowAnUploadMustFitInsideIsActuallyConsulted, which can
|
||||
// make each of them tightest in turn because it does not have to use the real constants. And "under
|
||||
// the window" is not enough either: what an upload does AFTER its body has to fit as well.
|
||||
func TestAnUploadDeadlineIsRefusedUnlessTheWholeUploadFitsTheTightestWindow(t *testing.T) {
|
||||
t.Setenv("TM_PLATFORM_LANGUAGE_PAIRS", "zh>ru")
|
||||
t.Setenv("TM_PLATFORM_BOOKS_DIR", "/srv/tm/books")
|
||||
t.Setenv("TM_PLATFORM_ENGINE_BIN", "/opt/engine/2026.08.01/tmctl")
|
||||
window := min(books.UploadGrace, pgstore.ClaimStale)
|
||||
for _, d := range []time.Duration{
|
||||
books.UploadGrace + time.Minute,
|
||||
pgstore.ClaimStale + time.Minute,
|
||||
// Inside the window and still wrong: the body ends a second before the claim is stealable and
|
||||
// the receipt is written after it.
|
||||
window - time.Second,
|
||||
window - books.UploadSettle,
|
||||
window := min(books.UploadGrace, books.ClaimGrace, pgstore.ClaimStale)
|
||||
for _, c := range []struct {
|
||||
window string
|
||||
deadline time.Duration
|
||||
}{
|
||||
{"the intake sweep's grace", books.UploadGrace + time.Minute},
|
||||
{"the idempotency claim window", pgstore.ClaimStale + time.Minute},
|
||||
// The case that matters for what this pack changed, even though the term above would also
|
||||
// catch it: at 21 minutes the sweep sees a book whose `added_at` — the stamp it falls back to,
|
||||
// written before the body arrived — is older than the claim's grace, while the upload that
|
||||
// created it is still walking.
|
||||
{"the parse claim's grace", books.ClaimGrace + time.Minute},
|
||||
// Inside the tightest window and still wrong: the body ends a second before the claim is
|
||||
// stealable, and everything the upload does afterwards happens after that.
|
||||
{"the tightest window, by one second", window - time.Second},
|
||||
{"the tightest window, once the tail is counted", window - books.UploadSettle},
|
||||
} {
|
||||
t.Setenv("TM_PLATFORM_UPLOAD_DEADLINE", d.String())
|
||||
t.Setenv("TM_PLATFORM_UPLOAD_DEADLINE", c.deadline.String())
|
||||
if _, err := Load(); err == nil {
|
||||
t.Errorf("an upload deadline of %s was accepted, though the whole upload does not fit %s", d, window)
|
||||
t.Errorf("an upload deadline of %s was accepted, though the whole upload does not fit %s (tightest: %s)",
|
||||
c.deadline, c.window, window)
|
||||
}
|
||||
}
|
||||
t.Setenv("TM_PLATFORM_UPLOAD_DEADLINE", (window - books.UploadSettle - time.Second).String())
|
||||
|
|
@ -315,6 +327,39 @@ func TestAnUploadDeadlineIsRefusedUnlessTheWholeUploadFitsTheTighterWindow(t *te
|
|||
}
|
||||
}
|
||||
|
||||
// The cap on concurrent cuts is a number an operator sets, it defaults to the figure the host was
|
||||
// sized for, and zero is refused rather than taken as "no limit" — an intake that cuts nothing would
|
||||
// accept every book `parsing` and look like it is working.
|
||||
//
|
||||
// ⚠ The refusal comes from the READER of counts (`loader.number`, which rejects every non-positive
|
||||
// value), not from a check of the intake's own. An earlier edition of this pack added a second check
|
||||
// beside it and this test appeared to pin it; it could not, because the first refusal fires and the
|
||||
// second line is unreachable. The behaviour is what is pinned here, not the layer.
|
||||
func TestTheCapOnConcurrentCutsIsConfiguredAndRefusesAnEmptyOne(t *testing.T) {
|
||||
t.Setenv("TM_PLATFORM_LANGUAGE_PAIRS", "zh>ru")
|
||||
t.Setenv("TM_PLATFORM_BOOKS_DIR", "/srv/tm/books")
|
||||
t.Setenv("TM_PLATFORM_ENGINE_BIN", "/opt/engine/2026.08.01/tmctl")
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("the default configuration was refused: %v", err)
|
||||
}
|
||||
if c.Intake.MaxCuts != books.DefaultMaxCuts {
|
||||
t.Errorf("an unset cap gave %d cuts, want the package default %d", c.Intake.MaxCuts, books.DefaultMaxCuts)
|
||||
}
|
||||
// Not the default, and not any other number in this deployment: a value the operator chose has to
|
||||
// arrive as the value the operator chose.
|
||||
t.Setenv("TM_PLATFORM_MAX_CUTS", "7")
|
||||
if c, err = Load(); err != nil || c.Intake.MaxCuts != 7 {
|
||||
t.Fatalf("a configured cap of 7 gave %d (err %v)", c.Intake.MaxCuts, err)
|
||||
}
|
||||
for _, v := range []string{"0", "-1"} {
|
||||
t.Setenv("TM_PLATFORM_MAX_CUTS", v)
|
||||
if _, err := Load(); err == nil {
|
||||
t.Errorf("a cap of %s was accepted, so this deployment would cut nothing and say nothing", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An intake with no declared pairs makes the wire and the intake say opposite things.
|
||||
func TestAnIntakeWithNoDeclaredPairsIsRefusedAtBoot(t *testing.T) {
|
||||
t.Setenv("TM_PLATFORM_BOOKS_DIR", "/srv/tm/books")
|
||||
|
|
@ -334,3 +379,26 @@ func TestAnIntakeWithNoDeclaredPairsIsRefusedAtBoot(t *testing.T) {
|
|||
t.Fatalf("a declared available pair was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Every window the gate is supposed to weigh is actually weighed.
|
||||
//
|
||||
// Built from values rather than from the real constants, and that is the whole point: with the real
|
||||
// ones the tightest window hides the other two, so a term deleted from the choice changes nothing any
|
||||
// boot-level case can observe (see the comment above). Here each window is made the tightest in turn,
|
||||
// with the other two far enough away that only the intended one can produce the answer.
|
||||
func TestEveryWindowAnUploadMustFitInsideIsActuallyConsulted(t *testing.T) {
|
||||
const tight, loose, looser = 5 * time.Minute, time.Hour, 2 * time.Hour
|
||||
for _, c := range []struct {
|
||||
window string
|
||||
sweepGrace, parseClaim, idempotencyKey time.Duration
|
||||
}{
|
||||
{"the intake sweep's grace", tight, loose, looser},
|
||||
{"the parse claim's grace", loose, tight, looser},
|
||||
{"the idempotency claim window", loose, looser, tight},
|
||||
} {
|
||||
if got := intakeWindow(c.sweepGrace, c.parseClaim, c.idempotencyKey); got != tight {
|
||||
t.Errorf("with %s the tightest at %s, the gate would hold an upload to %s: that window is not consulted at all",
|
||||
c.window, tight, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,17 +23,20 @@ import "net/http"
|
|||
// about members the wire already served. Prose and errata do not move a number; a served shape does.
|
||||
// And 0.11.0 already has its acceptance act (D39.208), which closes it. See the pack's report.
|
||||
//
|
||||
// ⚠ THE GATE IS RED WHILE THE CANON READS 0.11.0, and that is the ratified order and its true cause:
|
||||
// the CODE lands first and the canon follows in the second act. The reverse was refused deliberately
|
||||
// — errata 04.09-в is the day the canon moved first and the wire spent a working day announcing a
|
||||
// version it did not serve.
|
||||
// ⚠ THE GATE IS RED BETWEEN THE TWO LANDINGS, and which side is ahead says which of the two orders
|
||||
// is being followed. The ratified one is CODE FIRST: this constant is raised with the change that
|
||||
// implements the minor, the gate goes red because the canon still reads the EARLIER version, and the
|
||||
// canon's own landing clears it. The reverse — canon first, code behind, the gate red because the
|
||||
// canon reads a LATER minor — is refused deliberately, because in that window the wire announces a
|
||||
// version it does not serve: errata 04.09-в is the day that cost a working day, and 0.13.0 is the day
|
||||
// it happened again and stood for a full day, until the next session of this zone read its battery.
|
||||
//
|
||||
// ⛔ AND WHAT THE GATE CANNOT SEE, said here because this constant is where a reader comes looking: it
|
||||
// compares VERSIONS, not SHAPES. Three separate untruths in the 0.11.0 text passed it in one day —
|
||||
// the bar described in chapters while the wire sent units, `ordered_chapters: 0` promised where the
|
||||
// wire sent 2, three values of a vocabulary that had four — because the NUMBER matched each time. The
|
||||
// bump keeps the number from lying; it does not close the blind spot, which is unified backlog row 309.
|
||||
const ContractVersion = "0.12.0"
|
||||
const ContractVersion = "0.13.0"
|
||||
|
||||
// Capabilities is what this deployment can do: one flat document, the same for every account.
|
||||
type Capabilities struct {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"textmachine/platform/internal/books"
|
||||
"textmachine/platform/internal/pgstore"
|
||||
)
|
||||
|
||||
|
|
@ -166,8 +167,13 @@ func (i *idempotent) replayIfIdentical(w http.ResponseWriter, r *http.Request, d
|
|||
// settleCtx detaches a key's own write from the request: the client that will retry is precisely
|
||||
// the one that hung up, and on `r.Context()` the key then stays in flight for `ClaimStale` and the
|
||||
// retry re-does the work. Same rule as the intake's writeCtx, applied to the receipt.
|
||||
//
|
||||
// The budget is books.ReceiptBudget and not a number of this package's own, because it is a TERM of
|
||||
// what the boot compares against the windows an upload must finish inside (books.UploadSettle): the
|
||||
// receipt is the last thing an upload does, and a copy of its budget here is a copy that can drift
|
||||
// from the sum that has to contain it.
|
||||
func settleCtx(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), books.ReceiptBudget)
|
||||
}
|
||||
|
||||
// complete records what this attempt answered, so a repeat is given the same thing. `content` is the
|
||||
|
|
|
|||
|
|
@ -441,3 +441,32 @@ func TestAnOverlongKeyIsRefusedBeforeAnythingIsClaimed(t *testing.T) {
|
|||
t.Errorf("the store was asked about a key the surface had already refused: %+v", keys.claims)
|
||||
}
|
||||
}
|
||||
|
||||
// The receipt spends the share the INTAKE's budget set aside for it, not a number of this package's
|
||||
// own.
|
||||
//
|
||||
// The two live in different packages and the drift between them is silent. A literal here smaller
|
||||
// than the declared term is a receipt cut short; one LARGER is a tail that outlives what the boot
|
||||
// compared against the windows an upload must finish inside — and being short by exactly one step of
|
||||
// that sum is the defect this seam was drawn to end (books.UploadSettle, register row PD-464).
|
||||
func TestTheReceiptSpendsTheShareTheIntakeSetAsideForIt(t *testing.T) {
|
||||
before := time.Now()
|
||||
ctx, cancel := settleCtx(context.Background())
|
||||
defer cancel()
|
||||
d, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
t.Fatal("the receipt runs unbounded: a statement that hangs holds the goroutine of a request that is already over")
|
||||
}
|
||||
if got := d.Sub(before); got < books.ReceiptBudget || got > books.ReceiptBudget+time.Second {
|
||||
t.Errorf("the receipt was given %s, want the share the intake declared for it (%s)", got, books.ReceiptBudget)
|
||||
}
|
||||
// And detached, which is the other half of the same rule: the client that will retry is precisely
|
||||
// the one that hung up, so the receipt must outlive its request.
|
||||
gone, cancelGone := context.WithCancel(context.Background())
|
||||
cancelGone()
|
||||
kept, cancelKept := settleCtx(gone)
|
||||
defer cancelKept()
|
||||
if err := kept.Err(); err != nil {
|
||||
t.Errorf("the receipt died with its request (%v): the key stays in flight and the retry re-does the work", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ package jobs
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
|
@ -114,9 +115,33 @@ type parseWorker struct {
|
|||
}
|
||||
|
||||
func (w *parseWorker) Work(ctx context.Context, job *river.Job[ParseArgs]) error {
|
||||
return w.svc.Parse(ctx, job.Args.BookID)
|
||||
err := w.svc.Parse(ctx, job.Args.BookID)
|
||||
if errors.Is(err, ErrTryAgainLater) {
|
||||
// The pass established NOTHING and said so. Snoozing rather than failing, and the difference
|
||||
// matters because MaxAttempts is 1: a returned error consumes this job outright, and the book
|
||||
// would then wait out the intake sweep's whole grace over a host that was busy for a moment.
|
||||
// A snooze does not increment the attempt (river.JobSnooze), so it is the SAME recovery
|
||||
// mechanism coming back — not the second one this queue's policy refuses.
|
||||
return river.JobSnooze(RetryDelay)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// ErrTryAgainLater is a pass that did nothing and wants the job back rather than spent.
|
||||
//
|
||||
// Declared HERE rather than by the service, because what it selects is a QUEUE policy and this is the
|
||||
// package that owns one. A service wraps its own reason in it (books.giveBack); this package decides
|
||||
// what the queue does about it, and the two cannot drift into different opinions about whether the
|
||||
// job is finished.
|
||||
var ErrTryAgainLater = errors.New("jobs: this pass established nothing and the job should come back")
|
||||
|
||||
// RetryDelay is how long a job that established nothing waits before it is offered again.
|
||||
//
|
||||
// Shorter than a cut, because what it usually waits out is one: the ordinary producer of
|
||||
// ErrTryAgainLater is a host that has as many books under the engine as it will take, and a slot
|
||||
// frees when one of them finishes. Long enough that a saturated host is not re-asked in a tight loop.
|
||||
const RetryDelay = 30 * time.Second
|
||||
|
||||
type exportWorker struct {
|
||||
river.WorkerDefaults[ExportArgs]
|
||||
svc Exporter
|
||||
|
|
@ -134,6 +159,14 @@ func (w *exportWorker) Work(ctx context.Context, job *river.Job[ExportArgs]) err
|
|||
// constant plus a margin.
|
||||
const JobTimeout = 15 * time.Minute
|
||||
|
||||
// DefaultWorkers is how many jobs this queue runs at once when a deployment does not say.
|
||||
//
|
||||
// The number a host is sized for, and therefore the ONE place it is written: the intake's cap on
|
||||
// concurrent engine cuts is the same figure said for every way of starting one rather than only for
|
||||
// the way that goes through here (books.DefaultMaxCuts), and two literals coupled by prose is the
|
||||
// drift this package has already paid for elsewhere.
|
||||
const DefaultWorkers = 4
|
||||
|
||||
// Queue is the River client, wired to the services that do the work.
|
||||
type Queue struct {
|
||||
client *river.Client[pgstore.Tx]
|
||||
|
|
@ -146,7 +179,7 @@ type Queue struct {
|
|||
// by how many workers exist.
|
||||
func New(pool *pgxpool.Pool, spawner Spawner, parser Parser, exporter Exporter, log *slog.Logger, workers int) (*Queue, error) {
|
||||
if workers <= 0 {
|
||||
workers = 4
|
||||
workers = DefaultWorkers
|
||||
}
|
||||
w := river.NewWorkers()
|
||||
if spawner != nil {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/riverqueue/river"
|
||||
|
|
@ -36,3 +39,43 @@ func TestTheQueuesKindsAndRetryPolicyAreWhatTheRestOfTheSystemAssumes(t *testing
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stubParser answers with whatever the case wants, so the worker's own decision is what is judged.
|
||||
type stubParser struct{ err error }
|
||||
|
||||
func (p stubParser) Parse(context.Context, string) error { return p.err }
|
||||
|
||||
// A pass that established nothing gets its job BACK; anything else the pass says is the job's answer.
|
||||
//
|
||||
// ⛔ The distinction is the whole of it, and it exists because MaxAttempts is 1 (above). A returned
|
||||
// error consumes the single attempt outright, and the book then waits out the intake sweep's whole
|
||||
// grace — twenty minutes — over a host that was busy for a moment. A snooze does not increment the
|
||||
// attempt, so it is the SAME recovery coming back rather than the second mechanism this queue's
|
||||
// policy refuses. Nothing witnessed the mapping until this test: the service's own tests end at the
|
||||
// error it returns, and the queue's end at the options it declares.
|
||||
func TestAPassThatEstablishedNothingGetsItsJobBackInsteadOfSpendingIt(t *testing.T) {
|
||||
failed := errors.New("the store could not be reached")
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
err error
|
||||
snooze bool
|
||||
}{
|
||||
{"a pass that established nothing", fmt.Errorf("%w: the host is busy", ErrTryAgainLater), true},
|
||||
{"a pass that failed for its own reasons", failed, false},
|
||||
{"a pass that finished", nil, false},
|
||||
} {
|
||||
w := &parseWorker{svc: stubParser{err: c.err}}
|
||||
got := w.Work(t.Context(), &river.Job[ParseArgs]{Args: ParseArgs{BookID: "bk_1"}})
|
||||
var snoozed *river.JobSnoozeError
|
||||
switch {
|
||||
case c.snooze && !errors.As(got, &snoozed):
|
||||
t.Errorf("%s: the worker answered %v, want a snooze — this job is the book's only one, and spending it costs the sweep's whole grace", c.what, got)
|
||||
case c.snooze && snoozed.Duration != RetryDelay:
|
||||
t.Errorf("%s: the job comes back in %v, want %v", c.what, snoozed.Duration, RetryDelay)
|
||||
case !c.snooze && errors.As(got, &snoozed):
|
||||
t.Errorf("%s: the worker snoozed, so a job that is genuinely finished or genuinely broken would be offered again forever", c.what)
|
||||
case !c.snooze && !errors.Is(got, c.err):
|
||||
t.Errorf("%s: the worker answered %v, want the pass's own %v", c.what, got, c.err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,19 @@ type Metrics struct {
|
|||
// backup would sit under every alert ever written for this series.
|
||||
backupAge prometheus.Gauge
|
||||
|
||||
// The host's cap on concurrent engine cuts, and how hard it is being hit. Without these a cap is
|
||||
// indistinguishable from latency an operator has to guess the cause of — which is what the intake
|
||||
// looked like when it had no cap at all.
|
||||
cutsInFlight prometheus.Gauge
|
||||
cutsWaiting prometheus.Gauge
|
||||
cutSlots prometheus.Gauge
|
||||
cutWaits prometheus.Counter
|
||||
cutGiveUps prometheus.Counter
|
||||
// What the two counters above have already been told. The service counts cumulatively and a
|
||||
// Prometheus counter takes increments, so the difference is what is added; both are touched only
|
||||
// from the telemetry pass, which is one goroutine (cmd/tmplatformd, observe).
|
||||
countedCutWaits, countedCutGiveUps uint64
|
||||
|
||||
requests *prometheus.CounterVec
|
||||
latency *prometheus.HistogramVec
|
||||
}
|
||||
|
|
@ -139,9 +152,30 @@ func New() *Metrics {
|
|||
Help: "Time to serve a request, by route pattern and method.",
|
||||
Buckets: latencyBuckets,
|
||||
}, []string{"route", "method"})
|
||||
m.cutsInFlight = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: namespace, Name: "cuts_in_flight",
|
||||
Help: "Engine cuts running right now, across the upload that cuts its own book, the queue's workers and the backstop sweep.",
|
||||
})
|
||||
m.cutsWaiting = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: namespace, Name: "cuts_waiting",
|
||||
Help: "Cuts queued for a slot right now. Sustained above zero means the cap, not the engine, is what shapes intake latency.",
|
||||
})
|
||||
m.cutSlots = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: namespace, Name: "cut_slots",
|
||||
Help: "The cap itself (TM_PLATFORM_MAX_CUTS), published so saturation can be read without knowing the deployment's configuration.",
|
||||
})
|
||||
m.cutWaits = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: namespace, Name: "cut_waits_total",
|
||||
Help: "Cuts that found every slot taken and had to wait for one.",
|
||||
})
|
||||
m.cutGiveUps = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: namespace, Name: "cut_slot_timeouts_total",
|
||||
Help: "Cuts whose caller ran out of budget while waiting for a slot. An upload counted here is accepted `parsing` and finished by the queue, not refused.",
|
||||
})
|
||||
m.registry.MustRegister(m.queueDepth, m.oldestHold, m.quarantined, m.parked, m.liveRuns, m.tailerLag,
|
||||
m.booksInIntake, m.sweepDuration, m.sweepUnfinished, m.stalledRuns, m.abandonedSurfaces,
|
||||
m.backupAge, m.requests, m.latency,
|
||||
m.cutsInFlight, m.cutsWaiting, m.cutSlots, m.cutWaits, m.cutGiveUps,
|
||||
// The runtime and the process itself: memory, goroutines, file descriptors, CPU. They are
|
||||
// what answers "is this instance healthy" when none of the numbers above has moved.
|
||||
collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
|
||||
|
|
@ -206,6 +240,37 @@ func (m *Metrics) ObserveRunner(r Runner) {
|
|||
m.abandonedSurfaces.Set(float64(r.AbandonedSurfaces))
|
||||
}
|
||||
|
||||
// Cuts is one reading of the host's cap on concurrent engine cuts. Waited and GaveUp are cumulative
|
||||
// since the process started, which is what lets this publish increments to a counter.
|
||||
type Cuts struct {
|
||||
Limit int
|
||||
InFlight int
|
||||
Waiting int
|
||||
Waited uint64
|
||||
GaveUp uint64
|
||||
}
|
||||
|
||||
// ObserveCuts publishes where the host's cut capacity stands.
|
||||
func (m *Metrics) ObserveCuts(c Cuts) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.cutSlots.Set(float64(c.Limit))
|
||||
m.cutsInFlight.Set(float64(c.InFlight))
|
||||
m.cutsWaiting.Set(float64(c.Waiting))
|
||||
// Only ever forward. A reading below what was already counted would mean the process restarted
|
||||
// under the same registry, which cannot happen — but a counter that went backwards would break
|
||||
// every rate() over it, so the guard costs one comparison and removes the class.
|
||||
if c.Waited > m.countedCutWaits {
|
||||
m.cutWaits.Add(float64(c.Waited - m.countedCutWaits))
|
||||
m.countedCutWaits = c.Waited
|
||||
}
|
||||
if c.GaveUp > m.countedCutGiveUps {
|
||||
m.cutGiveUps.Add(float64(c.GaveUp - m.countedCutGiveUps))
|
||||
m.countedCutGiveUps = c.GaveUp
|
||||
}
|
||||
}
|
||||
|
||||
// ObserveTailerLag publishes how far the furthest-behind live run's cursor is from the end of its
|
||||
// journal, in bytes.
|
||||
func (m *Metrics) ObserveTailerLag(bytes int64) {
|
||||
|
|
|
|||
|
|
@ -183,3 +183,47 @@ func TestTheBackupAgeGaugeIsInfiniteWhenThereIsNoPoint(t *testing.T) {
|
|||
t.Errorf("age = %v seconds, want 5400 (the base unit is seconds, not minutes)", v)
|
||||
}
|
||||
}
|
||||
|
||||
// The cap's five series carry the five DIFFERENT numbers they are given.
|
||||
//
|
||||
// Five distinct values on purpose: with a fixture where any two coincide, a pair of gauges wired to
|
||||
// each other's readings is invisible, and «how saturated is the host» would be answered by «how deep
|
||||
// is the line» without anybody noticing. The counters are published as INCREMENTS from cumulative
|
||||
// readings, so a second observation is made and the counters have to have moved by the difference
|
||||
// while the gauges simply carry the latest.
|
||||
func TestTheCutCapacityIsExposedWithEveryFigureItWasGiven(t *testing.T) {
|
||||
m := New()
|
||||
m.ObserveCuts(Cuts{Limit: 6, InFlight: 4, Waiting: 3, Waited: 9, GaveUp: 2})
|
||||
body := scrape(t, m)
|
||||
for _, want := range []string{
|
||||
"tm_platform_cut_slots 6",
|
||||
"tm_platform_cuts_in_flight 4",
|
||||
"tm_platform_cuts_waiting 3",
|
||||
"tm_platform_cut_waits_total 9",
|
||||
"tm_platform_cut_slot_timeouts_total 2",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the exposition is missing %q", want)
|
||||
}
|
||||
}
|
||||
// A second reading: the gauges take the new value, the counters advance by the difference rather
|
||||
// than being set to it — a counter set to a cumulative reading would double-count on every scrape.
|
||||
m.ObserveCuts(Cuts{Limit: 6, InFlight: 1, Waiting: 0, Waited: 11, GaveUp: 2})
|
||||
body = scrape(t, m)
|
||||
for _, want := range []string{
|
||||
"tm_platform_cuts_in_flight 1",
|
||||
"tm_platform_cuts_waiting 0",
|
||||
"tm_platform_cut_waits_total 11",
|
||||
"tm_platform_cut_slot_timeouts_total 2",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("after a second reading the exposition is missing %q", want)
|
||||
}
|
||||
}
|
||||
// And a reading that went BACKWARDS — which cannot happen in this process and would break every
|
||||
// rate() over the series if it did — leaves the counter where it stood.
|
||||
m.ObserveCuts(Cuts{Limit: 6, InFlight: 0, Waiting: 0, Waited: 1, GaveUp: 0})
|
||||
if body = scrape(t, m); !strings.Contains(body, "tm_platform_cut_waits_total 11") {
|
||||
t.Error("a reading below what was already counted moved the counter backwards")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -206,6 +206,12 @@ func (s *Store) CreateUpload(ctx context.Context, id string, in NewUpload) (Book
|
|||
// enqueue joins the caller's transaction for the same reason the run admission's does: a status that
|
||||
// says "being parsed" with no job to do it waits for the backstop sweep's grace, and a job for a
|
||||
// book that never became `parsing` is a worker with nothing to claim.
|
||||
//
|
||||
// ⚠ It is NIL on the path a deployment actually takes, and that is deliberate rather than dead code
|
||||
// left behind. An intake that cuts the book inside the request must not have a job racing that cut
|
||||
// for the parse claim, so it enqueues at the END of the cut instead (books.cutsItsOwnUploads, and
|
||||
// ReleaseParseClaim below). What reaches this parameter is the other deployment — a queue and no
|
||||
// engine — which has nothing to wait for and hands the book over at once.
|
||||
func (s *Store) StartParsing(ctx context.Context, id string, characters int64,
|
||||
enqueue func(context.Context, Tx, string) error) (Book, error) {
|
||||
var b Book
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue