744 lines
32 KiB
Go
744 lines
32 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/chunk/chunktest"
|
||
"textmachine/backend/internal/config"
|
||
"textmachine/backend/internal/obs"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// reprice_model_test.go: the MODEL axis of the consent number (D39.150 п.1, backlog row 197 / ФЧ-5) and
|
||
// the disclosure of what the number is made of (ФЧ-4), end to end. Every assertion is on money the
|
||
// operator is SHOWN — the projection, the status document, the refusal text — and on the two surfaces
|
||
// saying one thing.
|
||
|
||
// newModelEchoProvider is newJSONProvider with one difference: the response names the model the REQUEST
|
||
// asked for, as a real endpoint does. The shared fake always answers «fake-model», which makes every call
|
||
// bill at fake-model's price whatever the stage asked — a premise the model axis cannot be tested on.
|
||
func newModelEchoProvider(rec *reqRec) *httptest.Server {
|
||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
body, _ := io.ReadAll(r.Body)
|
||
rec.record(string(body))
|
||
var req struct {
|
||
Model string `json:"model"`
|
||
}
|
||
_ = json.Unmarshal(body, &req)
|
||
text, _ := draftEdit(string(body))
|
||
tb, _ := json.Marshal(text)
|
||
fmt.Fprintf(w, `{"id":"fake","model":%q,"choices":[{"message":{"content":%s},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`,
|
||
req.Model, tb)
|
||
}))
|
||
}
|
||
|
||
// TestRebillProjectionPricesAtTheModelTheStageResolvesToNow is the acceptance's measurement made a pin:
|
||
// a stage moved to a model ten times the price, and the consent number quoting the retired model's
|
||
// table ($0.003640 for $0.036400 of work). The number must follow the model that WILL answer — in both
|
||
// directions of a move, because the answering model's price is a floor, never below.
|
||
func TestRebillProjectionPricesAtTheModelTheStageResolvesToNow(t *testing.T) {
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
for _, tc := range []struct {
|
||
name string
|
||
before string // the editor's model when the book was bought
|
||
after string // the editor's model now
|
||
}{
|
||
// The measured harm: bought on the cheap model, the stage now resolves to the dear one.
|
||
{"the editor moved to a model ten times the price", "fake-model", "fake-dear"},
|
||
// The other direction rounds up too: the tokens are the dear model's, so its price is the floor.
|
||
{"the editor moved to a model a tenth of the price", "fake-dear", "fake-model"},
|
||
} {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newModelEchoProvider(rec)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
addModel(t, bookPath, "fake-dear", 10.0)
|
||
if tc.before != "fake-model" {
|
||
setStageModel(t, bookPath, "edit", tc.before)
|
||
}
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// The premise, asserted: the editor's call was billed at the model it was bought on.
|
||
usage, err := r1.Store.CheckpointUsageForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
editCalls := 0
|
||
for _, cu := range usage {
|
||
if cu.Stage == "edit" {
|
||
editCalls++
|
||
if cu.ModelRequested != tc.before || cu.ModelActual != tc.before {
|
||
t.Fatalf("premise: the edit call must be recorded against %s, got requested=%s actual=%s", tc.before, cu.ModelRequested, cu.ModelActual)
|
||
}
|
||
}
|
||
}
|
||
if editCalls != 1 {
|
||
t.Fatalf("premise: exactly one edit call, got %d", editCalls)
|
||
}
|
||
|
||
// The model is in the per-stage snapshot slot, so moving it supersedes the EDIT wave alone: the
|
||
// draft row keeps resuming at $0 and only the editor's call is projected.
|
||
setStageModel(t, bookPath, "edit", tc.after)
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
statuses, err := r2.Store.ChunkStatusesForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
chunks := chunksOf(t, r2)
|
||
proj, err := r2.projectRebill(statuses, chunks, func() ([]chunk.Chunk, error) { return chunks, nil }, newRepricerT(t, r2))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if proj.Rows != 1 {
|
||
t.Fatalf("setup: only the edit row is superseded by a model move, got %d row(s)", proj.Rows)
|
||
}
|
||
// The dear model's price, whichever end of the move it sits at: never the cheap one's.
|
||
if want := 10 * fakeCallUSD; !nearUSD(proj.USD, want) {
|
||
t.Fatalf("the re-payment is projected at $%.6f; the model the stage resolves to (or answered with) makes the same tokens $%.6f — the smaller number is the silent under-quote D39.150 forbids", proj.USD, want)
|
||
}
|
||
if proj.ModelMovedRows != 1 || proj.HistoricalRows != 0 {
|
||
t.Fatalf("the row was priced across a model move and must say so: ModelMovedRows=%d HistoricalRows=%d", proj.ModelMovedRows, proj.HistoricalRows)
|
||
}
|
||
|
||
// The status document carries the same figure, the count that qualifies it, and both on the wire
|
||
// under their basis (ФЧ-4: the platform reads this document, not the refusal text).
|
||
rep, err := r2.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !nearUSD(rep.RebillUSD, proj.USD) || rep.RebillModelMovedRows != 1 || rep.RebillHistoricalRows != 0 {
|
||
t.Fatalf("status quotes $%.6f (moved=%d, historical=%d) where the gate projects $%.6f (moved=1, historical=0)",
|
||
rep.RebillUSD, rep.RebillModelMovedRows, rep.RebillHistoricalRows, proj.USD)
|
||
}
|
||
wire, err := json.Marshal(rep)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, field := range []string{`"rebill_model_moved_rows":1`, `"rebill_historical_rows":0`, `"rebill_basis":"pending"`} {
|
||
if !strings.Contains(string(wire), field) {
|
||
t.Errorf("status --json must carry %s beside the figure; got: %s", field, wire)
|
||
}
|
||
}
|
||
// The threshold's base is priced the same way, or the gate compares two currencies: draft at the
|
||
// cheap model plus the editor at the dear one, whichever direction the move went.
|
||
if want := 11 * fakeCallUSD; !nearUSD(rep.ProjectedBookUSD, want) {
|
||
t.Fatalf("projected_book_usd = $%.6f, want $%.6f (the same upward pricing as the amount)", rep.ProjectedBookUSD, want)
|
||
}
|
||
|
||
// The sentence the consent is given to: the amount, and the caveat that it is an upper estimate
|
||
// priced across a model move — in the refusal AND in the status renderer's own words.
|
||
caveat := "1 of the 1 unit(s) were bought from a model their stage no longer resolves to"
|
||
if basis := rep.RebillFigureBasis(); !strings.Contains(basis, caveat) || !strings.Contains(basis, "errs upward") {
|
||
t.Fatalf("the status basis must name the model move and the direction of error; got: %s", basis)
|
||
}
|
||
r2.Resnapshot = true
|
||
_, terr := r2.TranslateBook(ctx)
|
||
if terr == nil {
|
||
t.Fatal("the re-payment is over the threshold — the run must refuse")
|
||
}
|
||
msg := terr.Error()
|
||
for _, want := range []string{fmt.Sprintf("~$%.6f", proj.USD), "priced at the model each stage resolves to TODAY", "errs upward", caveat} {
|
||
if !strings.Contains(msg, want) {
|
||
t.Errorf("the refusal must carry %q; got: %s", want, msg)
|
||
}
|
||
}
|
||
if !strings.Contains(msg, rep.RebillFigureBasis()) {
|
||
t.Errorf("the refusal and `tmctl status` must describe the figure in ONE wording; refusal: %s\nstatus: %s", msg, rep.RebillFigureBasis())
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestTheScopedClauseOfTheRefusalCarriesItsOwnBasis is the threshold refusal under a volume ceiling: it
|
||
// prints the BOOK's figure and THIS run's slice, and each figure travels with the basis of the projection
|
||
// it came from — which of the slice's units are an upper estimate is not something the book's basis can
|
||
// say for them. Two chapters, the editor moved to a dearer model, a grant admitting one chapter: the slice
|
||
// differs from the book, so the scoped clause is printed and must carry the model-move caveat itself.
|
||
func TestTheScopedClauseOfTheRefusalCarriesItsOwnBasis(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newModelEchoProvider(rec)
|
||
defer srv.Close()
|
||
bookPath := volumeBook(t, srv.URL, 2)
|
||
addModel(t, bookPath, "fake-dear", 10.0)
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
setStageModel(t, bookPath, "edit", "fake-dear")
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
chunks := chunksOf(t, r2)
|
||
bounded := &volumeScope{
|
||
admitted: map[chunkKey]bool{{1, 0}: true},
|
||
leader: map[chunkKey]bool{{1, 0}: true},
|
||
stop: VolumeStop{MaxUnits: 1},
|
||
}
|
||
err := r2.checkRebillConsent(ctx, chunks, bounded)
|
||
if err == nil {
|
||
t.Fatal("a bounded run re-paying a moved row must still be refused above the threshold")
|
||
}
|
||
msg := err.Error()
|
||
if !strings.Contains(msg, "THIS run, bounded by --max-units, would re-pay 1 of them") {
|
||
t.Fatalf("the slice differs from the book, so the scoped clause must be printed: %s", msg)
|
||
}
|
||
if !strings.Contains(msg, "2 of the 2 unit(s) were bought from a model their stage no longer resolves to") {
|
||
t.Errorf("the book-wide clause must carry the book's caveat: %s", msg)
|
||
}
|
||
if !strings.Contains(msg, "1 of the 1 unit(s) were bought from a model their stage no longer resolves to") {
|
||
t.Errorf("the scoped clause must carry the SLICE's own caveat, not borrow the book's: %s", msg)
|
||
}
|
||
|
||
// ⚠ AND THE NAMED-CAP REFUSAL, which quotes the SLICE's figure and nothing else — so it must carry the
|
||
// slice's basis alone. Borrowing the book's there tells a caller deciding whether to raise their cap
|
||
// that twice as many units are an upper estimate as the number beside it covers. Its twin, the scoped
|
||
// clause above, has been pinned since it was written; this branch had no test anywhere.
|
||
r2.AcceptRebill = RebillConsent{Given: true, Capped: true, CapUSD: 0.000001}
|
||
cerr := r2.checkRebillConsent(ctx, chunks, bounded)
|
||
if cerr == nil || !strings.Contains(cerr.Error(), "caps consent at") {
|
||
t.Fatalf("a cap below the slice's figure must refuse naming the cap; got: %v", cerr)
|
||
}
|
||
cmsg := cerr.Error()
|
||
if !strings.Contains(cmsg, "1 of the 1 unit(s) were bought from a model their stage no longer resolves to") {
|
||
t.Errorf("the capped refusal quotes THIS run's figure and must carry THIS run's basis: %s", cmsg)
|
||
}
|
||
if strings.Contains(cmsg, "2 of the 2 unit(s) were bought from a model") {
|
||
t.Errorf("the capped refusal borrowed the BOOK's basis for a figure that is the slice's: %s", cmsg)
|
||
}
|
||
}
|
||
|
||
// addLocalModel adds a LOCAL provider on the same fake endpoint and a model on it priced at zero — the one
|
||
// shape the config admits a $0 model in (a non-local model must carry non-zero prices).
|
||
func addLocalModel(t *testing.T, bookPath, providerURL, name string) {
|
||
t.Helper()
|
||
path := filepath.Join(filepath.Dir(bookPath), "models.yaml")
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
withProvider := strings.Replace(string(raw), "providers:\n", fmt.Sprintf("providers:\n fakelocal:\n kind: local\n base_url: %q\n model: %s\n timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }\n", providerURL, name), 1)
|
||
if withProvider == string(raw) {
|
||
t.Fatal("setup: the fixture models.yaml has no providers block")
|
||
}
|
||
writeFile(t, path, withProvider+fmt.Sprintf(`
|
||
%s:
|
||
provider: fakelocal
|
||
price: { input_per_m: 0, cached_per_m: 0, cache_write_per_m: 0, output_per_m: 0 }
|
||
`, name))
|
||
}
|
||
|
||
// TestAZeroPricedRowMovedToAPaidModelIsNotQuotedAtZero is the model axis on the one row shape whose money
|
||
// says nothing about its generation: a stage run on a model priced at zero. Its row's cost_usd is $0, so a
|
||
// walk that decides membership from money alone visits no call and quotes $0 — after the stage moved to a
|
||
// paid model, the whole wave's re-run hidden at $0 and the consent gate never asked. The $0 row's
|
||
// generation is its trailing $0 calls, priced at the model that will answer.
|
||
//
|
||
// Found by the pack's own fresh-context review (lens 4), reproduced on a scratch copy before the fix.
|
||
func TestAZeroPricedRowMovedToAPaidModelIsNotQuotedAtZero(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newModelEchoProvider(rec)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
addLocalModel(t, bookPath, srv.URL, "fake-free") // priced at zero on every axis: a local model
|
||
setStageModel(t, bookPath, "edit", "fake-free")
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
statuses, err := r1.Store.ChunkStatusesForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
for _, cs := range statuses {
|
||
if cs.Stage == "edit" && cs.CostUSD != 0 {
|
||
t.Fatalf("premise: the edit row must have been billed $0, got $%.6f", cs.CostUSD)
|
||
}
|
||
}
|
||
|
||
setStageModel(t, bookPath, "edit", "fake-model")
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
chunks := chunksOf(t, r2)
|
||
proj, err := r2.projectRebill(statuses, chunks, func() ([]chunk.Chunk, error) { return chunks, nil }, newRepricerT(t, r2))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if proj.Rows != 1 {
|
||
t.Fatalf("setup: only the edit row is superseded by the model move, got %d", proj.Rows)
|
||
}
|
||
if !nearUSD(proj.USD, fakeCallUSD) || proj.ModelMovedRows != 1 {
|
||
t.Fatalf("a $0 row whose stage now resolves to a paid model must be quoted at that model and disclosed: got $%.6f moved=%d, want $%.6f moved=1 — a $0 quote here is the silent under-quote D39.150 forbids",
|
||
proj.USD, proj.ModelMovedRows, fakeCallUSD)
|
||
}
|
||
rep, err := r2.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !nearUSD(rep.RebillUSD, proj.USD) || rep.RebillModelMovedRows != 1 {
|
||
t.Fatalf("status must publish the same figure: $%.6f moved=%d", rep.RebillUSD, rep.RebillModelMovedRows)
|
||
}
|
||
// And the gate asks: a $0 figure would have slipped under any threshold.
|
||
r2.Resnapshot = true
|
||
if _, terr := r2.TranslateBook(ctx); terr == nil || !strings.Contains(terr.Error(), fmt.Sprintf("~$%.6f", proj.USD)) {
|
||
t.Fatalf("the consent gate must refuse quoting the paid model's figure; got: %v", terr)
|
||
}
|
||
}
|
||
|
||
// TestABilledDecodeRowIsDisclosedAsOldMoneyEndToEnd is ФЧ-4 on the production path: a row whose money
|
||
// cannot be re-priced (a billed 2xx with an unreadable body settles the reservation ESTIMATE against a
|
||
// `{}` usage, stagerun.go) makes the consent figure partly last season's money, and that fact must reach
|
||
// every surface the figure reaches — the projection, `status --json`, the status renderer's sentence and
|
||
// BOTH refusal texts — as one count in one wording.
|
||
//
|
||
// Mutation this catches: delete the `if !calls[i].priced { fromHistory = true }` mark in reprice.go's
|
||
// usd() — the amount stays right, and every disclosure below goes silent (cmd/tmmutate,
|
||
// FC1-unpriced-not-disclosed).
|
||
func TestABilledDecodeRowIsDisclosedAsOldMoneyEndToEnd(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||
body, _ := io.ReadAll(req.Body)
|
||
if !isEditBody(string(body)) {
|
||
fmt.Fprint(w, `NOT JSON AT ALL`) // 200 with a garbage body → BilledDecodeError, estimate settled
|
||
return
|
||
}
|
||
writeFakeCompletion(w, "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop")
|
||
}))
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatalf("a billed decode failure must flag, not crash: %v", err)
|
||
}
|
||
if res1.Chunks[0].FlagReason != FlagDecodeError {
|
||
t.Fatalf("premise: the draft must be flagged decode_error, got %+v", res1.Chunks[0])
|
||
}
|
||
committed, _, err := r1.Store.SpentUSD("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
if committed <= 0 {
|
||
t.Fatal("premise: the settled estimate is real money")
|
||
}
|
||
|
||
driftPipelineVersion(t, bookPath)
|
||
r2 := newRunner(t, bookPath)
|
||
statuses, err := r2.Store.ChunkStatusesForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
chunks := chunksOf(t, r2)
|
||
proj, err := r2.projectRebill(statuses, chunks, func() ([]chunk.Chunk, error) { return chunks, nil }, newRepricerT(t, r2))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// The flagged draft is billed and superseded, so it is projected; the skipped edit never reached a
|
||
// provider and is not. Its money is the settled estimate, carried as billed — and DISCLOSED.
|
||
if proj.Rows != 1 || !nearUSD(proj.USD, committed) {
|
||
t.Fatalf("setup: one billed row at its settled estimate $%.6f, got %d row(s) at $%.6f", committed, proj.Rows, proj.USD)
|
||
}
|
||
if proj.HistoricalRows != 1 {
|
||
t.Fatalf("a row with no usage to re-price must be counted as quoted from history: HistoricalRows=%d", proj.HistoricalRows)
|
||
}
|
||
|
||
rep, err := r2.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.RebillHistoricalRows != 1 || !nearUSD(rep.RebillUSD, proj.USD) {
|
||
t.Fatalf("status must carry the disclosure with the figure: rebill_historical_rows=%d rebill_usd=%.6f (want 1, %.6f)", rep.RebillHistoricalRows, rep.RebillUSD, proj.USD)
|
||
}
|
||
wire, err := json.Marshal(rep)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !strings.Contains(string(wire), `"rebill_historical_rows":1`) {
|
||
t.Fatalf("status --json must carry rebill_historical_rows beside rebill_usd; got: %s", wire)
|
||
}
|
||
disclosure := "1 of the 1 unit(s) carry, wholly or in part, the amount they were originally billed at"
|
||
if basis := rep.RebillFigureBasis(); !strings.Contains(basis, disclosure) {
|
||
t.Fatalf("the status sentence must disclose the old money; got: %s", basis)
|
||
}
|
||
|
||
// Both refusal texts — the threshold's and the named cap's — carry the same disclosure in the same
|
||
// words as status. The cap is set below the projection so that branch is the one that fires.
|
||
r2.Resnapshot = true
|
||
_, terr := r2.TranslateBook(ctx)
|
||
if terr == nil {
|
||
t.Fatal("the re-payment is over the threshold — the run must refuse")
|
||
}
|
||
if !strings.Contains(terr.Error(), disclosure) || !strings.Contains(terr.Error(), rep.RebillFigureBasis()) {
|
||
t.Errorf("the threshold refusal must disclose the old money in status's own words; got: %v", terr)
|
||
}
|
||
r2.Close() // the store's lock admits one writer per book
|
||
r3 := newRunner(t, bookPath)
|
||
defer r3.Close()
|
||
r3.Resnapshot = true
|
||
r3.AcceptRebill = RebillConsent{Given: true, Capped: true, CapUSD: proj.USD / 2}
|
||
_, cerr := r3.TranslateBook(ctx)
|
||
if cerr == nil || !strings.Contains(cerr.Error(), "caps consent at") {
|
||
t.Fatalf("a cap below the projection must refuse naming the cap; got: %v", cerr)
|
||
}
|
||
if !strings.Contains(cerr.Error(), disclosure) || !strings.Contains(cerr.Error(), rep.RebillFigureBasis()) {
|
||
t.Errorf("the capped refusal must disclose the old money in status's own words (ФЧ-4б); got: %v", cerr)
|
||
}
|
||
}
|
||
|
||
// TestEveryChunkOfAChapterIsRePricedFromItsOwnCalls pins the repricer's cell key at the POSITION —
|
||
// chapter and chunk — on a chapter the cut splits into several chunks whose calls cost different amounts.
|
||
// Every fixture of the money path used to have one chunk per chapter, so a cell keyed by chapter alone
|
||
// (pooling the neighbours' calls, and letting the newest-first walk eat the neighbour's money) left the
|
||
// whole battery green (backlog row 197 / ФЧ-1). Here the chunks cost differently on purpose, so pooled
|
||
// calls overshoot one row's money or leave another's cell empty, and both read as «quoted from history».
|
||
func TestEveryChunkOfAChapterIsRePricedFromItsOwnCalls(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||
body, _ := io.ReadAll(req.Body)
|
||
s := string(body)
|
||
text := "ЧЕРНОВИК ПЕРЕВОДА"
|
||
if isEditBody(s) {
|
||
text = "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД"
|
||
}
|
||
// The chunk that carries 鈴木 answers with a different token count, so its call costs a different
|
||
// amount from its neighbours' — the asymmetry a pooled cell cannot hide behind.
|
||
completion := 500
|
||
if strings.Contains(s, "鈴木") {
|
||
completion = 900
|
||
}
|
||
tb, _ := json.Marshal(text)
|
||
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":%d,"prompt_tokens_details":{"cached_tokens":200}}}`, tb, completion)
|
||
}))
|
||
defer srv.Close()
|
||
const seg = "\nsegmentation:\n draft_budget_out: 24\n edit_ceiling_out: 8000\n fertility: { cjk: 1.1978, other: 0.3852 }\n"
|
||
body := "<p>静かな図書館の朝。鈴木は本を読んだ。外では雨が降っていた。彼は窓を見た。時間は過ぎた。</p>"
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||
epub: []chunktest.Chapter{{ID: "c1", Href: "c1.xhtml", Body: body}},
|
||
spine: []string{"c1"}, regenerate: 0, waveWorkers: 1, gatesYAML: seg,
|
||
})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
rows, err := r1.Store.ChunkStatusesForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
committed, _, err := r1.Store.SpentUSD("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
// The premise, asserted: several DRAFT chunks in the one chapter, and not all at the same price.
|
||
costs := map[float64]bool{}
|
||
drafts := 0
|
||
for _, cs := range rows {
|
||
if cs.Stage == "draft" {
|
||
drafts++
|
||
costs[cs.CostUSD] = true
|
||
}
|
||
}
|
||
if drafts < 2 || len(costs) < 2 {
|
||
t.Fatalf("premise: the chapter must split into several draft chunks of different cost, got %d draft row(s) at %d distinct cost(s)", drafts, len(costs))
|
||
}
|
||
|
||
driftPipelineVersion(t, bookPath)
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
chunks := chunksOf(t, r2)
|
||
rp := newRepricerT(t, r2)
|
||
proj, err := r2.projectRebill(rows, chunks, func() ([]chunk.Chunk, error) { return chunks, nil }, rp)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if proj.Rows != len(rows) {
|
||
t.Fatalf("setup: every row is superseded, got %d of %d", proj.Rows, len(rows))
|
||
}
|
||
// Prices did not move, so each row re-prices to exactly its own billed amount, from its own calls.
|
||
if proj.HistoricalRows != 0 {
|
||
t.Fatalf("%d row(s) were quoted from history — a row's calls were found in another chunk's cell, or its own cell was empty", proj.HistoricalRows)
|
||
}
|
||
if !nearUSD(proj.USD, committed) {
|
||
t.Fatalf("the projection is $%.6f but the book was billed $%.6f at the same table", proj.USD, committed)
|
||
}
|
||
for _, cs := range rows {
|
||
if usd, fromHistory, _ := rp.usd(cs); fromHistory || !nearUSD(usd, cs.CostUSD) {
|
||
t.Errorf("ch%d/chunk%d/%s re-prices to $%.6f (fromHistory=%v), want its own $%.6f", cs.Chapter, cs.ChunkIdx, cs.Stage, usd, fromHistory, cs.CostUSD)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestAnotherBooksMoneyNeverEntersTheConsentNumber pins, on the pipeline path, the book filter of
|
||
// store.CheckpointUsageForBook — the only thing between one book's consent number and another book's
|
||
// checkpoints when two books share a project database (backlog row 197 / ФЧ-1, reproduced by the
|
||
// acceptance: removing `WHERE j.book_id = ?` left every money test green). The other book's calls sit at
|
||
// the SAME positions at a different price, so if they leak in, the newest-first walk overshoots this
|
||
// book's rows and the projection admits it is no longer re-priced from this book's own calls.
|
||
func TestAnotherBooksMoneyNeverEntersTheConsentNumber(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
committed, _, err := r1.Store.SpentUSD("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// A second book in the same database, billed AFTER this one at the same positions and ten times the
|
||
// price: exactly the rows a missing filter would hand to this book's repricer as its newest calls.
|
||
if err := r1.Store.UpsertSnapshot("snap-other", "brief-other", `{}`); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, stage := range []string{"draft", "edit"} {
|
||
job, err := r1.Store.EnsureJob("other-book", 1, stage, "snap-other")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
res, _, err := r1.Store.Reserve("other-book", 10*fakeCallUSD, store.Ceilings{BookUSD: 100, DayUSD: 100})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := r1.Store.SettleWithCheckpoint(res, 10*fakeCallUSD, store.Checkpoint{
|
||
RequestHash: "other-" + stage, JobID: job.ID, ChunkIdx: 0, Stage: stage, Role: "translator",
|
||
ModelRequested: "fake-model", ModelActual: "fake-model",
|
||
UsageJSON: `{"PromptTokens":10000,"CachedTokens":2000,"CompletionTokens":5000}`, CostUSD: 10 * fakeCallUSD,
|
||
}, nil); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
r1.Close()
|
||
|
||
driftPipelineVersion(t, bookPath)
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
statuses, err := r2.Store.ChunkStatusesForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
chunks := chunksOf(t, r2)
|
||
proj, err := r2.projectRebill(statuses, chunks, func() ([]chunk.Chunk, error) { return chunks, nil }, newRepricerT(t, r2))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if proj.Rows != 2 {
|
||
t.Fatalf("setup: both of this book's rows are superseded, got %d", proj.Rows)
|
||
}
|
||
if proj.HistoricalRows != 0 || !nearUSD(proj.USD, committed) {
|
||
t.Fatalf("this book's re-payment is $%.6f from its own calls; got $%.6f with %d row(s) quoted from history — another book's checkpoints reached the repricer",
|
||
committed, proj.USD, proj.HistoricalRows)
|
||
}
|
||
rep, err := r2.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// And the book's own cost projection is its own too — the other book cost ten times as much.
|
||
if !nearUSD(rep.ProjectedBookUSD, committed) {
|
||
t.Fatalf("projected_book_usd = $%.6f, want this book's own $%.6f", rep.ProjectedBookUSD, committed)
|
||
}
|
||
}
|
||
|
||
// TestAnEscalatedRowIsRePricedAtTheNewHopEndToEnd is the hop move through the STORE rather than through a
|
||
// hand-built CheckpointUsage: the escalation flag of a checkpoint has to reach the repricer for it to
|
||
// price the hop call at the stage's CURRENT escalate_to, and nothing else on the money path reads that
|
||
// column. A row that escalated once is projected as its primary call at the primary's price plus its hop
|
||
// call at the new hop's price — and disclosed as priced across a model move.
|
||
//
|
||
// Mutation this catches: read `0` instead of `c.escalation` in store.CheckpointUsageForBook — the hop call
|
||
// is then compared against the PRIMARY's slug, priced at the primary's table, and the ten-times figure
|
||
// below collapses to the old number.
|
||
func TestAnEscalatedRowIsRePricedAtTheNewHopEndToEnd(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
bookPath := setupEscalationProject(t, srv.URL, 1.0, nil)
|
||
addModel(t, bookPath, "fake-dear", 10.0)
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// The premise, from the store: the draft row carries a primary call and ONE escalation call to the
|
||
// fixture's fallback, and the flag that tells them apart is on file.
|
||
usage, err := r1.Store.CheckpointUsageForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
hops, primaries := 0, 0
|
||
for _, cu := range usage {
|
||
if cu.Stage != "draft" {
|
||
continue
|
||
}
|
||
if cu.Escalation {
|
||
hops++
|
||
if cu.ModelRequested != "fake-fallback" {
|
||
t.Fatalf("premise: the hop call must be recorded against fake-fallback, got %s", cu.ModelRequested)
|
||
}
|
||
} else {
|
||
primaries++
|
||
}
|
||
}
|
||
if hops != 1 || primaries != 1 {
|
||
t.Fatalf("premise: one primary and one escalation call on the draft row, got primaries=%d hops=%d", primaries, hops)
|
||
}
|
||
committed, _, err := r1.Store.SpentUSD("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
|
||
// The hop moves to a model ten times the price, and the book drifts so its rows are projected.
|
||
path := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
changed := strings.Replace(string(raw), "escalate_to: fake-fallback", "escalate_to: fake-dear", 1)
|
||
if changed == string(raw) {
|
||
t.Fatal("setup: the fixture's escalate_to was not found")
|
||
}
|
||
writeFile(t, path, changed)
|
||
driftPipelineVersion(t, bookPath)
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
statuses, err := r2.Store.ChunkStatusesForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
chunks := chunksOf(t, r2)
|
||
proj, err := r2.projectRebill(statuses, chunks, func() ([]chunk.Chunk, error) { return chunks, nil }, newRepricerT(t, r2))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if proj.Rows != 2 {
|
||
t.Fatalf("setup: the draft and the edit rows are superseded, got %d", proj.Rows)
|
||
}
|
||
// Everything re-prices to what it was billed, except the hop call, which is now nine calls dearer.
|
||
if want := committed + 9*fakeCallUSD; !nearUSD(proj.USD, want) {
|
||
t.Fatalf("the escalated row must be priced at the NEW hop for its hop call: projected $%.6f, want $%.6f (billed $%.6f)", proj.USD, want, committed)
|
||
}
|
||
if proj.ModelMovedRows != 1 || proj.HistoricalRows != 0 {
|
||
t.Fatalf("exactly the escalated row is priced across a model move: ModelMovedRows=%d HistoricalRows=%d", proj.ModelMovedRows, proj.HistoricalRows)
|
||
}
|
||
rep, err := r2.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !nearUSD(rep.RebillUSD, proj.USD) || rep.RebillModelMovedRows != 1 {
|
||
t.Fatalf("status must publish the same figure and count: $%.6f moved=%d, want $%.6f moved=1", rep.RebillUSD, rep.RebillModelMovedRows, proj.USD)
|
||
}
|
||
}
|
||
|
||
// setModelPrice re-prices ONE model in the fixture catalogue, leaving the others alone — so a test can
|
||
// make the difference between two slugs visible in dollars.
|
||
func setModelPrice(t *testing.T, bookPath, model string, factor float64) {
|
||
t.Helper()
|
||
path := filepath.Join(filepath.Dir(bookPath), "models.yaml")
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
lines := strings.Split(string(raw), "\n")
|
||
found := false
|
||
for i, l := range lines {
|
||
if !strings.HasPrefix(strings.TrimSpace(l), model+":") {
|
||
continue
|
||
}
|
||
for j := i + 1; j < len(lines) && j < i+5; j++ {
|
||
if !strings.Contains(lines[j], "price:") {
|
||
continue
|
||
}
|
||
lines[j] = fmt.Sprintf(" price: { input_per_m: %g, cached_per_m: %g, cache_write_per_m: 0, output_per_m: %g }",
|
||
1.0*factor, 0.1*factor, 2.0*factor)
|
||
found = true
|
||
break
|
||
}
|
||
break
|
||
}
|
||
if !found {
|
||
t.Fatalf("setup: model %q has no price line in the fixture catalogue", model)
|
||
}
|
||
writeFile(t, path, strings.Join(lines, "\n"))
|
||
}
|
||
|
||
// TestTheModelAxisReadsTheRESOLVEDSlotNotTheConfiguredOne pins WHICH of a stage's two model slots the
|
||
// re-pricing asks. A book that declares content labels is routed by `label_models` to a different model
|
||
// than its `model:` key (D39.26), and `ResolvedModel` is the one the run will actually call — so it is the
|
||
// one «the model this stage resolves to TODAY» has to mean. Reading the configured slug instead prices a
|
||
// labelled book's re-payment at a model the run never reaches, and claims no routing move while the
|
||
// routing is the whole point.
|
||
//
|
||
// Mutation this catches: `currentModelFor` reading `st.Model`/`st.EscalateTo` instead of
|
||
// `st.ResolvedModel`/`st.ResolvedHop` — the whole pipeline package stays green without this test.
|
||
func TestTheModelAxisReadsTheRESOLVEDSlotNotTheConfiguredOne(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newModelEchoProvider(rec)
|
||
defer srv.Close()
|
||
// The book declares `adult`; the policy routes the EDIT stage to fake-label, which the catalogue
|
||
// prices ten times the configured fake-model.
|
||
bookPath := setupEscalationProject(t, srv.URL, 1.0, &labelRouting{
|
||
acceptsLabels: []string{"adult"}, bookLabels: []string{"adult"},
|
||
policy: "adult", action: "route", editModel: "fake-label", chainModel: "fake-labelhop",
|
||
})
|
||
setModelPrice(t, bookPath, "fake-label", 10.0)
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
|
||
// The premise, asserted rather than assumed: the two slots genuinely differ for this stage.
|
||
var edit *config.Stage
|
||
for i := range r.Pipeline.Stages {
|
||
if r.Pipeline.Stages[i].Name == "edit" {
|
||
edit = &r.Pipeline.Stages[i]
|
||
}
|
||
}
|
||
if edit == nil {
|
||
t.Fatal("fixture drifted: no edit stage")
|
||
}
|
||
if edit.Model != "fake-model" || edit.ResolvedModel != "fake-label" {
|
||
t.Fatalf("premise: the label must route the edit stage away from its configured model; got model=%q resolved=%q",
|
||
edit.Model, edit.ResolvedModel)
|
||
}
|
||
|
||
const usage = `{"PromptTokens":1000,"CachedTokens":200,"CompletionTokens":500}`
|
||
// A call bought under the configured model, before the label routed the stage elsewhere.
|
||
usd, priced, moved := r.repriceCheckpoint(store.CheckpointUsage{
|
||
Stage: "edit", ModelRequested: "fake-model", ModelActual: "fake-model", UsageJSON: usage, CostUSD: 0.01})
|
||
if !priced {
|
||
t.Fatal("the call has its usage on file and must be re-priced")
|
||
}
|
||
if want := 10 * fakeCallUSD; !nearUSD(usd, want) || !moved {
|
||
t.Fatalf("the re-payment must be priced at the model the label ROUTES to: got ($%.6f, moved=%v), want ($%.6f, moved=true)",
|
||
usd, moved, want)
|
||
}
|
||
}
|