textmachine/platform/internal/httpapi/idempotency_test.go

443 lines
19 KiB
Go

package httpapi
import (
"bytes"
"context"
"errors"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"textmachine/platform/internal/books"
"textmachine/platform/internal/pgstore"
)
// idempotency_test.go: the HTTP half of `Idempotency-Key`.
//
// It had no test at all, and that is where three of the pack's defects lived: what goes INTO the
// fingerprint, and which context the key is settled on, are decidable only here — the store half
// sees an opaque digest and cannot tell a good one from a bad one.
// fakeKeys is a store that remembers what it was asked, so a test can observe the fingerprint the
// handler computed rather than only the status it answered.
type fakeKeys struct {
claims []pgstore.IdempotencyKey
completed []pgstore.IdempotencyKey
released []pgstore.IdempotencyKey
// byKey replays a stored answer, keyed by the fingerprint the handler computes.
byKey map[string]pgstore.IdempotentResponse
err error
// token is the claim the last granted attempt was given; settled is what each ending presented.
token pgstore.ClaimToken
settled []pgstore.ClaimToken
}
func newFakeKeys() *fakeKeys {
return &fakeKeys{byKey: map[string]pgstore.IdempotentResponse{}}
}
func (f *fakeKeys) id(k pgstore.IdempotencyKey) string {
return k.UserID + "|" + k.Method + "|" + k.Path + "|" + k.Key + "|" + string(k.Fingerprint)
}
func (f *fakeKeys) ClaimIdempotency(_ context.Context, k pgstore.IdempotencyKey, _ time.Time) (*pgstore.IdempotentResponse, pgstore.ClaimToken, error) {
f.claims = append(f.claims, k)
if f.err != nil {
return nil, "", f.err
}
if stored, ok := f.byKey[f.id(k)]; ok {
return &stored, "", nil
}
f.token = pgstore.NewClaimToken()
return nil, f.token, nil
}
func (f *fakeKeys) CompleteIdempotency(ctx context.Context, k pgstore.IdempotencyKey, token pgstore.ClaimToken, resp pgstore.IdempotentResponse, _ time.Time) error {
if err := ctx.Err(); err != nil {
return err // a cancelled context is exactly what the defect was about
}
f.settled = append(f.settled, token)
f.completed = append(f.completed, k)
f.byKey[f.id(k)] = resp
return nil
}
func (f *fakeKeys) ReleaseIdempotency(ctx context.Context, k pgstore.IdempotencyKey, token pgstore.ClaimToken) error {
if err := ctx.Err(); err != nil {
return err
}
f.settled = append(f.settled, token)
f.released = append(f.released, k)
return nil
}
// upload builds a well-formed intake body. `framing` changes the multipart boundary WITHOUT changing
// a single declared part, which is what a retry from a different client library looks like.
func idemUpload(t *testing.T, title, file, content, framing string) (string, *bytes.Buffer) {
t.Helper()
var body bytes.Buffer
w := multipart.NewWriter(&body)
if err := w.SetBoundary(framing); err != nil {
t.Fatal(err)
}
for name, value := range map[string]string{"title": title, "source_lang": "zh", "target_lang": "ru"} {
if err := w.WriteField(name, value); err != nil {
t.Fatal(err)
}
}
part, err := w.CreateFormFile("file", file)
if err != nil {
t.Fatal(err)
}
if _, err := part.Write([]byte(content)); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
return w.FormDataContentType(), &body
}
// The fingerprint is the DECLARED parts and only those: every one of them, and nothing that is not
// declared. It is what a claim can be taken on before a byte of the file has been read, and it does
// not settle identity — the file's digest does, afterwards.
//
// ⚠ The request's `Content-Length` used to be in here as a stand-in for the file's size. It counts
// the multipart framing, so a retry from another client library differed and was answered
// `key_reused`; a chunked body declared nothing at all, and two different files under one key were
// then indistinguishable (PD-262).
//
// Mutation caught: dropping any declared part; putting the request's length back in.
func TestTheFingerprintIsExactlyTheDeclaredParts(t *testing.T) {
decl := books.Intake{Title: "Книга", SourceLang: "zh", TargetLang: "ru", Filename: "roman.txt"}
base := intakeFingerprint(decl, true)
if !bytes.Equal(base, intakeFingerprint(decl, true)) {
t.Fatal("two identical declarations produced different fingerprints")
}
for _, tc := range []struct {
name string
with books.Intake
}{
{"another title", books.Intake{Title: "Другая", SourceLang: "zh", TargetLang: "ru", Filename: "roman.txt"}},
{"another source language", books.Intake{Title: "Книга", SourceLang: "ja", TargetLang: "ru", Filename: "roman.txt"}},
{"another target language", books.Intake{Title: "Книга", SourceLang: "zh", TargetLang: "en", Filename: "roman.txt"}},
{"another file name", books.Intake{Title: "Книга", SourceLang: "zh", TargetLang: "ru", Filename: "other.txt"}},
} {
if bytes.Equal(base, intakeFingerprint(tc.with, true)) {
t.Errorf("%s read as the same request", tc.name)
}
}
// An empty title GIVEN and no title at all are different requests: the first says "name it from
// the file", the second is a form that omitted the field.
empty := books.Intake{SourceLang: "zh", TargetLang: "ru", Filename: "roman.txt"}
if bytes.Equal(intakeFingerprint(empty, true), intakeFingerprint(empty, false)) {
t.Error("a declared empty title read as an absent one")
}
}
// The key is claimed with the scope the canon ratified — (principal, method, path) — so the HTTP
// layer must hand the store the REQUEST's own path.
//
// Mutation caught: passing the route pattern, or dropping method/path from the claim.
func TestTheClaimCarriesTheRequestsOwnOperation(t *testing.T) {
keys := newFakeKeys()
h := v0ServerWith(t, Deps{Library: &fakeLibrary{}, Keys: keys,
Intake: &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга", SourceLang: "zh",
TargetLang: "ru", Status: "parsing"}},
Capabilities: Capabilities{
Pairs: []LanguagePair{{Source: "zh", Target: "ru", Available: true}}, IntakeEnabled: true,
IntakeMaxBytes: 64 << 20, PageSizeDefault: 100,
}})
ct, body := idemUpload(t, "Книга", "roman.txt", "первая глава", "AAAABBBBCCCC")
r := httptest.NewRequest("POST", "/v0/books", body)
r.Header.Set("Content-Type", ct)
r.Header.Set("X-TM-Client", "probe")
r.Header.Set("Authorization", "Bearer t")
r.Header.Set("Idempotency-Key", "one-per-gesture")
h.ServeHTTP(httptest.NewRecorder(), r)
if len(keys.claims) != 1 {
t.Fatalf("the upload took %d claims, want 1", len(keys.claims))
}
got := keys.claims[0]
if got.Method != "POST" || got.Path != "/v0/books" || got.Key != "one-per-gesture" {
t.Errorf("claimed %+v, want the request's own method and path", got)
}
if len(got.Fingerprint) == 0 {
t.Error("the claim carried no fingerprint at all")
}
}
// …and on the route where the two differ. `POST /v0/books` has a path identical to its pattern, so
// the case above passed with `r.Pattern` in place of `r.URL.Path` — and under that substitution every
// book's run start shares one scope, so one key would collide across two books.
//
// Mutation caught: claiming the route pattern instead of the request's path.
func TestTheClaimOfARunCarriesTheBooksOwnPath(t *testing.T) {
keys := newFakeKeys()
rn := &fakeRuns{run: pgstore.Run{ID: "run_1", BookID: "bk_1", Status: "translating", CeilingChapters: 10}}
h := v0ServerWith(t, Deps{Library: &fakeLibrary{}, Runs: rn, Keys: keys})
r := httptest.NewRequest("POST", "/v0/books/bk_1/runs",
strings.NewReader(`{"stop_for_signing":false,"ceiling_chapters":10}`))
r.Header.Set("Content-Type", "application/json")
r.Header.Set("X-TM-Client", "probe")
r.Header.Set("Authorization", "Bearer t")
r.Header.Set("Idempotency-Key", "one-per-gesture")
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
if w.Code != http.StatusAccepted {
t.Fatalf("starting the run answered %d: %s", w.Code, w.Body.String())
}
if len(keys.claims) != 1 || keys.claims[0].Path != "/v0/books/bk_1/runs" {
t.Errorf("claimed %+v, want the path of THIS book's runs", keys.claims)
}
}
// Every ending of an attempt presents the claim it was granted, and that is what stops a superseded
// attempt from answering for a claim, or deleting the row of the successor already doing the work.
//
// Mutation caught: settling with the zero token; keeping the token out of idempotent.
func TestEveryEndingPresentsTheClaimItWasGranted(t *testing.T) {
for _, tc := range []struct {
name string
intake *fakeIntake
}{
{"a completed attempt", &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга",
SourceLang: "zh", TargetLang: "ru", Status: "parsing"}}},
{"an attempt that failed", &fakeIntake{err: errors.New("the storage is gone")}},
} {
t.Run(tc.name, func(t *testing.T) {
keys := newFakeKeys()
idemPost(t, idemServer(t, keys, tc.intake), "one-per-gesture", "AAAABBBBCCCC", nil)
if keys.token == "" {
t.Fatal("no claim was granted, so this fixture cannot observe the property")
}
if len(keys.settled) != 1 || keys.settled[0] != keys.token {
t.Errorf("the ending presented %q, want the claim %q it was granted", keys.settled, keys.token)
}
})
}
}
// idemServer is the intake surface with an idempotency store behind it.
func idemServer(t *testing.T, keys IdempotencyKeys, intake *fakeIntake) http.Handler {
t.Helper()
return v0ServerWith(t, Deps{Library: &fakeLibrary{}, Keys: keys, Intake: intake,
Capabilities: Capabilities{
Pairs: []LanguagePair{{Source: "zh", Target: "ru", Available: true}}, IntakeEnabled: true,
IntakeMaxBytes: 64 << 20, PageSizeDefault: 100,
}})
}
// idemPostTrailing is one upload with a field sent AFTER the file — the shape the contract refuses.
func idemPostTrailing(t *testing.T, h http.Handler, key, content string) *httptest.ResponseRecorder {
t.Helper()
var body bytes.Buffer
w := multipart.NewWriter(&body)
if err := w.SetBoundary("AAAABBBBCCCC"); err != nil {
t.Fatal(err)
}
for name, value := range map[string]string{"title": "Книга", "source_lang": "zh", "target_lang": "ru"} {
if err := w.WriteField(name, value); err != nil {
t.Fatal(err)
}
}
part, err := w.CreateFormFile("file", "roman.txt")
if err != nil {
t.Fatal(err)
}
if _, err := part.Write([]byte(content)); err != nil {
t.Fatal(err)
}
if err := w.WriteField("late", "after the file"); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
r := httptest.NewRequest("POST", "/v0/books", &body)
r.Header.Set("Content-Type", w.FormDataContentType())
r.Header.Set("X-TM-Client", "probe")
r.Header.Set("Authorization", "Bearer t")
r.Header.Set("Idempotency-Key", key)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, r)
return rec
}
// idemPostFile is idemPost with the FILE's bytes chosen by the caller and the framing fixed: what it
// varies is the one thing the declaration does not carry.
func idemPostFile(t *testing.T, h http.Handler, key, content string) *httptest.ResponseRecorder {
t.Helper()
return idemPostWith(t, h, key, "AAAABBBBCCCC", content, nil)
}
func idemPost(t *testing.T, h http.Handler, key, framing string, ctx context.Context) *httptest.ResponseRecorder {
t.Helper()
return idemPostWith(t, h, key, framing, "первая глава", ctx)
}
func idemPostWith(t *testing.T, h http.Handler, key, framing, content string, ctx context.Context) *httptest.ResponseRecorder {
t.Helper()
ct, body := idemUpload(t, "Книга", "roman.txt", content, framing)
r := httptest.NewRequest("POST", "/v0/books", body)
if ctx != nil {
r = r.WithContext(ctx)
}
r.Header.Set("Content-Type", ct)
r.Header.Set("X-TM-Client", "probe")
r.Header.Set("Authorization", "Bearer t")
r.Header.Set("Idempotency-Key", key)
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
return w
}
// The four answers the store can give, as the SURFACE renders them. Only this side decides the
// status, the cause and the Retry-After, and none of the four was observed by anything.
func TestTheClaimsFourAnswersReachTheClient(t *testing.T) {
t.Run("a repeat replays the first answer and does no work", func(t *testing.T) {
keys, intake := newFakeKeys(), &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга",
SourceLang: "zh", TargetLang: "ru", Status: "parsing"}}
h := idemServer(t, keys, intake)
first := idemPost(t, h, "one-per-gesture", "AAAABBBBCCCC", nil)
if first.Code != http.StatusCreated {
t.Fatalf("the first upload answered %d", first.Code)
}
again := idemPost(t, h, "one-per-gesture", "AAAABBBBCCCC", nil)
if again.Code != first.Code || again.Body.String() != first.Body.String() ||
again.Header().Get("Location") != first.Header().Get("Location") {
t.Errorf("the replay answered %d %q %q, want the first answer verbatim",
again.Code, again.Header().Get("Location"), again.Body.String())
}
if intake.accepted != 1 {
t.Errorf("the intake ran %d times under one key", intake.accepted)
}
})
t.Run("another FILE under the same key is a conflict, not a replay", func(t *testing.T) {
// The declaration is identical — same title, same languages, same file name — and only the
// bytes differ. Decided on the declaration alone, the second upload was answered with the
// first book's Location and the user never got the book they sent (PD-262).
keys, intake := newFakeKeys(), &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга",
SourceLang: "zh", TargetLang: "ru", Status: "parsing"}}
h := idemServer(t, keys, intake)
if w := idemPostFile(t, h, "one-per-gesture", "первая глава"); w.Code != http.StatusCreated {
t.Fatalf("the first upload answered %d", w.Code)
}
w := idemPostFile(t, h, "one-per-gesture", "совершенно другая книга")
if w.Code != http.StatusConflict || !contains(w.Body.String(), `"key_reused"`) {
t.Errorf("a second, different book under one key answered %d %s", w.Code, w.Body.String())
}
if intake.accepted != 1 {
t.Errorf("the intake ran %d times: the refusal must not create a book either", intake.accepted)
}
// …and a repeat that carries a part AFTER the file is refused exactly as a fresh key would be:
// the same bytes must not be legal or not depending on which key they wear.
late := idemPostTrailing(t, h, "one-per-gesture", "первая глава")
if late.Code != http.StatusBadRequest || !contains(late.Body.String(), `"missing_or_late"`) {
t.Errorf("a repeat with a part after the file answered %d %s", late.Code, late.Body.String())
}
// …and the FIRST file, re-sent, is still the replay it always was.
again := idemPostFile(t, h, "one-per-gesture", "первая глава")
if again.Code != http.StatusCreated {
t.Errorf("the same file re-sent answered %d, want the stored answer", again.Code)
}
if intake.accepted != 1 {
t.Errorf("the replay ran the intake again: %d", intake.accepted)
}
})
t.Run("another request under the same key is a conflict", func(t *testing.T) {
keys := newFakeKeys()
keys.err = pgstore.ErrKeyReused
w := idemPost(t, idemServer(t, keys, &fakeIntake{}), "one-per-gesture", "AAAABBBBCCCC", nil)
if w.Code != http.StatusConflict || !contains(w.Body.String(), `"key_reused"`) {
t.Errorf("a re-used key answered %d %s", w.Code, w.Body.String())
}
})
t.Run("a repeat while the first is running is told to wait", func(t *testing.T) {
keys := newFakeKeys()
keys.err = pgstore.ErrKeyInFlight
w := idemPost(t, idemServer(t, keys, &fakeIntake{}), "one-per-gesture", "AAAABBBBCCCC", nil)
if w.Code != http.StatusConflict || !contains(w.Body.String(), `"key_in_flight"`) {
t.Errorf("an in-flight key answered %d %s", w.Code, w.Body.String())
}
if w.Header().Get("Retry-After") == "" {
t.Error("no Retry-After on the answer that asks the client to wait")
}
})
t.Run("a claim that lost the race too often is told to wait, not answered 500", func(t *testing.T) {
// PD-369. Contention on one key is a legitimate request meeting other legitimate requests, and
// the answer must be one of the contract's own — 409 `key_in_flight` with a Retry-After — not
// an internal error the caller can do nothing with. The two paths share ONE writer
// (`keyInFlight`), and the case order matters: ErrKeyContended WRAPS ErrKeyInFlight, so the
// narrower case has to come first or this test reads the same as the one above it.
keys := newFakeKeys()
keys.err = pgstore.ErrKeyContended
w := idemPost(t, idemServer(t, keys, &fakeIntake{}), "one-per-gesture", "AAAABBBBCCCC", nil)
if w.Code != http.StatusConflict || !contains(w.Body.String(), `"key_in_flight"`) {
t.Errorf("a contended key answered %d %s, want 409 key_in_flight", w.Code, w.Body.String())
}
if w.Header().Get("Retry-After") == "" {
t.Error("no Retry-After on the answer that asks the client to wait")
}
if contains(w.Body.String(), `"internal_error"`) {
t.Error("a legitimate request under contention was answered with an internal error")
}
})
t.Run("an attempt that did not complete gives the key back", func(t *testing.T) {
keys := newFakeKeys()
h := idemServer(t, keys, &fakeIntake{err: errors.New("the storage is gone")})
if w := idemPost(t, h, "one-per-gesture", "AAAABBBBCCCC", nil); w.Code < 400 {
t.Fatalf("a failed intake answered %d", w.Code)
}
if len(keys.released) != 1 || len(keys.completed) != 0 {
t.Errorf("released %d, completed %d a failed attempt must give the key back",
len(keys.released), len(keys.completed))
}
})
}
// The receipt is written on a context of its OWN: the client that will retry is precisely the one
// that hung up, and on the request's context the key stays in flight for the whole claim window.
//
// Mutation caught: settleCtx without WithoutCancel.
func TestTheReceiptSurvivesAClientThatHungUp(t *testing.T) {
keys := newFakeKeys()
h := idemServer(t, keys, &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга",
SourceLang: "zh", TargetLang: "ru", Status: "parsing"}})
ctx, cancel := context.WithCancel(context.Background())
cancel()
idemPost(t, h, "one-per-gesture", "AAAABBBBCCCC", ctx)
if len(keys.completed) != 1 {
t.Errorf("the receipt was not recorded for a client that hung up: completed=%d", len(keys.completed))
}
}
// A key over the contract's bound is refused and never truncated — and the store is not touched.
func TestAnOverlongKeyIsRefusedBeforeAnythingIsClaimed(t *testing.T) {
keys := newFakeKeys()
h := v0ServerWith(t, Deps{Library: &fakeLibrary{}, Keys: keys,
Intake: &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга", SourceLang: "zh",
TargetLang: "ru", Status: "parsing"}},
Capabilities: Capabilities{
Pairs: []LanguagePair{{Source: "zh", Target: "ru", Available: true}}, IntakeEnabled: true,
IntakeMaxBytes: 64 << 20, PageSizeDefault: 100,
}})
ct, body := idemUpload(t, "Книга", "roman.txt", "текст", "AAAABBBBCCCC")
r := httptest.NewRequest("POST", "/v0/books", body)
r.Header.Set("Content-Type", ct)
r.Header.Set("X-TM-Client", "probe")
r.Header.Set("Authorization", "Bearer t")
r.Header.Set("Idempotency-Key", string(bytes.Repeat([]byte("k"), maxKeyLength+1)))
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
t.Errorf("an over-long key answered %d, want 400", w.Code)
}
if len(keys.claims) != 0 {
t.Errorf("the store was asked about a key the surface had already refused: %+v", keys.claims)
}
}