package httpapi import ( "bytes" "context" "errors" "mime/multipart" "net/http" "net/http/httptest" "strings" "testing" "time" "textmachine/platform/internal/books" "textmachine/platform/internal/pgstore" ) // idempotency_test.go: the HTTP half of `Idempotency-Key`. // // It had no test at all, and that is where three of the pack's defects lived: what goes INTO the // fingerprint, and which context the key is settled on, are decidable only here — the store half // sees an opaque digest and cannot tell a good one from a bad one. // fakeKeys is a store that remembers what it was asked, so a test can observe the fingerprint the // handler computed rather than only the status it answered. type fakeKeys struct { claims []pgstore.IdempotencyKey completed []pgstore.IdempotencyKey released []pgstore.IdempotencyKey // byKey replays a stored answer, keyed by the fingerprint the handler computes. byKey map[string]pgstore.IdempotentResponse err error // token is the claim the last granted attempt was given; settled is what each ending presented. token pgstore.ClaimToken settled []pgstore.ClaimToken } func newFakeKeys() *fakeKeys { return &fakeKeys{byKey: map[string]pgstore.IdempotentResponse{}} } func (f *fakeKeys) id(k pgstore.IdempotencyKey) string { return k.UserID + "|" + k.Method + "|" + k.Path + "|" + k.Key + "|" + string(k.Fingerprint) } func (f *fakeKeys) ClaimIdempotency(_ context.Context, k pgstore.IdempotencyKey, _ time.Time) (*pgstore.IdempotentResponse, pgstore.ClaimToken, error) { f.claims = append(f.claims, k) if f.err != nil { return nil, "", f.err } if stored, ok := f.byKey[f.id(k)]; ok { return &stored, "", nil } f.token = pgstore.NewClaimToken() return nil, f.token, nil } func (f *fakeKeys) CompleteIdempotency(ctx context.Context, k pgstore.IdempotencyKey, token pgstore.ClaimToken, resp pgstore.IdempotentResponse, _ time.Time) error { if err := ctx.Err(); err != nil { return err // a cancelled context is exactly what the defect was about } f.settled = append(f.settled, token) f.completed = append(f.completed, k) f.byKey[f.id(k)] = resp return nil } func (f *fakeKeys) ReleaseIdempotency(ctx context.Context, k pgstore.IdempotencyKey, token pgstore.ClaimToken) error { if err := ctx.Err(); err != nil { return err } f.settled = append(f.settled, token) f.released = append(f.released, k) return nil } // upload builds a well-formed intake body. `framing` changes the multipart boundary WITHOUT changing // a single declared part, which is what a retry from a different client library looks like. func idemUpload(t *testing.T, title, file, content, framing string) (string, *bytes.Buffer) { t.Helper() var body bytes.Buffer w := multipart.NewWriter(&body) if err := w.SetBoundary(framing); err != nil { t.Fatal(err) } for name, value := range map[string]string{"title": title, "source_lang": "zh", "target_lang": "ru"} { if err := w.WriteField(name, value); err != nil { t.Fatal(err) } } part, err := w.CreateFormFile("file", file) if err != nil { t.Fatal(err) } if _, err := part.Write([]byte(content)); err != nil { t.Fatal(err) } if err := w.Close(); err != nil { t.Fatal(err) } return w.FormDataContentType(), &body } // The fingerprint is the DECLARED parts and only those: every one of them, and nothing that is not // declared. It is what a claim can be taken on before a byte of the file has been read, and it does // not settle identity — the file's digest does, afterwards. // // ⚠ The request's `Content-Length` used to be in here as a stand-in for the file's size. It counts // the multipart framing, so a retry from another client library differed and was answered // `key_reused`; a chunked body declared nothing at all, and two different files under one key were // then indistinguishable (PD-262). // // Mutation caught: dropping any declared part; putting the request's length back in. func TestTheFingerprintIsExactlyTheDeclaredParts(t *testing.T) { decl := books.Intake{Title: "Книга", SourceLang: "zh", TargetLang: "ru", Filename: "roman.txt"} base := intakeFingerprint(decl, true) if !bytes.Equal(base, intakeFingerprint(decl, true)) { t.Fatal("two identical declarations produced different fingerprints") } for _, tc := range []struct { name string with books.Intake }{ {"another title", books.Intake{Title: "Другая", SourceLang: "zh", TargetLang: "ru", Filename: "roman.txt"}}, {"another source language", books.Intake{Title: "Книга", SourceLang: "ja", TargetLang: "ru", Filename: "roman.txt"}}, {"another target language", books.Intake{Title: "Книга", SourceLang: "zh", TargetLang: "en", Filename: "roman.txt"}}, {"another file name", books.Intake{Title: "Книга", SourceLang: "zh", TargetLang: "ru", Filename: "other.txt"}}, } { if bytes.Equal(base, intakeFingerprint(tc.with, true)) { t.Errorf("%s read as the same request", tc.name) } } // An empty title GIVEN and no title at all are different requests: the first says "name it from // the file", the second is a form that omitted the field. empty := books.Intake{SourceLang: "zh", TargetLang: "ru", Filename: "roman.txt"} if bytes.Equal(intakeFingerprint(empty, true), intakeFingerprint(empty, false)) { t.Error("a declared empty title read as an absent one") } } // The key is claimed with the scope the canon ratified — (principal, method, path) — so the HTTP // layer must hand the store the REQUEST's own path. // // Mutation caught: passing the route pattern, or dropping method/path from the claim. func TestTheClaimCarriesTheRequestsOwnOperation(t *testing.T) { keys := newFakeKeys() h := v0ServerWith(t, Deps{Library: &fakeLibrary{}, Keys: keys, Intake: &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга", SourceLang: "zh", TargetLang: "ru", Status: "parsing"}}, Capabilities: Capabilities{ Pairs: []LanguagePair{{Source: "zh", Target: "ru", Available: true}}, IntakeEnabled: true, IntakeMaxBytes: 64 << 20, PageSizeDefault: 100, }}) ct, body := idemUpload(t, "Книга", "roman.txt", "первая глава", "AAAABBBBCCCC") r := httptest.NewRequest("POST", "/v0/books", body) r.Header.Set("Content-Type", ct) r.Header.Set("X-TM-Client", "probe") r.Header.Set("Authorization", "Bearer t") r.Header.Set("Idempotency-Key", "one-per-gesture") h.ServeHTTP(httptest.NewRecorder(), r) if len(keys.claims) != 1 { t.Fatalf("the upload took %d claims, want 1", len(keys.claims)) } got := keys.claims[0] if got.Method != "POST" || got.Path != "/v0/books" || got.Key != "one-per-gesture" { t.Errorf("claimed %+v, want the request's own method and path", got) } if len(got.Fingerprint) == 0 { t.Error("the claim carried no fingerprint at all") } } // …and on the route where the two differ. `POST /v0/books` has a path identical to its pattern, so // the case above passed with `r.Pattern` in place of `r.URL.Path` — and under that substitution every // book's run start shares one scope, so one key would collide across two books. // // Mutation caught: claiming the route pattern instead of the request's path. func TestTheClaimOfARunCarriesTheBooksOwnPath(t *testing.T) { keys := newFakeKeys() rn := &fakeRuns{run: pgstore.Run{ID: "run_1", BookID: "bk_1", Status: "translating", OrderedChapters: ptr(10)}} h := v0ServerWith(t, Deps{Library: &fakeLibrary{}, Runs: rn, Keys: keys}) r := httptest.NewRequest("POST", "/v0/books/bk_1/runs", strings.NewReader(`{"stop_for_signing":false,"chapters":10}`)) r.Header.Set("Content-Type", "application/json") r.Header.Set("X-TM-Client", "probe") r.Header.Set("Authorization", "Bearer t") r.Header.Set("Idempotency-Key", "one-per-gesture") w := httptest.NewRecorder() h.ServeHTTP(w, r) if w.Code != http.StatusAccepted { t.Fatalf("starting the run answered %d: %s", w.Code, w.Body.String()) } if len(keys.claims) != 1 || keys.claims[0].Path != "/v0/books/bk_1/runs" { t.Errorf("claimed %+v, want the path of THIS book's runs", keys.claims) } } // Every ending of an attempt presents the claim it was granted, and that is what stops a superseded // attempt from answering for a claim, or deleting the row of the successor already doing the work. // // Mutation caught: settling with the zero token; keeping the token out of idempotent. func TestEveryEndingPresentsTheClaimItWasGranted(t *testing.T) { for _, tc := range []struct { name string intake *fakeIntake }{ {"a completed attempt", &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга", SourceLang: "zh", TargetLang: "ru", Status: "parsing"}}}, {"an attempt that failed", &fakeIntake{err: errors.New("the storage is gone")}}, } { t.Run(tc.name, func(t *testing.T) { keys := newFakeKeys() idemPost(t, idemServer(t, keys, tc.intake), "one-per-gesture", "AAAABBBBCCCC", nil) if keys.token == "" { t.Fatal("no claim was granted, so this fixture cannot observe the property") } if len(keys.settled) != 1 || keys.settled[0] != keys.token { t.Errorf("the ending presented %q, want the claim %q it was granted", keys.settled, keys.token) } }) } } // idemServer is the intake surface with an idempotency store behind it. func idemServer(t *testing.T, keys IdempotencyKeys, intake *fakeIntake) http.Handler { t.Helper() return v0ServerWith(t, Deps{Library: &fakeLibrary{}, Keys: keys, Intake: intake, Capabilities: Capabilities{ Pairs: []LanguagePair{{Source: "zh", Target: "ru", Available: true}}, IntakeEnabled: true, IntakeMaxBytes: 64 << 20, PageSizeDefault: 100, }}) } // idemPostTrailing is one upload with a field sent AFTER the file — the shape the contract refuses. func idemPostTrailing(t *testing.T, h http.Handler, key, content string) *httptest.ResponseRecorder { t.Helper() var body bytes.Buffer w := multipart.NewWriter(&body) if err := w.SetBoundary("AAAABBBBCCCC"); err != nil { t.Fatal(err) } for name, value := range map[string]string{"title": "Книга", "source_lang": "zh", "target_lang": "ru"} { if err := w.WriteField(name, value); err != nil { t.Fatal(err) } } part, err := w.CreateFormFile("file", "roman.txt") if err != nil { t.Fatal(err) } if _, err := part.Write([]byte(content)); err != nil { t.Fatal(err) } if err := w.WriteField("late", "after the file"); err != nil { t.Fatal(err) } if err := w.Close(); err != nil { t.Fatal(err) } r := httptest.NewRequest("POST", "/v0/books", &body) r.Header.Set("Content-Type", w.FormDataContentType()) r.Header.Set("X-TM-Client", "probe") r.Header.Set("Authorization", "Bearer t") r.Header.Set("Idempotency-Key", key) rec := httptest.NewRecorder() h.ServeHTTP(rec, r) return rec } // idemPostFile is idemPost with the FILE's bytes chosen by the caller and the framing fixed: what it // varies is the one thing the declaration does not carry. func idemPostFile(t *testing.T, h http.Handler, key, content string) *httptest.ResponseRecorder { t.Helper() return idemPostWith(t, h, key, "AAAABBBBCCCC", content, nil) } func idemPost(t *testing.T, h http.Handler, key, framing string, ctx context.Context) *httptest.ResponseRecorder { t.Helper() return idemPostWith(t, h, key, framing, "первая глава", ctx) } func idemPostWith(t *testing.T, h http.Handler, key, framing, content string, ctx context.Context) *httptest.ResponseRecorder { t.Helper() ct, body := idemUpload(t, "Книга", "roman.txt", content, framing) r := httptest.NewRequest("POST", "/v0/books", body) if ctx != nil { r = r.WithContext(ctx) } r.Header.Set("Content-Type", ct) r.Header.Set("X-TM-Client", "probe") r.Header.Set("Authorization", "Bearer t") r.Header.Set("Idempotency-Key", key) w := httptest.NewRecorder() h.ServeHTTP(w, r) return w } // The four answers the store can give, as the SURFACE renders them. Only this side decides the // status, the cause and the Retry-After, and none of the four was observed by anything. func TestTheClaimsFourAnswersReachTheClient(t *testing.T) { t.Run("a repeat replays the first answer and does no work", func(t *testing.T) { keys, intake := newFakeKeys(), &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга", SourceLang: "zh", TargetLang: "ru", Status: "parsing"}} h := idemServer(t, keys, intake) first := idemPost(t, h, "one-per-gesture", "AAAABBBBCCCC", nil) if first.Code != http.StatusCreated { t.Fatalf("the first upload answered %d", first.Code) } again := idemPost(t, h, "one-per-gesture", "AAAABBBBCCCC", nil) if again.Code != first.Code || again.Body.String() != first.Body.String() || again.Header().Get("Location") != first.Header().Get("Location") { t.Errorf("the replay answered %d %q %q, want the first answer verbatim", again.Code, again.Header().Get("Location"), again.Body.String()) } if intake.accepted != 1 { t.Errorf("the intake ran %d times under one key", intake.accepted) } }) t.Run("another FILE under the same key is a conflict, not a replay", func(t *testing.T) { // The declaration is identical — same title, same languages, same file name — and only the // bytes differ. Decided on the declaration alone, the second upload was answered with the // first book's Location and the user never got the book they sent (PD-262). keys, intake := newFakeKeys(), &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга", SourceLang: "zh", TargetLang: "ru", Status: "parsing"}} h := idemServer(t, keys, intake) if w := idemPostFile(t, h, "one-per-gesture", "первая глава"); w.Code != http.StatusCreated { t.Fatalf("the first upload answered %d", w.Code) } w := idemPostFile(t, h, "one-per-gesture", "совершенно другая книга") if w.Code != http.StatusConflict || !contains(w.Body.String(), `"key_reused"`) { t.Errorf("a second, different book under one key answered %d %s", w.Code, w.Body.String()) } if intake.accepted != 1 { t.Errorf("the intake ran %d times: the refusal must not create a book either", intake.accepted) } // …and a repeat that carries a part AFTER the file is refused exactly as a fresh key would be: // the same bytes must not be legal or not depending on which key they wear. late := idemPostTrailing(t, h, "one-per-gesture", "первая глава") if late.Code != http.StatusBadRequest || !contains(late.Body.String(), `"missing_or_late"`) { t.Errorf("a repeat with a part after the file answered %d %s", late.Code, late.Body.String()) } // …and the FIRST file, re-sent, is still the replay it always was. again := idemPostFile(t, h, "one-per-gesture", "первая глава") if again.Code != http.StatusCreated { t.Errorf("the same file re-sent answered %d, want the stored answer", again.Code) } if intake.accepted != 1 { t.Errorf("the replay ran the intake again: %d", intake.accepted) } }) t.Run("another request under the same key is a conflict", func(t *testing.T) { keys := newFakeKeys() keys.err = pgstore.ErrKeyReused w := idemPost(t, idemServer(t, keys, &fakeIntake{}), "one-per-gesture", "AAAABBBBCCCC", nil) if w.Code != http.StatusConflict || !contains(w.Body.String(), `"key_reused"`) { t.Errorf("a re-used key answered %d %s", w.Code, w.Body.String()) } }) t.Run("a repeat while the first is running is told to wait", func(t *testing.T) { keys := newFakeKeys() keys.err = pgstore.ErrKeyInFlight w := idemPost(t, idemServer(t, keys, &fakeIntake{}), "one-per-gesture", "AAAABBBBCCCC", nil) if w.Code != http.StatusConflict || !contains(w.Body.String(), `"key_in_flight"`) { t.Errorf("an in-flight key answered %d %s", w.Code, w.Body.String()) } if w.Header().Get("Retry-After") == "" { t.Error("no Retry-After on the answer that asks the client to wait") } }) t.Run("a claim that lost the race too often is told to wait, not answered 500", func(t *testing.T) { // PD-369. Contention on one key is a legitimate request meeting other legitimate requests, and // the answer must be one of the contract's own — 409 `key_in_flight` with a Retry-After — not // an internal error the caller can do nothing with. The two paths share ONE writer // (`keyInFlight`), and the case order matters: ErrKeyContended WRAPS ErrKeyInFlight, so the // narrower case has to come first or this test reads the same as the one above it. keys := newFakeKeys() keys.err = pgstore.ErrKeyContended w := idemPost(t, idemServer(t, keys, &fakeIntake{}), "one-per-gesture", "AAAABBBBCCCC", nil) if w.Code != http.StatusConflict || !contains(w.Body.String(), `"key_in_flight"`) { t.Errorf("a contended key answered %d %s, want 409 key_in_flight", w.Code, w.Body.String()) } if w.Header().Get("Retry-After") == "" { t.Error("no Retry-After on the answer that asks the client to wait") } if contains(w.Body.String(), `"internal_error"`) { t.Error("a legitimate request under contention was answered with an internal error") } }) t.Run("an attempt that did not complete gives the key back", func(t *testing.T) { keys := newFakeKeys() h := idemServer(t, keys, &fakeIntake{err: errors.New("the storage is gone")}) if w := idemPost(t, h, "one-per-gesture", "AAAABBBBCCCC", nil); w.Code < 400 { t.Fatalf("a failed intake answered %d", w.Code) } if len(keys.released) != 1 || len(keys.completed) != 0 { t.Errorf("released %d, completed %d — a failed attempt must give the key back", len(keys.released), len(keys.completed)) } }) } // The receipt is written on a context of its OWN: the client that will retry is precisely the one // that hung up, and on the request's context the key stays in flight for the whole claim window. // // Mutation caught: settleCtx without WithoutCancel. func TestTheReceiptSurvivesAClientThatHungUp(t *testing.T) { keys := newFakeKeys() h := idemServer(t, keys, &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга", SourceLang: "zh", TargetLang: "ru", Status: "parsing"}}) ctx, cancel := context.WithCancel(context.Background()) cancel() idemPost(t, h, "one-per-gesture", "AAAABBBBCCCC", ctx) if len(keys.completed) != 1 { t.Errorf("the receipt was not recorded for a client that hung up: completed=%d", len(keys.completed)) } } // A key over the contract's bound is refused and never truncated — and the store is not touched. func TestAnOverlongKeyIsRefusedBeforeAnythingIsClaimed(t *testing.T) { keys := newFakeKeys() h := v0ServerWith(t, Deps{Library: &fakeLibrary{}, Keys: keys, Intake: &fakeIntake{book: pgstore.Book{ID: "bk_1", Title: "Книга", SourceLang: "zh", TargetLang: "ru", Status: "parsing"}}, Capabilities: Capabilities{ Pairs: []LanguagePair{{Source: "zh", Target: "ru", Available: true}}, IntakeEnabled: true, IntakeMaxBytes: 64 << 20, PageSizeDefault: 100, }}) ct, body := idemUpload(t, "Книга", "roman.txt", "текст", "AAAABBBBCCCC") r := httptest.NewRequest("POST", "/v0/books", body) r.Header.Set("Content-Type", ct) r.Header.Set("X-TM-Client", "probe") r.Header.Set("Authorization", "Bearer t") r.Header.Set("Idempotency-Key", string(bytes.Repeat([]byte("k"), maxKeyLength+1))) w := httptest.NewRecorder() h.ServeHTTP(w, r) if w.Code != http.StatusBadRequest { t.Errorf("an over-long key answered %d, want 400", w.Code) } if len(keys.claims) != 0 { t.Errorf("the store was asked about a key the surface had already refused: %+v", keys.claims) } }