414 lines
16 KiB
Go
414 lines
16 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"log/slog"
|
||
"os"
|
||
"path/filepath"
|
||
"reflect"
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/miner"
|
||
"textmachine/backend/internal/obs"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// identitySeed is deliberately hostile to the fold: every ordered axis is presented in the WRONG order.
|
||
// The terms descend by src where the store ascends, one term carries two senses whose order is inverted,
|
||
// two terms share a src and differ only by spoiler window (presented later-window-first), the aliases of
|
||
// one term are listed in reverse, and the voice/address rows are scrambled against their own ORDER BY.
|
||
// A fold that "keeps the order it was given" therefore cannot accidentally pass this fixture.
|
||
const identitySeed = `
|
||
terms:
|
||
- src: 鈴木
|
||
dst: Судзуки
|
||
type: name
|
||
status: approved
|
||
since_ch: 5
|
||
decl: { invariant: true, forms: ["Судзуки"] }
|
||
aliases:
|
||
- { alias: すずき, type: name }
|
||
- { alias: SUZUKI, type: name }
|
||
- { alias: Suzuki, type: name }
|
||
- src: 鈴木
|
||
dst: Судзуки-старший
|
||
type: name
|
||
status: approved
|
||
since_ch: 1
|
||
until_ch: 4
|
||
decl: { invariant: true, forms: ["Судзуки-старший"] }
|
||
- src: 図書館
|
||
dst: библиотека
|
||
type: term
|
||
status: approved
|
||
sense: здание
|
||
since_ch: 3
|
||
decl: { invariant: false, forms: ["библиотека", "библиотеки"] }
|
||
- src: 図書館
|
||
dst: книгохранилище
|
||
type: term
|
||
status: draft
|
||
sense: а-архаизм
|
||
since_ch: 1
|
||
until_ch: 2
|
||
- src: 朝
|
||
dst: утро
|
||
type: term
|
||
status: approved
|
||
voices:
|
||
- src: 図書館
|
||
sense: здание
|
||
register: нейтральный
|
||
self_ref: я
|
||
address_default: formal
|
||
since_ch: 3
|
||
- src: 鈴木
|
||
register: разговорный
|
||
self_ref: я
|
||
address_default: informal
|
||
since_ch: 1
|
||
until_ch: 4
|
||
- src: 鈴木
|
||
register: книжный
|
||
self_ref: я
|
||
address_default: formal
|
||
since_ch: 5
|
||
addresses:
|
||
- speaker: 鈴木
|
||
addressee: 図書館
|
||
addressee_sense: здание
|
||
register: formal
|
||
form: вы
|
||
closeness: далеко
|
||
since_ch: 5
|
||
- speaker: 図書館
|
||
speaker_sense: здание
|
||
addressee: 鈴木
|
||
register: informal
|
||
form: ты
|
||
closeness: близко
|
||
since_ch: 1
|
||
`
|
||
|
||
// identityDelta is the owner's decision file — the state that only exists AFTER a `bank-apply` and that
|
||
// the stored glossary therefore does not hold. Its rows sort INTO the middle of the seed's, so a fold
|
||
// that appended them instead of merging them by value would be caught.
|
||
const identityDelta = `
|
||
terms:
|
||
- src: 静か
|
||
dst: тихий
|
||
type: term
|
||
status: approved
|
||
- src: 元海
|
||
dst: Изначальное море
|
||
type: term
|
||
status: approved
|
||
`
|
||
|
||
// identityAutoBank is the engine's own unsigned proposal file (Source:"mined", base-excluded), included
|
||
// so the fixture exercises the BASE/ENRICHED split as well as the plain ordering.
|
||
const identityAutoBank = `
|
||
terms:
|
||
- src: 行った
|
||
dst: пошёл
|
||
type: term
|
||
status: auto
|
||
`
|
||
|
||
// bankFoldFixture builds a project whose bank is folded from all four sources at once — curated seed,
|
||
// ruby readings, the owner's mined-delta and the engine's auto-bank — and returns the book path.
|
||
func bankFoldFixture(t *testing.T, providerURL string) string {
|
||
t.Helper()
|
||
bookPath := setupProjectOpts(t, providerURL, projectOpts{
|
||
regenerate: 1, source: suzukiSource, glossarySeed: identitySeed,
|
||
})
|
||
dir := filepath.Dir(bookPath)
|
||
writeFile(t, filepath.Join(dir, "test-book.mined-delta.yaml"), identityDelta)
|
||
writeFile(t, filepath.Join(dir, "test-book.db.auto-bank.yaml"), identityAutoBank)
|
||
return bookPath
|
||
}
|
||
|
||
// stripIDs zeroes the store's autoincrement id, the ONE field the round-trip adds and the fold never
|
||
// hashes (membank.ComputeVersion excludes it by construction). Everything else must match exactly.
|
||
func stripIDs(rows []store.GlossaryEntry) []store.GlossaryEntry {
|
||
out := append([]store.GlossaryEntry(nil), rows...)
|
||
for i := range out {
|
||
out[i].ID = 0
|
||
}
|
||
return out
|
||
}
|
||
|
||
// TestTheInMemoryFoldIsIdenticalToTheStoreRoundTrip is the proof the free estimate rests on, and it is a
|
||
// proof by EXECUTION rather than by argument: both folds are run over the same inputs and every quantity
|
||
// that money depends on is compared.
|
||
//
|
||
// What is at stake. The canonical fold goes THROUGH the store — ReplaceBank writes, GlossaryForBook reads
|
||
// back ORDER BY src, sense, since_ch, until_ch, status, dst — and that order decides the bank's content
|
||
// hash and, through Select's stable budget sort, the literal bytes of the rendered glossary block, which
|
||
// are hashed into chunk_status.content_hash. A read-only surface cannot write, so it must reproduce the
|
||
// order without the store. If it reproduced it only approximately, the free projection would name one
|
||
// number and the paid run would charge another — which is precisely the defect the projection is sold as
|
||
// fixing.
|
||
//
|
||
// The comparison is made at three depths on purpose: the rows themselves (so a divergence is localised),
|
||
// the two bank versions (the snapshot component), and the per-position rendered content hashes (the wire
|
||
// bytes the resume fast-path and the re-bill projection actually compare).
|
||
func TestTheInMemoryFoldIsIdenticalToTheStoreRoundTrip(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := bankFoldFixture(t, srv.URL)
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
// A RUN MUST HAVE HAPPENED before either fold is measured, and it is load-bearing rather than
|
||
// scene-setting. renderedContentHashes reproduces an EDIT position only when every member's stored
|
||
// draft is retrievable (repin.go), so on a book with no chunk_status rows the hash map comes back
|
||
// draft-only — and the edit wave is exactly the half the mined-delta and auto-bank rows reach, since
|
||
// Source:"mined" folds into the ENRICHED bank alone. Without the run this comparison would be blind to
|
||
// every possible defect in the half it was written to prove.
|
||
r0 := newRunner(t, bookPath)
|
||
if err := r0.Store.ReplaceRubyReadings("test-book", []store.RubyReading{
|
||
{BookID: "test-book", Base: "鈴木", Reading: "すずき", FirstChapter: 1, Occurrences: 3},
|
||
{BookID: "test-book", Base: "図書館", Reading: "としょかん", FirstChapter: 1, Occurrences: 2},
|
||
}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := r0.TranslateBook(ctx); err != nil {
|
||
t.Fatalf("the run this comparison rests on did not complete: %v", err)
|
||
}
|
||
r0.Close()
|
||
|
||
// --- Path A: the canonical fold, through the store ---
|
||
ra := newRunner(t, bookPath)
|
||
if err := ra.seedGlossary(ctx); err != nil {
|
||
t.Fatalf("canonical fold: %v", err)
|
||
}
|
||
rowsA, err := ra.Store.GlossaryForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
voicesA, err := ra.Store.VoiceProfilesForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
pairsA, err := ra.Store.AddressPairsForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
verA, baseA := ra.memory.Version(), ra.baseMemory.Version()
|
||
hashesA := foldRenderedHashes(t, ra)
|
||
// The project flock is exclusive, so the second runner cannot open until the first lets go.
|
||
ra.Close()
|
||
|
||
// --- Path B: the same fold with the write removed ---
|
||
rb := newRunner(t, bookPath)
|
||
defer rb.Close()
|
||
if err := rb.projectFoldedMemory(); err != nil {
|
||
t.Fatalf("in-memory fold: %v", err)
|
||
}
|
||
in, err := rb.gatherBankInputs()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
rowsB, voicesB, pairsB, err := storeOrder(in)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
verB, baseB := rb.memory.Version(), rb.baseMemory.Version()
|
||
hashesB := foldRenderedHashes(t, rb)
|
||
|
||
// The fixture must actually exercise the ordering — a fold that never had to sort anything would
|
||
// pass this test while proving nothing.
|
||
if len(rowsA) < 8 {
|
||
t.Fatalf("fixture too thin to prove an ordering: %d rows", len(rowsA))
|
||
}
|
||
if reflect.DeepEqual(stripIDs(rowsA), stripIDs(in.entries)) {
|
||
t.Fatalf("fixture is degenerate: the BUILD order already equals the store order, so sorting is untested")
|
||
}
|
||
|
||
if !reflect.DeepEqual(stripIDs(rowsA), stripIDs(rowsB)) {
|
||
for i := range rowsA {
|
||
if i >= len(rowsB) {
|
||
t.Fatalf("row %d: store has %+v, memory ran out (%d rows)", i, rowsA[i], len(rowsB))
|
||
}
|
||
a, b := rowsA[i], rowsB[i]
|
||
a.ID, b.ID = 0, 0
|
||
if !reflect.DeepEqual(a, b) {
|
||
t.Fatalf("row %d differs:\n store = %+v\n memory = %+v", i, a, b)
|
||
}
|
||
}
|
||
t.Fatalf("row counts differ: store %d, memory %d", len(rowsA), len(rowsB))
|
||
}
|
||
if !reflect.DeepEqual(voicesA, voicesB) {
|
||
t.Fatalf("voice profiles differ:\n store = %+v\n memory = %+v", voicesA, voicesB)
|
||
}
|
||
if !reflect.DeepEqual(pairsA, pairsB) {
|
||
t.Fatalf("address pairs differ:\n store = %+v\n memory = %+v", pairsA, pairsB)
|
||
}
|
||
if verA != verB {
|
||
t.Fatalf("enriched bank version differs: store %s, memory %s", verA, verB)
|
||
}
|
||
if baseA != baseB {
|
||
t.Fatalf("base bank version differs: store %s, memory %s", baseA, baseB)
|
||
}
|
||
if baseA == verA {
|
||
t.Fatalf("fixture is degenerate: the auto-bank row must make base ≠ enriched, both are %s", verA)
|
||
}
|
||
if !reflect.DeepEqual(hashesA, hashesB) {
|
||
t.Fatalf("rendered content hashes differ — the free estimate would name a number the paid run does not charge:\n store = %+v\n memory = %+v", hashesA, hashesB)
|
||
}
|
||
// ⚠ NOT merely "non-empty". Two empty maps compare equal, and so do two DRAFT-ONLY maps — and the
|
||
// draft wave is the half the fixture's mined rows do not even reach (Source:"mined" folds into the
|
||
// ENRICHED bank only). The comparison is only a proof if it actually covers the edit wave, so the
|
||
// coverage is asserted rather than assumed.
|
||
waves := map[string]int{}
|
||
for _, byStage := range hashesA {
|
||
for stage := range byStage {
|
||
waves[stage]++
|
||
}
|
||
}
|
||
if waves["draft"] == 0 || waves["edit"] == 0 {
|
||
t.Fatalf("the byte-level comparison must cover BOTH waves — the mined rows this fixture carries reach only the edit one; got %v", waves)
|
||
}
|
||
}
|
||
|
||
// foldRenderedHashes reproduces the per-position wire hashes exactly as the re-bill projection does
|
||
// (rebill.go), so the comparison is over the quantity money is decided by rather than over a proxy.
|
||
func foldRenderedHashes(t *testing.T, r *Runner) map[chunkKey]map[string]string {
|
||
t.Helper()
|
||
chunks, withText, err := r.readModelChunks()
|
||
if err != nil {
|
||
t.Fatalf("read-model chunks: %v", err)
|
||
}
|
||
_ = chunks
|
||
full, err := withText()
|
||
if err != nil {
|
||
t.Fatalf("re-chunk with text: %v", err)
|
||
}
|
||
return r.renderedContentHashes(full, precomputeSticky(full, r.baseMemory, r.Pipeline.Context.GlossaryTokenBudget))
|
||
}
|
||
|
||
// TestTheInMemoryFoldRefusesWhatTheStoreWouldRefuse pins the other half of the identity: where the
|
||
// round-trip would ABORT, the projection must abort too. A fold that quietly de-duplicated a repeated
|
||
// UNIQUE key would answer with a tidy number for a book whose next run cannot even start.
|
||
func TestTheInMemoryFoldRefusesWhatTheStoreWouldRefuse(t *testing.T) {
|
||
dup := store.GlossaryEntry{BookID: "b", Src: "鈴木", Dst: "Судзуки", Status: "approved"}
|
||
if _, _, _, err := storeOrder(bankInputs{entries: []store.GlossaryEntry{dup, dup}}); err == nil {
|
||
t.Fatalf("a repeated glossary UNIQUE key must refuse, not de-duplicate")
|
||
}
|
||
aliased := store.GlossaryEntry{BookID: "b", Src: "図書館", Dst: "библиотека", Status: "approved",
|
||
Aliases: []store.GlossaryAlias{{Alias: "としょかん"}, {Alias: "としょかん"}}}
|
||
if _, _, _, err := storeOrder(bankInputs{entries: []store.GlossaryEntry{aliased}}); err == nil {
|
||
t.Fatalf("a repeated alias must refuse: glossary_aliases carries UNIQUE (book_id, term_id, alias)")
|
||
}
|
||
v := store.VoiceProfile{Src: "鈴木", Register: "разговорный"}
|
||
if _, _, _, err := storeOrder(bankInputs{voices: []store.VoiceProfile{v, v}}); err == nil {
|
||
t.Fatalf("a repeated voice-profile UNIQUE key must refuse")
|
||
}
|
||
p := store.AddressPair{SpeakerSrc: "鈴木", AddresseeSrc: "図書館"}
|
||
if _, _, _, err := storeOrder(bankInputs{pairs: []store.AddressPair{p, p}}); err == nil {
|
||
t.Fatalf("a repeated address-pair UNIQUE key must refuse")
|
||
}
|
||
}
|
||
|
||
// TestAFailedFoldStillReportsWhatItSkipped pins a regression this pack introduced and then fixed. Before
|
||
// the gather was extracted from seedGlossary, its non-fatal warnings were emitted inline as they were
|
||
// made, so a book that died on a LATER check had still told the operator what the earlier steps skipped.
|
||
// The first version of the extraction logged the collected remarks only after the error return — losing
|
||
// the diagnostics of exactly the run that needs them most.
|
||
func TestAFailedFoldStillReportsWhatItSkipped(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
// The seed holds a signed term, and a voice row for a character the bank does NOT have — the latter is
|
||
// fatal, and it is checked AFTER the auto-bank load.
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||
regenerate: 1, source: suzukiSource, glossarySeed: `
|
||
terms:
|
||
- src: 鈴木
|
||
dst: Судзуки
|
||
type: name
|
||
status: approved
|
||
decl: { invariant: true, forms: ["Судзуки"] }
|
||
voices:
|
||
- src: 存在しない
|
||
register: книжный
|
||
self_ref: я
|
||
address_default: formal
|
||
`})
|
||
dir := filepath.Dir(bookPath)
|
||
// An auto-bank row on the key the signed seed term already holds: dropped with a REMARK, and the drop
|
||
// happens BEFORE the fatal voice check.
|
||
writeFile(t, filepath.Join(dir, "test-book.db.auto-bank.yaml"), `
|
||
terms:
|
||
- src: 鈴木
|
||
dst: Сузуки
|
||
type: name
|
||
status: auto
|
||
`)
|
||
var logBuf bytes.Buffer
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||
|
||
err := r.seedGlossary(context.Background())
|
||
if err == nil {
|
||
t.Fatalf("precondition: a voice row naming an absent character must fail the fold")
|
||
}
|
||
if !strings.Contains(err.Error(), "absent from the bank") {
|
||
t.Fatalf("precondition: expected the voice-character refusal, got: %v", err)
|
||
}
|
||
if !strings.Contains(logBuf.String(), "auto-bank rows dropped") {
|
||
t.Fatalf("the fold died and took its own diagnostics with it; the dropped auto-bank row was never reported.\nlog was:\n%s", logBuf.String())
|
||
}
|
||
}
|
||
|
||
// TestTheAutoBankIsWrittenAtomically pins a hole THIS pack opened in someone else's file. Until the
|
||
// read-only surfaces began folding the bank from the decision documents themselves, the auto-bank was
|
||
// read only by the run that had just written it, so a plain truncating os.WriteFile was safe. The moment
|
||
// `status` and `export` started reading it, it became one of the concurrently-read sidecars artifact.go
|
||
// exists for — and a truncating write hands a concurrent reader either zero bytes (a parse error, so the
|
||
// buyer sees stale figures) or a valid PREFIX of the term list, which is worse: a fold of a bank that
|
||
// never existed and a rebill_units no run will ever charge.
|
||
//
|
||
// The instrument is the inode, the same one bank-apply's tests use: a rename ALWAYS changes it, and a
|
||
// truncate-in-place never does. It is a stronger assertion than comparing bytes, because rewriting the
|
||
// same bytes non-atomically would pass a byte comparison and fail this.
|
||
func TestTheAutoBankIsWrittenAtomically(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource})
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
ctx := context.Background()
|
||
|
||
first := []miner.Term{{Src: "図書館", Type: "term", SinceCh: 1, Freq: 3}}
|
||
if err := r.writeAutoBank(ctx, first, nil, nil); err != nil {
|
||
t.Fatalf("write auto-bank: %v", err)
|
||
}
|
||
before := inode(t, r.autoBankPath())
|
||
|
||
second := []miner.Term{
|
||
{Src: "図書館", Type: "term", SinceCh: 1, Freq: 3},
|
||
{Src: "鈴木", Type: "name", SinceCh: 1, Freq: 5},
|
||
}
|
||
if err := r.writeAutoBank(ctx, second, nil, nil); err != nil {
|
||
t.Fatalf("rewrite auto-bank: %v", err)
|
||
}
|
||
if after := inode(t, r.autoBankPath()); after == before {
|
||
t.Fatalf("the auto-bank was rewritten IN PLACE (inode %d unchanged): a concurrent status/export read can catch it truncated or half-written, and a valid prefix of the term list folds a bank that never existed", before)
|
||
}
|
||
// And the document that landed is the whole new one, not a prefix of it.
|
||
body, err := os.ReadFile(r.autoBankPath())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, want := range []string{"図書館", "鈴木"} {
|
||
if !strings.Contains(string(body), want) {
|
||
t.Fatalf("the replacement document is incomplete: %q missing from\n%s", want, body)
|
||
}
|
||
}
|
||
}
|