package books import ( "errors" "os" "os/exec" "path/filepath" "strings" "testing" "time" "gopkg.in/yaml.v3" "textmachine/platform/internal/ingest" "textmachine/platform/internal/runner" ) // A template with the shape an operator deploys: absolute paths, ceilings, a comment, and a couple of // keys this platform has never heard of. const templateYAML = `# The deployment's starting book configuration. pipeline: /srv/tm/configs/pipeline.yaml models: /srv/tm/configs/models.yaml langpack_root: /srv/tm/configs/langpacks ceilings: book_usd: 25 day_usd: 5 venuti: 0.4 honorifics: keep a_key_this_build_has_never_heard_of: [1, 2, 3] ` func render(t *testing.T, template string, b bookConfig) map[string]any { t.Helper() out, err := renderBookConfig([]byte(template), b) if err != nil { t.Fatalf("render: %v", err) } var got map[string]any if err := yaml.Unmarshal(out, &got); err != nil { t.Fatalf("the rendered configuration does not parse: %v\n%s", err, out) } return got } // The division of ownership, which is the whole of form Б: the platform fills in what only it knows // and carries EVERYTHING else through untouched — including keys it has never heard of, because the // engine's schema grows without this zone and a template that lost a key on every render would make // every engine release a platform release. func TestTheRenderFillsInWhatThePlatformKnowsAndCarriesTheRestThrough(t *testing.T) { got := render(t, templateYAML, bookConfig{ ID: "bk_ABC", Title: "蛊真人", SourceLang: "zh", TargetLang: "ru", SourceFile: "source.txt", }) for key, want := range map[string]any{ "book_id": "bk_ABC", "title": "蛊真人", "source_lang": "zh", "target_lang": "ru", "source_file": "source.txt", } { if got[key] != want { t.Errorf("%s = %v, want %v", key, got[key], want) } } // ⚠ `genre` is NOT written any more: it left the contract and the intake form in 0.3.0 (Б-23, // owner 16.08), so the platform no longer knows a book's genre and the key belongs entirely to // the operator's template. A render that still invented one would write an empty string over // whatever they had put there. if _, ok := got["genre"]; ok { t.Errorf("the render wrote a genre the platform no longer knows: %+v", got["genre"]) } // The operator's own settings, untouched — including the ceilings, which are the reason the // template exists at all, and a key with no meaning to this platform. if got["pipeline"] != "/srv/tm/configs/pipeline.yaml" || got["honorifics"] != "keep" { t.Errorf("the template's own wiring did not survive: %+v", got) } ceilings, ok := got["ceilings"].(map[string]any) if !ok || ceilings["book_usd"] != 25 || ceilings["day_usd"] != 5 { t.Errorf("the operator's ceilings did not survive: %+v", got["ceilings"]) } if _, ok := got["a_key_this_build_has_never_heard_of"]; !ok { t.Error("a key this platform does not know was dropped from the operator's template") } // The comment survives too. Not cosmetic: the file belongs to the operator from here on, and a // render that strips what they wrote about it is a render they cannot maintain. out, err := renderBookConfig([]byte(templateYAML), bookConfig{ID: "bk_ABC"}) if err != nil { t.Fatal(err) } if !strings.Contains(string(out), "# The deployment's starting book configuration.") { t.Errorf("the template's comments were stripped:\n%s", out) } } // A template that already carries one of the platform's own keys has it REPLACED, not duplicated. // The engine's decoder is strict and a duplicate key is a hard parse error, so a text-templated // render would turn an operator's helpful placeholder into an unloadable book. func TestATemplateThatAlreadyCarriesAPlatformKeyHasItReplacedOnce(t *testing.T) { const withPlaceholders = "book_id: REPLACE_ME\nsource_lang: xx\npipeline: /p.yaml\n" out, err := renderBookConfig([]byte(withPlaceholders), bookConfig{ID: "bk_1", SourceLang: "ja"}) if err != nil { t.Fatal(err) } if n := strings.Count(string(out), "book_id:"); n != 1 { t.Fatalf("book_id appears %d times:\n%s", n, out) } var strict struct { BookID string `yaml:"book_id"` SourceLang string `yaml:"source_lang"` TargetLang string `yaml:"target_lang"` Title string `yaml:"title"` Genre string `yaml:"genre"` SourceFile string `yaml:"source_file"` Pipeline string `yaml:"pipeline"` } dec := yaml.NewDecoder(strings.NewReader(string(out))) dec.KnownFields(true) if err := dec.Decode(&strict); err != nil { t.Fatalf("the rendered configuration does not decode strictly: %v\n%s", err, out) } if strict.BookID != "bk_1" || strict.SourceLang != "ja" || strict.Pipeline != "/p.yaml" { t.Errorf("decoded %+v", strict) } } // Every value the platform writes is a STRING to the engine, and the values come from a user's // filename and from a form field. Written without the explicit tag, a title of "123" is emitted as a // number and the engine's strict decoder then refuses to put an int in a string field — a book that // can never be parsed, produced by a title nobody would think twice about. // // Mutation caught: dropping Tag: "!!str" from setString. func TestValuesThatLookLikeOtherTypesStayStrings(t *testing.T) { for _, v := range []string{ "123", "0755", "1.5", "true", "false", "yes", "no", "null", "~", "", "-", "*anchor", "&anchor", "!tag", "#not-a-comment", "a: b", "a\nb", " padded ", "[1,2]", "{a: 1}", "@reserved", "`backtick", `"quoted"`, "'single'", } { out, err := renderBookConfig([]byte("pipeline: /p.yaml\n"), bookConfig{Title: v, ID: "bk_1"}) if err != nil { t.Fatalf("title %q: %v", v, err) } var back struct { Title string `yaml:"title"` BookID string `yaml:"book_id"` } dec := yaml.NewDecoder(strings.NewReader(string(out))) dec.KnownFields(false) if err := dec.Decode(&back); err != nil { t.Errorf("title %q made the configuration undecodable: %v\n%s", v, err, out) continue } if back.Title != v { t.Errorf("title %q came back as %q", v, back.Title) } if back.BookID != "bk_1" { t.Errorf("title %q corrupted the rest of the document: book_id = %q\n%s", v, back.BookID, out) } } } // A template that is not a configuration is a DEPLOYMENT fault, and every one of them has to reach // the caller as the class that WAITS. The alternative is a host that rejects every upload it // receives — and a rejection over a broken source is what deletes the file. func TestATemplateThatIsNotAConfigurationIsADeploymentFault(t *testing.T) { for _, tc := range []struct{ name, body string }{ {"empty", ""}, {"a list", "- a\n- b\n"}, {"a scalar", "just a string\n"}, {"not yaml at all", "{{{\n"}, {"two documents", "a: 1\n---\nb: 2\n"}, } { if _, err := renderBookConfig([]byte(tc.body), bookConfig{ID: "bk_1"}); !errors.Is(err, ErrNoTemplate) { t.Errorf("%s: %v, want ErrNoTemplate", tc.name, err) } } } // The whole seam through the intake: a book uploaded to a deployment WITH a template gets its // configuration rendered by the first parse pass, and the operator is never involved. func TestAnUploadedBookIsProvisionedFromTheDeploymentTemplate(t *testing.T) { f := newFixture(t) tpl := filepath.Join(t.TempDir(), "book.yaml") if err := os.WriteFile(tpl, []byte(templateYAML), 0o600); err != nil { t.Fatal(err) } f.svc.Cfg.BookTemplate = tpl book := f.accept(t, "蛊真人.txt", "первая глава") if err := f.svc.Parse(f.ctx, book.ID); err != nil { t.Fatal(err) } if got := f.card(t, book.ID); got.Status != "not_started" { t.Fatalf("a book on a deployment with a template is %q; nobody had to touch the host", got.Status) } raw, err := os.ReadFile(filepath.Join(f.root, book.ID, runner.ConfigFile)) if err != nil { t.Fatal(err) } var got map[string]any if err := yaml.Unmarshal(raw, &got); err != nil { t.Fatal(err) } // The book's identity is the PLATFORM's, and the source is the file the intake actually wrote — // extension included, because the engine dispatches its reader by it. if got["book_id"] != book.ID || got["source_file"] != SourceName+".txt" { t.Errorf("rendered book_id=%v source_file=%v", got["book_id"], got["source_file"]) } if got["source_lang"] != "zh" || got["target_lang"] != "ru" { t.Errorf("the user's declared intake did not reach the configuration: %+v", got) } if got["title"] != "蛊真人" { t.Errorf("title = %v", got["title"]) } } // Once the file exists it is the OPERATOR's, and the platform never writes it again. An operator who // lowered a ceiling or pointed a book at a different pipeline must find their change still there the // next time anything touches the book (D39.110 §2b). // // ⚠ What this pins is the SHORT-CIRCUIT — provision sees a configuration and returns before it reads // anything — and that is stated precisely because the `O_EXCL` two lines further down is NOT what it // catches: with an existing file the open is never reached, so a landing that swaps it for `O_TRUNC` // survives this test. `O_EXCL` guards a different case, two passes of the intake rendering at once, // and it has no test: both would write identical bytes, so there is nothing for an assertion to see. // Named rather than claimed. // // Mutation caught: removing the `os.Stat` short-circuit from provision. func TestARenderedConfigurationIsNeverRewritten(t *testing.T) { f := newFixture(t) tpl := filepath.Join(t.TempDir(), "book.yaml") if err := os.WriteFile(tpl, []byte(templateYAML), 0o600); err != nil { t.Fatal(err) } f.svc.Cfg.BookTemplate = tpl book := f.accept(t, "book.txt", "первая глава") // The first walk renders the configuration and then fails on the ENGINE, so the book stays in // intake and a second walk really happens. A book that parsed on the first pass is never looked at // again, which is what made the earlier version of this test assert nothing at all. f.engine.set(ingest.Manifest{}, &exec.Error{Name: "tmctl", Err: os.ErrNotExist}) if err := f.svc.Parse(f.ctx, book.ID); err != nil { t.Fatal(err) } path := filepath.Join(f.root, book.ID, runner.ConfigFile) if _, err := os.Stat(path); err != nil { t.Fatalf("the first walk rendered nothing: %v", err) } edited := "# an operator was here\nbook_id: " + book.ID + "\nsource_lang: zh\ntarget_lang: ru\nsource_file: source.txt\npipeline: /other.yaml\n" if err := os.WriteFile(path, []byte(edited), 0o600); err != nil { t.Fatal(err) } // The template is REMOVED before the second walk. If anything reaches for it, the book stops being // parseable — which is what turns "the file was not rewritten" into "the file was not even // considered". if err := os.Remove(tpl); err != nil { t.Fatal(err) } f.engine.set(wholeManifest(3, 1), nil) // Another walk of the same book — a re-parse after the claim went stale. f.now = f.now.Add(ClaimGrace + time.Minute) if err := f.svc.Parse(f.ctx, book.ID); err != nil { t.Fatal(err) } if got := f.card(t, book.ID); got.Status != "not_started" { t.Fatalf("the second walk went looking for a template for a book that already had a configuration: %q", got.Status) } raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if string(raw) != edited { t.Fatalf("the platform rewrote a configuration that was not its own:\n%s", raw) } } // A deployment whose template is missing or broken must WAIT, not reject: the fault applies to every // book on the host at once, and rejecting is what deletes a user's upload. // // Mutation caught: returning any error other than ErrNotProvisioned from provision when the template // cannot be used. func TestABrokenTemplateStopsNoIntakeAndSpendsNoBudget(t *testing.T) { for _, tc := range []struct{ name, body string }{ {"missing", ""}, {"unparseable", "{{{\n"}, } { t.Run(tc.name, func(t *testing.T) { f := newFixture(t) tpl := filepath.Join(t.TempDir(), "book.yaml") if tc.body != "" { if err := os.WriteFile(tpl, []byte(tc.body), 0o600); err != nil { t.Fatal(err) } } f.svc.Cfg.BookTemplate = tpl book := f.accept(t, "book.txt", "первая глава") for range parseAttempts * 2 { if err := f.svc.Parse(f.ctx, book.ID); err != nil { t.Fatal(err) } f.now = f.now.Add(ClaimGrace + time.Minute) } if got := f.card(t, book.ID); got.Status != "parsing" { t.Fatalf("a book on a host with a broken template is %q, want parsing", got.Status) } var attempts int if err := f.store.Pool().QueryRow(f.ctx, `select parse_attempts from books where id = $1`, book.ID).Scan(&attempts); err != nil { t.Fatal(err) } if attempts != 0 { t.Errorf("waiting for a template spent %d attempts; the first real answer after that "+ "would be terminal on arrival", attempts) } if f.engine.called() != 0 { t.Errorf("the engine was asked %d times for a book with no configuration", f.engine.called()) } // And the moment the operator fixes it, the very next pass renders and parses. if err := os.WriteFile(tpl, []byte(templateYAML), 0o600); err != nil { t.Fatal(err) } f.engine.set(wholeManifest(3, 1), nil) f.now = f.now.Add(ClaimGrace + time.Minute) if err := f.svc.Parse(f.ctx, book.ID); err != nil { t.Fatal(err) } if got := f.card(t, book.ID); got.Status != "not_started" { t.Fatalf("after the template was fixed the book is %q", got.Status) } }) } } // A deployment with NO template configured keeps the behaviour it had before form Б: the operator // drops the file in by hand and the book waits, unharmed, until they do. func TestADeploymentWithoutATemplateStillWaitsForItsOperator(t *testing.T) { f := newFixture(t) // no BookTemplate book := f.accept(t, "book.txt", "первая глава") if err := f.svc.Parse(f.ctx, book.ID); err != nil { t.Fatal(err) } if got := f.card(t, book.ID); got.Status != "parsing" { t.Fatalf("book %q on a deployment that provisions by hand", got.Status) } if _, err := os.Stat(filepath.Join(f.root, book.ID, runner.ConfigFile)); !os.IsNotExist(err) { t.Errorf("a deployment with no template wrote a configuration anyway: %v", err) } } // The source file name is a fact about the DIRECTORY, and it is what `source_file` has to say. It is // found by name rather than stored because the intake is the only writer of that directory — but the // finding has to survive the neighbours it legitimately has. func TestTheSourceFileIsFoundNextToTheConfiguration(t *testing.T) { dir := t.TempDir() for _, name := range []string{runner.ConfigFile, "notes.txt", SourceName + ".epub"} { if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o600); err != nil { t.Fatal(err) } } if err := os.Mkdir(filepath.Join(dir, SourceName), 0o750); err != nil { t.Fatal(err) // a DIRECTORY called source is not the source } got, err := sourceFileIn(dir) if err != nil { t.Fatal(err) } if got != SourceName+".epub" { t.Errorf("source file %q, want %s.epub", got, SourceName) } if _, err := sourceFileIn(t.TempDir()); err == nil { t.Error("an empty directory answered with a source file") } }