369 lines
15 KiB
Go
369 lines
15 KiB
Go
package books
|
|
|
|
import (
|
|
"errors"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
|
|
"textmachine/platform/internal/ingest"
|
|
"textmachine/platform/internal/runner"
|
|
)
|
|
|
|
// A template with the shape an operator deploys: absolute paths, ceilings, a comment, and a couple of
|
|
// keys this platform has never heard of.
|
|
const templateYAML = `# The deployment's starting book configuration.
|
|
pipeline: /srv/tm/configs/pipeline.yaml
|
|
models: /srv/tm/configs/models.yaml
|
|
langpack_root: /srv/tm/configs/langpacks
|
|
ceilings:
|
|
book_usd: 25
|
|
day_usd: 5
|
|
venuti: 0.4
|
|
honorifics: keep
|
|
a_key_this_build_has_never_heard_of: [1, 2, 3]
|
|
`
|
|
|
|
func render(t *testing.T, template string, b bookConfig) map[string]any {
|
|
t.Helper()
|
|
out, err := renderBookConfig([]byte(template), b)
|
|
if err != nil {
|
|
t.Fatalf("render: %v", err)
|
|
}
|
|
var got map[string]any
|
|
if err := yaml.Unmarshal(out, &got); err != nil {
|
|
t.Fatalf("the rendered configuration does not parse: %v\n%s", err, out)
|
|
}
|
|
return got
|
|
}
|
|
|
|
// The division of ownership, which is the whole of form Б: the platform fills in what only it knows
|
|
// and carries EVERYTHING else through untouched — including keys it has never heard of, because the
|
|
// engine's schema grows without this zone and a template that lost a key on every render would make
|
|
// every engine release a platform release.
|
|
func TestTheRenderFillsInWhatThePlatformKnowsAndCarriesTheRestThrough(t *testing.T) {
|
|
got := render(t, templateYAML, bookConfig{
|
|
ID: "bk_ABC", Title: "蛊真人", SourceLang: "zh", TargetLang: "ru", SourceFile: "source.txt",
|
|
})
|
|
for key, want := range map[string]any{
|
|
"book_id": "bk_ABC", "title": "蛊真人", "source_lang": "zh", "target_lang": "ru",
|
|
"source_file": "source.txt",
|
|
} {
|
|
if got[key] != want {
|
|
t.Errorf("%s = %v, want %v", key, got[key], want)
|
|
}
|
|
}
|
|
// ⚠ `genre` is NOT written any more: it left the contract and the intake form in 0.3.0 (Б-23,
|
|
// owner 16.08), so the platform no longer knows a book's genre and the key belongs entirely to
|
|
// the operator's template. A render that still invented one would write an empty string over
|
|
// whatever they had put there.
|
|
if _, ok := got["genre"]; ok {
|
|
t.Errorf("the render wrote a genre the platform no longer knows: %+v", got["genre"])
|
|
}
|
|
// The operator's own settings, untouched — including the ceilings, which are the reason the
|
|
// template exists at all, and a key with no meaning to this platform.
|
|
if got["pipeline"] != "/srv/tm/configs/pipeline.yaml" || got["honorifics"] != "keep" {
|
|
t.Errorf("the template's own wiring did not survive: %+v", got)
|
|
}
|
|
ceilings, ok := got["ceilings"].(map[string]any)
|
|
if !ok || ceilings["book_usd"] != 25 || ceilings["day_usd"] != 5 {
|
|
t.Errorf("the operator's ceilings did not survive: %+v", got["ceilings"])
|
|
}
|
|
if _, ok := got["a_key_this_build_has_never_heard_of"]; !ok {
|
|
t.Error("a key this platform does not know was dropped from the operator's template")
|
|
}
|
|
// The comment survives too. Not cosmetic: the file belongs to the operator from here on, and a
|
|
// render that strips what they wrote about it is a render they cannot maintain.
|
|
out, err := renderBookConfig([]byte(templateYAML), bookConfig{ID: "bk_ABC"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(out), "# The deployment's starting book configuration.") {
|
|
t.Errorf("the template's comments were stripped:\n%s", out)
|
|
}
|
|
}
|
|
|
|
// A template that already carries one of the platform's own keys has it REPLACED, not duplicated.
|
|
// The engine's decoder is strict and a duplicate key is a hard parse error, so a text-templated
|
|
// render would turn an operator's helpful placeholder into an unloadable book.
|
|
func TestATemplateThatAlreadyCarriesAPlatformKeyHasItReplacedOnce(t *testing.T) {
|
|
const withPlaceholders = "book_id: REPLACE_ME\nsource_lang: xx\npipeline: /p.yaml\n"
|
|
out, err := renderBookConfig([]byte(withPlaceholders), bookConfig{ID: "bk_1", SourceLang: "ja"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n := strings.Count(string(out), "book_id:"); n != 1 {
|
|
t.Fatalf("book_id appears %d times:\n%s", n, out)
|
|
}
|
|
var strict struct {
|
|
BookID string `yaml:"book_id"`
|
|
SourceLang string `yaml:"source_lang"`
|
|
TargetLang string `yaml:"target_lang"`
|
|
Title string `yaml:"title"`
|
|
Genre string `yaml:"genre"`
|
|
SourceFile string `yaml:"source_file"`
|
|
Pipeline string `yaml:"pipeline"`
|
|
}
|
|
dec := yaml.NewDecoder(strings.NewReader(string(out)))
|
|
dec.KnownFields(true)
|
|
if err := dec.Decode(&strict); err != nil {
|
|
t.Fatalf("the rendered configuration does not decode strictly: %v\n%s", err, out)
|
|
}
|
|
if strict.BookID != "bk_1" || strict.SourceLang != "ja" || strict.Pipeline != "/p.yaml" {
|
|
t.Errorf("decoded %+v", strict)
|
|
}
|
|
}
|
|
|
|
// Every value the platform writes is a STRING to the engine, and the values come from a user's
|
|
// filename and from a form field. Written without the explicit tag, a title of "123" is emitted as a
|
|
// number and the engine's strict decoder then refuses to put an int in a string field — a book that
|
|
// can never be parsed, produced by a title nobody would think twice about.
|
|
//
|
|
// Mutation caught: dropping Tag: "!!str" from setString.
|
|
func TestValuesThatLookLikeOtherTypesStayStrings(t *testing.T) {
|
|
for _, v := range []string{
|
|
"123", "0755", "1.5", "true", "false", "yes", "no", "null", "~", "", "-",
|
|
"*anchor", "&anchor", "!tag", "#not-a-comment", "a: b", "a\nb", " padded ",
|
|
"[1,2]", "{a: 1}", "@reserved", "`backtick", `"quoted"`, "'single'",
|
|
} {
|
|
out, err := renderBookConfig([]byte("pipeline: /p.yaml\n"), bookConfig{Title: v, ID: "bk_1"})
|
|
if err != nil {
|
|
t.Fatalf("title %q: %v", v, err)
|
|
}
|
|
var back struct {
|
|
Title string `yaml:"title"`
|
|
BookID string `yaml:"book_id"`
|
|
}
|
|
dec := yaml.NewDecoder(strings.NewReader(string(out)))
|
|
dec.KnownFields(false)
|
|
if err := dec.Decode(&back); err != nil {
|
|
t.Errorf("title %q made the configuration undecodable: %v\n%s", v, err, out)
|
|
continue
|
|
}
|
|
if back.Title != v {
|
|
t.Errorf("title %q came back as %q", v, back.Title)
|
|
}
|
|
if back.BookID != "bk_1" {
|
|
t.Errorf("title %q corrupted the rest of the document: book_id = %q\n%s", v, back.BookID, out)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A template that is not a configuration is a DEPLOYMENT fault, and every one of them has to reach
|
|
// the caller as the class that WAITS. The alternative is a host that rejects every upload it
|
|
// receives — and a rejection over a broken source is what deletes the file.
|
|
func TestATemplateThatIsNotAConfigurationIsADeploymentFault(t *testing.T) {
|
|
for _, tc := range []struct{ name, body string }{
|
|
{"empty", ""},
|
|
{"a list", "- a\n- b\n"},
|
|
{"a scalar", "just a string\n"},
|
|
{"not yaml at all", "{{{\n"},
|
|
{"two documents", "a: 1\n---\nb: 2\n"},
|
|
} {
|
|
if _, err := renderBookConfig([]byte(tc.body), bookConfig{ID: "bk_1"}); !errors.Is(err, ErrNoTemplate) {
|
|
t.Errorf("%s: %v, want ErrNoTemplate", tc.name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The whole seam through the intake: a book uploaded to a deployment WITH a template gets its
|
|
// configuration rendered by the first parse pass, and the operator is never involved.
|
|
func TestAnUploadedBookIsProvisionedFromTheDeploymentTemplate(t *testing.T) {
|
|
f := newFixture(t)
|
|
tpl := filepath.Join(t.TempDir(), "book.yaml")
|
|
if err := os.WriteFile(tpl, []byte(templateYAML), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.svc.Cfg.BookTemplate = tpl
|
|
book := f.accept(t, "蛊真人.txt", "первая глава")
|
|
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := f.card(t, book.ID); got.Status != "not_started" {
|
|
t.Fatalf("a book on a deployment with a template is %q; nobody had to touch the host", got.Status)
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(f.root, book.ID, runner.ConfigFile))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got map[string]any
|
|
if err := yaml.Unmarshal(raw, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The book's identity is the PLATFORM's, and the source is the file the intake actually wrote —
|
|
// extension included, because the engine dispatches its reader by it.
|
|
if got["book_id"] != book.ID || got["source_file"] != SourceName+".txt" {
|
|
t.Errorf("rendered book_id=%v source_file=%v", got["book_id"], got["source_file"])
|
|
}
|
|
if got["source_lang"] != "zh" || got["target_lang"] != "ru" {
|
|
t.Errorf("the user's declared intake did not reach the configuration: %+v", got)
|
|
}
|
|
if got["title"] != "蛊真人" {
|
|
t.Errorf("title = %v", got["title"])
|
|
}
|
|
}
|
|
|
|
// Once the file exists it is the OPERATOR's, and the platform never writes it again. An operator who
|
|
// lowered a ceiling or pointed a book at a different pipeline must find their change still there the
|
|
// next time anything touches the book (D39.110 §2b).
|
|
//
|
|
// ⚠ What this pins is the SHORT-CIRCUIT — provision sees a configuration and returns before it reads
|
|
// anything — and that is stated precisely because the `O_EXCL` two lines further down is NOT what it
|
|
// catches: with an existing file the open is never reached, so a landing that swaps it for `O_TRUNC`
|
|
// survives this test. `O_EXCL` guards a different case, two passes of the intake rendering at once,
|
|
// and it has no test: both would write identical bytes, so there is nothing for an assertion to see.
|
|
// Named rather than claimed.
|
|
//
|
|
// Mutation caught: removing the `os.Stat` short-circuit from provision.
|
|
func TestARenderedConfigurationIsNeverRewritten(t *testing.T) {
|
|
f := newFixture(t)
|
|
tpl := filepath.Join(t.TempDir(), "book.yaml")
|
|
if err := os.WriteFile(tpl, []byte(templateYAML), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.svc.Cfg.BookTemplate = tpl
|
|
book := f.accept(t, "book.txt", "первая глава")
|
|
// The first walk renders the configuration and then fails on the ENGINE, so the book stays in
|
|
// intake and a second walk really happens. A book that parsed on the first pass is never looked at
|
|
// again, which is what made the earlier version of this test assert nothing at all.
|
|
f.engine.set(ingest.Manifest{}, &exec.Error{Name: "tmctl", Err: os.ErrNotExist})
|
|
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(f.root, book.ID, runner.ConfigFile)
|
|
if _, err := os.Stat(path); err != nil {
|
|
t.Fatalf("the first walk rendered nothing: %v", err)
|
|
}
|
|
edited := "# an operator was here\nbook_id: " + book.ID +
|
|
"\nsource_lang: zh\ntarget_lang: ru\nsource_file: source.txt\npipeline: /other.yaml\n"
|
|
if err := os.WriteFile(path, []byte(edited), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The template is REMOVED before the second walk. If anything reaches for it, the book stops being
|
|
// parseable — which is what turns "the file was not rewritten" into "the file was not even
|
|
// considered".
|
|
if err := os.Remove(tpl); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.engine.set(ingest.Manifest{Version: "tm-manifest-v2", ChaptersTotal: 3,
|
|
SourceSHA256: strings.Repeat("ab", 32), ChunkerVersion: "chunk-2026.07"}, nil)
|
|
// Another walk of the same book — a re-parse after the claim went stale.
|
|
f.now = f.now.Add(claimGrace + time.Minute)
|
|
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := f.card(t, book.ID); got.Status != "not_started" {
|
|
t.Fatalf("the second walk went looking for a template for a book that already had a configuration: %q", got.Status)
|
|
}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(raw) != edited {
|
|
t.Fatalf("the platform rewrote a configuration that was not its own:\n%s", raw)
|
|
}
|
|
}
|
|
|
|
// A deployment whose template is missing or broken must WAIT, not reject: the fault applies to every
|
|
// book on the host at once, and rejecting is what deletes a user's upload.
|
|
//
|
|
// Mutation caught: returning any error other than ErrNotProvisioned from provision when the template
|
|
// cannot be used.
|
|
func TestABrokenTemplateStopsNoIntakeAndSpendsNoBudget(t *testing.T) {
|
|
for _, tc := range []struct{ name, body string }{
|
|
{"missing", ""},
|
|
{"unparseable", "{{{\n"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
f := newFixture(t)
|
|
tpl := filepath.Join(t.TempDir(), "book.yaml")
|
|
if tc.body != "" {
|
|
if err := os.WriteFile(tpl, []byte(tc.body), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
f.svc.Cfg.BookTemplate = tpl
|
|
book := f.accept(t, "book.txt", "первая глава")
|
|
for range parseAttempts * 2 {
|
|
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.now = f.now.Add(claimGrace + time.Minute)
|
|
}
|
|
if got := f.card(t, book.ID); got.Status != "parsing" {
|
|
t.Fatalf("a book on a host with a broken template is %q, want parsing", got.Status)
|
|
}
|
|
var attempts int
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select parse_attempts from books where id = $1`, book.ID).Scan(&attempts); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if attempts != 0 {
|
|
t.Errorf("waiting for a template spent %d attempts; the first real answer after that "+
|
|
"would be terminal on arrival", attempts)
|
|
}
|
|
if f.engine.called() != 0 {
|
|
t.Errorf("the engine was asked %d times for a book with no configuration", f.engine.called())
|
|
}
|
|
// And the moment the operator fixes it, the very next pass renders and parses.
|
|
if err := os.WriteFile(tpl, []byte(templateYAML), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.engine.set(ingest.Manifest{Version: "tm-manifest-v2", ChaptersTotal: 3,
|
|
SourceSHA256: strings.Repeat("ab", 32), ChunkerVersion: "chunk-2026.07"}, nil)
|
|
f.now = f.now.Add(claimGrace + time.Minute)
|
|
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := f.card(t, book.ID); got.Status != "not_started" {
|
|
t.Fatalf("after the template was fixed the book is %q", got.Status)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A deployment with NO template configured keeps the behaviour it had before form Б: the operator
|
|
// drops the file in by hand and the book waits, unharmed, until they do.
|
|
func TestADeploymentWithoutATemplateStillWaitsForItsOperator(t *testing.T) {
|
|
f := newFixture(t) // no BookTemplate
|
|
book := f.accept(t, "book.txt", "первая глава")
|
|
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := f.card(t, book.ID); got.Status != "parsing" {
|
|
t.Fatalf("book %q on a deployment that provisions by hand", got.Status)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(f.root, book.ID, runner.ConfigFile)); !os.IsNotExist(err) {
|
|
t.Errorf("a deployment with no template wrote a configuration anyway: %v", err)
|
|
}
|
|
}
|
|
|
|
// The source file name is a fact about the DIRECTORY, and it is what `source_file` has to say. It is
|
|
// found by name rather than stored because the intake is the only writer of that directory — but the
|
|
// finding has to survive the neighbours it legitimately has.
|
|
func TestTheSourceFileIsFoundNextToTheConfiguration(t *testing.T) {
|
|
dir := t.TempDir()
|
|
for _, name := range []string{runner.ConfigFile, "notes.txt", SourceName + ".epub"} {
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := os.Mkdir(filepath.Join(dir, SourceName), 0o750); err != nil {
|
|
t.Fatal(err) // a DIRECTORY called source is not the source
|
|
}
|
|
got, err := sourceFileIn(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got != SourceName+".epub" {
|
|
t.Errorf("source file %q, want %s.epub", got, SourceName)
|
|
}
|
|
if _, err := sourceFileIn(t.TempDir()); err == nil {
|
|
t.Error("an empty directory answered with a source file")
|
|
}
|
|
}
|