textmachine/backend/internal/pipeline/promptcomments_test.go

package pipeline

import (
	"crypto/sha256"
	"encoding/hex"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

// promptcomments_test.go pins backlog 14b: an editorial <!-- … --> note in a prompt file is for the
// human who edits the template, and must never be paid for as system tokens on every call.

func rawSHA(s string) string {
	sum := sha256.Sum256([]byte(s))
	return hex.EncodeToString(sum[:])
}

// rawTemplateFixture carries one canary comment per section, so a strip that covers only some of
// them is visible rather than silently partial.
const rawTemplateFixture = "<!-- СЕКРЕТ-система -->\nЯдро системы.\n" +
	"---FEWSHOT---\n<!-- СЕКРЕТ-фьюшот -->\nПример.\n" +
	"---USER---\n<!-- СЕКРЕТ-юзер -->\nТекст: {{text}}"

// A comment must not survive into ANY message — system, few-shot or user. Checking all three
// catches a partial strip (e.g. one applied only to the system half after the split).
func TestPromptCommentsNeverReachTheWire(t *testing.T) {
	tpl := writeTemplate(t, rawTemplateFixture)

	// Production flattens the few-shot block into System before rendering (runner.go), so the test
	// must too — otherwise the few-shot canary is never actually carried into a message and the
	// assertion below passes vacuously.
	flat := *tpl
	flat.System = tpl.SystemFor(true)
	msgs, err := MessagesWithInjection(&flat, RenderVars{Book: testBook(), Text: "исходник"}, "инъекция")
	if err != nil {
		t.Fatal(err)
	}
	// Premise check: the canaries must be present in the RAW file, or this test proves nothing.
	for _, canary := range []string{"СЕКРЕТ-система", "СЕКРЕТ-фьюшот", "СЕКРЕТ-юзер"} {
		if !strings.Contains(rawTemplateFixture, canary) {
			t.Fatalf("fixture lost canary %q — the test would pass vacuously", canary)
		}
	}
	for _, m := range msgs {
		for _, bad := range []string{"СЕКРЕТ", commentOpen, commentClose} {
			if strings.Contains(m.Content, bad) {
				t.Fatalf("%s message leaked %q on the wire: %q", m.Role, bad, m.Content)
			}
		}
	}
	// The surrounding prompt text itself must survive the strip intact — in every section.
	if !strings.Contains(flat.System, "Ядро системы.") || !strings.Contains(flat.System, "Пример.") {
		t.Fatalf("stripping ate real prompt text: system = %q", flat.System)
	}
	if !strings.Contains(tpl.User, "Текст: {{text}}") {
		t.Fatalf("stripping ate real prompt text: user = %q", tpl.User)
	}
}

// The "no book re-snapshots" guarantee: a file with no comments must hash EXACTLY as the raw
// bytes did before this change, so every book already in flight keeps its snapshot id.
func TestPromptCommentFreeFileKeepsRawSHA(t *testing.T) {
	const content = "Система без комментариев.\n---USER---\n{{text}}"
	tpl := writeTemplate(t, content)
	if tpl.SHA256 != rawSHA(content) {
		t.Fatalf("a comment-free prompt must keep the raw-bytes SHA (no book may move)\n got  %s\n want %s",
			tpl.SHA256, rawSHA(content))
	}
}

// The mutation-killer for "hash the raw file instead of the canonical form": with a comment
// present the two hashes DIFFER, and only the canonical one is correct. It also pins the payoff —
// editing a comment no longer moves the hash, so it no longer costs money.
func TestPromptSHAIsOverTheCanonicalForm(t *testing.T) {
	const withComment = "<!-- заметка редактора -->\nСистема.\n---USER---\n{{text}}"
	const stripped = "\nСистема.\n---USER---\n{{text}}"
	const otherComment = "<!-- СОВСЕМ другая заметка, длиннее -->\nСистема.\n---USER---\n{{text}}"

	tpl := writeTemplate(t, withComment)
	if tpl.SHA256 == rawSHA(withComment) {
		t.Fatal("SHA is over the RAW file: a comment edit would still re-bill the book")
	}
	if tpl.SHA256 != rawSHA(stripped) {
		t.Fatalf("SHA must be over the comment-stripped form\n got  %s\n want %s", tpl.SHA256, rawSHA(stripped))
	}
	if other := writeTemplate(t, otherComment); other.SHA256 != tpl.SHA256 {
		t.Fatalf("two files differing ONLY in comment text must share a SHA: %s vs %s", tpl.SHA256, other.SHA256)
	}
}

// Stripping must happen BEFORE the ---USER--- split: a separator sitting inside a comment is not a
// separator. Stripping after the split would cut the file at a line the model never sees.
func TestPromptSeparatorInsideCommentDoesNotSplit(t *testing.T) {
	tpl := writeTemplate(t, "Система.\n<!-- отключено:\n---USER---\nстарый юзер-блок\n-->\n---USER---\nнастоящий {{text}}")
	if strings.Contains(tpl.System, "старый юзер-блок") || strings.Contains(tpl.User, "старый юзер-блок") {
		t.Fatalf("commented-out block survived: system=%q user=%q", tpl.System, tpl.User)
	}
	if tpl.System != "Система." {
		t.Fatalf("system = %q, want the text before the comment only", tpl.System)
	}
	if tpl.User != "настоящий {{text}}" {
		t.Fatalf("user = %q — the split took the COMMENTED separator", tpl.User)
	}
}

// An unterminated comment is a malformed template: fail loud at LOAD time, before any billing,
// rather than silently swallowing the rest of the file (or shipping the note to the model).
func TestPromptUnterminatedCommentFailsLoud(t *testing.T) {
	path := filepath.Join(t.TempDir(), "tpl.md")
	if err := os.WriteFile(path, []byte("Система.\n<!-- забыли закрыть\n---USER---\n{{text}}"), 0o644); err != nil {
		t.Fatal(err)
	}
	_, err := LoadPromptTemplate(path)
	if err == nil {
		t.Fatal("an unterminated <!-- must fail loud at load")
	}
	if !strings.Contains(err.Error(), "unterminated") {
		t.Fatalf("the error must name the cause, got: %v", err)
	}
}

// The shipped pair prompts must not carry a comment onto the wire — the concrete regression that
// sent terminologist.md's meta header to the model in the system message of every batch. Pins the
// STRUCTURE (no comment markers, no prompt-file wording), never the prompt text itself.
func TestShippedPromptsCarryNoCommentsOnTheWire(t *testing.T) {
	root := filepath.Join("..", "..", "prompts")
	if _, err := os.Stat(root); err != nil {
		t.Skipf("pair prompts not present in this checkout: %v", err)
	}
	// Walk, not ReadDir: the repair/ class prompts live in a subdirectory and are loaded by the
	// same loader, so a non-recursive guard would silently cover only part of the shipped set.
	var files []string
	if err := filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !fi.IsDir() && strings.HasSuffix(p, ".md") {
			files = append(files, p)
		}
		return nil
	}); err != nil {
		t.Fatal(err)
	}
	for _, path := range files {
		tpl, err := LoadPromptTemplate(path)
		if err != nil {
			t.Fatalf("%s: %v", path, err)
		}
		for part, s := range map[string]string{"system": tpl.SystemFor(true), "user": tpl.User} {
			if strings.Contains(s, commentOpen) || strings.Contains(s, commentClose) {
				t.Errorf("%s: a comment marker reached the %s message", path, part)
			}
		}
	}
	// A floor, so a mis-pointed path or a changed layout cannot make the guard silently inert.
	if len(files) < 10 {
		t.Fatalf("expected at least 10 shipped prompt files, walked %d: %v", len(files), files)
	}
}