From bb1a1fa56f6210687e64e7a231b73bb0bdaadfbd Mon Sep 17 00:00:00 2001 From: Alex Verkhovsky Date: Thu, 20 Aug 2026 04:33:18 -0700 Subject: [PATCH] chore(test): rewrite snapshot renderer tests in Python The Node suite still installed skills under `_bmad/bmm/`. Cover the current host-skill layout from render_skill.py itself, and keep a few internals tests for publish paths that are awkward to hit through a full skill. --- package.json | 2 +- .../bmad/scripts/tests/test_render_skill.py | 516 +++++++++++++++++ test/test-build-auto-renderer.js | 530 ------------------ 3 files changed, 517 insertions(+), 531 deletions(-) create mode 100644 skills/bmad/scripts/tests/test_render_skill.py delete mode 100644 test/test-build-auto-renderer.js diff --git a/package.json b/package.json index 7285a21ca..0fd4dccf4 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "test:npx-skills": "uv run --python 3.11 python -m unittest tools/tests/test_bmad_setup.py", "test:implementation-model": "node test/test-validate-published-implementation-model.mjs", "test:refs": "uv run --python 3.11 --with \"pyyaml>=6.0.2,<7\" python -m unittest tools/tests/test_validate_file_refs.py", - "test:renderer": "uv run --python 3.11 python -m unittest skills/bmad/scripts/tests/test_config_utils.py skills/bmad/scripts/tests/test_resolve_config.py skills/bmad/scripts/tests/test_resolve_customization.py && node test/test-build-auto-renderer.js", + "test:renderer": "uv run --python 3.11 python -m unittest skills/bmad/scripts/tests/test_config_utils.py skills/bmad/scripts/tests/test_resolve_config.py skills/bmad/scripts/tests/test_resolve_customization.py skills/bmad/scripts/tests/test_render_skill.py", "test:retrospective": "uv run --python 3.11 skills/bmad-retrospective/scripts/tests/test_git_evidence.py && uv run --python 3.11 skills/bmad-retrospective/scripts/tests/test_sprint_status.py", "test:site-url": "node website/test/test-site-url.mjs", "test:skills": "uv run --python 3.11 python -m unittest tools/tests/test_validate_skills.py", diff --git a/skills/bmad/scripts/tests/test_render_skill.py b/skills/bmad/scripts/tests/test_render_skill.py new file mode 100644 index 000000000..bed1fce83 --- /dev/null +++ b/skills/bmad/scripts/tests/test_render_skill.py @@ -0,0 +1,516 @@ +"""Snapshot renderer tests against the current install layout and shipped skills. + +Host skills live outside `_bmad/`. `_bmad/` is the project runtime setup +materializes: shared scripts, team config, custom overlays, and published +snapshots. Call `render()` for the success path. Use the installed CLI for +the agent-facing dispatch/HALT contract and for anything that needs a +separate process. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace + + +SCRIPTS_SRC = Path(__file__).resolve().parents[1] +REPO = SCRIPTS_SRC.parents[2] +SKILLS_SRC = REPO / "skills" +CONFIG_TEMPLATE = (SCRIPTS_SRC.parent / "assets" / "config.template.toml").read_text( + encoding="utf-8" +) +SHARED_SCRIPTS = ( + "config_utils.py", + "memlog.py", + "render_skill.py", + "resolve_config.py", + "resolve_customization.py", +) +SHIPPED_SKILLS = ("bmad-build-auto", "bmad-build") +COMPILE_TOKEN = re.compile(r"\{\{(?:\.|config\.)|\{workflow\.|\[\[bmad-snapshot:") +DISPATCH_PREFIX = "read and follow " + +sys.path.insert(0, str(SCRIPTS_SRC)) +import render_skill as rs # noqa: E402 + + +def _team_config(project: Path) -> str: + return CONFIG_TEMPLATE.replace("{directory_name}", project.name) + + +def _copy_skill(dest: Path, name: str) -> Path: + shutil.copytree( + SKILLS_SRC / name, + dest, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), + ) + return dest + + +def _files(directory: Path) -> dict[str, bytes]: + files = { + path.relative_to(directory).as_posix(): path.read_bytes() + for path in directory.rglob("*") + if path.is_file() + } + return dict(sorted(files.items())) + + +def _markdown(directory: Path) -> str: + return "\n".join( + content.decode("utf-8") + for name, content in _files(directory).items() + if name.endswith(".md") + ) + + +def _namespace_dir(project: Path, skill_name: str) -> Path: + root = str(project.resolve()) + slug = re.sub(r"[^a-z0-9]+", "-", project.name.lower()).strip("-") or "project" + slug = slug[:80].rstrip("-") or "project" + root_hash = hashlib.sha256(root.encode("utf-8")).hexdigest()[:12] + return project / "_bmad" / "render" / skill_name / f"{slug}-{root_hash}" + + +class PublishInternalsTests(unittest.TestCase): + """Corruption and reuse branches of `_publish` without rendering a whole skill.""" + + def test_identical_publish_reuses_and_rejects_each_corruption_mode(self): + with tempfile.TemporaryDirectory() as temp_dir: + dest = Path(temp_dir) / "generation" + outputs = {"workflow.md": b"hello\n"} + manifest = { + "schema_version": 1, + "outputs": {"workflow.md": rs._hash_bytes(b"hello\n")}, + } + rs._publish(dest, outputs, manifest) + rs._publish(dest, outputs, manifest) + self.assertEqual((dest / "workflow.md").read_bytes(), b"hello\n") + + with self.assertRaisesRegex(rs.RenderError, "collision or corruption"): + rs._publish(dest, outputs, {**manifest, "extra": True}) + + (dest / "extra.md").write_text("stray\n", encoding="utf-8") + with self.assertRaisesRegex(rs.RenderError, "unexpected or missing"): + rs._publish(dest, outputs, manifest) + (dest / "extra.md").unlink() + + (dest / "workflow.md").write_bytes(b"hello\ncorrupt") + with self.assertRaisesRegex(rs.RenderError, "hash mismatch"): + rs._publish(dest, outputs, manifest) + (dest / "workflow.md").write_bytes(b"hello\n") + + (dest / "manifest.json").write_text("{", encoding="utf-8") + with self.assertRaisesRegex(rs.RenderError, "corrupt existing"): + rs._publish(dest, outputs, manifest) + + +class RenderSkillTests(unittest.TestCase): + def _workspace( + self, + *, + name: str = "project", + shared_bmad: Path | None = None, + config: str | None = None, + ) -> SimpleNamespace: + outer = Path(tempfile.mkdtemp(prefix="bmad-render-")) + self.addCleanup(shutil.rmtree, outer, True) + project = outer / name + project.mkdir(parents=True) + (project / "nested" / "cwd").mkdir(parents=True) + if shared_bmad is None: + bmad = project / "_bmad" + scripts = bmad / "scripts" + scripts.mkdir(parents=True) + for script in SHARED_SCRIPTS: + shutil.copy2(SCRIPTS_SRC / script, scripts / script) + (bmad / "custom").mkdir() + (bmad / "config.toml").write_text( + config if config is not None else _team_config(project), + encoding="utf-8", + ) + else: + (project / "_bmad").symlink_to(shared_bmad) + bmad = shared_bmad + return SimpleNamespace(outer=outer, project=project, bmad=bmad) + + def _skill(self, ws: SimpleNamespace, name: str) -> Path: + return _copy_skill(ws.outer / "skills" / name, name) + + def _cli( + self, project: Path, skill: Path, *, cwd: Path | None = None + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(project / "_bmad" / "scripts" / "render_skill.py"), + "--project-root", + str(project), + "--skill", + str(skill), + ], + cwd=cwd or project, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + def _entry(self, result: subprocess.CompletedProcess[str]) -> Path: + self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr) + lines = result.stdout.strip().split("\n") + self.assertEqual(len(lines), 1, msg=result.stdout) + self.assertTrue(lines[0].startswith(DISPATCH_PREFIX), msg=result.stdout) + output = Path(lines[0][len(DISPATCH_PREFIX) :]) + self.assertTrue(output.is_absolute()) + return output + + def _assert_snapshot(self, workflow: Path, project: Path, skill_name: str) -> Path: + snap = workflow.parent + self.assertEqual(workflow.name, "workflow.md") + self.assertIn(f"{os.sep}render{os.sep}{skill_name}{os.sep}", str(workflow)) + self.assertFalse((snap / "SKILL.md").exists()) + manifest = json.loads((snap / "manifest.json").read_text(encoding="utf-8")) + self.assertEqual(manifest["project_root"], str(project.resolve())) + self.assertEqual(manifest["skill"], skill_name) + actual = _files(snap) + expected = [*manifest["outputs"], "manifest.json"] + self.assertEqual(sorted(actual), sorted(expected)) + for name, digest in manifest["outputs"].items(): + self.assertEqual(rs._hash_bytes(actual[name]), digest, name) + markdown = _markdown(snap) + self.assertIsNone(COMPILE_TOKEN.search(markdown), markdown) + self.assertNotIn("{skill-root}", markdown) + artifacts = str(project.resolve() / "_bmad-output" / "implementation-artifacts") + self.assertIn(artifacts, markdown) + return snap + + def test_unsupported_customization_default_type_is_rejected(self): + # No shipped skill uses a boolean customization default; arranging one + # through customize.toml would only exist to reach this branch. + with self.assertRaisesRegex(rs.RenderError, "unsupported default type"): + rs._resolve_customization_value( + True, True, "customization.workflow.flag" + ) + + def test_shipped_skills_publish_root_bound_snapshots(self): + for name in SHIPPED_SKILLS: + with self.subTest(name): + ws = self._workspace() + skill = self._skill(ws, name) + workflow = rs.render(ws.project, skill) + snap = self._assert_snapshot(workflow, ws.project, name) + self.assertIn("{spec_file}", _markdown(snap)) + hunter = snap / "review-prompts" / "edge-case-hunter.md" + self.assertTrue(hunter.is_file()) + self.assertIn(str(hunter), _markdown(snap)) + + def test_cli_from_nested_cwd_dispatches_one_absolute_workflow(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + workflow = self._entry( + self._cli(ws.project, skill, cwd=ws.project / "nested" / "cwd") + ) + self._assert_snapshot(workflow, ws.project, "bmad-build") + self.assertFalse((ws.bmad / "scripts" / "__pycache__").exists()) + self.assertFalse((skill / "__pycache__").exists()) + + def test_identical_input_and_unreferenced_config_reuse_bytes(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + first = rs.render(ws.project, skill) + first_files = _files(first.parent) + self.assertEqual(rs.render(ws.project, skill), first) + with (ws.bmad / "config.toml").open("a", encoding="utf-8") as handle: + handle.write('\nunreferenced_value = "ignored"\n') + self.assertEqual(rs.render(ws.project, skill), first) + current = _files(first.parent) + for name, content in first_files.items(): + self.assertEqual(current[name], content, name) + + def test_referenced_config_and_source_changes_publish_new_generations(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build-auto") + before = rs.render(ws.project, skill) + before_files = _files(before.parent) + (ws.bmad / "config.user.toml").write_text( + '[modules.bmm]\nimplementation_artifacts = "{project-root}/impl-v2"\n', + encoding="utf-8", + ) + after_config = rs.render(ws.project, skill) + self.assertNotEqual(after_config, before) + self.assertIn("/impl-v2/", after_config.read_text(encoding="utf-8")) + self.assertTrue(before.exists()) + + (skill / "compile-epic-context.md").write_text( + (skill / "compile-epic-context.md").read_text(encoding="utf-8") + + "\n\n", + encoding="utf-8", + ) + after_source = rs.render(ws.project, skill) + self.assertNotEqual(after_source, after_config) + current = _files(before.parent) + for name, content in before_files.items(): + self.assertEqual(current[name], content, name) + + def test_shared_runtime_keeps_distinct_root_bound_snapshots(self): + first = self._workspace() + skill = self._skill(first, "bmad-build") + second = self._workspace(name="other", shared_bmad=first.bmad) + one = rs.render(first.project, skill) + two = rs.render(second.project, skill) + self.assertNotEqual(one, two) + self.assertIn(str(first.project.resolve()), one.read_text(encoding="utf-8")) + self.assertIn(str(second.project.resolve()), two.read_text(encoding="utf-8")) + + def test_concurrent_cli_renderers_reuse_one_complete_generation(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + with ThreadPoolExecutor(max_workers=2) as pool: + results = list( + pool.map(lambda _: self._cli(ws.project, skill), range(2)) + ) + entries = [self._entry(result) for result in results] + self.assertEqual(entries[0], entries[1]) + self.assertTrue((entries[0].parent / "manifest.json").is_file()) + + def test_malformed_config_and_customization_halt_without_traceback(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + (ws.bmad / "custom" / "config.toml").write_text("[core\nbad", encoding="utf-8") + result = self._cli(ws.project, skill) + self.assertNotEqual(result.returncode, 0) + self.assertTrue(result.stdout.startswith("HALT:"), result.stdout) + self.assertNotIn(DISPATCH_PREFIX, result.stdout) + self.assertNotIn("Traceback", result.stdout + result.stderr) + + (ws.bmad / "custom" / "config.toml").unlink() + (ws.bmad / "custom" / f"{skill.name}.toml").write_text( + "[workflow\nbad", encoding="utf-8" + ) + result = self._cli(ws.project, skill) + self.assertNotEqual(result.returncode, 0) + self.assertIn("failed to parse", result.stdout) + + def test_missing_wrong_type_and_non_string_layer_id_halt(self): + template = _team_config(Path("project")) + missing = template.replace( + 'implementation_artifacts = "{project-root}/_bmad-output/implementation-artifacts"\n', + "", + ) + ws = self._workspace(config=missing) + skill = self._skill(ws, "bmad-build") + result = self._cli(ws.project, skill) + self.assertIn("missing config value", result.stdout) + + wrong = template.replace( + 'implementation_artifacts = "{project-root}/_bmad-output/implementation-artifacts"', + "implementation_artifacts = 42", + ) + ws = self._workspace(config=wrong) + skill = self._skill(ws, "bmad-build") + result = self._cli(ws.project, skill) + self.assertIn("must be a string", result.stdout) + + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + (ws.bmad / "custom" / f"{skill.name}.toml").write_text( + '[[workflow.review_layers]]\nid = 42\nname = "bad"\ninstruction = "bad"\n', + encoding="utf-8", + ) + result = self._cli(ws.project, skill) + self.assertIn("identifier `id` must be a string", result.stdout) + + def test_customization_prose_is_not_rescanned_as_source_tokens(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + literal = "[[bmad-snapshot:step-04-review.md]]" + compile_literal = "{workflow.implementation_handoff}" + (ws.bmad / "custom" / f"{skill.name}.user.toml").write_text( + f'[workflow]\non_complete = "Preserve {literal} and {compile_literal} as prose"\n', + encoding="utf-8", + ) + markdown = _markdown(rs.render(ws.project, skill).parent) + self.assertIn(literal, markdown) + self.assertIn(compile_literal, markdown) + + def test_review_layer_override_guard_and_empty_layer_halt(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + (ws.bmad / "custom" / f"{skill.name}.toml").write_text( + "\n".join( + [ + "[[workflow.review_layers]]", + 'id = "blind-hunter"', + 'name = "Replacement"', + 'instruction = "Run replacement review."', + 'when = "the replacement condition holds"', + "", + ] + ), + encoding="utf-8", + ) + review = (rs.render(ws.project, skill).parent / "step-04-review.md").read_text( + encoding="utf-8" + ) + self.assertIn("Replacement (`blind-hunter`)", review) + self.assertIn("Run only when: the replacement condition holds", review) + self.assertIn("Run replacement review.", review) + + defaults = tomllib.loads((skill / "customize.toml").read_text(encoding="utf-8")) + disabled = "\n".join( + '[[workflow.review_layers]]\n' + f'id = "{layer["id"]}"\n' + 'name = "disabled"\n' + 'instruction = ""\n' + for layer in defaults["workflow"]["review_layers"] + ) + (ws.bmad / "custom" / f"{skill.name}.toml").write_text( + disabled, encoding="utf-8" + ) + review = (rs.render(ws.project, skill).parent / "step-04-review.md").read_text( + encoding="utf-8" + ) + self.assertIn("No active review layers. HALT", review) + + def test_empty_open_spec_override_clears_the_shipped_default(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + (ws.bmad / "custom" / f"{skill.name}.user.toml").write_text( + '[workflow]\nopen_spec = ""\n', encoding="utf-8" + ) + snap = rs.render(ws.project, skill).parent + for name in ("step-05-present.md", "step-oneshot.md"): + rendered = (snap / name).read_text(encoding="utf-8") + self.assertNotIn("code -r", rendered) + self.assertIn("Suggested Review Order", rendered) + + def test_installed_renderer_identity_change_publishes_a_new_generation(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + original = self._entry(self._cli(ws.project, skill)) + with (ws.bmad / "scripts" / "render_skill.py").open("a", encoding="utf-8") as handle: + handle.write("\n# renderer identity change\n") + changed = self._entry(self._cli(ws.project, skill)) + self.assertNotEqual(changed, original) + self.assertTrue(original.exists()) + + def test_convention_only_skill_renders_without_customization(self): + ws = self._workspace() + skill = ws.outer / "skills" / "plain-workflow" + skill.mkdir(parents=True) + (skill / "workflow.md").write_text( + "Read `[[bmad-snapshot:step.md]]`.\n", encoding="utf-8" + ) + (skill / "step.md").write_text("No rendered values required.\n", encoding="utf-8") + workflow = rs.render(ws.project, skill) + self.assertIn(f"{os.sep}render{os.sep}plain-workflow{os.sep}", str(workflow)) + self.assertTrue((workflow.parent / "step.md").is_file()) + self.assertIn(str(workflow.parent / "step.md"), workflow.read_text(encoding="utf-8")) + + def test_ambiguous_shorthand_and_source_symlink_escape_halt(self): + config = _team_config(Path("project")).replace( + "[core]\n", + '[core]\nimplementation_artifacts = "{project-root}/dup"\n', + 1, + ) + ws = self._workspace(config=config) + skill = self._skill(ws, "bmad-build") + result = self._cli(ws.project, skill) + self.assertIn("ambiguous config value", result.stdout) + + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + outside = ws.outer / "outside.md" + outside.write_text("outside\n", encoding="utf-8") + (skill / "workflow.md").unlink() + (skill / "workflow.md").symlink_to(outside) + result = self._cli(ws.project, skill) + self.assertIn("escapes skill directory", result.stdout) + + def test_long_project_basename_is_bounded_in_the_snapshot_namespace(self): + ws = self._workspace(name="project-" + ("x" * 220)) + skill = self._skill(ws, "bmad-build") + workflow = rs.render(ws.project, skill) + self.assertLessEqual(len(workflow.parent.parent.name), 93) + + def test_snapshot_paths_stay_opaque_when_the_project_name_looks_like_tokens(self): + ws = self._workspace(name="{workflow.on_complete}-{{.planning_artifacts}}") + skill = self._skill(ws, "bmad-build") + workflow = rs.render(ws.project, skill) + text = workflow.read_text(encoding="utf-8") + match = re.search(r"`([^`]*step-01-clarify-and-route\.md)`", text) + self.assertIsNotNone(match, text) + self.assertTrue(match.group(1).startswith(str(ws.project.resolve()))) + self.assertTrue(Path(match.group(1)).is_file()) + + def test_publication_failure_does_not_dispatch_or_alter_another_root(self): + stable = self._workspace() + skill = self._skill(stable, "bmad-build") + original = rs.render(stable.project, skill) + original_files = _files(original.parent) + broken = self._workspace(name="broken", shared_bmad=stable.bmad) + namespace = _namespace_dir(broken.project, skill.name) + namespace.parent.mkdir(parents=True, exist_ok=True) + namespace.write_text("not a directory\n", encoding="utf-8") + result = self._cli(broken.project, skill) + self.assertNotEqual(result.returncode, 0) + self.assertTrue(result.stdout.startswith("HALT:"), result.stdout) + self.assertNotIn(DISPATCH_PREFIX, result.stdout) + current = _files(original.parent) + for name, content in original_files.items(): + self.assertEqual(current[name], content, name) + + def test_corrupt_existing_destination_is_never_overwritten(self): + ws = self._workspace() + skill = self._skill(ws, "bmad-build") + workflow = rs.render(ws.project, skill) + workflow.write_text(workflow.read_text(encoding="utf-8") + "corrupt", encoding="utf-8") + result = self._cli(ws.project, skill) + self.assertNotEqual(result.returncode, 0) + self.assertIn("hash mismatch", result.stdout) + self.assertTrue(workflow.read_text(encoding="utf-8").endswith("corrupt")) + + def test_shipped_skill_md_command_dispatches_for_both_skills(self): + for name in SHIPPED_SKILLS: + with self.subTest(name): + ws = self._workspace() + skill = self._skill(ws, name) + text = (skill / "SKILL.md").read_text(encoding="utf-8") + fenced = re.search(r"```bash\n(.*?)```", text, re.S) + self.assertIsNotNone(fenced, f"{name}: SKILL.md ships no bash command") + command = ( + fenced.group(1) + .strip() + .replace("{project-root}", str(ws.project)) + .replace("{skill-root}", str(skill)) + ) + self.assertNotIn("{", command) + dispatched = self._entry( + subprocess.run( + command, + cwd=ws.project / "nested" / "cwd", + shell=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + ) + self.assertEqual(dispatched.name, "workflow.md") + self.assertTrue(dispatched.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test-build-auto-renderer.js b/test/test-build-auto-renderer.js deleted file mode 100644 index 161d68a99..000000000 --- a/test/test-build-auto-renderer.js +++ /dev/null @@ -1,530 +0,0 @@ -// Test only deterministic renderer behavior. -// Do not test model inference or assert prose copied verbatim from skill sources. -/** Black-box tests for the shared immutable snapshot renderer, covering both bmad-build-auto and bmad-build. */ -'use strict'; - -const crypto = require('node:crypto'); -const fs = require('node:fs'); -const os = require('node:os'); -const path = require('node:path'); -const { spawn, spawnSync } = require('node:child_process'); - -const REPO = path.resolve(__dirname, '..'); -const SCRIPT_SRC = path.join(REPO, 'skills', 'bmad', 'scripts'); -const SKILLS_SRC = path.join(REPO, 'skills'); -const DEFAULT_SKILL = 'bmad-build-auto'; -const tempDirs = []; -let total = 0; -let passed = 0; - -function assert(condition, message) { - if (!condition) throw new Error(message); -} - -function test(name, fn) { - total++; - try { - fn(); - passed++; - console.log(` PASS ${name}`); - } catch (error) { - console.error(` FAIL ${name}: ${error.message}`); - } -} - -async function asyncTest(name, fn) { - total++; - try { - await fn(); - passed++; - console.log(` PASS ${name}`); - } catch (error) { - console.error(` FAIL ${name}: ${error.message}`); - } -} - -function copyDir(src, dest) { - fs.mkdirSync(dest, { recursive: true }); - for (const entry of fs.readdirSync(src, { withFileTypes: true })) { - const from = path.join(src, entry.name); - const to = path.join(dest, entry.name); - if (entry.isDirectory()) copyDir(from, to); - else fs.copyFileSync(from, to); - } -} - -function baseConfig(extra = '') { - return [ - '[core]', - extra, - '', - '[modules.bmm]', - 'planning_artifacts = "{project-root}/planning"', - 'implementation_artifacts = "{project-root}/implementation"', - '', - ].join('\n'); -} - -function fixture({ sharedBmad, config = baseConfig(), projectName = 'project', skillName = DEFAULT_SKILL } = {}) { - const outer = fs.mkdtempSync(path.join(os.tmpdir(), 'bmad-build-auto-render-')); - tempDirs.push(outer); - const project = path.join(outer, projectName); - const bmad = sharedBmad || path.join(outer, 'installed-bmad'); - fs.mkdirSync(project, { recursive: true }); - if (!sharedBmad) { - fs.mkdirSync(path.join(bmad, 'scripts'), { recursive: true }); - for (const name of ['config_utils.py', 'render_skill.py']) { - fs.copyFileSync(path.join(SCRIPT_SRC, name), path.join(bmad, 'scripts', name)); - } - copyDir(path.join(SKILLS_SRC, skillName), path.join(bmad, 'bmm', skillName)); - fs.writeFileSync(path.join(bmad, 'config.toml'), config, 'utf8'); - } - fs.symlinkSync(bmad, path.join(project, '_bmad'), process.platform === 'win32' ? 'junction' : 'dir'); - fs.mkdirSync(path.join(project, 'nested', 'cwd'), { recursive: true }); - return { outer, project, bmad, skillName, skill: path.join(bmad, 'bmm', skillName) }; -} - -function run(fix, cwd = fix.project) { - return spawnSync( - 'uv', - ['run', '--python', '3.11', path.join(fix.bmad, 'scripts', 'render_skill.py'), '--project-root', fix.project, '--skill', fix.skill], - { - cwd, - encoding: 'utf8', - }, - ); -} - -function runAsync(fix) { - return new Promise((resolve) => { - const child = spawn( - 'uv', - ['run', '--python', '3.11', path.join(fix.bmad, 'scripts', 'render_skill.py'), '--project-root', fix.project, '--skill', fix.skill], - { cwd: fix.project }, - ); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (chunk) => (stdout += chunk)); - child.stderr.on('data', (chunk) => (stderr += chunk)); - child.on('error', (error) => resolve({ status: null, stdout, stderr: `${stderr}${error.message}` })); - child.on('close', (status) => resolve({ status, stdout, stderr })); - }); -} - -function entry(result) { - assert(result.status === 0, `renderer failed: ${result.stdout}${result.stderr}`); - const lines = result.stdout.trim().split('\n'); - const prefix = 'read and follow '; - const outputPath = lines[0]?.slice(prefix.length); - assert(lines.length === 1 && lines[0].startsWith(prefix) && path.isAbsolute(outputPath), `bad dispatch: ${result.stdout}`); - return outputPath; -} - -function bytesByName(directory) { - const files = {}; - - function visit(current, relative = '') { - for (const entry of fs.readdirSync(current, { withFileTypes: true })) { - const name = relative ? path.posix.join(relative, entry.name) : entry.name; - const fullPath = path.join(current, entry.name); - if (entry.isDirectory()) visit(fullPath, name); - else files[name] = fs.readFileSync(fullPath); - } - } - - visit(directory); - return Object.fromEntries(Object.entries(files).sort(([left], [right]) => left.localeCompare(right))); -} - -function hash(content) { - return crypto.createHash('sha256').update(content).digest('hex'); -} - -async function main() { - const fix = fixture(); - let firstEntry; - let firstBytes; - - test('one uv call from a nested cwd publishes one absolute dispatch', () => { - const result = run(fix, path.join(fix.project, 'nested', 'cwd')); - firstEntry = entry(result); - assert(path.isAbsolute(firstEntry), 'entry is not absolute'); - assert(fs.existsSync(firstEntry), 'entry does not exist'); - firstBytes = bytesByName(path.dirname(firstEntry)); - }); - - test('renderer execution leaves no bytecode cache beside installed files', () => { - assert(!fs.existsSync(path.join(fix.bmad, 'scripts', '__pycache__')), 'shared-script bytecode cache was created'); - assert(!fs.existsSync(path.join(fix.skill, '__pycache__')), 'skill bytecode cache was created'); - }); - - test('snapshot excludes SKILL.md and manifest hashes every rendered output', () => { - const dir = path.dirname(firstEntry); - const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8')); - const actualNames = Object.keys(bytesByName(dir)); - const expectedNames = [...Object.keys(manifest.outputs), 'manifest.json'].sort(); - assert(JSON.stringify(actualNames) === JSON.stringify(expectedNames), 'snapshot file set differs from manifest'); - assert(!fs.existsSync(path.join(dir, 'SKILL.md')), 'SKILL.md was published'); - assert(manifest.project_root === fs.realpathSync(fix.project), 'manifest root is wrong'); - for (const [name, expected] of Object.entries(manifest.outputs)) { - assert(hash(fs.readFileSync(path.join(dir, name))) === expected, `bad output hash for ${name}`); - } - }); - - test('compile tokens disappear, runtime placeholders survive, and references stay in generation', () => { - const dir = path.dirname(firstEntry); - const markdown = fs - .readdirSync(dir) - .filter((name) => name.endsWith('.md')) - .map((name) => fs.readFileSync(path.join(dir, name), 'utf8')) - .join('\n'); - assert(!/\{\{(?:\.|config\.)|\{workflow\.|\[\[bmad-snapshot:/.test(markdown), 'compile token survived'); - assert(!markdown.includes('{skill-root}'), 'mutable skill-root reference survived'); - assert(markdown.includes('{spec_file}'), 'runtime placeholder was removed'); - assert(markdown.includes('/implementation/bmad-build-auto-result-'), 'implementation_artifacts value missing'); - // Blind hunter is inlined; only file-backed reviewers ship under review-prompts/. - for (const prompt of ['edge-case-hunter.md', 'verification-gap.md']) { - const promptPath = path.join(dir, 'review-prompts', prompt); - assert(markdown.includes(promptPath), `snapshot reviewer path missing: ${prompt}`); - assert(fs.existsSync(promptPath), `snapshot reviewer missing: ${prompt}`); - } - for (const match of markdown.matchAll(/`(\/[^`]+\/step-[^`]+\.md)`/g)) { - assert(path.dirname(match[1]) === dir, `cross-generation reference: ${match[1]}`); - } - }); - - test('identical input and irrelevant config changes reuse immutable bytes', () => { - const second = entry(run(fix)); - assert(second === firstEntry, 'identical input created a generation'); - fs.appendFileSync(path.join(fix.bmad, 'config.toml'), '\nunreferenced_value = "ignored"\n'); - const third = entry(run(fix)); - assert(third === firstEntry, 'unreferenced config changed generation identity'); - const current = bytesByName(path.dirname(firstEntry)); - for (const name of Object.keys(firstBytes)) { - assert(firstBytes[name].equals(current[name]), `immutable file changed: ${name}`); - } - }); - - test('effective source changes publish a new generation and preserve the old one', () => { - fs.appendFileSync(path.join(fix.skill, 'compile-epic-context.md'), '\n\n'); - const next = entry(run(fix)); - assert(next !== firstEntry, 'effective change reused generation'); - const current = bytesByName(path.dirname(firstEntry)); - for (const name of Object.keys(firstBytes)) { - assert(firstBytes[name].equals(current[name]), `old generation changed: ${name}`); - } - }); - - test('a referenced resolved value publishes a new generation', () => { - const configured = fixture(); - const before = entry(run(configured)); - fs.writeFileSync( - path.join(configured.bmad, 'config.user.toml'), - '[modules.bmm]\nimplementation_artifacts = "{project-root}/impl-v2"\n', - 'utf8', - ); - const after = entry(run(configured)); - assert(after !== before, 'referenced config change reused generation'); - assert(fs.readFileSync(after, 'utf8').includes('/impl-v2/bmad-build-auto-result-'), 'new value was not rendered'); - assert(fs.existsSync(before), 'prior generation disappeared'); - }); - - test('two project roots sharing _bmad get distinct root-bound snapshots', () => { - const other = fixture({ sharedBmad: fix.bmad }); - const one = entry(run(fix)); - const two = entry(run(other)); - assert(one !== two, 'shared roots collided'); - assert(fs.readFileSync(two, 'utf8').includes(other.project), 'second root was not baked'); - assert(fs.readFileSync(one, 'utf8').includes(fix.project), 'first root lost its binding'); - }); - - await asyncTest('concurrent identical renderers both reuse one complete generation', async () => { - const concurrent = fixture(); - const results = await Promise.all([runAsync(concurrent), runAsync(concurrent)]); - const entries = results.map(entry); - assert(entries[0] === entries[1], 'concurrent renderers returned different generations'); - assert(fs.existsSync(path.join(path.dirname(entries[0]), 'manifest.json')), 'manifest missing'); - }); - - test('malformed present config and customization layers HALT without traceback or dispatch', () => { - const invalid = fixture(); - fs.mkdirSync(path.join(invalid.bmad, 'custom'), { recursive: true }); - fs.writeFileSync(path.join(invalid.bmad, 'custom', 'config.toml'), '[core\nbad', 'utf8'); - let result = run(invalid); - assert(result.status !== 0 && result.stdout.startsWith('HALT:'), 'malformed config did not HALT'); - assert(!result.stdout.includes('read and follow') && !result.stderr.includes('Traceback'), 'failure leaked dispatch/traceback'); - fs.rmSync(path.join(invalid.bmad, 'custom', 'config.toml')); - fs.writeFileSync(path.join(invalid.bmad, 'custom', `${invalid.skillName}.toml`), '[workflow\nbad', 'utf8'); - result = run(invalid); - assert(result.status !== 0 && result.stdout.includes('failed to parse'), 'malformed customization did not HALT'); - }); - - test('missing, wrong-type, and non-string keyed values HALT cleanly', () => { - const missing = fixture({ config: baseConfig().replace('implementation_artifacts = "{project-root}/implementation"\n', '') }); - assert(run(missing).stdout.includes('missing config value'), 'missing value accepted'); - const wrong = fixture({ - config: baseConfig().replace('implementation_artifacts = "{project-root}/implementation"', 'implementation_artifacts = 42'), - }); - assert(run(wrong).stdout.includes('must be a string'), 'wrong type accepted'); - const keyed = fixture(); - fs.mkdirSync(path.join(keyed.bmad, 'custom'), { recursive: true }); - fs.writeFileSync( - path.join(keyed.bmad, 'custom', `${keyed.skillName}.toml`), - '[[workflow.review_layers]]\nid = 42\nname = "bad"\ninstruction = "bad"\n', - 'utf8', - ); - assert(run(keyed).stdout.includes('identifier `id` must be a string'), 'non-string id accepted'); - }); - - test('snapshot-like text inside customization prose is preserved', () => { - const custom = fixture(); - fs.mkdirSync(path.join(custom.bmad, 'custom'), { recursive: true }); - const literal = '[[bmad-snapshot:step-04-review.md]]'; - const compileLiteral = '{workflow.implementation_handoff}'; - fs.writeFileSync( - path.join(custom.bmad, 'custom', `${custom.skillName}.user.toml`), - `[workflow]\non_complete = "Preserve ${literal} and ${compileLiteral} as prose"\n`, - 'utf8', - ); - const output = fs.readFileSync(entry(run(custom)), 'utf8'); - assert(output.includes(literal), 'customization prose was globally rewritten'); - assert(output.includes(compileLiteral), 'customization compile-token prose was rewritten'); - }); - - test('review layer overrides, guards, disabling, and the empty-layer HALT are rendered', () => { - const reviewed = fixture(); - fs.mkdirSync(path.join(reviewed.bmad, 'custom'), { recursive: true }); - fs.writeFileSync( - path.join(reviewed.bmad, 'custom', `${reviewed.skillName}.toml`), - [ - '[[workflow.review_layers]]', - 'id = "blind-hunter"', - 'name = "Replacement"', - 'instruction = "Run replacement review."', - 'when = "the replacement condition holds"', - '', - ].join('\n'), - 'utf8', - ); - let review = fs.readFileSync(path.join(path.dirname(entry(run(reviewed))), 'step-04-review.md'), 'utf8'); - assert(review.includes('Replacement (`blind-hunter`)'), 'keyed review override missing'); - assert(review.includes('Run only when: the replacement condition holds'), 'review guard missing'); - assert(review.includes('Run replacement review.'), 'review instruction missing'); - - const ids = ['blind-hunter', 'edge-case-hunter', 'verification-gap', 'intent-alignment']; - fs.writeFileSync( - path.join(reviewed.bmad, 'custom', `${reviewed.skillName}.toml`), - ids.map((id) => `[[workflow.review_layers]]\nid = "${id}"\nname = "disabled"\ninstruction = ""\n`).join('\n'), - 'utf8', - ); - review = fs.readFileSync(path.join(path.dirname(entry(run(reviewed))), 'step-04-review.md'), 'utf8'); - assert(review.includes('No active review layers. HALT'), 'all-disabled review HALT missing'); - }); - - test('renderer identity changes create a new immutable generation', () => { - const identity = fixture(); - const original = entry(run(identity)); - fs.appendFileSync(path.join(identity.bmad, 'scripts', 'render_skill.py'), '\n# renderer identity change\n'); - const rendererChanged = entry(run(identity)); - assert(rendererChanged !== original, 'renderer change reused generation'); - assert(fs.existsSync(original), 'prior identity generation disappeared'); - }); - - test('Markdown sources are discovered without a duplicate render contract', () => { - const discovered = fixture(); - fs.writeFileSync(path.join(discovered.skill, 'extra.md'), 'Artifacts under {{config.modules.bmm.planning_artifacts}}.\n', 'utf8'); - const output = entry(run(discovered)); - const extra = path.join(path.dirname(output), 'extra.md'); - assert(fs.readFileSync(extra, 'utf8').includes('/planning.'), 'discovered source was not rendered'); - assert(!fs.existsSync(path.join(discovered.skill, 'render.toml')), 'duplicate render contract exists'); - }); - - test('shared renderer accepts a convention-only skill without customization', () => { - const generic = fixture(); - const skill = path.join(generic.bmad, 'core', 'plain-workflow'); - fs.mkdirSync(skill, { recursive: true }); - fs.writeFileSync(path.join(skill, 'workflow.md'), 'Read [[bmad-snapshot:step.md]].\n', 'utf8'); - fs.writeFileSync(path.join(skill, 'step.md'), 'No rendered values required.\n', 'utf8'); - const output = entry(run({ ...generic, skill })); - assert(output.includes(`${path.sep}render${path.sep}plain-workflow${path.sep}`), 'generic skill namespace missing'); - assert(fs.existsSync(path.join(path.dirname(output), 'step.md')), 'generic skill source missing'); - }); - - test('ambiguous shorthand config and source symlink escapes HALT', () => { - const invalid = fixture({ config: baseConfig('implementation_artifacts = "{project-root}/dup"') }); - let result = run(invalid); - assert(result.status !== 0 && result.stdout.includes('ambiguous config value'), 'ambiguous config accepted'); - - const escaped = fixture(); - const outside = path.join(escaped.outer, 'outside.md'); - fs.writeFileSync(outside, 'outside', 'utf8'); - fs.rmSync(path.join(escaped.skill, 'workflow.md')); - fs.symlinkSync(outside, path.join(escaped.skill, 'workflow.md'), 'file'); - result = run(escaped); - assert(result.stdout.includes('escapes skill directory'), 'source symlink escape accepted'); - }); - - test('long project basenames are bounded in the snapshot namespace', () => { - const outer = fs.mkdtempSync(path.join(os.tmpdir(), 'bmad-build-auto-long-')); - tempDirs.push(outer); - const long = 'project-' + 'x'.repeat(220); - const project = path.join(outer, long); - const source = fixture(); - fs.mkdirSync(project); - fs.symlinkSync(source.bmad, path.join(project, '_bmad'), process.platform === 'win32' ? 'junction' : 'dir'); - const longFix = { project, bmad: source.bmad, skill: source.skill }; - const output = entry(run(longFix)); - assert(path.basename(path.dirname(path.dirname(output))).length <= 93, 'namespace component was not bounded'); - }); - - test('snapshot paths remain opaque when the project root resembles render tokens', () => { - const special = fixture({ projectName: '{workflow.on_complete}-{{.communication_language}}' }); - const output = entry(run(special)); - const workflow = fs.readFileSync(output, 'utf8'); - const match = workflow.match(/`([^`]*step-01-clarify-and-route\.md)`/); - assert(match, 'rendered workflow has no first-step reference'); - assert(match[1].startsWith(fs.realpathSync(special.project)), 'project root was rewritten as a source token'); - assert(fs.existsSync(match[1]), 'rendered first-step reference does not exist'); - }); - - test('publication failure does not dispatch or alter another root snapshot', () => { - const stable = fixture(); - const original = entry(run(stable)); - const originalBytes = bytesByName(path.dirname(original)); - const broken = fixture({ sharedBmad: stable.bmad }); - const slug = path - .basename(broken.project) - .toLowerCase() - .replaceAll(/[^a-z0-9]+/g, '-'); - const rootHash = hash(Buffer.from(fs.realpathSync(broken.project))).slice(0, 12); - const namespace = path.join(stable.bmad, 'render', stable.skillName, `${slug}-${rootHash}`); - fs.writeFileSync(namespace, 'not a directory', 'utf8'); - const result = run(broken); - assert(result.status !== 0 && result.stdout.startsWith('HALT:'), 'publication failure did not HALT'); - assert(!result.stdout.includes('read and follow'), 'failed publication dispatched'); - const current = bytesByName(path.dirname(original)); - for (const name of Object.keys(originalBytes)) { - assert(originalBytes[name].equals(current[name]), `stable snapshot changed: ${name}`); - } - }); - - test('corrupt existing destination is never overwritten or dispatched', () => { - const corrupt = fixture(); - const output = entry(run(corrupt)); - const workflow = path.join(path.dirname(output), 'workflow.md'); - fs.appendFileSync(workflow, 'corrupt'); - const result = run(corrupt); - assert(result.status !== 0 && result.stdout.includes('hash mismatch'), 'corruption was reused'); - assert(fs.readFileSync(workflow, 'utf8').endsWith('corrupt'), 'corrupt generation was overwritten'); - }); - - test('bmad-build renders through the same shared snapshot contract', () => { - const build = fixture({ skillName: 'bmad-build' }); - const output = entry(run(build)); - const dir = path.dirname(output); - assert(path.basename(output) === 'workflow.md', `dispatch is not a snapshot workflow.md: ${output}`); - assert(output.includes(`${path.sep}render${path.sep}bmad-build${path.sep}`), 'bmad-build snapshot namespace missing'); - - const markdown = Object.entries(bytesByName(dir)) - .filter(([name]) => name.endsWith('.md')) - .map(([, content]) => content.toString('utf8')) - .join('\n'); - assert(!markdown.includes('{{.'), 'config token survived'); - assert(!markdown.includes('{workflow.'), 'customization token survived'); - assert(!markdown.includes('[[bmad-snapshot:'), 'snapshot token survived'); - assert(!/`\.{1,2}\/[^`]*\.md`/.test(markdown), 'relative skill-root reference survived'); - assert(!markdown.includes('resolve_customization.py'), 'legacy renderer script referenced'); - assert(!markdown.includes('main_config'), 'legacy config variable referenced'); - - const renderRoot = path.join(fs.realpathSync(build.project), '_bmad', 'render'); - const referenced = new Set(); - for (const match of markdown.matchAll(/`(\/[^`]+\.md)`/g)) { - const target = match[1]; - if (!target.startsWith(`${renderRoot}${path.sep}`)) continue; - assert(target.startsWith(`${dir}${path.sep}`), `cross-generation reference: ${target}`); - assert(fs.existsSync(target), `snapshot reference does not resolve: ${target}`); - referenced.add(path.relative(dir, target)); - } - // Every published step must be reachable, which also keeps the loop above non-vacuous. - for (const name of Object.keys(bytesByName(dir))) { - if (!/^(?:step-|sync-sprint-status)/.test(name)) continue; - assert(referenced.has(name), `published step is unreachable from the snapshot: ${name}`); - } - - const prompt = path.join(dir, 'review-prompts', 'edge-case-hunter.md'); - assert(fs.existsSync(prompt), 'review prompt was not published into the snapshot'); - assert(markdown.includes(prompt), 'snapshot reviewer path missing'); - - const review = fs.readFileSync(path.join(dir, 'step-04-review.md'), 'utf8'); - for (const heading of [ - '#### Blind Hunter (`blind-hunter`)', - '#### Edge Case Hunter (`edge-case-hunter`)', - '#### Verification Gap Reviewer (`verification-gap`)', - ]) { - assert(review.includes(heading), `default review layer missing: ${heading}`); - } - assert(review.includes('{diff_file}'), 'runtime placeholder was removed from review layers'); - assert(review.includes('{claims_file}'), 'claims placeholder was removed from review layers'); - assert(!review.includes('{diff_output}'), 'stale inline-diff placeholder survived'); - - const oneshot = fs.readFileSync(path.join(dir, 'step-oneshot.md'), 'utf8'); - assert(oneshot.includes('#### Blind Hunter (`blind-hunter`)'), 'oneshot review layer block missing'); - - // The spec editor handoff must reach both terminal routes (#2652). - const present = fs.readFileSync(path.join(dir, 'step-05-present.md'), 'utf8'); - assert(present.includes('code -r'), 'open_spec default missing from step-05-present.md'); - assert(oneshot.includes('code -r'), 'open_spec default missing from step-oneshot.md'); - assert(/^Offer to push\b/m.test(present), 'standalone "Offer to push" line was lost'); - - const artifacts = `${fs.realpathSync(build.project)}/implementation`; - assert(markdown.includes(`${artifacts}/sprint-status.yaml`), 'sprint-status path was not baked absolute'); - assert(markdown.includes(`${artifacts}/deferred-work.md`), 'deferred-work path was not baked absolute'); - - for (const name of ['step-01-clarify-and-route.md', 'step-02-plan.md', 'step-04-review.md', 'step-oneshot.md']) { - const site = fs.readFileSync(path.join(dir, name), 'utf8'); - assert(site.includes(`${artifacts}/deferred-work.md`), `${name} does not contain the deferred-work path`); - } - - const shipped = fs.readFileSync(path.join(SKILLS_SRC, 'bmad-build', 'customize.toml'), 'utf8'); - assert( - !shipped.includes('{absolute-root}') && !shipped.includes('{absolute-spec-file}'), - 'legacy absolute-path token in customize.toml', - ); - }); - - test('empty open_spec override disables automatic opening', () => { - const build = fixture({ skillName: 'bmad-build' }); - fs.mkdirSync(path.join(build.bmad, 'custom'), { recursive: true }); - fs.writeFileSync(path.join(build.bmad, 'custom', `${build.skillName}.user.toml`), '[workflow]\nopen_spec = ""\n', 'utf8'); - const dir = path.dirname(entry(run(build))); - for (const name of ['step-05-present.md', 'step-oneshot.md']) { - const rendered = fs.readFileSync(path.join(dir, name), 'utf8'); - assert(!rendered.includes('code -r'), `open_spec default survived in ${name}`); - assert(!rendered.includes('spec was sent'), `opening summary survived in ${name}`); - assert(rendered.includes('Suggested Review Order'), `review trail generation disappeared from ${name}`); - } - }); - - test('the command shipped in SKILL.md dispatches for both skills', () => { - for (const skillName of [DEFAULT_SKILL, 'bmad-build']) { - const fix = fixture({ skillName }); - const fenced = fs.readFileSync(path.join(fix.skill, 'SKILL.md'), 'utf8').match(/```bash\n([\s\S]*?)```/); - assert(fenced, `${skillName}: SKILL.md ships no bash command block`); - const command = fenced[1].trim().replaceAll('{project-root}', fix.project).replaceAll('{skill-root}', fix.skill); - assert(!command.includes('{'), `${skillName}: unsubstituted placeholder in shipped command: ${command}`); - // Run it verbatim from a nested cwd — no --python pin, exactly as an agent would. - const dispatched = entry(spawnSync(command, { cwd: path.join(fix.project, 'nested', 'cwd'), shell: true, encoding: 'utf8' })); - assert(path.basename(dispatched) === 'workflow.md', `${skillName}: shipped command did not dispatch workflow.md`); - assert(fs.existsSync(dispatched), `${skillName}: dispatched entry does not exist`); - } - }); - - for (const dir of tempDirs) fs.rmSync(dir, { recursive: true, force: true }); - console.log(`\n${passed}/${total} shared renderer tests passed`); - process.exitCode = passed === total ? 0 : 1; -} - -main().catch((error) => { - console.error(error); - process.exitCode = 1; -});