mirror of
https://github.com/Canner/WrenAI.git
synced 2026-08-28 15:52:53 +08:00
feat(wren): v5 project layout — knowledge/ first-class, memory decoupled from LanceDB (#2399)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jax Liu
@@ -607,14 +607,14 @@ app.add_typer(cube_app)
|
||||
app.add_typer(utils_app)
|
||||
app.add_typer(skills_app)
|
||||
|
||||
from importlib.util import find_spec # noqa: E402
|
||||
# The `memory` subcommand group is always registered: `wren memory store`
|
||||
# writes knowledge/sql/*.md without the optional `memory` extra, and the
|
||||
# lancedb-backed commands degrade with a clear "install wren[memory]" message.
|
||||
# Importing memory_app stays light — lancedb / the heavy ML stack are imported
|
||||
# lazily inside the commands that need them, never at CLI startup.
|
||||
from wren.memory.cli import memory_app # noqa: E402
|
||||
|
||||
# Detect the optional `memory` extra without importing it: a real import would
|
||||
# eagerly pull lancedb (and the heavy ML stack it loads) into every CLI startup.
|
||||
if find_spec("lancedb") and find_spec("sentence_transformers"):
|
||||
from wren.memory.cli import memory_app # noqa: PLC0415
|
||||
|
||||
app.add_typer(memory_app)
|
||||
app.add_typer(memory_app)
|
||||
|
||||
from wren.genbi.cli import genbi_app # noqa: PLC0415, E402
|
||||
from wren.profile_cli import profile_app # noqa: PLC0415, E402
|
||||
|
||||
+185
-16
@@ -43,6 +43,10 @@ When the user wants to add models, change schema, or onboard a new table:
|
||||
3. `wren context build` — compile to `target/mdl.json`
|
||||
4. `wren memory index` — re-index schema for search
|
||||
|
||||
## Capturing business context
|
||||
|
||||
Rules the schema can't express — canonical tables, default filters, units, enum meanings — go in `knowledge/rules/*.md` (read by `wren context instructions`). Confirmed NL→SQL examples are saved with `wren memory store` (step 5 above), which writes them to `knowledge/sql/`. Both live in the project and are committed with it.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
This project requires the `wren` CLI. Install with your data source extra:
|
||||
@@ -51,7 +55,7 @@ This project requires the `wren` CLI. Install with your data source extra:
|
||||
pip install "wrenai[postgres,memory,ui]"
|
||||
```
|
||||
|
||||
Replace `postgres` with your data source (`mysql`, `bigquery`, `snowflake`, `clickhouse`, `trino`, `mssql`, `databricks`, `redshift`, `spark`, `athena`, `oracle`). The `memory` extra enables semantic search; `ui` enables the interactive UI.
|
||||
Replace `postgres` with your data source (`mysql`, `bigquery`, `snowflake`, `clickhouse`, `trino`, `mssql`, `databricks`, `redshift`, `spark`, `athena`, `oracle`). The `memory` extra upgrades recall to semantic (embedding) search — without it, `memory store` / `index` / `recall` still work over the `knowledge/` files. `ui` enables the interactive UI.
|
||||
|
||||
See https://docs.getwren.ai/oss/engine/get_started/installation for full setup.
|
||||
|
||||
@@ -149,11 +153,13 @@ def convert_mdl_to_project(mdl_json: dict) -> list[ProjectFile]:
|
||||
files: list[ProjectFile] = []
|
||||
|
||||
# ── wren_project.yml ──────────────────────────────────────
|
||||
# Map layoutVersion back to schema_version
|
||||
# Map engine layoutVersion to the schema_version an import should land on.
|
||||
# layoutVersion 3 covers both v4 (composite PK) and v5; a fresh import
|
||||
# targets the current layout (5, knowledge/-native).
|
||||
layout_version = mdl_json.get("layoutVersion", 1)
|
||||
_LAYOUT_TO_SCHEMA = {1: 2, 2: 3, 3: 4}
|
||||
_LAYOUT_TO_SCHEMA = {1: 2, 2: 3, 3: 5}
|
||||
schema_version = _LAYOUT_TO_SCHEMA.get(
|
||||
layout_version, 4 if layout_version >= 3 else (3 if layout_version >= 2 else 2)
|
||||
layout_version, 5 if layout_version >= 3 else (3 if layout_version >= 2 else 2)
|
||||
)
|
||||
project_config: dict[str, Any] = {"schema_version": schema_version}
|
||||
if "name" in mdl_json:
|
||||
@@ -260,12 +266,17 @@ def convert_mdl_to_project(mdl_json: dict) -> list[ProjectFile]:
|
||||
)
|
||||
)
|
||||
|
||||
# ── Instructions ──────────────────────────────────────────
|
||||
# ── knowledge/ (v5: business rules under knowledge/rules/) ──
|
||||
files.append(
|
||||
ProjectFile(
|
||||
relative_path="knowledge/knowledge.yml", content="schema_version: 1\n"
|
||||
)
|
||||
)
|
||||
instructions = mdl_json.get("_instructions")
|
||||
if instructions:
|
||||
files.append(
|
||||
ProjectFile(
|
||||
relative_path="instructions.md",
|
||||
relative_path="knowledge/rules/general.md",
|
||||
content=instructions.strip() + "\n",
|
||||
)
|
||||
)
|
||||
@@ -439,6 +450,14 @@ _SUPPORTED_SCHEMA_VERSIONS = {1, 2, 3, 4, 5}
|
||||
# it adds no engine-facing MDL JSON, so it reuses v4's engine layoutVersion 3.
|
||||
_LAYOUT_VERSION_MAP = {1: 1, 2: 1, 3: 2, 4: 3, 5: 3}
|
||||
|
||||
# knowledge/ layout (v5). Single source of truth — reused by the v4→v5 upgrade
|
||||
# step and by project init. The knowledge axis has its own schema_version in
|
||||
# knowledge.yml, decoupled from the MDL schema_version in wren_project.yml.
|
||||
_KNOWLEDGE_SUBDIRS = ("rules", "glossary", "metrics", "caveats", "sql")
|
||||
_KNOWLEDGE_CONFIG_FILE = "knowledge/knowledge.yml"
|
||||
_KNOWLEDGE_SCHEMA_VERSION = 1
|
||||
_SUPPORTED_KNOWLEDGE_VERSIONS = {1}
|
||||
|
||||
# Valid dialect values (matches Rust DataSource enum)
|
||||
_VALID_DIALECTS = {
|
||||
"athena",
|
||||
@@ -494,11 +513,12 @@ def load_models(project_path: Path) -> list[dict]:
|
||||
"""Load models — dispatches on schema_version.
|
||||
|
||||
v1 (legacy): models/*.yml (flat files)
|
||||
v2: models/<name>/metadata.yml + optional ref_sql.sql
|
||||
v2-v5: models/<name>/metadata.yml + optional ref_sql.sql (directory-per-model)
|
||||
"""
|
||||
sv = get_schema_version(project_path)
|
||||
if sv == 1:
|
||||
return _load_models_v1(project_path)
|
||||
# sv in {2, 3, 4, 5}: directory-per-model layout
|
||||
return _load_models_v2(project_path)
|
||||
|
||||
|
||||
@@ -553,11 +573,12 @@ def load_views(project_path: Path) -> list[dict]:
|
||||
"""Load views — dispatches on schema_version.
|
||||
|
||||
v1 (legacy): views.yml (single file with `views:` list)
|
||||
v2: views/<name>/metadata.yml + optional sql.yml
|
||||
v2-v5: views/<name>/metadata.yml + optional sql.yml (directory-per-view)
|
||||
"""
|
||||
sv = get_schema_version(project_path)
|
||||
if sv == 1:
|
||||
return _load_views_v1(project_path)
|
||||
# sv in {2, 3, 4, 5}: directory-per-view layout
|
||||
return _load_views_v2(project_path)
|
||||
|
||||
|
||||
@@ -607,11 +628,12 @@ def load_cubes(project_path: Path) -> list[dict]:
|
||||
"""Load cubes — dispatches on schema_version.
|
||||
|
||||
v1 (legacy): cubes/*.yml
|
||||
v2: cubes/<name>/metadata.yml
|
||||
v2-v5: cubes/<name>/metadata.yml (directory-per-cube)
|
||||
"""
|
||||
sv = get_schema_version(project_path)
|
||||
if sv == 1:
|
||||
return _load_cubes_v1(project_path)
|
||||
# sv in {2, 3, 4, 5}: directory-per-cube layout
|
||||
return _load_cubes_v2(project_path)
|
||||
|
||||
|
||||
@@ -667,13 +689,79 @@ def load_relationships(project_path: Path) -> list[dict]:
|
||||
|
||||
|
||||
def load_instructions(project_path: Path) -> str | None:
|
||||
"""Load instructions.md as a string."""
|
||||
"""Load the legacy instructions.md as a string.
|
||||
|
||||
Deprecated in favour of knowledge/rules/ — see load_rules().
|
||||
"""
|
||||
inst_file = project_path / "instructions.md"
|
||||
if not inst_file.exists():
|
||||
return None
|
||||
return inst_file.read_text(encoding="utf-8").strip() or None
|
||||
|
||||
|
||||
def load_knowledge_rules(project_path: Path) -> str | None:
|
||||
"""Concatenate knowledge/rules/*.md (sorted). None if there are none."""
|
||||
rules_dir = project_path / "knowledge" / "rules"
|
||||
if not rules_dir.is_dir():
|
||||
return None
|
||||
parts = [
|
||||
text
|
||||
for f in sorted(rules_dir.glob("*.md"))
|
||||
if (text := f.read_text(encoding="utf-8").strip())
|
||||
]
|
||||
return "\n\n".join(parts) if parts else None
|
||||
|
||||
|
||||
def load_rules(project_path: Path) -> tuple[str | None, bool]:
|
||||
"""Load business rules from knowledge/rules/ and the legacy instructions.md.
|
||||
|
||||
Returns ``(content, used_legacy)`` where ``used_legacy`` is True when the
|
||||
deprecated instructions.md contributed content, so callers can warn.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
rules = load_knowledge_rules(project_path)
|
||||
if rules:
|
||||
parts.append(rules)
|
||||
legacy = load_instructions(project_path)
|
||||
if legacy:
|
||||
parts.append(legacy)
|
||||
content = "\n\n".join(parts) if parts else None
|
||||
# Presence-based: an existing (even empty) instructions.md is the deprecated
|
||||
# pattern worth flagging, regardless of whether it currently has content.
|
||||
used_legacy = (project_path / "instructions.md").exists()
|
||||
return content, used_legacy
|
||||
|
||||
|
||||
def load_knowledge_config(project_path: Path) -> dict:
|
||||
"""Load knowledge/knowledge.yml (knowledge version axis). Empty dict if absent."""
|
||||
kfile = project_path / _KNOWLEDGE_CONFIG_FILE
|
||||
if not kfile.exists():
|
||||
return {}
|
||||
data = yaml.safe_load(kfile.read_text(encoding="utf-8")) or {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def get_knowledge_schema_version(project_path: Path) -> int:
|
||||
"""Return the knowledge-axis schema_version (default 1). Decoupled from MDL.
|
||||
|
||||
Returns 0 when there is no knowledge/ at all.
|
||||
"""
|
||||
if not (project_path / "knowledge").is_dir():
|
||||
return 0
|
||||
try:
|
||||
cfg = load_knowledge_config(project_path)
|
||||
except yaml.YAMLError as e:
|
||||
raise SystemExit(f"Error: invalid YAML in {_KNOWLEDGE_CONFIG_FILE}: {e}")
|
||||
raw = cfg.get("schema_version", _KNOWLEDGE_SCHEMA_VERSION)
|
||||
try:
|
||||
return int(raw)
|
||||
except (TypeError, ValueError):
|
||||
raise SystemExit(
|
||||
f"Error: invalid schema_version {raw!r} in {_KNOWLEDGE_CONFIG_FILE}. "
|
||||
"Expected an integer."
|
||||
)
|
||||
|
||||
|
||||
# ── Build ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -804,6 +892,33 @@ def validate_project(project_path: Path) -> list[ValidationError]:
|
||||
if any(e.path == PROJECT_FILE and "schema_version" in e.message for e in errors):
|
||||
return errors
|
||||
|
||||
# knowledge/ version axis — independent of the MDL schema_version above.
|
||||
if (project_path / "knowledge").is_dir():
|
||||
try:
|
||||
kcfg = load_knowledge_config(project_path)
|
||||
kv = int(kcfg.get("schema_version", _KNOWLEDGE_SCHEMA_VERSION))
|
||||
except yaml.YAMLError as e:
|
||||
errors.append(
|
||||
ValidationError("error", _KNOWLEDGE_CONFIG_FILE, f"invalid YAML: {e}")
|
||||
)
|
||||
except (TypeError, ValueError) as e:
|
||||
errors.append(
|
||||
ValidationError(
|
||||
"error",
|
||||
_KNOWLEDGE_CONFIG_FILE,
|
||||
f"schema_version must be an integer ({e})",
|
||||
)
|
||||
)
|
||||
else:
|
||||
if kv not in _SUPPORTED_KNOWLEDGE_VERSIONS:
|
||||
errors.append(
|
||||
ValidationError(
|
||||
"error",
|
||||
_KNOWLEDGE_CONFIG_FILE,
|
||||
f"unsupported knowledge schema_version {kv} — please upgrade wren CLI",
|
||||
)
|
||||
)
|
||||
|
||||
# Load data (snake_case)
|
||||
models = load_models(project_path)
|
||||
views = load_views(project_path)
|
||||
@@ -1160,6 +1275,36 @@ class UpgradeError(Exception):
|
||||
"""Raised when a project upgrade cannot proceed."""
|
||||
|
||||
|
||||
def _knowledge_skeleton_targets() -> list[str]:
|
||||
"""Canonical relative paths of a fresh knowledge/ skeleton.
|
||||
|
||||
Empty subdirectories carry a .gitkeep so the layout survives in git.
|
||||
"""
|
||||
paths = [f"knowledge/{sub}/.gitkeep" for sub in _KNOWLEDGE_SUBDIRS]
|
||||
paths.append(_KNOWLEDGE_CONFIG_FILE)
|
||||
return paths
|
||||
|
||||
|
||||
def create_knowledge_skeleton(project_path: Path) -> list[str]:
|
||||
"""Create any missing parts of the knowledge/ skeleton. Idempotent.
|
||||
|
||||
Returns the relative paths actually created (empty if already complete).
|
||||
Existing files are never overwritten.
|
||||
"""
|
||||
created: list[str] = []
|
||||
for rel in _knowledge_skeleton_targets():
|
||||
dest = project_path / rel
|
||||
if dest.exists():
|
||||
continue
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if rel == _KNOWLEDGE_CONFIG_FILE:
|
||||
dest.write_text(f"schema_version: {_KNOWLEDGE_SCHEMA_VERSION}\n")
|
||||
else:
|
||||
dest.write_text("") # .gitkeep
|
||||
created.append(rel)
|
||||
return created
|
||||
|
||||
|
||||
def plan_upgrade(
|
||||
project_path: Path,
|
||||
target_version: int | None = None,
|
||||
@@ -1196,6 +1341,10 @@ def plan_upgrade(
|
||||
created, deleted = _plan_v1_to_v2(project_path)
|
||||
files_created.extend(created)
|
||||
files_deleted.extend(deleted)
|
||||
elif version == 4:
|
||||
created, deleted = _plan_v4_to_v5(project_path)
|
||||
files_created.extend(created)
|
||||
files_deleted.extend(deleted)
|
||||
# v2→v3 (dialect) and v3→v4 (composite primary_key): no file layout
|
||||
# changes needed — only wren_project.yml is restamped.
|
||||
|
||||
@@ -1267,13 +1416,28 @@ def _plan_v1_to_v2(project_path: Path) -> tuple[list[str], list[str]]:
|
||||
return created, deleted
|
||||
|
||||
|
||||
def _plan_v4_to_v5(project_path: Path) -> tuple[list[str], list[str]]:
|
||||
"""Plan v4→v5: create the knowledge/ skeleton if absent.
|
||||
|
||||
First file-creating step since v1→v2. Idempotent — lists only the
|
||||
skeleton paths that don't already exist.
|
||||
"""
|
||||
created = [
|
||||
rel
|
||||
for rel in _knowledge_skeleton_targets()
|
||||
if not (project_path / rel).exists()
|
||||
]
|
||||
return created, []
|
||||
|
||||
|
||||
def apply_upgrade(project_path: Path, result: UpgradeResult) -> None:
|
||||
"""Write upgrade changes to disk."""
|
||||
if not result.files_created and not result.files_deleted:
|
||||
# No-op (e.g. v2→v3, only wren_project.yml changes)
|
||||
pass
|
||||
else:
|
||||
_apply_v1_to_v2(project_path)
|
||||
"""Write upgrade changes to disk, replaying each version step in order."""
|
||||
for version in range(result.from_version, result.to_version):
|
||||
if version == 1:
|
||||
_apply_v1_to_v2(project_path)
|
||||
elif version == 4:
|
||||
_apply_v4_to_v5(project_path)
|
||||
# v2→v3, v3→v4: only the wren_project.yml stamp changes (handled below)
|
||||
|
||||
# Update wren_project.yml
|
||||
config = load_project_config(project_path)
|
||||
@@ -1368,6 +1532,11 @@ def _apply_v1_to_v2(project_path: Path) -> None:
|
||||
old_file.unlink()
|
||||
|
||||
|
||||
def _apply_v4_to_v5(project_path: Path) -> None:
|
||||
"""Execute v4→v5: create the knowledge/ skeleton (idempotent)."""
|
||||
create_knowledge_skeleton(project_path)
|
||||
|
||||
|
||||
# ── Semantic validation (view dry-plan + description completeness) ─────────
|
||||
|
||||
_VALID_LEVELS = frozenset({"error", "warning", "strict"})
|
||||
|
||||
@@ -255,8 +255,7 @@ def init(
|
||||
|
||||
project_file = project_path / PROJECT_FILE
|
||||
agents_file = project_path / "AGENTS.md"
|
||||
queries_file = project_path / "queries.yml"
|
||||
conflicts = [f for f in (project_file, agents_file, queries_file) if f.exists()]
|
||||
conflicts = [f for f in (project_file, agents_file) if f.exists()]
|
||||
if conflicts and not force:
|
||||
names = ", ".join(f"'{c.name}'" for c in conflicts)
|
||||
typer.echo(
|
||||
@@ -272,7 +271,7 @@ def init(
|
||||
|
||||
# wren_project.yml
|
||||
project_yml = (
|
||||
"schema_version: 3\n"
|
||||
"schema_version: 5\n"
|
||||
"name: my_project\n"
|
||||
'version: "1.0"\n'
|
||||
"\n"
|
||||
@@ -342,26 +341,25 @@ def init(
|
||||
"statement: >\n SELECT * FROM example LIMIT 100\n"
|
||||
)
|
||||
|
||||
# Instructions placeholder
|
||||
(project_path / "instructions.md").write_text(
|
||||
"# User Instructions\n\n"
|
||||
"Add custom rules or guidelines for LLM-based query generation here.\n"
|
||||
# ── knowledge/ skeleton (first-class business context) ──
|
||||
from wren.context import ( # noqa: PLC0415
|
||||
_AGENTS_MD_TEMPLATE,
|
||||
create_knowledge_skeleton,
|
||||
)
|
||||
|
||||
create_knowledge_skeleton(project_path)
|
||||
general_rules = project_path / "knowledge" / "rules" / "general.md"
|
||||
if force or not general_rules.exists():
|
||||
general_rules.write_text(
|
||||
"# Business rules\n\n"
|
||||
"Add custom rules or guidelines for LLM-based query generation here.\n"
|
||||
)
|
||||
|
||||
# ── AGENTS.md ──
|
||||
from wren.context import _AGENTS_MD_TEMPLATE # noqa: PLC0415
|
||||
|
||||
(project_path / "AGENTS.md").write_text(_AGENTS_MD_TEMPLATE)
|
||||
|
||||
# Curated NL-SQL pairs (auto-loaded by `wren memory index`)
|
||||
(project_path / "queries.yml").write_text(
|
||||
"# Curated NL-SQL pairs for this project.\n"
|
||||
"# These are auto-loaded into memory on `wren memory index`.\n"
|
||||
"# Use `wren memory dump` to export pairs from memory to this file.\n"
|
||||
"# Format: same as `wren memory dump` output.\n"
|
||||
"version: 1\n"
|
||||
"pairs: []\n"
|
||||
)
|
||||
# NL→SQL pairs live in knowledge/sql/ (written by `wren memory store`),
|
||||
# so no queries.yml is scaffolded.
|
||||
|
||||
typer.echo(f"Wren project initialized: {project_path}")
|
||||
typer.echo(" wren_project.yml — project metadata (edit data_source)")
|
||||
@@ -374,9 +372,22 @@ def init(
|
||||
typer.echo(" models/ — (empty; add your own models)")
|
||||
typer.echo(" views/ — (empty; add your own views)")
|
||||
typer.echo(" relationships.yml — define joins between models")
|
||||
typer.echo(" instructions.md — LLM instructions")
|
||||
typer.echo(
|
||||
" knowledge/rules/ — business rules for LLM query generation"
|
||||
)
|
||||
typer.echo(
|
||||
" knowledge/sql/ — confirmed NL-SQL pairs (wren memory store)"
|
||||
)
|
||||
typer.echo(" AGENTS.md — AI agent workflow guidance")
|
||||
typer.echo(" queries.yml — curated NL-SQL pairs for memory")
|
||||
# A pre-existing legacy queries.yml is still auto-loaded by `wren memory
|
||||
# index`; surface it so v4 and v5 pair sources don't silently mix.
|
||||
if (project_path / "queries.yml").exists():
|
||||
typer.echo(
|
||||
"Note: a legacy queries.yml is present. It's still loaded on "
|
||||
"`wren memory index`, but is deprecated — migrate its pairs into "
|
||||
"knowledge/sql/ (see the migration reference).",
|
||||
err=True,
|
||||
)
|
||||
typer.echo("")
|
||||
typer.echo(
|
||||
"Next: Install agent skills via "
|
||||
@@ -601,7 +612,7 @@ def build(
|
||||
"""Build into target/mdl.json for the engine.
|
||||
|
||||
Default mode: reads wren_project.yml, models/*/metadata.yml (+ref_sql.sql),
|
||||
views/*/metadata.yml (+sql.yml), relationships.yml, and instructions.md.
|
||||
views/*/metadata.yml (+sql.yml), relationships.yml, and knowledge/.
|
||||
|
||||
With --from-osi: reads an Open Semantic Interchange YAML file and emits
|
||||
MDL JSON directly. The OSI file stays as the single source of truth; no
|
||||
@@ -693,8 +704,8 @@ def show(
|
||||
build_json,
|
||||
build_manifest,
|
||||
discover_project_path,
|
||||
load_instructions,
|
||||
load_project_config,
|
||||
load_rules,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -722,7 +733,7 @@ def show(
|
||||
models = manifest.get("models", [])
|
||||
views = manifest.get("views", [])
|
||||
rels = manifest.get("relationships", [])
|
||||
instr_content = load_instructions(project_path)
|
||||
instr_content, used_legacy_instructions = load_rules(project_path)
|
||||
|
||||
typer.echo(
|
||||
f"Project: {config.get('name', '?')} (v{config.get('version', '?')})"
|
||||
@@ -752,7 +763,11 @@ def show(
|
||||
|
||||
if instr_content:
|
||||
lines = instr_content.strip().split("\n")
|
||||
typer.echo(f"\nInstructions: {len(lines)} lines")
|
||||
typer.echo(f"\nBusiness rules: {len(lines)} lines")
|
||||
if used_legacy_instructions:
|
||||
typer.echo(
|
||||
" (instructions.md is deprecated — move it into knowledge/rules/*.md)"
|
||||
)
|
||||
|
||||
if not models and not views:
|
||||
typer.echo("Empty project. Run `wren context init` to get started.")
|
||||
@@ -762,8 +777,8 @@ def show(
|
||||
def instructions(
|
||||
path: ProjectPathOpt = None,
|
||||
) -> None:
|
||||
"""Print user instructions for LLM consumption."""
|
||||
from wren.context import discover_project_path, load_instructions # noqa: PLC0415
|
||||
"""Print business rules (knowledge/rules/ + legacy instructions.md) for LLM consumption."""
|
||||
from wren.context import discover_project_path, load_rules # noqa: PLC0415
|
||||
|
||||
try:
|
||||
project_path = discover_project_path(path)
|
||||
@@ -771,7 +786,13 @@ def instructions(
|
||||
typer.echo(str(e), err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
content = load_instructions(project_path)
|
||||
content, used_legacy = load_rules(project_path)
|
||||
if used_legacy:
|
||||
typer.echo(
|
||||
"Warning: instructions.md is deprecated — move its content into "
|
||||
"knowledge/rules/*.md.",
|
||||
err=True,
|
||||
)
|
||||
if content:
|
||||
typer.echo(content)
|
||||
|
||||
|
||||
+27
-10
@@ -356,7 +356,7 @@ def convert_dbt_project_to_wren_project(
|
||||
|
||||
dbt_binding_dir = _relative_or_absolute_path(target.project_dir, project_root)
|
||||
project_config = {
|
||||
"schema_version": 2,
|
||||
"schema_version": 5,
|
||||
"name": artifacts.manifest.get("metadata", {}).get(
|
||||
"project_name", target.project.get("name", "dbt_project")
|
||||
),
|
||||
@@ -391,7 +391,7 @@ def convert_dbt_project_to_wren_project(
|
||||
),
|
||||
),
|
||||
ProjectFile(
|
||||
relative_path="instructions.md",
|
||||
relative_path="knowledge/rules/general.md",
|
||||
content=_build_base_instructions(
|
||||
target,
|
||||
model_count,
|
||||
@@ -401,18 +401,35 @@ def convert_dbt_project_to_wren_project(
|
||||
artifacts.run_results is not None,
|
||||
),
|
||||
),
|
||||
ProjectFile(relative_path="AGENTS.md", content=_AGENTS_MD_TEMPLATE),
|
||||
ProjectFile(
|
||||
relative_path="queries.yml",
|
||||
content=yaml.dump(
|
||||
{"version": 1, "pairs": query_pairs},
|
||||
default_flow_style=False,
|
||||
sort_keys=False,
|
||||
allow_unicode=True,
|
||||
),
|
||||
relative_path="knowledge/knowledge.yml", content="schema_version: 1\n"
|
||||
),
|
||||
ProjectFile(relative_path="AGENTS.md", content=_AGENTS_MD_TEMPLATE),
|
||||
]
|
||||
|
||||
# NL→SQL pairs live one-per-file under knowledge/sql/ (v5 source of truth).
|
||||
from wren.memory.markdown import render_query_markdown, slugify # noqa: PLC0415
|
||||
|
||||
used_slugs: set[str] = set()
|
||||
for pair in query_pairs:
|
||||
base = slugify(pair["nl"])
|
||||
slug, n = base, 1
|
||||
while slug in used_slugs:
|
||||
n += 1
|
||||
slug = f"{base}-{n}"
|
||||
used_slugs.add(slug)
|
||||
files.append(
|
||||
ProjectFile(
|
||||
relative_path=f"knowledge/sql/{slug}.md",
|
||||
content=render_query_markdown(
|
||||
pair["nl"],
|
||||
pair["sql"],
|
||||
source=pair.get("source", "dbt"),
|
||||
datasource=pair.get("datasource"),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
files.extend(
|
||||
ProjectFile(
|
||||
relative_path=f"models/{model['name']}/metadata.yml",
|
||||
|
||||
@@ -167,18 +167,45 @@ def index(
|
||||
typer.Option("--no-queries", help="Skip auto-loading project queries.yml."),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Index MDL schema into LanceDB (and optionally seed example queries)."""
|
||||
"""Index the project for recall.
|
||||
|
||||
With the ``memory`` extra: builds the LanceDB semantic index (schema + seed
|
||||
+ knowledge/sql pairs). Without it: the grep backend reads knowledge/sql/*.md
|
||||
directly, so there is nothing to build.
|
||||
"""
|
||||
from wren.memory.index_backend import resolve_backend # noqa: PLC0415
|
||||
|
||||
if resolve_backend() == "grep":
|
||||
from wren.context import discover_project_path # noqa: PLC0415
|
||||
from wren.memory.markdown import load_query_pairs # noqa: PLC0415
|
||||
|
||||
try:
|
||||
project_path = discover_project_path()
|
||||
except SystemExit as e:
|
||||
typer.echo(str(e), err=True)
|
||||
raise typer.Exit(1)
|
||||
n = len(load_query_pairs(project_path))
|
||||
typer.echo(
|
||||
f"grep backend: {n} pair(s) in knowledge/sql/ — no index build needed."
|
||||
)
|
||||
typer.echo(
|
||||
"`wren memory recall` works over grep; semantic schema search "
|
||||
"(`wren memory fetch`) needs `wren[memory]`.",
|
||||
err=True,
|
||||
)
|
||||
return
|
||||
|
||||
manifest = _load_manifest(mdl)
|
||||
|
||||
if include_instructions and mdl is None:
|
||||
try:
|
||||
from wren.context import ( # noqa: I001, PLC0415
|
||||
discover_project_path,
|
||||
load_instructions,
|
||||
load_rules,
|
||||
)
|
||||
|
||||
project_path = discover_project_path()
|
||||
instr = load_instructions(project_path)
|
||||
instr, _ = load_rules(project_path)
|
||||
if instr:
|
||||
manifest["_instructions"] = instr
|
||||
except (
|
||||
@@ -200,12 +227,25 @@ def index(
|
||||
+ "."
|
||||
)
|
||||
|
||||
# ── Auto-load project queries.yml ──
|
||||
# ── Rebuild query history from knowledge/sql/*.md (source of truth) ──
|
||||
# Legacy queries.yml is still loaded when present, for the transition.
|
||||
if not no_queries:
|
||||
try:
|
||||
from wren.context import discover_project_path # noqa: PLC0415
|
||||
from wren.memory.markdown import load_query_pairs # noqa: PLC0415
|
||||
|
||||
project_path = discover_project_path(explicit=None)
|
||||
|
||||
md_pairs = load_query_pairs(project_path)
|
||||
if md_pairs:
|
||||
# upsert → re-running index converges on the markdown content.
|
||||
res = mem_store.load_queries(md_pairs, upsert=True)
|
||||
typer.echo(
|
||||
f"Indexed {res['loaded'] + res['updated']} pair(s) from "
|
||||
f"knowledge/sql/.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
queries_file = project_path / "queries.yml"
|
||||
if queries_file.exists():
|
||||
raw = queries_file.read_text(encoding="utf-8")
|
||||
@@ -216,7 +256,7 @@ def index(
|
||||
skipped = load_result["skipped"]
|
||||
if loaded:
|
||||
typer.echo(
|
||||
f"Loaded {loaded} pair(s) from queries.yml"
|
||||
f"Loaded {loaded} pair(s) from queries.yml (legacy)"
|
||||
f" ({skipped} skipped).",
|
||||
err=True,
|
||||
)
|
||||
@@ -317,10 +357,42 @@ def store(
|
||||
tags: Annotated[Optional[str], typer.Option("--tags")] = None,
|
||||
path: PathOpt = None,
|
||||
) -> None:
|
||||
"""Store a NL→SQL pair for future few-shot retrieval."""
|
||||
mem_store = _get_store(path)
|
||||
mem_store.store_query(nl, sql, datasource=datasource, tags=tags)
|
||||
typer.echo("Query stored.")
|
||||
"""Store a NL→SQL pair as knowledge/sql/<slug>.md (source of truth), then index it.
|
||||
|
||||
The markdown file is always written (no extra required). When the ``memory``
|
||||
extra is installed, the pair is also indexed into LanceDB for semantic recall.
|
||||
"""
|
||||
from wren.context import discover_project_path # noqa: PLC0415
|
||||
from wren.memory.markdown import write_query_markdown # noqa: PLC0415
|
||||
|
||||
try:
|
||||
project_path = discover_project_path()
|
||||
except SystemExit as e:
|
||||
typer.echo(str(e), err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
tag_list = [t.strip() for t in tags.split(",") if t.strip()] if tags else None
|
||||
md_path = write_query_markdown(
|
||||
project_path, nl, sql, datasource=datasource, tags=tag_list
|
||||
)
|
||||
typer.echo(f"Stored: {md_path}")
|
||||
|
||||
# Best-effort: index into LanceDB when the memory extra is available.
|
||||
try:
|
||||
from wren.memory.store import MemoryStore # noqa: PLC0415
|
||||
|
||||
resolved = path or str(_default_memory_path())
|
||||
MemoryStore(path=resolved).store_query(
|
||||
nl, sql, datasource=datasource, tags=tags
|
||||
)
|
||||
except ModuleNotFoundError as e:
|
||||
if (e.name or "").split(".")[0] not in {
|
||||
"lancedb",
|
||||
"sentence_transformers",
|
||||
"pyarrow",
|
||||
}:
|
||||
raise
|
||||
# memory extra not installed — markdown-only; run `wren memory index` later.
|
||||
|
||||
|
||||
@memory_app.command()
|
||||
@@ -331,20 +403,63 @@ def recall(
|
||||
path: PathOpt = None,
|
||||
output: OutputOpt = "table",
|
||||
) -> None:
|
||||
"""Search past NL→SQL pairs by semantic similarity."""
|
||||
mem_store = _get_store(path)
|
||||
results = mem_store.recall_queries(query, limit=limit, datasource=datasource)
|
||||
"""Search past NL→SQL pairs over knowledge/sql/.
|
||||
|
||||
Uses semantic search with the ``memory`` extra, or dependency-free token
|
||||
matching (grep backend) without it.
|
||||
"""
|
||||
from wren.context import discover_project_path # noqa: PLC0415
|
||||
from wren.memory.index_backend import get_index # noqa: PLC0415
|
||||
|
||||
try:
|
||||
project = discover_project_path()
|
||||
except SystemExit as e:
|
||||
typer.echo(str(e), err=True)
|
||||
raise typer.Exit(1)
|
||||
idx = get_index(project, path or str(_default_memory_path()))
|
||||
results = idx.search(query, limit=limit, datasource=datasource)
|
||||
_annotate_markdown_paths(results)
|
||||
_print_results(results, output)
|
||||
|
||||
|
||||
def _annotate_markdown_paths(results: list[dict]) -> None:
|
||||
"""Best-effort: point each recall result at its knowledge/sql/*.md source.
|
||||
|
||||
Matches on the exact NL (not a derived slug), so collision-suffixed files
|
||||
are attributed correctly.
|
||||
"""
|
||||
try:
|
||||
from wren.context import discover_project_path # noqa: PLC0415
|
||||
from wren.memory.markdown import load_query_pairs # noqa: PLC0415
|
||||
|
||||
project = discover_project_path()
|
||||
except (SystemExit, Exception): # noqa: BLE001 — annotation is optional
|
||||
return
|
||||
nl_to_path = {p["nl"]: p["path"] for p in load_query_pairs(project)}
|
||||
for r in results:
|
||||
nl = r.get("nl_query") or r.get("nl")
|
||||
if nl and nl in nl_to_path:
|
||||
r["path"] = nl_to_path[nl]
|
||||
|
||||
|
||||
@memory_app.command()
|
||||
def status(
|
||||
path: PathOpt = None,
|
||||
) -> None:
|
||||
"""Show memory index statistics."""
|
||||
mem_store = _get_store(path)
|
||||
info = mem_store.status()
|
||||
typer.echo(f"Path: {info['path']}")
|
||||
"""Show memory backend and index statistics."""
|
||||
from wren.context import discover_project_path # noqa: PLC0415
|
||||
from wren.memory.index_backend import get_index # noqa: PLC0415
|
||||
|
||||
try:
|
||||
project = discover_project_path()
|
||||
except SystemExit as e:
|
||||
typer.echo(str(e), err=True)
|
||||
raise typer.Exit(1)
|
||||
info = get_index(project, path or str(_default_memory_path())).status()
|
||||
typer.echo(f"Backend: {info['backend']}")
|
||||
if info["backend"] == "grep":
|
||||
typer.echo(f" knowledge/sql: {info['pairs']} pair(s)")
|
||||
return
|
||||
tables = info.get("tables", {})
|
||||
if not tables:
|
||||
typer.echo("No tables indexed yet.")
|
||||
@@ -360,14 +475,143 @@ def reset(
|
||||
bool, typer.Option("--force", "-f", help="Skip confirmation")
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Drop all memory tables and start fresh."""
|
||||
"""Drop the derived memory index. knowledge/sql/*.md is preserved.
|
||||
|
||||
The LanceDB index is a derived artifact — after reset, run `wren memory
|
||||
index` to rebuild it from the markdown source of truth.
|
||||
"""
|
||||
from wren.context import discover_project_path # noqa: PLC0415
|
||||
from wren.memory.index_backend import get_index # noqa: PLC0415
|
||||
|
||||
try:
|
||||
project = discover_project_path()
|
||||
except SystemExit as e:
|
||||
typer.echo(str(e), err=True)
|
||||
raise typer.Exit(1)
|
||||
idx = get_index(project, path or str(_default_memory_path()))
|
||||
if idx.name == "grep":
|
||||
typer.echo("grep backend has no derived index — knowledge/sql/ is the source.")
|
||||
return
|
||||
if not force:
|
||||
confirm = typer.confirm("This will delete all indexed memory. Continue?")
|
||||
confirm = typer.confirm(
|
||||
"This drops the derived memory index. Your knowledge/sql/*.md "
|
||||
"source files are kept. Continue?"
|
||||
)
|
||||
if not confirm:
|
||||
raise typer.Abort()
|
||||
mem_store = _get_store(path)
|
||||
mem_store.reset()
|
||||
typer.echo("Memory reset.")
|
||||
idx.reset()
|
||||
typer.echo(
|
||||
"Memory index reset. Run `wren memory index` to rebuild from knowledge/sql/."
|
||||
)
|
||||
|
||||
|
||||
@memory_app.command()
|
||||
def check(
|
||||
path: PathOpt = None,
|
||||
) -> None:
|
||||
"""Report drift between knowledge/sql/*.md (source) and the derived index."""
|
||||
from wren.context import discover_project_path # noqa: PLC0415
|
||||
from wren.memory.index_backend import get_index # noqa: PLC0415
|
||||
from wren.memory.markdown import load_query_pairs # noqa: PLC0415
|
||||
|
||||
try:
|
||||
project = discover_project_path()
|
||||
except SystemExit as e:
|
||||
typer.echo(str(e), err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
idx = get_index(project, path or str(_default_memory_path()))
|
||||
if idx.name == "grep":
|
||||
n = len(load_query_pairs(project))
|
||||
typer.echo(
|
||||
f"grep backend: knowledge/sql/ is the index ({n} pair(s)) — always in sync."
|
||||
)
|
||||
return
|
||||
|
||||
md_nls = {p["nl"] for p in load_query_pairs(project)}
|
||||
mem_store = idx.store
|
||||
indexed, _ = mem_store.list_queries(limit=1_000_000)
|
||||
indexed_nls = {r.get("nl_query") for r in indexed}
|
||||
# Only user-sourced pairs come from markdown; seeds/views are derived from
|
||||
# the manifest and are not expected to have a knowledge/sql/ file.
|
||||
indexed_user = {
|
||||
r.get("nl_query")
|
||||
for r in indexed
|
||||
if _parse_source(r.get("tags")) not in ("seed", "view")
|
||||
}
|
||||
|
||||
# Compare user pairs only — seed/view rows aren't markdown-backed.
|
||||
missing = md_nls - indexed_user # in markdown but not indexed as a user pair
|
||||
stale = indexed_user - md_nls # user-indexed but no longer in markdown
|
||||
|
||||
typer.echo(
|
||||
f"knowledge/sql: {len(md_nls)} pair(s); index: {len(indexed_nls)} pair(s)"
|
||||
)
|
||||
if not missing and not stale:
|
||||
typer.echo("In sync.")
|
||||
return
|
||||
if missing:
|
||||
typer.echo(f" {len(missing)} not indexed — run `wren memory index`.")
|
||||
if stale:
|
||||
typer.echo(
|
||||
f" {len(stale)} user pair(s) indexed without markdown — "
|
||||
"stale index, run `wren memory index`."
|
||||
)
|
||||
|
||||
|
||||
@memory_app.command()
|
||||
def export(
|
||||
path: PathOpt = None,
|
||||
include_seed: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--include-seed",
|
||||
help="Also export auto-generated seed pairs (normally regenerated on index).",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""One-time migration: export the LanceDB query_history into knowledge/sql/*.md.
|
||||
|
||||
Reads the existing index (requires the ``memory`` extra) and writes each
|
||||
NL→SQL pair to the markdown source of truth, preserving source and
|
||||
timestamp. The same NL updates one file (dedup). LanceDB is left intact —
|
||||
run `wren memory index` to rebuild, then `wren memory reset` once verified.
|
||||
"""
|
||||
from wren.context import discover_project_path # noqa: PLC0415
|
||||
from wren.memory.markdown import write_query_markdown # noqa: PLC0415
|
||||
|
||||
try:
|
||||
project = discover_project_path()
|
||||
except SystemExit as e:
|
||||
typer.echo(str(e), err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
mem_store = _get_store(path) # requires the memory extra to read LanceDB
|
||||
rows = mem_store.dump_queries()
|
||||
exported, skipped = 0, 0
|
||||
for r in rows:
|
||||
source = _parse_source(r.get("tags", "")) or "user"
|
||||
nl, sql = r.get("nl_query"), r.get("sql_query")
|
||||
if (source == "seed" and not include_seed) or not nl or not sql:
|
||||
skipped += 1
|
||||
continue
|
||||
created = r.get("created_at")
|
||||
created_at = created.isoformat() if hasattr(created, "isoformat") else None
|
||||
write_query_markdown(
|
||||
project,
|
||||
nl,
|
||||
sql,
|
||||
datasource=r.get("datasource") or None,
|
||||
source=source,
|
||||
created_at=created_at,
|
||||
)
|
||||
exported += 1
|
||||
|
||||
typer.echo(f"Exported {exported} pair(s) to knowledge/sql/ ({skipped} skipped).")
|
||||
typer.echo(
|
||||
"Run `wren memory index` to rebuild, then `wren memory reset` once verified.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
|
||||
# ── List / Forget / Dump / Load ──────────────────────────────────────────
|
||||
@@ -506,9 +750,9 @@ def forget(
|
||||
# ── Dump / Load helpers ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _parse_source(tags: str) -> str:
|
||||
"""Extract source value from tags string."""
|
||||
for part in tags.split():
|
||||
def _parse_source(tags: str | None) -> str:
|
||||
"""Extract source value from a (possibly null/empty) tags string."""
|
||||
for part in (tags or "").split():
|
||||
if part.startswith("source:"):
|
||||
return part[len("source:") :]
|
||||
return "user"
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Pluggable NL→SQL recall backends over ``knowledge/sql/*.md``.
|
||||
|
||||
The markdown files are the source of truth. A backend is just a query interface
|
||||
over them:
|
||||
|
||||
- ``GrepIndex`` — dependency-free token/substring search. The default when the
|
||||
``memory`` extra is absent; ``knowledge/sql/`` *is* the index (nothing to build).
|
||||
- ``LanceDBIndex`` — semantic search via the ``memory`` extra (lancedb +
|
||||
sentence-transformers), with LanceDB as a derived index.
|
||||
|
||||
Backend selection: ``WREN_MEMORY_BACKEND=grep|lancedb`` forces a choice;
|
||||
otherwise LanceDB is used when its extra is importable, else Grep.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from wren.memory.markdown import load_query_pairs
|
||||
|
||||
_TOKEN_RE = re.compile(r"[a-z0-9]+")
|
||||
|
||||
|
||||
def _tokens(text: str) -> set[str]:
|
||||
return {t for t in _TOKEN_RE.findall((text or "").lower()) if len(t) >= 2}
|
||||
|
||||
|
||||
def _pair_to_result(pair: dict, *, score: int | None = None) -> dict:
|
||||
"""Shape a knowledge/sql pair like a recall row (parity with LanceDB)."""
|
||||
tags = pair.get("tags")
|
||||
row = {
|
||||
"nl_query": pair["nl"],
|
||||
"sql_query": pair["sql"],
|
||||
"datasource": pair.get("datasource", ""),
|
||||
"tags": ",".join(tags) if isinstance(tags, list) else (tags or ""),
|
||||
"path": pair.get("path"),
|
||||
}
|
||||
if score is not None:
|
||||
row["score"] = score
|
||||
return row
|
||||
|
||||
|
||||
class MemoryIndex(ABC):
|
||||
"""Recall interface over knowledge/sql/. Implementations never persist the
|
||||
markdown — they only build/search a (possibly derived) index over it."""
|
||||
|
||||
name: str
|
||||
|
||||
@abstractmethod
|
||||
def rebuild(self) -> dict:
|
||||
"""(Re)build the index from knowledge/sql/. Returns a small summary."""
|
||||
|
||||
@abstractmethod
|
||||
def search(
|
||||
self, query: str, *, limit: int = 3, datasource: str | None = None
|
||||
) -> list[dict]:
|
||||
"""Return up to *limit* NL→SQL pairs relevant to *query*."""
|
||||
|
||||
@abstractmethod
|
||||
def reset(self) -> None:
|
||||
"""Drop any derived index. The markdown source is never touched."""
|
||||
|
||||
@abstractmethod
|
||||
def status(self) -> dict:
|
||||
"""Return backend + size info."""
|
||||
|
||||
|
||||
class GrepIndex(MemoryIndex):
|
||||
"""Dependency-free recall: token-overlap + substring over knowledge/sql/."""
|
||||
|
||||
name = "grep"
|
||||
|
||||
def __init__(self, project_path: Path):
|
||||
self._project = project_path
|
||||
|
||||
def rebuild(self) -> dict:
|
||||
# The markdown is the index — nothing to build.
|
||||
return {"backend": self.name, "pairs": len(load_query_pairs(self._project))}
|
||||
|
||||
def reset(self) -> None:
|
||||
return # no derived index to drop
|
||||
|
||||
def status(self) -> dict:
|
||||
return {"backend": self.name, "pairs": len(load_query_pairs(self._project))}
|
||||
|
||||
def search(
|
||||
self, query: str, *, limit: int = 3, datasource: str | None = None
|
||||
) -> list[dict]:
|
||||
q_tokens = _tokens(query)
|
||||
q_lower = query.strip().lower()
|
||||
scored: list[tuple[int, dict]] = []
|
||||
for pair in load_query_pairs(self._project):
|
||||
if datasource and pair.get("datasource") != datasource:
|
||||
continue
|
||||
score = len(q_tokens & (_tokens(pair["nl"]) | _tokens(pair["sql"])))
|
||||
if q_lower and q_lower in pair["nl"].lower():
|
||||
score += 5 # whole-query substring match in the NL ranks highest
|
||||
if score > 0:
|
||||
scored.append((score, pair))
|
||||
# Highest score first; stable tie-break by NL for determinism.
|
||||
scored.sort(key=lambda s: (-s[0], s[1]["nl"]))
|
||||
return [_pair_to_result(p, score=score) for score, p in scored[:limit]]
|
||||
|
||||
|
||||
class LanceDBIndex(MemoryIndex):
|
||||
"""Semantic recall via the ``memory`` extra; LanceDB is a derived index."""
|
||||
|
||||
name = "lancedb"
|
||||
|
||||
def __init__(self, project_path: Path, path: str):
|
||||
from wren.memory.store import MemoryStore # noqa: PLC0415
|
||||
|
||||
self._project = project_path
|
||||
self._store = MemoryStore(path=path)
|
||||
|
||||
@property
|
||||
def store(self):
|
||||
return self._store
|
||||
|
||||
def rebuild(self) -> dict:
|
||||
pairs = load_query_pairs(self._project)
|
||||
if not pairs:
|
||||
return {"backend": self.name, "loaded": 0, "updated": 0}
|
||||
res = self._store.load_queries(pairs, upsert=True)
|
||||
return {"backend": self.name, **res}
|
||||
|
||||
def reset(self) -> None:
|
||||
self._store.reset()
|
||||
|
||||
def status(self) -> dict:
|
||||
return {"backend": self.name, **self._store.status()}
|
||||
|
||||
def search(
|
||||
self, query: str, *, limit: int = 3, datasource: str | None = None
|
||||
) -> list[dict]:
|
||||
return self._store.recall_queries(query, limit=limit, datasource=datasource)
|
||||
|
||||
|
||||
def _extra_available() -> bool:
|
||||
return bool(find_spec("lancedb")) and bool(find_spec("sentence_transformers"))
|
||||
|
||||
|
||||
def resolve_backend(env: str | None = None) -> str:
|
||||
"""Return the backend that will actually be used.
|
||||
|
||||
Honors an explicit ``WREN_MEMORY_BACKEND=grep|lancedb`` (or *env*) override,
|
||||
else auto-detects. ``lancedb`` is downgraded to ``grep`` whenever its extra
|
||||
is unavailable — so the result always reflects what ``get_index`` will build.
|
||||
"""
|
||||
choice = (
|
||||
(env if env is not None else os.environ.get("WREN_MEMORY_BACKEND", ""))
|
||||
.strip()
|
||||
.lower()
|
||||
)
|
||||
if choice == "grep":
|
||||
return "grep"
|
||||
# explicit "lancedb", an unrecognized value, or empty → prefer lancedb when
|
||||
# its extra is importable, otherwise grep.
|
||||
return "lancedb" if _extra_available() else "grep"
|
||||
|
||||
|
||||
def get_index(
|
||||
project_path: Path, path: str, *, backend: str | None = None
|
||||
) -> MemoryIndex:
|
||||
"""Construct the resolved MemoryIndex for *project_path*.
|
||||
|
||||
An explicit *backend* is normalized (``" LanceDB "`` → ``lancedb``); an
|
||||
unrecognized value falls back to auto-detection. LanceDB downgrades to
|
||||
GrepIndex when its extra is missing.
|
||||
"""
|
||||
name = (backend or "").strip().lower()
|
||||
if name not in {"grep", "lancedb"}:
|
||||
name = resolve_backend()
|
||||
if name == "lancedb" and _extra_available():
|
||||
return LanceDBIndex(project_path, path)
|
||||
return GrepIndex(project_path)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Markdown source-of-truth for NL→SQL memory pairs (``knowledge/sql/<slug>.md``).
|
||||
|
||||
Dependency-free: no LanceDB / pyarrow / sentence-transformers. The markdown file
|
||||
is the source of truth; the LanceDB index (when the ``memory`` extra is
|
||||
installed) is a derived artifact built from it — mirroring how ``wren context
|
||||
build`` compiles YAML into ``target/mdl.json``.
|
||||
|
||||
File format — YAML frontmatter, optional markdown body for notes::
|
||||
|
||||
---
|
||||
nl: What is the total revenue across all orders?
|
||||
sql: |
|
||||
SELECT SUM(amount) AS total_revenue FROM orders
|
||||
datasource: postgres
|
||||
tags:
|
||||
- revenue
|
||||
source: user
|
||||
---
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
_KNOWLEDGE_SQL_SUBDIR = ("knowledge", "sql")
|
||||
_MAX_SLUG_LEN = 60
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
"""Normalize NL text into a filesystem-safe, deterministic slug."""
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text.strip().lower()).strip("-")
|
||||
if len(text) > _MAX_SLUG_LEN:
|
||||
text = text[:_MAX_SLUG_LEN].rstrip("-")
|
||||
return text or "query"
|
||||
|
||||
|
||||
def knowledge_sql_dir(project_path: Path) -> Path:
|
||||
return project_path.joinpath(*_KNOWLEDGE_SQL_SUBDIR)
|
||||
|
||||
|
||||
def parse_query_markdown(path: Path) -> dict:
|
||||
"""Parse a knowledge/sql/*.md file into its frontmatter dict.
|
||||
|
||||
Returns the frontmatter mapping with an extra ``_body`` key (stripped
|
||||
markdown body). Returns {} when the file has no frontmatter.
|
||||
"""
|
||||
lines = path.read_text(encoding="utf-8").splitlines(keepends=True)
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return {}
|
||||
# The closing delimiter is a line that is exactly "---" at column 0.
|
||||
# Frontmatter values are indented (e.g. block-scalar sql), so a "---" line
|
||||
# inside a value is indented and never matches here.
|
||||
for i in range(1, len(lines)):
|
||||
if lines[i].rstrip("\n") == "---":
|
||||
try:
|
||||
data = yaml.safe_load("".join(lines[1:i])) or {}
|
||||
except yaml.YAMLError:
|
||||
return {} # malformed frontmatter — treat as no pair, don't crash callers
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
data["_body"] = "".join(lines[i + 1 :]).strip()
|
||||
return data
|
||||
return {}
|
||||
|
||||
|
||||
def load_query_pairs(project_path: Path) -> list[dict]:
|
||||
"""Load every NL→SQL pair from ``knowledge/sql/*.md`` (the source of truth).
|
||||
|
||||
Returns dicts shaped for ``MemoryStore.load_queries``: ``nl``, ``sql``,
|
||||
plus ``datasource`` / ``tags`` / ``source`` when present and ``path`` (the
|
||||
source file, relative to the project). Files without a parseable ``nl``+``sql``
|
||||
frontmatter are skipped.
|
||||
"""
|
||||
sql_dir = knowledge_sql_dir(project_path)
|
||||
if not sql_dir.is_dir():
|
||||
return []
|
||||
pairs: list[dict] = []
|
||||
for md in sorted(sql_dir.glob("*.md")):
|
||||
fm = parse_query_markdown(md)
|
||||
nl, sql = fm.get("nl"), fm.get("sql")
|
||||
if not nl or not sql:
|
||||
continue
|
||||
pair: dict = {"nl": nl, "sql": sql, "source": fm.get("source", "user")}
|
||||
if fm.get("datasource"):
|
||||
pair["datasource"] = fm["datasource"]
|
||||
if fm.get("tags"):
|
||||
pair["tags"] = fm["tags"]
|
||||
pair["path"] = str(md.relative_to(project_path))
|
||||
pairs.append(pair)
|
||||
return pairs
|
||||
|
||||
|
||||
def _resolve_slug(base: str, nl: str, sql_dir: Path) -> str:
|
||||
"""Deterministic slug; reuse the file for the same NL, suffix on collision."""
|
||||
candidate = base
|
||||
n = 1
|
||||
while True:
|
||||
dest = sql_dir / f"{candidate}.md"
|
||||
if not dest.exists():
|
||||
return candidate
|
||||
# Same NL → same logical pair → reuse (update in place).
|
||||
existing = parse_query_markdown(dest).get("nl")
|
||||
if existing == nl:
|
||||
return candidate
|
||||
n += 1
|
||||
candidate = f"{base}-{n}"
|
||||
|
||||
|
||||
def render_query_markdown(
|
||||
nl: str,
|
||||
sql: str,
|
||||
*,
|
||||
datasource: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
source: str = "user",
|
||||
created_at: str | None = None,
|
||||
body: str | None = None,
|
||||
) -> str:
|
||||
"""Render the frontmatter document for a NL→SQL pair.
|
||||
|
||||
An optional *body* (user notes below the frontmatter) is preserved verbatim.
|
||||
"""
|
||||
front: dict = {"nl": nl.strip(), "sql": sql.strip(), "source": source}
|
||||
if created_at:
|
||||
front["created_at"] = created_at
|
||||
if datasource:
|
||||
front["datasource"] = datasource
|
||||
if tags:
|
||||
front["tags"] = tags
|
||||
front_yaml = yaml.safe_dump(
|
||||
front, sort_keys=False, allow_unicode=True, default_flow_style=False
|
||||
)
|
||||
doc = f"---\n{front_yaml}---\n"
|
||||
if body:
|
||||
doc += f"\n{body.strip()}\n"
|
||||
return doc
|
||||
|
||||
|
||||
def write_query_markdown(
|
||||
project_path: Path,
|
||||
nl: str,
|
||||
sql: str,
|
||||
*,
|
||||
datasource: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
source: str = "user",
|
||||
created_at: str | None = None,
|
||||
) -> Path:
|
||||
"""Write a NL→SQL pair to ``knowledge/sql/<slug>.md``. Returns the path.
|
||||
|
||||
Deterministic: the same NL updates the same file; a different NL that
|
||||
slugs to an existing name gets a numeric suffix.
|
||||
"""
|
||||
nl = nl.strip() # canonical form — stored, slugged, and matched consistently
|
||||
sql_dir = knowledge_sql_dir(project_path)
|
||||
sql_dir.mkdir(parents=True, exist_ok=True)
|
||||
slug = _resolve_slug(slugify(nl), nl, sql_dir)
|
||||
dest = sql_dir / f"{slug}.md"
|
||||
# Preserve any user-authored notes below the frontmatter when updating in place.
|
||||
existing_body = parse_query_markdown(dest).get("_body") if dest.exists() else None
|
||||
dest.write_text(
|
||||
render_query_markdown(
|
||||
nl,
|
||||
sql,
|
||||
datasource=datasource,
|
||||
tags=tags,
|
||||
source=source,
|
||||
created_at=created_at,
|
||||
body=existing_body or None,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return dest
|
||||
@@ -588,7 +588,10 @@ def _process_metrics(
|
||||
wren_cfg: WrenConfig,
|
||||
dataset_names: set[str],
|
||||
) -> tuple[str | None, list[ValidationError]]:
|
||||
"""Render OSI top-level metrics as a markdown block for instructions.md.
|
||||
"""Render OSI top-level metrics as a markdown block of business rules.
|
||||
|
||||
Surfaced via the manifest's ``_instructions`` carrier, which the importer
|
||||
writes to ``knowledge/rules/`` (or indexes as rules on build).
|
||||
|
||||
Wren has no first-class equivalent of OSI's free-floating metrics
|
||||
(cubes are bound to a single base_object). For v1 we surface them as
|
||||
|
||||
@@ -146,7 +146,7 @@ This script:
|
||||
- Filters out dlt metadata columns (`_dlt_id`, `_dlt_load_id`, `_dlt_list_idx`) from model definitions
|
||||
- Detects parent-child relationships from `_dlt_parent_id` columns and table naming conventions
|
||||
- **Normalizes column types using `wren.type_mapping.parse_type()`** (sqlglot-based)
|
||||
- Generates a complete v2 YAML project (wren_project.yml, models/, relationships.yml, instructions.md)
|
||||
- Generates a complete v5 YAML project (wren_project.yml, models/, relationships.yml, knowledge/rules/)
|
||||
|
||||
After running, show the user what was generated:
|
||||
|
||||
|
||||
@@ -275,7 +275,7 @@ def generate_project_files(
|
||||
|
||||
# -- wren_project.yml --
|
||||
project_config = {
|
||||
"schema_version": 2,
|
||||
"schema_version": 5,
|
||||
"name": project_name,
|
||||
"version": "1.0",
|
||||
"catalog": "",
|
||||
@@ -342,10 +342,10 @@ def generate_project_files(
|
||||
else:
|
||||
files["relationships.yml"] = "relationships: []\n"
|
||||
|
||||
# -- instructions.md --
|
||||
# -- knowledge/ (v5: business rules live under knowledge/rules/) --
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
instructions = (
|
||||
"# Instructions\n\n"
|
||||
rules = (
|
||||
"# Project rules\n\n"
|
||||
f"This Wren project was auto-generated from a dlt DuckDB pipeline.\n\n"
|
||||
f"- **Source DuckDB:** `{duckdb_path}`\n"
|
||||
f"- **Generated:** {timestamp}\n"
|
||||
@@ -354,7 +354,9 @@ def generate_project_files(
|
||||
"dlt metadata columns (`_dlt_id`, `_dlt_parent_id`, etc.) are hidden from models\n"
|
||||
"but still present in the underlying DuckDB tables.\n"
|
||||
)
|
||||
files["instructions.md"] = instructions
|
||||
files["knowledge/rules/general.md"] = rules
|
||||
# knowledge axis version marker (decoupled from the MDL schema_version)
|
||||
files["knowledge/knowledge.yml"] = "schema_version: 1\n"
|
||||
|
||||
return files
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: enrich-context
|
||||
description: "Augment a Wren project with business context that DB schema cannot carry — enum value meanings, units (USD vs cents, ms vs sec), NULL semantics, magic sentinels (-1 = unknown), soft-delete default filters, business synonyms, time-grain / TZ conventions, cross-system identifiers, currency rules, canonical-table preferences, AND named aggregation metrics (ARR, churn, DAU, WAU, NRR) proposed as cubes. Runs in one of two modes selected at session start: `grill` (one question at a time, user-driven) or `auto-pilot` (agent infers and applies, escalates only on conflicts and high-blast-radius additions like new cubes / views / relationships). Reads everything under <project>/raw/ (PDFs, glossaries, handbooks, code, data dictionaries) and optionally samples low-cardinality columns from the live DB (grill mode), compares against the current MDL / cubes / instructions.md / queries.yml / memory pairs, then fills gaps via the ten-category gap catalog and the cube proposal flow. Confirmed findings are written back to the right sink. Use when: user says 'enrich context', 'augment my project', 'grill me on this project', 'auto-fill my context', 'agent doesn't understand our docs / enum values / units / null meanings', 'business context is missing', 'what does status=A mean', 'is this amount in USD or cents', 'we keep getting wrong aggregations', 'add cubes for ARR / DAU / churn', 'we have a handbook / glossary / data dictionary the agent should know'; or after generating an MDL and noticing the agent lacks business semantics."
|
||||
description: "Augment a Wren project with business context that DB schema cannot carry — enum value meanings, units (USD vs cents, ms vs sec), NULL semantics, magic sentinels (-1 = unknown), soft-delete default filters, business synonyms, time-grain / TZ conventions, cross-system identifiers, currency rules, canonical-table preferences, AND named aggregation metrics (ARR, churn, DAU, WAU, NRR) proposed as cubes. Runs in one of two modes selected at session start: `grill` (one question at a time, user-driven) or `auto-pilot` (agent infers and applies, escalates only on conflicts and high-blast-radius additions like new cubes / views / relationships). Reads everything under <project>/raw/ (PDFs, glossaries, handbooks, code, data dictionaries) and optionally samples low-cardinality columns from the live DB (grill mode), compares against the current MDL / cubes / knowledge (rules + NL→SQL pairs), then fills gaps via the ten-category gap catalog and the cube proposal flow. Confirmed findings are written back to the right sink. Use when: user says 'enrich context', 'augment my project', 'grill me on this project', 'auto-fill my context', 'agent doesn't understand our docs / enum values / units / null meanings', 'business context is missing', 'what does status=A mean', 'is this amount in USD or cents', 'we keep getting wrong aggregations', 'add cubes for ARR / DAU / churn', 'we have a handbook / glossary / data dictionary the agent should know'; or after generating an MDL and noticing the agent lacks business semantics."
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: wren-engine
|
||||
@@ -8,7 +8,7 @@ metadata:
|
||||
|
||||
# Wren Enrich Context — Fill the Business-Context Gap
|
||||
|
||||
This skill exists because most business context never lives in a DB schema — it lives in handbooks, glossaries, finance reports, support playbooks, code comments, Slack rules-of-thumb. The agent reads those raw artifacts, finds what's missing from the Wren project, and **either grills the user one question at a time (grill mode) or applies its best inferences directly and hands over an audit (auto-pilot mode)** before writing back. The output lands in three (or four) sinks each project already has — no new artifact, no new tooling.
|
||||
This skill exists because most business context never lives in a DB schema — it lives in handbooks, glossaries, finance reports, support playbooks, code comments, Slack rules-of-thumb. The agent reads those raw artifacts, finds what's missing from the Wren project, and **either grills the user one question at a time (grill mode) or applies its best inferences directly and hands over an audit (auto-pilot mode)** before writing back. The output lands in the sinks each project already has — MDL, `cubes/`, `knowledge/rules/`, and `knowledge/sql/` — no new artifact, no new tooling.
|
||||
|
||||
## Hard rules — READ FIRST
|
||||
|
||||
@@ -29,7 +29,7 @@ This skill exists because most business context never lives in a DB schema — i
|
||||
7. **Drop into grill for three cases.** Always interrupt auto-pilot and ask the user when:
|
||||
- (a) **Lane 2 conflict** — raw and current MDL disagree.
|
||||
- (b) **High-blast-radius proposal (any lane)** — new cube, new view, new relationship, or new MDL metric/calculated column. These become public artifacts visible to every future agent session, so blast radius doesn't depend on whether the trigger was raw evidence (Lane 2) or inference (Lane 3).
|
||||
- (c) **Lane 2 routing ambiguity** — you can't confidently pick a sink (MDL / `instructions.md` / `queries.yml` / `cubes/`).
|
||||
- (c) **Lane 2 routing ambiguity** — you can't confidently pick a sink (MDL / `knowledge/rules/` / `knowledge/sql/` / `cubes/`).
|
||||
|
||||
Everything else: apply directly and log to the audit list.
|
||||
|
||||
@@ -88,14 +88,18 @@ If either check fails, stop and tell the user — suggest `wren skills get onboa
|
||||
|
||||
From this point on, **every command and file path in this skill is relative to the chosen project root**. Do not switch projects mid-session — if the user wants to work a different project, end this session and re-run.
|
||||
|
||||
### Step 2 — Detect memory availability
|
||||
### Step 2 — Detect semantic-memory availability
|
||||
|
||||
```bash
|
||||
wren memory --help >/dev/null 2>&1
|
||||
wren memory status 2>/dev/null | grep -q "Backend: lancedb"
|
||||
```
|
||||
|
||||
- Exit 0 → set `MEMORY_AVAILABLE = true`. The fourth sink (direct `wren memory store`) is open.
|
||||
- Exit non-zero → set `MEMORY_AVAILABLE = false`. Skip the memory-only paths below.
|
||||
Writing NL→SQL pairs (`wren memory store` → `knowledge/sql/*.md`) works **regardless** — that
|
||||
sink is always open. This checks which recall backend is active, i.e. whether the optional
|
||||
`memory` extra is installed (LanceDB = semantic; grep = dependency-free fallback):
|
||||
|
||||
- Match (`Backend: lancedb`) → set `MEMORY_AVAILABLE = true`. Semantic recall and `wren memory fetch` are usable, and `wren memory index` builds an embedding index in Step 8.
|
||||
- No match (grep backend) → set `MEMORY_AVAILABLE = false`. Skip the semantic read/index paths below; pair writeback still happens via `wren memory store`, and grep recall still works.
|
||||
|
||||
### Step 3 — Ensure raw/ folder exists
|
||||
|
||||
@@ -128,10 +132,10 @@ Read every file under `raw/`. Use whatever capability your agent has natively (t
|
||||
| Source | Command |
|
||||
|---|---|
|
||||
| MDL (full) | `wren context show --output json` |
|
||||
| Project instructions | `wren context instructions` |
|
||||
| Business rules | `wren context instructions` (reads `knowledge/rules/` + any legacy `instructions.md`) |
|
||||
| Existing cubes (names) | `wren cube list` |
|
||||
| Existing cubes (measures + dimensions) | `wren cube describe <cube>` for each name above |
|
||||
| Curated NL-SQL pairs | read `queries.yml` directly |
|
||||
| NL→SQL pairs | read `knowledge/sql/*.md` directly |
|
||||
| (Memory) stored pairs | `wren memory list -n 200 --output json` |
|
||||
| (Memory) schema as text | `wren memory describe` |
|
||||
|
||||
@@ -190,16 +194,16 @@ Scan the current MDL and check:
|
||||
- Every column has a description (at least for non-PK, non-FK ones)?
|
||||
- Every model has a `primary_key`?
|
||||
- Every model has at least one relationship (orphan models are suspicious)?
|
||||
- `instructions.md` is more than the scaffold default?
|
||||
- `queries.yml` has at least a few canonical pairs?
|
||||
- `knowledge/rules/` has real content beyond the scaffold default?
|
||||
- `knowledge/sql/` has at least a few canonical NL→SQL pairs?
|
||||
|
||||
Plus, walk every column / model against `gap_catalog` triggers:
|
||||
|
||||
- For each column matching catalog #1 / #2 / #3 / #5 / #7 triggers → is the corresponding `[tag]` line present in `properties.description`?
|
||||
- For each model with a soft-delete column (`deleted_at`, `is_active`, `archived_at`, etc.) → is there a `## Default filters` rule in `instructions.md` covering it (catalog #4)?
|
||||
- For each model with a soft-delete column (`deleted_at`, `is_active`, `archived_at`, etc.) → is there a `## Default filters` rule in `knowledge/rules/` covering it (catalog #4)?
|
||||
- For each lookalike table pair (e.g. `users` / `users_v3`) → is there a `## Canonical tables` rule (catalog #10)?
|
||||
- For each `*_currency` / `fx_rate` / external-system ID column → is the matching `## Currency` (#9) or `## External identifiers` (#8) section present?
|
||||
- Business terms in `instructions.md` or raw that don't map verbatim to model / column names → catalog #6 `## Naming conventions` rule missing.
|
||||
- Business terms in `knowledge/rules/` or raw that don't map verbatim to model / column names → catalog #6 `## Naming conventions` rule missing.
|
||||
|
||||
Each unsatisfied check is a candidate. Combine with Step 4.5 probe results (if available) before moving to Lane 2.
|
||||
|
||||
@@ -222,7 +226,7 @@ After reading raw and the current MDL, propose additions the user did **not** li
|
||||
- "Your support handbook keeps mentioning `core users` without defining it. Is this `users WHERE tier = 'premium'`? Want me to make a view?"
|
||||
- "The data dictionary says `events.payload` is JSON but the column has no description — let me draft one."
|
||||
|
||||
For any aggregation-shaped proposal (`SUM`, `COUNT`, `AVG`, "by month / by status / per customer" patterns), **default to a cube**. Run `wren cube list` + `wren cube describe` first to confirm no existing cube already covers the measure expression; if one does, skip the proposal and add a `queries.yml` example pointing at the existing cube instead. The full decision tree, naming rules, and validation flow live in `cube_proposals`.
|
||||
For any aggregation-shaped proposal (`SUM`, `COUNT`, `AVG`, "by month / by status / per customer" patterns), **default to a cube**. Run `wren cube list` + `wren cube describe` first to confirm no existing cube already covers the measure expression; if one does, skip the proposal and add a `knowledge/sql/` example pointing at the existing cube instead. The full decision tree, naming rules, and validation flow live in `cube_proposals`.
|
||||
|
||||
**In grill mode, open every Lane 3 question with "I'm guessing — ".** In auto-pilot, tag the audit entry with `agent inference` so the user sees you extrapolated.
|
||||
|
||||
@@ -244,7 +248,7 @@ For every grill turn:
|
||||
|
||||
1. State the gap and where it came from (Lane 1 / 2 / 3, and for Lane 2 quote the raw file + a short excerpt).
|
||||
2. Propose the concrete answer — draft the description, the rule, the SQL pair, the relationship.
|
||||
3. **Propose the sink** ("I'll add this to `instructions.md` as a rule" / "I'll add this to the `users` model description in MDL").
|
||||
3. **Propose the sink** ("I'll add this to `knowledge/rules/` as a rule" / "I'll add this to the `users` model description in MDL").
|
||||
4. Let the user accept / edit / skip.
|
||||
5. On accept: write back (Step 7).
|
||||
6. On edit: apply their wording, then write back.
|
||||
@@ -278,11 +282,10 @@ Decide the sink as part of the proposal (Step 6.3 in grill mode; Step 6.2 in aut
|
||||
|---|---|---|
|
||||
| Schema structure / relationship / view / model or column description | **MDL YAML** under `models/`, `views/`, `relationships.yml` | Edit the YAML file directly. For catalog #1 / #2 / #3 / #5 / #7 / PII, append a `[tag]` line to `properties.description` (prose first, then one tag per category). See `gap_catalog` for the exact tag format and triggers. |
|
||||
| Aggregation metric / named measure (with measures + dimensions) | **`cubes/<name>/metadata.yml`** | New file per cube. Default sink for any `SUM` / `COUNT` / `AVG` / ratio metric raw defines or Lane 3 infers. See `cube_proposals` for the YAML template, naming policy, duplication guard, and validation flow. Run `wren context validate` + `wren cube query --cube <name> --sql-only` after writing; revert on either failure. **Always escalates to grill in auto-pilot** (Universal Rule 7b). |
|
||||
| Default filter / implicit rule / business convention / naming convention / external mapping / currency / canonical table | **`instructions.md`** | Append under the catalog-specified `##` section heading (#4 → `## Default filters`, #6 → `## Naming conventions`, #8 → `## External identifiers`, #9 → `## Currency`, #10 → `## Canonical tables`). Create the heading if absent; never modify existing text. |
|
||||
| Canonical NL→SQL example the team should share | **`queries.yml`** | Append a new entry under `pairs:` |
|
||||
| Ad-hoc NL→SQL pair (user-local, not for the repo) — **only if `MEMORY_AVAILABLE = true`** | **`wren memory store`** | `wren memory store --nl "..." --sql "..." --tags "source:enrich"` |
|
||||
| Default filter / implicit rule / business convention / naming convention / external mapping / currency / canonical table | **`knowledge/rules/`** | Append under the catalog-specified `##` section heading (#4 → `## Default filters`, #6 → `## Naming conventions`, #8 → `## External identifiers`, #9 → `## Currency`, #10 → `## Canonical tables`) inside a topic file under `knowledge/rules/` (e.g. `knowledge/rules/conventions.md`). Create the file/heading if absent; never modify existing text. |
|
||||
| NL→SQL example (canonical or ad-hoc) | **`knowledge/sql/`** | `wren memory store --nl "..." --sql "..." --tags "source:enrich"` — writes `knowledge/sql/<slug>.md` (committable) and indexes it when the extra is present. Works whether or not `MEMORY_AVAILABLE`. |
|
||||
|
||||
Catalog-driven routing means every column-local proposal goes to the column's `properties.description` with a `[tag]` line; every cross-model rule goes to `instructions.md` under a fixed heading. This keeps re-enrichment deterministic (greppable) and avoids inventing new sink locations.
|
||||
Catalog-driven routing means every column-local proposal goes to the column's `properties.description` with a `[tag]` line; every cross-model rule goes to `knowledge/rules/` under a fixed heading. This keeps re-enrichment deterministic (greppable) and avoids inventing new sink locations.
|
||||
|
||||
### After every MDL edit
|
||||
|
||||
@@ -298,9 +301,9 @@ If it fails:
|
||||
|
||||
### Format reminders
|
||||
|
||||
- `queries.yml` schema follows `wren memory dump` output: `version: 1`, `pairs:` list of `{nl, sql, source}` (use `source: enrich`).
|
||||
- NL→SQL pairs are written by `wren memory store`; each becomes a `knowledge/sql/<slug>.md` with YAML frontmatter (`nl`, `sql`, `source`, optional `datasource`/`tags`). Don't hand-write the files — use `wren memory store --tags "source:enrich"`.
|
||||
- MDL YAML uses snake_case keys (e.g. `primary_key`, `is_calculated`, `not_null`). `wren context build` converts to camelCase for `target/mdl.json`.
|
||||
- `instructions.md` is free-form markdown. Group rules by topic with headings.
|
||||
- `knowledge/rules/` holds free-form markdown, one file per topic. Group rules by topic with `##` headings.
|
||||
|
||||
## Step 8 — Session finalize
|
||||
|
||||
@@ -318,7 +321,8 @@ If `MEMORY_AVAILABLE = true`:
|
||||
wren memory index
|
||||
```
|
||||
|
||||
This re-embeds the new schema items, the updated `instructions.md`, and the new `queries.yml` entries.
|
||||
This rebuilds the index from the MDL schema items and the `knowledge/sql/` pairs. (Business
|
||||
rules in `knowledge/rules/` are read by `wren context instructions`, not embedded by `index`.)
|
||||
|
||||
## Step 9 — Summary
|
||||
|
||||
@@ -333,10 +337,9 @@ Added:
|
||||
MDL : N model descriptions, N column descriptions, N relationships, N views
|
||||
by tag: [enum]=N [unit]=N [null]=N [magic]=N [time]=N [pii]=N
|
||||
cubes : N new (names: <list>) via cubes/<name>/metadata.yml
|
||||
instructions.md : N new rules across sections
|
||||
knowledge/rules/ : N new rules across sections
|
||||
by section: Default filters=N | Naming conventions=N | External identifiers=N | Currency=N | Canonical tables=N
|
||||
queries.yml : N new NL→SQL pairs
|
||||
memory store : N ad-hoc pairs (only if MEMORY_AVAILABLE)
|
||||
knowledge/sql/ : N new NL→SQL pairs via wren memory store
|
||||
Probe : N columns sampled, M failed (grill mode only)
|
||||
|
||||
Please fix manually (we don't edit existing fields):
|
||||
@@ -359,10 +362,10 @@ Append a detailed audit so the user can sanity-check inferences:
|
||||
|
||||
```text
|
||||
Inferred items (please review):
|
||||
high | MDL model:orders.description | from raw/glossary.pdf p.2 — "Order = ..."
|
||||
high | instructions.md rule | from raw/handbook.md §4 — "default tier ..."
|
||||
med | MDL column:users.signup_source.desc | agent inference from raw/onboarding.md
|
||||
low | queries.yml: "weekly active customers" | agent inference, no direct raw evidence
|
||||
high | MDL model:orders.description | from raw/glossary.pdf p.2 — "Order = ..."
|
||||
high | knowledge/rules/ rule | from raw/handbook.md §4 — "default tier ..."
|
||||
med | MDL column:users.signup_source.desc | agent inference from raw/onboarding.md
|
||||
low | knowledge/sql/: "weekly active customers" | agent inference, no direct raw evidence
|
||||
|
||||
Validation:
|
||||
K successful applies, M reverted after wren context validate failed:
|
||||
@@ -377,20 +380,20 @@ The user should be encouraged to skim the audit and either accept it as-is, manu
|
||||
## Things to avoid
|
||||
|
||||
- Do not write a `gaps.yml`, `state.yml`, or any other tracking artifact. The session lives entirely in conversation.
|
||||
- Do not modify any existing MDL field, instructions rule, or queries.yml entry — only append / add. Surface mismatches on the manual-fix list.
|
||||
- Do not modify any existing MDL field, `knowledge/rules/` rule, or `knowledge/sql/` pair — only append / add. Surface mismatches on the manual-fix list.
|
||||
- Do not install new Python packages (`pypdf`, `docling`, …) to read raw. Use what your agent already has; ask the user to convert files you can't open.
|
||||
- Do not auto-resolve a conflict between raw and current MDL — always grill the user, in **both** modes.
|
||||
- Do not present Lane 3 inferences as if they were quoted from raw. Open with "I'm guessing — " (grill) or tag `agent inference` (auto-pilot).
|
||||
- Do not call `wren memory store` when `MEMORY_AVAILABLE = false` — write to `queries.yml` instead so the pair survives a future `wren memory index`.
|
||||
- `wren memory store` works whether or not `MEMORY_AVAILABLE` — it always writes the `knowledge/sql/*.md` pair (and indexes it only when the extra is present). No need to fall back to another sink.
|
||||
- Do not commit anything to git. The user owns the commit decision.
|
||||
- Do not nag about skipped questions. Skip is skip for this session (grill mode only — auto-pilot has no skip concept).
|
||||
- Do not run `wren context build` after every single MDL edit — once at the end is enough. Do run `wren context validate` after every edit.
|
||||
- Do not assume `raw/` was created by `wren context init` — it isn't. This skill creates it.
|
||||
- Do not switch modes mid-session. The user re-runs to change mode.
|
||||
- Do not append a `[tag]` line if the same category tag already exists for that column — Universal Rule 1. Surface contradictions on the manual-fix list instead.
|
||||
- Do not invent new `instructions.md` section headings. Stick to the five catalog-defined headings (`## Default filters`, `## Naming conventions`, `## External identifiers`, `## Currency`, `## Canonical tables`). Anything that doesn't fit goes on the manual-fix list.
|
||||
- Do not invent new `knowledge/rules/` section headings. Stick to the five catalog-defined headings (`## Default filters`, `## Naming conventions`, `## External identifiers`, `## Currency`, `## Canonical tables`). Anything that doesn't fit goes on the manual-fix list.
|
||||
- Do not probe the live DB in auto-pilot mode. Step 4.5 is grill-only by default.
|
||||
- Do not propose a cube whose measure expression already exists in another cube on the same `base_object` — write a `queries.yml` example pointing at the existing cube instead. See `cube_proposals` duplication guard.
|
||||
- Do not propose a cube whose measure expression already exists in another cube on the same `base_object` — write a `knowledge/sql/` example pointing at the existing cube instead. See `cube_proposals` duplication guard.
|
||||
- Do not modify an existing cube YAML even when raw contradicts it — Universal Rule 1. Surface on the manual-fix list.
|
||||
- Do not write a new cube alongside an old MDL `metrics:` entry that already covers the same logic. Surface as "consider migrating to cube" on the manual-fix list.
|
||||
- Do not skip `wren cube query --cube <name> --sql-only` after creating a cube. Structural `wren context validate` doesn't catch unresolvable measure / dimension expressions.
|
||||
|
||||
@@ -31,7 +31,7 @@ wren cube describe <cube_name> # measures + expressions per cube
|
||||
|
||||
For each measure you're about to propose:
|
||||
|
||||
- **Same expression already exists in another cube** (e.g. `SUM(amount)` for the same `base_object`) → do **not** propose a new cube. Add a `queries.yml` example pointing at the existing cube instead, so the agent learns to reach for it.
|
||||
- **Same expression already exists in another cube** (e.g. `SUM(amount)` for the same `base_object`) → do **not** propose a new cube. Store a `knowledge/sql/` example (via `wren memory store`) pointing at the existing cube instead, so the agent learns to reach for it.
|
||||
- **Same name in `wren cube list`** but different `base_object` → name collision. Either fall back to `<name>_v2` (auto-pilot) or grill the user for a better name (grill mode).
|
||||
- **Old MDL `metrics:` already defines this** (visible in `wren context show --output json` under each model's `metrics:` array) → do not propose. Surface on the Step 9 "please fix manually" list with the note "old metrics: entry — consider migrating to a cube".
|
||||
|
||||
@@ -155,22 +155,18 @@ properties:
|
||||
|
||||
Note no `time_dimensions` because raw didn't ask for time-bucketing. Add one when raw also says "monthly ARR trend" or similar.
|
||||
|
||||
### Example 2 — raw mentions a measure already covered (skip, write queries.yml)
|
||||
### Example 2 — raw mentions a measure already covered (skip, store an NL→SQL pair)
|
||||
|
||||
Raw `support_handbook.md`: *"DAU = distinct active users per day."*
|
||||
|
||||
Existing cube `daily_engagement` already has measure `dau` with expression `COUNT(DISTINCT user_id)`.
|
||||
|
||||
Action: skip the cube proposal. Add to `queries.yml`:
|
||||
Action: skip the cube proposal. Store a pair pointing at the existing cube (lands in `knowledge/sql/`):
|
||||
|
||||
```yaml
|
||||
- nl: "daily active users for last week"
|
||||
sql: |
|
||||
-- via cube
|
||||
SELECT day, dau FROM (
|
||||
<result of: wren cube query --cube daily_engagement --measures dau --time-dimension "day:day:2024-01-01,2024-01-08">
|
||||
)
|
||||
source: enrich
|
||||
```bash
|
||||
wren memory store --tags "source:enrich" \
|
||||
--nl "daily active users for last week" \
|
||||
--sql "SELECT day, dau FROM (<result of: wren cube query --cube daily_engagement --measures dau --time-dimension 'day:day:2024-01-01,2024-01-08'>)"
|
||||
```
|
||||
|
||||
(Or simpler — log it in the Step 9 audit and let the agent reach for `wren cube query` directly at usage time.)
|
||||
|
||||
@@ -10,7 +10,7 @@ Ten business-semantic categories that the schema alone cannot carry. The main `S
|
||||
| **Lane 2** (claim-diff) | For each atomic claim extracted from raw, classify it under one of the 10 categories before deciding the sink. |
|
||||
| **Lane 3** (inference) | If raw is silent but a trigger from this catalog fires AND the slot is empty, propose an inference (open with "I'm guessing — " in grill mode, tag `agent inference` in auto-pilot). |
|
||||
|
||||
Categories 1, 2, 3, 5, 7 write to **column `properties.description`** (prose + `[tag]` line). Categories 4, 6, 8, 9, 10 write to **`instructions.md`** (new `##` section appended). All sinks are append-only — never modify what's there.
|
||||
Categories 1, 2, 3, 5, 7 write to **column `properties.description`** (prose + `[tag]` line). Categories 4, 6, 8, 9, 10 write to **`knowledge/rules/`** (new `##` section appended in a topic file). All sinks are append-only — never modify what's there.
|
||||
|
||||
## Description write format (column-local categories)
|
||||
|
||||
@@ -59,7 +59,7 @@ Use lowercase tag names exactly as listed below — Lane 1 greps these for re-en
|
||||
### 4. Soft-delete / active filters
|
||||
|
||||
- **Trigger:** model has any of `deleted_at`, `is_deleted`, `archived_at`, `is_active`, `is_internal`, `tombstone_at` column OR raw mentions "soft delete", "tombstone", "active rows only", "exclude internal".
|
||||
- **Sink:** `instructions.md` under heading `## Default filters` (create if absent, append rule if present).
|
||||
- **Sink:** `knowledge/rules/` under heading `## Default filters` (create if absent, append rule if present).
|
||||
- **Write format:**
|
||||
```markdown
|
||||
## Default filters
|
||||
@@ -80,7 +80,7 @@ Use lowercase tag names exactly as listed below — Lane 1 greps these for re-en
|
||||
|
||||
- **Trigger:** raw uses a business term that maps to a model / column / metric, but the term doesn't appear verbatim in MDL names or descriptions.
|
||||
- Examples: "customer" → `customers` (vs `accounts`, `customers_v3`); "ARR" → `mrr * 12`; "DAU" → distinct active users per day.
|
||||
- **Sink:** `instructions.md` under heading `## Naming conventions`.
|
||||
- **Sink:** `knowledge/rules/` under heading `## Naming conventions`.
|
||||
- **Write format:**
|
||||
```markdown
|
||||
## Naming conventions
|
||||
@@ -102,7 +102,7 @@ Use lowercase tag names exactly as listed below — Lane 1 greps these for re-en
|
||||
### 8. Cross-system identifiers
|
||||
|
||||
- **Trigger:** column name contains an external-system tag (`stripe_*`, `salesforce_*`, `intercom_*`, `hubspot_*`, `*_external_id`, `*_external_ref`) OR raw maps an internal ID to an external system.
|
||||
- **Sink:** `instructions.md` under heading `## External identifiers`.
|
||||
- **Sink:** `knowledge/rules/` under heading `## External identifiers`.
|
||||
- **Write format:**
|
||||
```markdown
|
||||
## External identifiers
|
||||
@@ -114,7 +114,7 @@ Use lowercase tag names exactly as listed below — Lane 1 greps these for re-en
|
||||
### 9. Currency / locale
|
||||
|
||||
- **Trigger:** any model has `currency`, `locale`, `country`, `region`, `fx_rate`, `original_amount` column OR raw mentions FX rates, multi-currency, or non-USD reporting.
|
||||
- **Sink:** `instructions.md` under heading `## Currency`.
|
||||
- **Sink:** `knowledge/rules/` under heading `## Currency`.
|
||||
- **Write format:**
|
||||
```markdown
|
||||
## Currency
|
||||
@@ -126,7 +126,7 @@ Use lowercase tag names exactly as listed below — Lane 1 greps these for re-en
|
||||
### 10. Canonical table preferences
|
||||
|
||||
- **Trigger:** schema has lookalike tables (`users` / `users_v3`, `orders` / `orders_archive` / `orders_summary`) OR raw says "use X not Y" / "deprecated" / "raw mirror".
|
||||
- **Sink:** `instructions.md` under heading `## Canonical tables`.
|
||||
- **Sink:** `knowledge/rules/` under heading `## Canonical tables`.
|
||||
- **Write format:**
|
||||
```markdown
|
||||
## Canonical tables
|
||||
@@ -143,8 +143,8 @@ To check what a previous enrich run already covered before adding more:
|
||||
# Column-local tags
|
||||
grep -rE '\[(enum|unit|null|magic|time|pii)\]' models/
|
||||
|
||||
# instructions.md section headings written by enrich
|
||||
grep -E '^## (Default filters|Naming conventions|External identifiers|Currency|Canonical tables)' instructions.md
|
||||
# knowledge/rules/ section headings written by enrich
|
||||
grep -rE '^## (Default filters|Naming conventions|External identifiers|Currency|Canonical tables)' knowledge/rules/
|
||||
```
|
||||
|
||||
Any existing `[tag]` line or `##` section means that category has been touched on that target — **do not rewrite by Universal Rule 1**. Surface contradictions on the manual-fix list instead.
|
||||
|
||||
@@ -31,7 +31,7 @@ Check whether `wren_project.yml` exists in the current working directory
|
||||
1. Tell the user that an existing wren project was detected and show its path.
|
||||
2. Ask:
|
||||
- **Reset** — wipe the existing project (`models/`, `views/`,
|
||||
`relationships.yml`, `instructions.md`, and rebuild `wren_project.yml`)
|
||||
`relationships.yml`, `knowledge/`, and rebuild `wren_project.yml`)
|
||||
and regenerate from scratch in the same directory.
|
||||
- **New path** — keep the existing project untouched and choose a
|
||||
different directory for the new project. Ask the user for the new path,
|
||||
@@ -160,7 +160,7 @@ project/
|
||||
├── views/ # named SQL statements
|
||||
├── cubes/ # pre-aggregation cubes (measures + dimensions)
|
||||
├── relationships.yml
|
||||
└── instructions.md
|
||||
└── knowledge/ # business rules (rules/) + NL→SQL pairs (sql/)
|
||||
```
|
||||
|
||||
> **When to define cubes:** If the user asks aggregation questions like
|
||||
|
||||
@@ -113,7 +113,7 @@ Validation runs automatically. The CLI overwrites profiles silently — there is
|
||||
wren context init --empty
|
||||
```
|
||||
|
||||
Refuses to overwrite an existing `wren_project.yml`. Creates the project directory layout (`models/`, `views/`, `relationships.yml`, `instructions.md`, `AGENTS.md`, `queries.yml`).
|
||||
Refuses to overwrite an existing `wren_project.yml`. Creates the project directory layout (`models/`, `views/`, `relationships.yml`, `knowledge/` (rules + sql), `AGENTS.md`).
|
||||
|
||||
## Step 3.6 — Bind the profile to the project
|
||||
|
||||
|
||||
@@ -44,9 +44,9 @@ def test_memory_subcommand_registered_when_extra_present():
|
||||
assert "memory" in names, "memory subcommand not registered despite extra installed"
|
||||
|
||||
|
||||
@pytest.mark.skipif(MEMORY_INSTALLED, reason="memory extra IS installed")
|
||||
def test_memory_subcommand_absent_when_extra_missing():
|
||||
"""The `memory` subcommand group is not registered when the extra is missing."""
|
||||
def test_memory_subcommand_always_registered():
|
||||
"""`memory` is always registered — `wren memory store` writes knowledge/sql/*.md
|
||||
without the extra; lancedb-backed commands degrade with a clear message."""
|
||||
cli = importlib.import_module("wren.cli")
|
||||
names = {g.typer_instance.info.name for g in cli.app.registered_groups}
|
||||
assert "memory" not in names, "memory subcommand registered without the extra"
|
||||
assert "memory" in names, "memory subcommand should always be registered"
|
||||
|
||||
@@ -280,6 +280,94 @@ def test_load_instructions_missing(tmp_path):
|
||||
assert load_instructions(tmp_path) is None
|
||||
|
||||
|
||||
# ── knowledge/ rules + version axis (O3) ──────────────────────────────────
|
||||
|
||||
|
||||
def test_load_knowledge_rules_concatenates_sorted(tmp_path):
|
||||
from wren.context import load_knowledge_rules # noqa: PLC0415
|
||||
|
||||
_make_v2_project(tmp_path)
|
||||
rdir = tmp_path / "knowledge" / "rules"
|
||||
rdir.mkdir(parents=True)
|
||||
(rdir / "b_units.md").write_text("Amounts are USD.\n")
|
||||
(rdir / "a_filters.md").write_text("Exclude soft-deleted rows.\n")
|
||||
result = load_knowledge_rules(tmp_path)
|
||||
# sorted by filename: a_filters before b_units
|
||||
assert result == "Exclude soft-deleted rows.\n\nAmounts are USD."
|
||||
|
||||
|
||||
def test_load_knowledge_rules_missing(tmp_path):
|
||||
from wren.context import load_knowledge_rules # noqa: PLC0415
|
||||
|
||||
_make_v2_project(tmp_path)
|
||||
assert load_knowledge_rules(tmp_path) is None
|
||||
|
||||
|
||||
def test_load_rules_combines_knowledge_and_legacy(tmp_path):
|
||||
from wren.context import load_rules # noqa: PLC0415
|
||||
|
||||
_make_v2_project(tmp_path)
|
||||
(tmp_path / "knowledge" / "rules").mkdir(parents=True)
|
||||
(tmp_path / "knowledge" / "rules" / "general.md").write_text("From knowledge.\n")
|
||||
(tmp_path / "instructions.md").write_text("From legacy.\n")
|
||||
content, used_legacy = load_rules(tmp_path)
|
||||
assert content == "From knowledge.\n\nFrom legacy."
|
||||
assert used_legacy is True
|
||||
|
||||
|
||||
def test_load_rules_knowledge_only_no_legacy_flag(tmp_path):
|
||||
from wren.context import load_rules # noqa: PLC0415
|
||||
|
||||
_make_v2_project(tmp_path)
|
||||
(tmp_path / "knowledge" / "rules").mkdir(parents=True)
|
||||
(tmp_path / "knowledge" / "rules" / "general.md").write_text("Only knowledge.\n")
|
||||
content, used_legacy = load_rules(tmp_path)
|
||||
assert content == "Only knowledge."
|
||||
assert used_legacy is False
|
||||
|
||||
|
||||
def test_load_rules_flags_empty_legacy_file(tmp_path):
|
||||
"""An existing-but-empty instructions.md still flags the deprecated pattern."""
|
||||
from wren.context import load_rules # noqa: PLC0415
|
||||
|
||||
_make_v2_project(tmp_path)
|
||||
(tmp_path / "instructions.md").write_text("")
|
||||
content, used_legacy = load_rules(tmp_path)
|
||||
assert content is None # empty file contributes no content
|
||||
assert used_legacy is True
|
||||
|
||||
|
||||
def test_get_knowledge_schema_version(tmp_path):
|
||||
from wren.context import create_knowledge_skeleton, get_knowledge_schema_version # noqa: PLC0415
|
||||
|
||||
_make_v2_project(tmp_path)
|
||||
assert get_knowledge_schema_version(tmp_path) == 0 # no knowledge/ yet
|
||||
create_knowledge_skeleton(tmp_path)
|
||||
assert get_knowledge_schema_version(tmp_path) == 1
|
||||
|
||||
|
||||
def test_validate_reports_malformed_knowledge_yml(tmp_path):
|
||||
"""A malformed knowledge.yml surfaces as a ValidationError, not a crash."""
|
||||
_make_v2_project(tmp_path, schema_version=5)
|
||||
(tmp_path / "knowledge").mkdir()
|
||||
(tmp_path / "knowledge" / "knowledge.yml").write_text(
|
||||
"schema_version: [unterminated\n"
|
||||
)
|
||||
errors = validate_project(tmp_path)
|
||||
assert any("knowledge" in e.path and "invalid YAML" in e.message for e in errors)
|
||||
|
||||
|
||||
def test_validate_rejects_unsupported_knowledge_version(tmp_path):
|
||||
_make_v2_project(tmp_path, schema_version=5)
|
||||
(tmp_path / "knowledge").mkdir()
|
||||
(tmp_path / "knowledge" / "knowledge.yml").write_text("schema_version: 99\n")
|
||||
errors = validate_project(tmp_path)
|
||||
assert any(
|
||||
"knowledge" in e.path and "unsupported knowledge schema_version" in e.message
|
||||
for e in errors
|
||||
)
|
||||
|
||||
|
||||
# ── build_manifest / build_json ───────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -1308,6 +1396,73 @@ def test_upgrade_preserves_instructions(tmp_path):
|
||||
assert "Rule 1" in content
|
||||
|
||||
|
||||
# ── v4 → v5 (knowledge/ skeleton) ─────────────────────────────────────────
|
||||
|
||||
_KNOWLEDGE_SKELETON = [
|
||||
"knowledge/rules/.gitkeep",
|
||||
"knowledge/glossary/.gitkeep",
|
||||
"knowledge/metrics/.gitkeep",
|
||||
"knowledge/caveats/.gitkeep",
|
||||
"knowledge/sql/.gitkeep",
|
||||
"knowledge/knowledge.yml",
|
||||
]
|
||||
|
||||
|
||||
def test_plan_upgrade_v4_to_v5_lists_knowledge(tmp_path):
|
||||
_make_v2_project(tmp_path, schema_version=4)
|
||||
result = plan_upgrade(tmp_path, target_version=5)
|
||||
assert result.from_version == 4
|
||||
assert result.to_version == 5
|
||||
assert set(result.files_created) == set(_KNOWLEDGE_SKELETON)
|
||||
assert "wren_project.yml" in result.files_modified
|
||||
# plan must not touch disk
|
||||
assert not (tmp_path / "knowledge").exists()
|
||||
|
||||
|
||||
def test_apply_upgrade_v4_to_v5_creates_knowledge(tmp_path):
|
||||
_make_v2_project(tmp_path, schema_version=4)
|
||||
apply_upgrade(tmp_path, plan_upgrade(tmp_path, target_version=5))
|
||||
assert get_schema_version(tmp_path) == 5
|
||||
for rel in _KNOWLEDGE_SKELETON:
|
||||
assert (tmp_path / rel).exists(), rel
|
||||
# knowledge axis has its own schema_version, decoupled from MDL
|
||||
import yaml as _yaml # noqa: PLC0415
|
||||
|
||||
kcfg = _yaml.safe_load((tmp_path / "knowledge" / "knowledge.yml").read_text())
|
||||
assert kcfg["schema_version"] == 1
|
||||
|
||||
|
||||
def test_upgrade_v4_to_v5_idempotent(tmp_path):
|
||||
_make_v2_project(tmp_path, schema_version=4)
|
||||
apply_upgrade(tmp_path, plan_upgrade(tmp_path, target_version=5))
|
||||
# second pass: already at latest → no-op plan, knowledge untouched
|
||||
again = plan_upgrade(tmp_path, target_version=5)
|
||||
assert again.from_version == again.to_version == 5
|
||||
assert again.files_created == []
|
||||
|
||||
|
||||
def test_upgrade_v4_to_v5_preserves_existing_knowledge(tmp_path):
|
||||
"""An existing knowledge file is never overwritten by the upgrade."""
|
||||
_make_v2_project(tmp_path, schema_version=4)
|
||||
(tmp_path / "knowledge" / "rules").mkdir(parents=True)
|
||||
(tmp_path / "knowledge" / "rules" / "house.md").write_text("# keep me\n")
|
||||
apply_upgrade(tmp_path, plan_upgrade(tmp_path, target_version=5))
|
||||
assert (tmp_path / "knowledge" / "rules" / "house.md").read_text() == "# keep me\n"
|
||||
assert (tmp_path / "knowledge" / "knowledge.yml").exists()
|
||||
|
||||
|
||||
def test_apply_upgrade_v2_to_v5_full_chain(tmp_path):
|
||||
"""v2 → v5 restamps through and builds the knowledge skeleton; models still load."""
|
||||
_make_v2_project(tmp_path)
|
||||
d = tmp_path / "models" / "orders"
|
||||
d.mkdir(parents=True)
|
||||
(d / "metadata.yml").write_text("name: orders\ntable_reference:\n table: orders\n")
|
||||
apply_upgrade(tmp_path, plan_upgrade(tmp_path, target_version=5))
|
||||
assert get_schema_version(tmp_path) == 5
|
||||
assert (tmp_path / "knowledge" / "knowledge.yml").exists()
|
||||
assert load_models(tmp_path)[0]["name"] == "orders"
|
||||
|
||||
|
||||
_PROJECT_FILE = "wren_project.yml"
|
||||
|
||||
|
||||
|
||||
@@ -223,7 +223,10 @@ def test_init_creates_scaffold(tmp_path):
|
||||
assert (tmp_path / "views" / "example_view" / "metadata.yml").exists()
|
||||
assert (tmp_path / "views" / "example_view" / "sql.yml").exists()
|
||||
assert (tmp_path / "relationships.yml").exists()
|
||||
assert (tmp_path / "instructions.md").exists()
|
||||
# knowledge/ is first-class: rules live here, not in the legacy instructions.md
|
||||
assert (tmp_path / "knowledge" / "knowledge.yml").exists()
|
||||
assert (tmp_path / "knowledge" / "rules" / "general.md").exists()
|
||||
assert not (tmp_path / "instructions.md").exists()
|
||||
|
||||
# Verify wren_project.yml contains namespace clarification comments and defaults
|
||||
project_yml = (tmp_path / "wren_project.yml").read_text()
|
||||
@@ -304,11 +307,125 @@ def test_init_empty_skips_example_model_and_view(tmp_path):
|
||||
assert (tmp_path / "wren_project.yml").exists()
|
||||
assert (tmp_path / "relationships.yml").exists()
|
||||
assert (tmp_path / "AGENTS.md").exists()
|
||||
assert (tmp_path / "queries.yml").exists()
|
||||
assert (tmp_path / "knowledge" / "knowledge.yml").exists()
|
||||
# v5 no longer scaffolds a legacy queries.yml (pairs live in knowledge/sql/)
|
||||
assert not (tmp_path / "queries.yml").exists()
|
||||
# Summary mentions empty rather than the example paths
|
||||
assert "empty" in result.output
|
||||
|
||||
|
||||
# ── knowledge/ as first-class (O3) ────────────────────────────────────────
|
||||
|
||||
|
||||
def test_init_builds_knowledge_skeleton(tmp_path):
|
||||
result = runner.invoke(app, ["context", "init", "--path", str(tmp_path)])
|
||||
assert result.exit_code == 0, result.output
|
||||
for sub in ("rules", "glossary", "metrics", "caveats", "sql"):
|
||||
assert (tmp_path / "knowledge" / sub).is_dir()
|
||||
assert (tmp_path / "knowledge" / "knowledge.yml").exists()
|
||||
assert (tmp_path / "knowledge" / "rules" / "general.md").exists()
|
||||
# legacy single-file instructions.md is no longer scaffolded
|
||||
assert not (tmp_path / "instructions.md").exists()
|
||||
|
||||
|
||||
def test_init_warns_on_legacy_queries_yml(tmp_path):
|
||||
"""A pre-existing legacy queries.yml is surfaced as deprecated at init."""
|
||||
(tmp_path / "queries.yml").write_text("version: 1\npairs: []\n")
|
||||
result = runner.invoke(app, ["context", "init", "--empty", "--path", str(tmp_path)])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "queries.yml" in result.output and "deprecated" in result.output
|
||||
|
||||
|
||||
def test_init_does_not_clobber_existing_rules(tmp_path):
|
||||
"""Re-running init must not overwrite existing business rules without --force."""
|
||||
runner.invoke(app, ["context", "init", "--empty", "--path", str(tmp_path)])
|
||||
general = tmp_path / "knowledge" / "rules" / "general.md"
|
||||
general.write_text("My real rules.\n")
|
||||
# init again with --force (clears the project-file conflict guard)
|
||||
result = runner.invoke(
|
||||
app, ["context", "init", "--empty", "--force", "--path", str(tmp_path)]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
# --force intentionally re-seeds the starter content
|
||||
assert "Add custom rules" in general.read_text()
|
||||
|
||||
# but without --force, an existing general.md is preserved
|
||||
general.write_text("My real rules.\n")
|
||||
(tmp_path / "wren_project.yml").unlink() # avoid the conflict-guard early exit
|
||||
runner.invoke(app, ["context", "init", "--empty", "--path", str(tmp_path)])
|
||||
assert general.read_text() == "My real rules.\n"
|
||||
|
||||
|
||||
def test_instructions_cmd_reads_knowledge_rules(tmp_path):
|
||||
runner.invoke(app, ["context", "init", "--empty", "--path", str(tmp_path)])
|
||||
(tmp_path / "knowledge" / "rules" / "units.md").write_text("Amounts are USD.\n")
|
||||
result = runner.invoke(app, ["context", "instructions", "--path", str(tmp_path)])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Amounts are USD." in result.output
|
||||
|
||||
|
||||
def test_instructions_cmd_warns_on_legacy(tmp_path):
|
||||
runner.invoke(app, ["context", "init", "--empty", "--path", str(tmp_path)])
|
||||
(tmp_path / "instructions.md").write_text("Legacy rule.\n")
|
||||
result = runner.invoke(app, ["context", "instructions", "--path", str(tmp_path)])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Legacy rule." in result.output
|
||||
assert "deprecated" in result.output
|
||||
|
||||
|
||||
# ── v5 is the default init layout (O1) ───────────────────────────────────
|
||||
|
||||
|
||||
def test_init_writes_v5(tmp_path):
|
||||
"""`wren context init` stamps the latest layout, schema_version 5."""
|
||||
result = runner.invoke(app, ["context", "init", "--path", str(tmp_path)])
|
||||
assert result.exit_code == 0, result.output
|
||||
config = yaml.safe_load((tmp_path / "wren_project.yml").read_text())
|
||||
assert config["schema_version"] == 5
|
||||
# per-model / per-view directory layout (unchanged since v2)
|
||||
assert (tmp_path / "models" / "example" / "metadata.yml").exists()
|
||||
assert (tmp_path / "views" / "example_view" / "metadata.yml").exists()
|
||||
|
||||
|
||||
def test_v5_build_roundtrip(tmp_path):
|
||||
"""init → build produces a valid mdl.json stamped layoutVersion 3."""
|
||||
assert (
|
||||
runner.invoke(app, ["context", "init", "--path", str(tmp_path)]).exit_code == 0
|
||||
)
|
||||
result = runner.invoke(app, ["context", "build", "--path", str(tmp_path)])
|
||||
assert result.exit_code == 0, result.output
|
||||
mdl = json.loads((tmp_path / "target" / "mdl.json").read_text())
|
||||
assert mdl["layoutVersion"] == 3
|
||||
assert any(m["name"] == "example" for m in mdl["models"])
|
||||
|
||||
|
||||
def test_v5_uses_v2_reader(tmp_path):
|
||||
"""A v5 project reads models/views identically to the same project at v3."""
|
||||
from wren.context import load_models, load_views # noqa: PLC0415
|
||||
|
||||
def _populate(root: Path, sv: int) -> Path:
|
||||
root.mkdir(parents=True)
|
||||
(root / "wren_project.yml").write_text(
|
||||
f"schema_version: {sv}\nname: t\ndata_source: postgres\n"
|
||||
"catalog: wren\nschema: public\n"
|
||||
)
|
||||
md = root / "models" / "orders"
|
||||
md.mkdir(parents=True)
|
||||
(md / "metadata.yml").write_text(
|
||||
"name: orders\ntable_reference:\n table: orders\n"
|
||||
"columns:\n - name: id\n type: INTEGER\n"
|
||||
)
|
||||
vd = root / "views" / "summary"
|
||||
vd.mkdir(parents=True)
|
||||
(vd / "metadata.yml").write_text("name: summary\nstatement: SELECT 1\n")
|
||||
return root
|
||||
|
||||
v3 = _populate(tmp_path / "v3", 3)
|
||||
v5 = _populate(tmp_path / "v5", 5)
|
||||
assert load_models(v5) == load_models(v3)
|
||||
assert load_views(v5) == load_views(v3)
|
||||
|
||||
|
||||
# ── wren context validate ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -511,6 +628,35 @@ def test_upgrade_cli_explicit_to_version(tmp_path):
|
||||
assert config["schema_version"] == 2
|
||||
|
||||
|
||||
def test_upgrade_cli_v4_to_v5_builds_knowledge(tmp_path):
|
||||
"""Upgrading a v4 project to latest creates the knowledge/ skeleton."""
|
||||
(tmp_path / "wren_project.yml").write_text(
|
||||
"schema_version: 4\nname: test\ndata_source: postgres\n"
|
||||
)
|
||||
result = runner.invoke(app, ["context", "upgrade", "--path", str(tmp_path)])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Upgrade complete" in result.output
|
||||
config = yaml.safe_load((tmp_path / "wren_project.yml").read_text())
|
||||
assert config["schema_version"] == 5
|
||||
assert (tmp_path / "knowledge" / "knowledge.yml").exists()
|
||||
assert (tmp_path / "knowledge" / "rules" / ".gitkeep").exists()
|
||||
|
||||
|
||||
def test_upgrade_cli_v4_to_v5_dry_run_no_write(tmp_path):
|
||||
"""Dry-run lists the knowledge skeleton but writes nothing."""
|
||||
(tmp_path / "wren_project.yml").write_text(
|
||||
"schema_version: 4\nname: test\ndata_source: postgres\n"
|
||||
)
|
||||
result = runner.invoke(
|
||||
app, ["context", "upgrade", "--path", str(tmp_path), "--dry-run"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "knowledge/knowledge.yml" in result.output
|
||||
assert not (tmp_path / "knowledge").exists()
|
||||
config = yaml.safe_load((tmp_path / "wren_project.yml").read_text())
|
||||
assert config["schema_version"] == 4
|
||||
|
||||
|
||||
# ── wren context import dbt ───────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -564,10 +710,13 @@ def test_import_dbt_writes_project_and_builds(tmp_path):
|
||||
assert (output_dir / "wren_project.yml").exists()
|
||||
assert (output_dir / "models" / "fct_orders" / "metadata.yml").exists()
|
||||
assert (output_dir / "relationships.yml").exists()
|
||||
assert (output_dir / "queries.yml").exists()
|
||||
assert (output_dir / "knowledge" / "rules" / "general.md").exists()
|
||||
assert list((output_dir / "knowledge" / "sql").glob("*.md")) # seeded NL→SQL pairs
|
||||
assert not (output_dir / "queries.yml").exists()
|
||||
assert "skipped 1 ephemeral" in result.output
|
||||
|
||||
config = yaml.safe_load((output_dir / "wren_project.yml").read_text())
|
||||
assert config["schema_version"] == 5
|
||||
assert config["data_source"] == "duckdb"
|
||||
assert config["dbt"]["profile"] == "jaffle_shop"
|
||||
relationships = yaml.safe_load((output_dir / "relationships.yml").read_text())
|
||||
@@ -594,7 +743,6 @@ def test_import_dbt_force_overwrites_managed_files(tmp_path):
|
||||
output_dir = tmp_path / "wren_project"
|
||||
output_dir.mkdir()
|
||||
(output_dir / "wren_project.yml").write_text("name: old\n")
|
||||
(output_dir / "queries.yml").write_text("version: 1\npairs: []\n")
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
@@ -630,7 +778,10 @@ def test_import_dbt_force_overwrites_managed_files(tmp_path):
|
||||
)
|
||||
assert forced.exit_code == 0, forced.output
|
||||
assert "jaffle_shop" in (output_dir / "wren_project.yml").read_text()
|
||||
assert "source: dbt" in (output_dir / "queries.yml").read_text()
|
||||
sql_files = list((output_dir / "knowledge" / "sql").glob("*.md"))
|
||||
contents = [f.read_text() for f in sql_files]
|
||||
assert any("source: dbt" in c for c in contents)
|
||||
assert any("datasource: duckdb" in c for c in contents) # metadata preserved
|
||||
|
||||
|
||||
def test_write_project_files_force_preserves_queries_without_replacement(tmp_path):
|
||||
@@ -852,7 +1003,7 @@ def test_set_profile_preserves_other_fields(tmp_path, monkeypatch):
|
||||
assert config["name"] == "my_project"
|
||||
assert config["catalog"] == "wren"
|
||||
assert config["schema"] == "public"
|
||||
assert config["schema_version"] == 3
|
||||
assert config["schema_version"] == 5
|
||||
|
||||
|
||||
def test_set_profile_preserves_custom_fields(tmp_path, monkeypatch):
|
||||
|
||||
@@ -150,7 +150,7 @@ def test_convert_mdl_to_project():
|
||||
assert "views/monthly_revenue/metadata.yml" in file_map
|
||||
assert "views/monthly_revenue/sql.yml" in file_map
|
||||
assert "relationships.yml" in file_map
|
||||
assert "instructions.md" in file_map
|
||||
assert "knowledge/rules/general.md" in file_map
|
||||
assert "AGENTS.md" in file_map
|
||||
assert file_map["AGENTS.md"] == _AGENTS_MD_TEMPLATE
|
||||
|
||||
@@ -192,8 +192,8 @@ def test_convert_mdl_to_project():
|
||||
assert rels["relationships"][0]["join_type"] == "MANY_TO_ONE"
|
||||
assert rels["relationships"][0]["name"] == "orders_customers"
|
||||
|
||||
# Instructions
|
||||
assert "Always use UTC" in file_map["instructions.md"]
|
||||
# Business rules → knowledge/rules/
|
||||
assert "Always use UTC" in file_map["knowledge/rules/general.md"]
|
||||
|
||||
|
||||
# ── write_project_files ────────────────────────────────────────────────────
|
||||
@@ -208,7 +208,7 @@ def test_write_project_files(tmp_path: Path):
|
||||
assert (tmp_path / "models" / "revenue_summary" / "ref_sql.sql").exists()
|
||||
assert (tmp_path / "views" / "monthly_revenue" / "sql.yml").exists()
|
||||
assert (tmp_path / "relationships.yml").exists()
|
||||
assert (tmp_path / "instructions.md").exists()
|
||||
assert (tmp_path / "knowledge" / "rules" / "general.md").exists()
|
||||
assert (tmp_path / "AGENTS.md").exists()
|
||||
assert (tmp_path / "AGENTS.md").read_text() == _AGENTS_MD_TEMPLATE
|
||||
|
||||
@@ -256,11 +256,11 @@ def test_convert_then_build_roundtrip(tmp_path: Path):
|
||||
|
||||
|
||||
def test_empty_mdl():
|
||||
"""Empty models/views/relationships — only wren_project.yml and AGENTS.md are produced."""
|
||||
"""Empty models/views/relationships — only the project, AGENTS.md, and the knowledge marker."""
|
||||
mdl = {"catalog": "wren", "schema": "public"}
|
||||
files = convert_mdl_to_project(mdl)
|
||||
paths = {f.relative_path for f in files}
|
||||
assert paths == {"wren_project.yml", "AGENTS.md"}
|
||||
assert paths == {"wren_project.yml", "AGENTS.md", "knowledge/knowledge.yml"}
|
||||
assert "instructions.md" not in paths
|
||||
|
||||
|
||||
@@ -273,10 +273,11 @@ def test_no_data_source():
|
||||
|
||||
|
||||
def test_no_instructions():
|
||||
"""No _instructions — instructions.md is not produced."""
|
||||
"""No _instructions — no business-rules file is produced."""
|
||||
mdl = {"catalog": "wren", "schema": "public"}
|
||||
files = convert_mdl_to_project(mdl)
|
||||
assert not any(f.relative_path == "instructions.md" for f in files)
|
||||
assert not any(f.relative_path == "knowledge/rules/general.md" for f in files)
|
||||
|
||||
|
||||
def test_unknown_camel_key_preserved():
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""O6 — pluggable recall backend; grep works without the `memory` extra.
|
||||
|
||||
Runs in the unit job. The grep backend is dependency-free; CLI tests force
|
||||
WREN_MEMORY_BACKEND=grep so they exercise the no-extra path even when the
|
||||
extra happens to be installed locally.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from wren.cli import app
|
||||
from wren.memory.index_backend import GrepIndex, get_index, resolve_backend
|
||||
from wren.memory.markdown import write_query_markdown
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
# ── GrepIndex (dependency-free) ────────────────────────────────────────────
|
||||
|
||||
|
||||
def _seed(tmp_path):
|
||||
write_query_markdown(
|
||||
tmp_path, "Total revenue by month", "SELECT month, SUM(amount) FROM orders"
|
||||
)
|
||||
write_query_markdown(
|
||||
tmp_path, "Number of customers", "SELECT COUNT(*) FROM customers"
|
||||
)
|
||||
|
||||
|
||||
def test_grep_search_token_overlap(tmp_path):
|
||||
_seed(tmp_path)
|
||||
hits = GrepIndex(tmp_path).search("monthly revenue", limit=3)
|
||||
assert hits
|
||||
assert hits[0]["nl_query"] == "Total revenue by month"
|
||||
assert hits[0]["path"] == "knowledge/sql/total-revenue-by-month.md"
|
||||
|
||||
|
||||
def test_grep_search_no_match(tmp_path):
|
||||
_seed(tmp_path)
|
||||
assert GrepIndex(tmp_path).search("xyzzy unrelated", limit=3) == []
|
||||
|
||||
|
||||
def test_grep_search_datasource_filter(tmp_path):
|
||||
write_query_markdown(tmp_path, "Revenue pg", "SELECT 1", datasource="postgres")
|
||||
write_query_markdown(tmp_path, "Revenue bq", "SELECT 2", datasource="bigquery")
|
||||
hits = GrepIndex(tmp_path).search("revenue", limit=5, datasource="postgres")
|
||||
assert [h["nl_query"] for h in hits] == ["Revenue pg"]
|
||||
|
||||
|
||||
def test_grep_rebuild_and_status_and_reset(tmp_path):
|
||||
_seed(tmp_path)
|
||||
idx = GrepIndex(tmp_path)
|
||||
assert idx.rebuild()["pairs"] == 2
|
||||
assert idx.status() == {"backend": "grep", "pairs": 2}
|
||||
idx.reset() # no-op, must not raise
|
||||
assert idx.status()["pairs"] == 2
|
||||
|
||||
|
||||
def test_resolve_backend_env_override():
|
||||
from wren.memory.index_backend import _extra_available # noqa: PLC0415
|
||||
|
||||
assert resolve_backend("grep") == "grep"
|
||||
# explicit lancedb is honored only when its extra is importable, else grep
|
||||
assert resolve_backend("lancedb") == ("lancedb" if _extra_available() else "grep")
|
||||
# unrecognized values fall back to auto-detection
|
||||
assert resolve_backend("bogus") in {"grep", "lancedb"}
|
||||
|
||||
|
||||
def test_get_index_normalizes_explicit_backend(tmp_path):
|
||||
# explicit backend is stripped/lowercased before selection
|
||||
assert isinstance(get_index(tmp_path, "x", backend=" GREP "), GrepIndex)
|
||||
|
||||
|
||||
# ── CLI commands over the grep backend (no extra needed) ───────────────────
|
||||
|
||||
|
||||
def test_cli_recall_grep_without_semantic(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("WREN_PROJECT_HOME", str(tmp_path))
|
||||
monkeypatch.setenv("WREN_MEMORY_BACKEND", "grep")
|
||||
_seed(tmp_path)
|
||||
result = runner.invoke(
|
||||
app, ["memory", "recall", "-q", "revenue by month", "-o", "json"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Total revenue by month" in result.output
|
||||
assert "knowledge/sql/total-revenue-by-month.md" in result.output
|
||||
|
||||
|
||||
def test_cli_index_grep_is_noop(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("WREN_PROJECT_HOME", str(tmp_path))
|
||||
monkeypatch.setenv("WREN_MEMORY_BACKEND", "grep")
|
||||
_seed(tmp_path)
|
||||
result = runner.invoke(app, ["memory", "index"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "grep backend" in result.output
|
||||
assert "2 pair(s)" in result.output
|
||||
|
||||
|
||||
def test_cli_status_and_reset_and_check_grep(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("WREN_PROJECT_HOME", str(tmp_path))
|
||||
monkeypatch.setenv("WREN_MEMORY_BACKEND", "grep")
|
||||
_seed(tmp_path)
|
||||
|
||||
status = runner.invoke(app, ["memory", "status"])
|
||||
assert status.exit_code == 0, status.output
|
||||
assert "Backend: grep" in status.output
|
||||
|
||||
reset = runner.invoke(app, ["memory", "reset", "--force"])
|
||||
assert reset.exit_code == 0, reset.output
|
||||
assert "no derived index" in reset.output
|
||||
|
||||
check = runner.invoke(app, ["memory", "check"])
|
||||
assert check.exit_code == 0, check.output
|
||||
assert "always in sync" in check.output
|
||||
@@ -857,3 +857,91 @@ class TestYamlRoundTrip:
|
||||
# All should be skipped as duplicates
|
||||
assert result["skipped"] == 2
|
||||
assert result["loaded"] == 0
|
||||
|
||||
|
||||
# ── O5: knowledge/sql markdown is the source of truth for the index ────────
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestMarkdownSourcedIndex:
|
||||
"""index/recall/reset treat LanceDB as a derived index over knowledge/sql/."""
|
||||
|
||||
def test_load_query_pairs_index_is_convergent(self, memory_store, tmp_path):
|
||||
from wren.memory.markdown import ( # noqa: PLC0415
|
||||
load_query_pairs,
|
||||
write_query_markdown,
|
||||
)
|
||||
|
||||
write_query_markdown(tmp_path, "Total revenue", "SELECT SUM(amount) FROM o")
|
||||
write_query_markdown(tmp_path, "Count orders", "SELECT COUNT(*) FROM o")
|
||||
pairs = load_query_pairs(tmp_path)
|
||||
|
||||
memory_store.load_queries(pairs, upsert=True)
|
||||
first, _ = memory_store.list_queries(limit=100)
|
||||
# re-running index converges (no duplication)
|
||||
memory_store.load_queries(load_query_pairs(tmp_path), upsert=True)
|
||||
second, _ = memory_store.list_queries(limit=100)
|
||||
assert len(first) == len(second) == 2
|
||||
|
||||
def test_reset_then_reindex_restores_from_markdown(self, memory_store, tmp_path):
|
||||
from wren.memory.markdown import ( # noqa: PLC0415
|
||||
load_query_pairs,
|
||||
write_query_markdown,
|
||||
)
|
||||
|
||||
write_query_markdown(tmp_path, "Total revenue", "SELECT SUM(amount) FROM o")
|
||||
memory_store.load_queries(load_query_pairs(tmp_path), upsert=True)
|
||||
|
||||
memory_store.reset() # derived index dropped
|
||||
assert memory_store.status()["tables"] == {}
|
||||
# markdown source survives the reset
|
||||
assert (tmp_path / "knowledge" / "sql" / "total-revenue.md").exists()
|
||||
|
||||
# rebuild from markdown → recall works again
|
||||
memory_store.load_queries(load_query_pairs(tmp_path), upsert=True)
|
||||
hits = memory_store.recall_queries("revenue", limit=3)
|
||||
assert any("SUM(amount)" in h["sql_query"] for h in hits)
|
||||
|
||||
def test_lancedb_backend_via_get_index(self, tmp_path, monkeypatch):
|
||||
"""With the extra, get_index resolves to LanceDBIndex and recalls semantically."""
|
||||
pytest.importorskip("lancedb", reason="wren[memory] extras not installed")
|
||||
pytest.importorskip(
|
||||
"sentence_transformers", reason="wren[memory] extras not installed"
|
||||
)
|
||||
monkeypatch.setenv("WREN_MEMORY_BACKEND", "lancedb")
|
||||
from wren.memory.index_backend import get_index # noqa: PLC0415
|
||||
from wren.memory.markdown import write_query_markdown # noqa: PLC0415
|
||||
|
||||
write_query_markdown(tmp_path, "Total revenue", "SELECT SUM(amount) FROM o")
|
||||
idx = get_index(tmp_path, str(tmp_path / ".wren" / "memory"))
|
||||
assert idx.name == "lancedb"
|
||||
idx.rebuild()
|
||||
hits = idx.search("revenue", limit=3)
|
||||
assert any("SUM(amount)" in h["sql_query"] for h in hits)
|
||||
|
||||
def test_cli_export_migrates_query_history_to_markdown(self, tmp_path, monkeypatch):
|
||||
"""`wren memory export` writes existing LanceDB pairs to knowledge/sql/."""
|
||||
pytest.importorskip("lancedb", reason="wren[memory] extras not installed")
|
||||
pytest.importorskip(
|
||||
"sentence_transformers", reason="wren[memory] extras not installed"
|
||||
)
|
||||
from typer.testing import CliRunner # noqa: PLC0415
|
||||
|
||||
from wren.cli import app # noqa: PLC0415
|
||||
from wren.memory.markdown import parse_query_markdown # noqa: PLC0415
|
||||
from wren.memory.store import MemoryStore # noqa: PLC0415
|
||||
|
||||
monkeypatch.setenv("WREN_PROJECT_HOME", str(tmp_path))
|
||||
store = MemoryStore(path=str(tmp_path / ".wren" / "memory"))
|
||||
store.store_query("Top revenue", "SELECT SUM(amount) FROM o", tags="source:user")
|
||||
store.store_query("A seed query", "SELECT 1", tags="source:seed")
|
||||
|
||||
result = CliRunner().invoke(app, ["memory", "export"])
|
||||
assert result.exit_code == 0, result.output
|
||||
|
||||
user_md = tmp_path / "knowledge" / "sql" / "top-revenue.md"
|
||||
assert user_md.exists()
|
||||
assert not (tmp_path / "knowledge" / "sql" / "a-seed-query.md").exists() # seed skipped
|
||||
fm = parse_query_markdown(user_md)
|
||||
assert fm["source"] == "user"
|
||||
assert "created_at" in fm # timestamp preserved
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
"""O4 — knowledge/sql/*.md is the source of truth for NL→SQL memory.
|
||||
|
||||
These run in the unit job (no `memory` extra): markdown writing is
|
||||
dependency-free and `wren memory store` must work without LanceDB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import yaml
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from wren.cli import app
|
||||
from wren.memory.markdown import (
|
||||
load_query_pairs,
|
||||
parse_query_markdown,
|
||||
slugify,
|
||||
write_query_markdown,
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def test_slugify_normalizes_and_truncates():
|
||||
assert slugify("What is the total revenue?") == "what-is-the-total-revenue"
|
||||
assert slugify(" Trim/These ") == "trim-these"
|
||||
assert slugify("!!!") == "query"
|
||||
assert len(slugify("word " * 40)) <= 60
|
||||
|
||||
|
||||
def test_write_query_markdown_frontmatter(tmp_path):
|
||||
dest = write_query_markdown(
|
||||
tmp_path,
|
||||
"Total revenue?",
|
||||
"SELECT SUM(amount) FROM orders",
|
||||
datasource="postgres",
|
||||
tags=["revenue", "kpi"],
|
||||
)
|
||||
assert dest == tmp_path / "knowledge" / "sql" / "total-revenue.md"
|
||||
fm = parse_query_markdown(dest)
|
||||
assert fm["nl"] == "Total revenue?"
|
||||
assert "SUM(amount)" in fm["sql"]
|
||||
assert fm["datasource"] == "postgres"
|
||||
assert fm["tags"] == ["revenue", "kpi"]
|
||||
assert fm["source"] == "user"
|
||||
|
||||
|
||||
def test_write_query_markdown_minimal(tmp_path):
|
||||
dest = write_query_markdown(tmp_path, "Count orders", "SELECT COUNT(*) FROM orders")
|
||||
fm = parse_query_markdown(dest)
|
||||
assert fm["nl"] == "Count orders"
|
||||
assert "datasource" not in fm
|
||||
assert "tags" not in fm
|
||||
|
||||
|
||||
def test_same_nl_updates_same_file(tmp_path):
|
||||
a = write_query_markdown(tmp_path, "Total revenue?", "SELECT 1")
|
||||
b = write_query_markdown(
|
||||
tmp_path, "Total revenue?", "SELECT SUM(amount) FROM orders"
|
||||
)
|
||||
assert a == b # same slug, updated in place
|
||||
files = list((tmp_path / "knowledge" / "sql").glob("*.md"))
|
||||
assert len(files) == 1
|
||||
assert "SUM(amount)" in parse_query_markdown(b)["sql"]
|
||||
|
||||
|
||||
def test_same_nl_with_whitespace_reuses_file(tmp_path):
|
||||
"""NLs differing only by surrounding whitespace map to the same file."""
|
||||
a = write_query_markdown(tmp_path, "Total revenue?", "SELECT 1")
|
||||
b = write_query_markdown(tmp_path, " Total revenue? ", "SELECT 2")
|
||||
assert a == b
|
||||
assert len(list((tmp_path / "knowledge" / "sql").glob("*.md"))) == 1
|
||||
|
||||
|
||||
def test_update_preserves_user_body(tmp_path):
|
||||
"""Re-storing the same NL keeps user-authored notes below the frontmatter."""
|
||||
dest = write_query_markdown(tmp_path, "Total revenue", "SELECT 1")
|
||||
dest.write_text(dest.read_text() + "\nNote: excludes refunds.\n")
|
||||
write_query_markdown(tmp_path, "Total revenue", "SELECT SUM(amount) FROM o")
|
||||
text = dest.read_text()
|
||||
assert "SUM(amount)" in text # sql updated
|
||||
assert "Note: excludes refunds." in text # body preserved
|
||||
|
||||
|
||||
def test_sql_containing_dashes_roundtrips(tmp_path):
|
||||
"""SQL whose body contains a '---' line must not break frontmatter parsing."""
|
||||
sql = "SELECT 1\n---\nUNION ALL\nSELECT 2"
|
||||
dest = write_query_markdown(tmp_path, "Dashy query", sql)
|
||||
fm = parse_query_markdown(dest)
|
||||
assert fm["nl"] == "Dashy query"
|
||||
assert fm["sql"] == sql
|
||||
|
||||
|
||||
def test_slug_collision_gets_suffix(tmp_path):
|
||||
a = write_query_markdown(tmp_path, "Revenue?!", "SELECT 1")
|
||||
b = write_query_markdown(tmp_path, "Revenue???", "SELECT 2") # same slug base
|
||||
assert a != b
|
||||
assert b.name == "revenue-2.md"
|
||||
assert len(list((tmp_path / "knowledge" / "sql").glob("*.md"))) == 2
|
||||
|
||||
|
||||
def test_parse_source_handles_null_tags():
|
||||
"""A null/empty tags value must not crash export's source parsing."""
|
||||
from wren.memory.cli import _parse_source # noqa: PLC0415
|
||||
|
||||
assert _parse_source(None) == "user"
|
||||
assert _parse_source("") == "user"
|
||||
assert _parse_source("source:seed") == "seed"
|
||||
|
||||
|
||||
def test_write_query_markdown_created_at(tmp_path):
|
||||
"""created_at is written when provided and ignored by the pair loader."""
|
||||
dest = write_query_markdown(
|
||||
tmp_path, "Q", "SELECT 1", created_at="2026-01-02T03:04:05+00:00"
|
||||
)
|
||||
fm = parse_query_markdown(dest)
|
||||
assert fm["created_at"] == "2026-01-02T03:04:05+00:00"
|
||||
pairs = load_query_pairs(tmp_path)
|
||||
assert pairs[0]["nl"] == "Q" and "created_at" not in pairs[0]
|
||||
|
||||
|
||||
def test_load_query_pairs(tmp_path):
|
||||
write_query_markdown(
|
||||
tmp_path,
|
||||
"Total revenue",
|
||||
"SELECT SUM(amount) FROM orders",
|
||||
datasource="postgres",
|
||||
tags=["revenue"],
|
||||
)
|
||||
write_query_markdown(tmp_path, "Count orders", "SELECT COUNT(*) FROM orders")
|
||||
pairs = load_query_pairs(tmp_path)
|
||||
assert len(pairs) == 2
|
||||
by_nl = {p["nl"]: p for p in pairs}
|
||||
assert by_nl["Total revenue"]["sql"].startswith("SELECT SUM")
|
||||
assert by_nl["Total revenue"]["datasource"] == "postgres"
|
||||
assert by_nl["Total revenue"]["tags"] == ["revenue"]
|
||||
assert by_nl["Total revenue"]["source"] == "user"
|
||||
assert by_nl["Count orders"]["path"] == "knowledge/sql/count-orders.md"
|
||||
|
||||
|
||||
def test_load_query_pairs_empty(tmp_path):
|
||||
assert load_query_pairs(tmp_path) == []
|
||||
|
||||
|
||||
def test_load_query_pairs_skips_unparseable(tmp_path):
|
||||
sql_dir = tmp_path / "knowledge" / "sql"
|
||||
sql_dir.mkdir(parents=True)
|
||||
(sql_dir / "notes.md").write_text("# just a note, no frontmatter\n")
|
||||
(sql_dir / "partial.md").write_text("---\nnl: only nl, no sql\n---\n")
|
||||
# malformed YAML frontmatter must be skipped, not crash the whole load
|
||||
(sql_dir / "broken.md").write_text("---\nnl: [unterminated\nsql: x\n---\n")
|
||||
write_query_markdown(tmp_path, "Good one", "SELECT 1")
|
||||
pairs = load_query_pairs(tmp_path)
|
||||
assert [p["nl"] for p in pairs] == ["Good one"]
|
||||
|
||||
|
||||
def test_recall_path_annotation_handles_collisions(tmp_path, monkeypatch):
|
||||
"""recall path annotation matches on exact NL, not a derived slug."""
|
||||
from wren.memory.cli import _annotate_markdown_paths # noqa: PLC0415
|
||||
|
||||
monkeypatch.setenv("WREN_PROJECT_HOME", str(tmp_path))
|
||||
write_query_markdown(tmp_path, "Revenue?!", "SELECT 1") # -> revenue.md
|
||||
write_query_markdown(tmp_path, "Revenue???", "SELECT 2") # -> revenue-2.md
|
||||
results = [{"nl_query": "Revenue???", "sql_query": "SELECT 2"}]
|
||||
_annotate_markdown_paths(results)
|
||||
assert results[0]["path"] == "knowledge/sql/revenue-2.md"
|
||||
|
||||
|
||||
def test_cli_store_writes_markdown_without_extra(tmp_path, monkeypatch):
|
||||
"""`wren memory store` works without the memory extra — markdown only."""
|
||||
monkeypatch.setenv("WREN_PROJECT_HOME", str(tmp_path))
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"memory",
|
||||
"store",
|
||||
"--nl",
|
||||
"Top customers by revenue",
|
||||
"--sql",
|
||||
"SELECT customer_id, SUM(amount) FROM orders GROUP BY 1",
|
||||
"--tags",
|
||||
"revenue,customers",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Stored:" in result.output
|
||||
md = tmp_path / "knowledge" / "sql" / "top-customers-by-revenue.md"
|
||||
assert md.exists()
|
||||
fm = yaml.safe_load(md.read_text().split("---")[1])
|
||||
assert fm["tags"] == ["revenue", "customers"]
|
||||
@@ -774,7 +774,7 @@ def test_cli_init_from_osi_scaffolds_project(tmp_path: Path):
|
||||
assert (proj / "models" / "orders" / "metadata.yml").exists()
|
||||
assert (proj / "models" / "customers" / "metadata.yml").exists()
|
||||
assert (proj / "relationships.yml").exists()
|
||||
assert (proj / "instructions.md").exists()
|
||||
assert (proj / "knowledge" / "rules" / "general.md").exists()
|
||||
assert (proj / "AGENTS.md").exists()
|
||||
# The OSI semantic_model.name flowed into wren_project.yml
|
||||
import yaml as _yaml # noqa: PLC0415
|
||||
|
||||
@@ -20,7 +20,7 @@ Wren AI runs the agent through two beats whenever you set up a new project.
|
||||
|
||||
**Beat 2: Enrich deep.** Structure is only the start. The hard business meaning lives in docs, decks, Slack threads, and analyst SQL. The `enrich-context` workflow brings that meaning in through two modes:
|
||||
|
||||
- **Grill mode**: the agent asks one focused question at a time ("which is the canonical `orders` table?", "what does `status = 4` mean?", "should `active customer` exclude internal users?"). You answer; the agent patches MDL, `instructions.md`, `queries.yml`, or memory.
|
||||
- **Grill mode**: the agent asks one focused question at a time ("which is the canonical `orders` table?", "what does `status = 4` mean?", "should `active customer` exclude internal users?"). You answer; the agent patches MDL or `knowledge/` (rules and NL→SQL pairs).
|
||||
- **Auto-pilot mode**: drop PDFs, glossaries, handbooks, and SQL history into `<project>/raw/`. The agent reads them, proposes context changes with evidence, and waits for review.
|
||||
|
||||
Both modes write to reviewable, version-controlled artifacts. Nothing is silently absorbed into a black box.
|
||||
@@ -32,11 +32,12 @@ Four artifacts capture different layers of learning:
|
||||
| Artifact | What it stores | Updated by |
|
||||
|---|---|---|
|
||||
| **MDL** (`models/`, `views/`, `relationships.yml`) | Structural and semantic contract: what data exists, how it relates, which calculations are reusable | `wren context build`, manual edits, agent-proposed changes |
|
||||
| **`instructions.md`** | Operational guidance: preferred terminology, default filters, table selection rules, caveats | Manual edits or agent-proposed changes |
|
||||
| **Memory** (`.wren/memory/`) | Retrieval index over MDL + instructions, plus a record of confirmed natural-language-to-SQL pairs | `wren memory index`, `wren memory store` |
|
||||
| **`queries.yml`** | Curated, committable seed of natural-language-to-SQL examples | `wren memory dump` from accumulated memory |
|
||||
| **`knowledge/rules/`** | Operational guidance: preferred terminology, default filters, table selection rules, caveats | Manual edits or agent-proposed changes |
|
||||
| **`knowledge/sql/`** | Confirmed natural-language-to-SQL pairs — committable, the source of truth for recall | `wren memory store`, manual edits |
|
||||
| **Memory index** (`.wren/memory/`) | Derived retrieval index over MDL + `knowledge/` (optional LanceDB, else grep) | `wren memory index` |
|
||||
|
||||
The agent reads from all four when it gathers context for a new question. The first three change rarely; memory and queries grow with use.
|
||||
The agent reads from all of these when it gathers context for a new question. MDL and rules
|
||||
change rarely; the NL→SQL pairs grow with use, and the index is rebuilt from them.
|
||||
|
||||
## The query workflow in practice
|
||||
|
||||
@@ -66,7 +67,7 @@ Wren AI lets the system compound:
|
||||
- Recurring metrics reuse accepted SQL patterns.
|
||||
- Schema retrieval narrows as the project grows.
|
||||
- Corrections become future grounding instead of disappearing at session end.
|
||||
- Teams can commit `queries.yml` so new environments inherit the learning.
|
||||
- Teams can commit `knowledge/sql/` so new environments inherit the learning.
|
||||
|
||||
The agent is not getting smarter. The context layer it reads from is getting richer, and it is reviewable every step of the way.
|
||||
|
||||
|
||||
@@ -20,16 +20,16 @@ a prompt or locked inside a UI:
|
||||
| Artifact | What it holds | Example |
|
||||
| --- | --- | --- |
|
||||
| **MDL** (semantic models) | Structure and semantics: models, columns, relationships, calculated fields, views, cubes | `loyalty_v3` is the canonical loyalty table; `revenue = price * qty - refunds` |
|
||||
| **`instructions.md`** | Business rules and operating policy the schema can't carry | "active customer excludes service accounts"; "always filter `is_deleted = false`" |
|
||||
| **Memory index** | Behavioral retrieval state: which schema items were relevant, which SQL answered a similar question, confirmed examples | local runtime state under `.wren/memory/`, usually gitignored |
|
||||
| **`queries.yml`** | Curated question→SQL pairs exported from memory to keep and share | the version-controlled surface for shared examples |
|
||||
| **`knowledge/rules/`** | Business rules and operating policy the schema can't carry | "active customer excludes service accounts"; "always filter `is_deleted = false`" |
|
||||
| **`knowledge/sql/`** | Confirmed question→SQL pairs, one markdown file each — kept and shared | the version-controlled source of truth for recall |
|
||||
| **Memory index** | Derived retrieval state over MDL + `knowledge/`: which items were relevant, which SQL answered a similar question | local, rebuildable index under `.wren/memory/`, gitignored |
|
||||
|
||||
MDL says what the data *means*. Instructions say how your team wants it *used*.
|
||||
Memory says what has *worked*. MDL, `instructions.md`, and `queries.yml` are
|
||||
version-controlled files in your repo; the memory index under `.wren/memory/` is
|
||||
local runtime state (usually gitignored), and you share what it learns by
|
||||
exporting curated pairs to `queries.yml`. Together they are the context layer the
|
||||
rest of the docs refer to. See [What does Wren AI mean by context?](/oss/concepts/what_is_context).
|
||||
MDL says what the data *means*. `knowledge/rules/` says how your team wants it *used*.
|
||||
`knowledge/sql/` records what has *worked*. MDL and everything under `knowledge/` are
|
||||
version-controlled files in your repo; the memory index under `.wren/memory/` is a
|
||||
derived artifact (usually gitignored) that is rebuilt from them with `wren memory index`.
|
||||
Together they are the context layer the rest of the docs refer to. See
|
||||
[What does Wren AI mean by context?](/oss/concepts/what_is_context).
|
||||
|
||||
## How knowledge gets in: scaffold fast, enrich deep
|
||||
|
||||
|
||||
@@ -21,22 +21,36 @@ That is what Wren AI memory provides.
|
||||
|
||||
## What memory stores
|
||||
|
||||
Memory is local to a Wren project. It is stored under `.wren/memory/`, indexed with [LanceDB](https://lancedb.com/), and never leaves your machine unless you choose to share or commit it.
|
||||
The **source of truth is markdown in your project**: confirmed natural-language-to-SQL
|
||||
pairs live in `knowledge/sql/*.md`, and business rules in `knowledge/rules/`. These are
|
||||
plain files you commit and review like any other source.
|
||||
|
||||
The memory layer has two main collections:
|
||||
On top of that markdown, Wren builds a **derived index** for retrieval. With the optional
|
||||
`memory` extra it's a [LanceDB](https://lancedb.com/) embedding index under `.wren/memory/`
|
||||
(gitignored, rebuildable any time); without it, a dependency-free grep backend searches the
|
||||
markdown directly. Either way the index is disposable — `knowledge/` is the durable layer.
|
||||
|
||||
| Collection | What it stores | Why it matters |
|
||||
The index covers two kinds of content:
|
||||
|
||||
| Content | Source | Why it matters |
|
||||
| --- | --- | --- |
|
||||
| `schema_items` | Models, columns, relationships, views, cubes, and indexed instructions | Lets the agent retrieve the right context for a question without sending the entire project into the prompt. |
|
||||
| `query_history` | Confirmed natural-language-to-SQL pairs | Gives the agent few-shot examples from your actual business, not generic examples. |
|
||||
| Schema items — models, columns, relationships, views, cubes, business rules | MDL + `knowledge/rules/` | Lets the agent retrieve the right context for a question without sending the entire project into the prompt. |
|
||||
| NL→SQL pairs | `knowledge/sql/*.md` | Gives the agent few-shot examples from your actual business, not generic examples. |
|
||||
|
||||
Memory may include:
|
||||
## With and without the `memory` extra
|
||||
|
||||
- schema and column descriptions extracted from MDL
|
||||
- relevant content from `instructions.md`
|
||||
- successful natural-language-to-SQL pairs
|
||||
- imported examples from `queries.yml`
|
||||
- query history stored after successful agent workflows
|
||||
The source of truth is the same either way — only the retrieval engine differs:
|
||||
|
||||
| | With `memory` extra (LanceDB) | Without it (grep, default) |
|
||||
| --- | --- | --- |
|
||||
| NL→SQL `recall` | **Semantic** — embedding similarity, so paraphrases match (store *"monthly revenue"*, recall *"sales per month"*) | **Lexical** — token overlap + substring over `knowledge/sql/*.md`, read directly at query time. Paraphrases with no shared words won't match |
|
||||
| Persistent index | LanceDB under `.wren/memory/` (built by `index`/`store`) | None — the markdown *is* the index, so `index` is a no-op |
|
||||
| Schema search (`fetch`) | Available (embedding retrieval over schema items) | **Not available** — needs embeddings; large schemas should install the extra |
|
||||
|
||||
The grep backend is the zero-dependency fallback: it works out of the box and keeps
|
||||
`store`/`recall` available, at lower recall quality. Install the extra (or set
|
||||
`WREN_MEMORY_BACKEND=lancedb`) for semantic recall and schema search; nothing about your
|
||||
`knowledge/` files changes when you switch.
|
||||
|
||||
## How memory is used
|
||||
|
||||
@@ -61,7 +75,7 @@ This loop gives the agent two kinds of grounding:
|
||||
|
||||
Memory does not define your semantic layer. MDL does.
|
||||
|
||||
Memory helps agents find and reuse context, but the durable contract still lives in project files: models, relationships, views, cubes, and instructions. If a definition is important enough to govern future behavior, put it in MDL or `instructions.md`, then re-index memory.
|
||||
Memory helps agents find and reuse context, but the durable contract still lives in project files: models, relationships, views, cubes, and `knowledge/`. If a definition is important enough to govern future behavior, put it in MDL or `knowledge/rules/`, then re-index memory.
|
||||
|
||||
Think of memory as the retrieval and learning layer on top of the contract.
|
||||
|
||||
@@ -75,13 +89,14 @@ Wren AI memory lets the system compound:
|
||||
- recurring metrics reuse accepted SQL patterns
|
||||
- schema retrieval becomes more targeted on large projects
|
||||
- corrections can become future grounding instead of disappearing after the chat
|
||||
- teams can seed memory with known-good `queries.yml` examples
|
||||
- teams can seed memory by committing known-good `knowledge/sql/*.md` pairs
|
||||
|
||||
The goal is not to memorize every answer. The goal is to make the agent better at finding the right context before it reasons.
|
||||
|
||||
## When to re-index
|
||||
|
||||
`wren memory store` adds a new confirmed NL-SQL pair to query history. But the schema and instruction index is rebuilt with:
|
||||
`wren memory store` writes the confirmed NL→SQL pair to `knowledge/sql/` and indexes it. The
|
||||
schema/rules side of the index is rebuilt with:
|
||||
|
||||
```bash
|
||||
wren memory index
|
||||
@@ -90,22 +105,27 @@ wren memory index
|
||||
Re-index after:
|
||||
|
||||
- editing model descriptions, columns, relationships, views, or cubes
|
||||
- changing `instructions.md`
|
||||
- importing or editing seed examples in `queries.yml`
|
||||
- changing `knowledge/rules/`
|
||||
- adding or editing pairs in `knowledge/sql/`
|
||||
- running a major context enrichment pass
|
||||
|
||||
See the [Refine answer quality](/oss/guides/refine) recipe and [CLI reference](/oss/reference/cli#wren-memory--schema--query-memory) for command details.
|
||||
|
||||
## Sharing memory
|
||||
|
||||
By default, `.wren/memory/` is local runtime state and is usually gitignored.
|
||||
Sharing is just committing `knowledge/`. The NL→SQL pairs in `knowledge/sql/*.md` are
|
||||
plain, reviewable files — commit them and every environment picks them up; the next
|
||||
`wren memory index` rebuilds the local index from them. The derived index under
|
||||
`.wren/memory/` stays gitignored, because it's reproducible from the markdown rather than
|
||||
the collaboration surface itself.
|
||||
|
||||
If your team wants to share confirmed examples, prefer exporting them to `queries.yml` with `wren memory dump`, reviewing them like source files, and loading them back into memory in each environment. This keeps the useful behavioral context portable without turning binary index files into the main collaboration surface.
|
||||
(Have an older project whose history is still in a LanceDB index? `wren memory export`
|
||||
writes it out to `knowledge/sql/*.md` — see [Migration](/oss/reference/migration).)
|
||||
|
||||
## In short
|
||||
|
||||
- **MDL** defines the business meaning.
|
||||
- **Instructions** define guidance and policy.
|
||||
- **Memory** retrieves relevant context and recalls proven examples.
|
||||
- **`knowledge/`** captures business rules and confirmed NL→SQL pairs — committed and reviewable.
|
||||
- **Memory** is the derived index that retrieves relevant context and recalls proven examples.
|
||||
|
||||
Memory is how Wren AI gets better with use while keeping the source of truth inspectable and versionable.
|
||||
|
||||
@@ -208,11 +208,11 @@ For `--from-osi`, see the dedicated [OSI guide](./osi.md) — it covers the alte
|
||||
|
||||
## Upgrade an existing project
|
||||
|
||||
When new MDL features ship (the `dialect` field, new cube semantics), upgrade with:
|
||||
When a new layout `schema_version` ships, upgrade with:
|
||||
|
||||
```bash
|
||||
wren context upgrade # bumps to the latest schema_version
|
||||
wren context upgrade --to 3 # bump to a specific version
|
||||
wren context upgrade --to 5 # bump to a specific version
|
||||
wren context upgrade --dry-run # preview without writing
|
||||
```
|
||||
|
||||
@@ -223,6 +223,11 @@ wren context validate
|
||||
wren context build
|
||||
```
|
||||
|
||||
Some versions add content beyond the automatic restamp — e.g. **v5** introduces
|
||||
`knowledge/` and makes it the home for business rules and NL→SQL memory, so there are extra
|
||||
steps to move `instructions.md` and an existing LanceDB index across. Per-version steps live
|
||||
in the [Migration reference](/oss/reference/migration).
|
||||
|
||||
## When to come back here
|
||||
|
||||
- Adding a new environment (staging / preview / customer X)
|
||||
|
||||
+47
-33
@@ -8,35 +8,39 @@ Scaffolding gives you a baseline MDL. This recipe is how you close the loop. Bri
|
||||
|
||||
## What you'll end up with
|
||||
|
||||
- An `instructions.md` that captures business rules, canonical tables, and team conventions
|
||||
- A memory index over MDL + instructions so agents retrieve relevant context per question
|
||||
- A `queries.yml` of confirmed natural-language-to-SQL pairs, committable to your repo
|
||||
- Business rules under `knowledge/rules/` capturing canonical tables and team conventions
|
||||
- Confirmed natural-language-to-SQL pairs under `knowledge/sql/`, committed to your repo
|
||||
- A memory index over MDL + `knowledge/` so agents retrieve relevant context per question
|
||||
- An agent that gets better at your business each time someone confirms an answer
|
||||
|
||||
## Prerequisite: install the `memory` extra
|
||||
## Optional: the `memory` extra
|
||||
|
||||
The memory layer is an optional extra. It is **not** included in the base CLI. Install it before running any `wren memory ...` command:
|
||||
Refinement works out of the box — `wren memory store`, `index`, and `recall` operate over
|
||||
the markdown in `knowledge/` with no extra dependency (token/substring matching).
|
||||
|
||||
Install the `memory` extra only when you want **semantic** (embedding) recall and schema
|
||||
search (`wren memory fetch`):
|
||||
|
||||
```bash
|
||||
pip install "wrenai[memory]"
|
||||
```
|
||||
|
||||
Combine with your data source extra as needed:
|
||||
|
||||
```bash
|
||||
# combine with your data source extra as needed:
|
||||
pip install "wrenai[memory,postgres]"
|
||||
pip install "wrenai[memory,bigquery]"
|
||||
```
|
||||
|
||||
Without the `memory` extra, the memory commands below will not be available.
|
||||
With the extra, recall is embedding-based and LanceDB caches the index; without it, the
|
||||
same commands fall back to the dependency-free grep backend.
|
||||
|
||||
## The flow today
|
||||
|
||||
The day-to-day refinement loop runs on the `usage` guide plus a few `wren memory` and `instructions.md` edits. When you need to go deeper than incremental edits, such as backfilling enum meanings, units, default filters, synonyms, or named metrics across a whole project, reach for the [`enrich-context`](#enrich-context) guide described below.
|
||||
The day-to-day refinement loop runs on the `usage` guide plus a few `wren memory` and
|
||||
`knowledge/rules/` edits. When you need to go deeper than incremental edits, such as
|
||||
backfilling enum meanings, units, default filters, synonyms, or named metrics across a
|
||||
whole project, reach for the [`enrich-context`](#enrich-context) guide described below.
|
||||
|
||||
### 1. Capture business rules in `instructions.md`
|
||||
### 1. Capture business rules in `knowledge/rules/`
|
||||
|
||||
`instructions.md` is the place to write down the rules that are not visible from the schema:
|
||||
`knowledge/rules/` is where you write down the rules that are not visible from the schema.
|
||||
Use one markdown file per topic (e.g. `knowledge/rules/revenue.md`):
|
||||
|
||||
```markdown
|
||||
## Business rules
|
||||
@@ -51,7 +55,10 @@ The day-to-day refinement loop runs on the `usage` guide plus a few `wren memory
|
||||
- Timestamps are stored in UTC.
|
||||
```
|
||||
|
||||
Organize by topic with `##` headings. Each heading and its body becomes a retrievable chunk in memory. Edit by hand, or have your agent propose changes when it spots a recurring confusion.
|
||||
Each file (and `##` heading within it) becomes a retrievable chunk in memory. Edit by hand,
|
||||
or have your agent propose changes when it spots a recurring confusion. (Older projects: a
|
||||
top-level `instructions.md` is still read, but it's deprecated — move it into
|
||||
`knowledge/rules/`; see [Migration](/oss/reference/migration).)
|
||||
|
||||
### 2. Let `usage` compound from every confirmed answer
|
||||
|
||||
@@ -62,32 +69,38 @@ User asks a question
|
||||
→ wren memory recall (find similar past pairs)
|
||||
→ wren memory fetch (retrieve relevant schema)
|
||||
→ write SQL, dry-plan, execute
|
||||
→ wren memory store (persist the confirmed pair)
|
||||
→ wren memory store (write the confirmed pair to knowledge/sql/ + index)
|
||||
```
|
||||
|
||||
Each stored pair makes future similar questions faster and more accurate. The loop runs on every turn, with no separate enrichment phase needed.
|
||||
Each `store` writes a `knowledge/sql/<slug>.md` file — that markdown is the durable record;
|
||||
the index is built from it. Future similar questions get faster and more accurate, with no
|
||||
separate enrichment phase needed.
|
||||
|
||||
### 3. Re-index after each change
|
||||
|
||||
Whenever you edit `instructions.md`, MDL, or `queries.yml`, rebuild the memory index so the agent's retrieval reflects the new context:
|
||||
Whenever you edit MDL or `knowledge/`, rebuild the memory index so the agent's retrieval
|
||||
reflects the new context:
|
||||
|
||||
```bash
|
||||
wren memory index
|
||||
```
|
||||
|
||||
This re-reads MDL + `instructions.md` + `queries.yml` into the memory store. Targeted retrieval (`wren memory fetch -q "..."`) and recall (`wren memory recall -q "..."`) now see the new context.
|
||||
This re-reads MDL + `knowledge/rules/` + `knowledge/sql/` into the index. Targeted
|
||||
retrieval (`wren memory fetch -q "..."`) and recall (`wren memory recall -q "..."`) now see
|
||||
the new context. Run `wren memory check` to see whether the index is in sync with the
|
||||
markdown.
|
||||
|
||||
### 4. Export learned context to your repo
|
||||
### 4. Commit the learned context
|
||||
|
||||
Curate the team's accumulated learning into `queries.yml`:
|
||||
The pairs are already files — committing `knowledge/` *is* the export:
|
||||
|
||||
```bash
|
||||
wren memory dump --source user -o queries.yml
|
||||
git add queries.yml
|
||||
git commit -m "curate query pairs from this sprint"
|
||||
git add knowledge/
|
||||
git commit -m "curate query pairs and rules from this sprint"
|
||||
```
|
||||
|
||||
A new environment picks them up automatically on the next `wren memory index`.
|
||||
A new environment picks them up automatically on the next `wren memory index` (which
|
||||
rebuilds the local index from the committed markdown).
|
||||
|
||||
## When to come back here
|
||||
|
||||
@@ -98,23 +111,24 @@ A new environment picks them up automatically on the next `wren memory index`.
|
||||
|
||||
## Memory hygiene
|
||||
|
||||
Three commands to keep memory tidy:
|
||||
Because the pairs are files under `knowledge/sql/`, hygiene is mostly ordinary file edits:
|
||||
|
||||
| Command | When |
|
||||
| Action | How |
|
||||
|---|---|
|
||||
| `wren memory list` | Browse stored pairs |
|
||||
| `wren memory forget --id <n> --force` | Remove an incorrect pair |
|
||||
| `wren memory dump --source user` | Export confirmed pairs to `queries.yml` for commit |
|
||||
| Browse stored pairs | `wren memory list`, or read `knowledge/sql/*.md` |
|
||||
| Fix an incorrect pair | edit (or delete) its `knowledge/sql/<slug>.md`, then `wren memory index` |
|
||||
| Share confirmed pairs | commit `knowledge/sql/` (no export step) |
|
||||
| Check index vs. markdown | `wren memory check` |
|
||||
|
||||
See the [CLI reference](/oss/reference/cli) for the full memory command surface.
|
||||
|
||||
## `enrich-context`
|
||||
|
||||
The `enrich-context` guide goes deeper than incremental `instructions.md` edits. It reads everything you drop into `<project>/raw/` (PDFs, glossaries, handbooks, analyst SQL, data dictionaries), compares it against the current MDL / `instructions.md` / `queries.yml` / memory, and fills the gaps, writing back only to reviewable, version-controlled artifacts. It works from a ten-category gap catalog: enum value meanings, units, NULL semantics, magic sentinels, default filters, synonyms, time conventions, cross-system identifiers, currency rules, and canonical-table preferences. Named aggregation metrics (ARR, churn, DAU) are proposed as cubes.
|
||||
The `enrich-context` guide goes deeper than incremental `knowledge/rules/` edits. It reads everything you drop into `<project>/raw/` (PDFs, glossaries, handbooks, analyst SQL, data dictionaries), compares it against the current MDL and `knowledge/`, and fills the gaps, writing back only to reviewable, version-controlled artifacts. It works from a ten-category gap catalog: enum value meanings, units, NULL semantics, magic sentinels, default filters, synonyms, time conventions, cross-system identifiers, currency rules, and canonical-table preferences. Named aggregation metrics (ARR, churn, DAU) are proposed as cubes.
|
||||
|
||||
Pick one of two modes at session start:
|
||||
|
||||
- **Grill mode**: the agent walks each gap one question at a time and asks focused questions ("Which of `customers`, `customers_v3`, `loyalty_v3` is canonical?", "What does `status = 4` mean?"). You answer in plain language; the agent drafts the change and patches MDL, `instructions.md`, `queries.yml`, or memory based on the answer category. With your OK, it can also sample low-cardinality columns from the live DB to discover enum and sentinel values.
|
||||
- **Grill mode**: the agent walks each gap one question at a time and asks focused questions ("Which of `customers`, `customers_v3`, `loyalty_v3` is canonical?", "What does `status = 4` mean?"). You answer in plain language; the agent drafts the change and patches MDL or `knowledge/` (rules and NL→SQL pairs) based on the answer category. With your OK, it can also sample low-cardinality columns from the live DB to discover enum and sentinel values.
|
||||
- **Auto-pilot mode**: drop docs, glossaries, SQL history, or a metric handbook into `<project>/raw/` and the agent reads them, applies its best inferences directly, and escalates to grill only on raw-vs-MDL conflicts and high-blast-radius additions (new cubes / views / relationships). It hands you a confidence-tagged audit at the end.
|
||||
|
||||
Both modes only **add**. They never modify an existing field; contradictions are surfaced on a "please fix manually" list. With the `wren` skill installed (`npx skills add Canner/WrenAI`), trigger it by saying "enrich context" or "grill me on this project". The stub fetches the guide with `wren skills get enrich-context`. See the [skills reference](/oss/reference/skills#enrich-context) for the full breakdown.
|
||||
|
||||
@@ -102,10 +102,9 @@ A Wren project is the portable context package for one business data layer.
|
||||
It includes:
|
||||
|
||||
- **MDL source files** - models, relationships, views, cubes, and project metadata.
|
||||
- **`instructions.md`** - business and operational guidance for agents.
|
||||
- **`queries.yml`** - reviewed natural-language-to-SQL examples that can seed memory.
|
||||
- **`knowledge/`** - business rules (`rules/`) and confirmed NL→SQL pairs (`sql/`), the source of truth for memory.
|
||||
- **`target/mdl.json`** - compiled MDL manifest used by the engine.
|
||||
- **`.wren/memory/`** - local LanceDB indexes for schema retrieval and query recall.
|
||||
- **`.wren/memory/`** - derived, optional LanceDB index rebuilt from `knowledge/` for semantic retrieval.
|
||||
|
||||
Connection profiles live separately in `~/.wren/profiles.yml` so credentials stay environment-specific.
|
||||
|
||||
|
||||
@@ -96,6 +96,22 @@ Requires `target/manifest.json` and `target/catalog.json`; run `dbt build` and `
|
||||
|
||||
---
|
||||
|
||||
## `wren context upgrade`
|
||||
|
||||
Upgrade a project to the latest layout (`schema_version` 5). Forward-only and idempotent;
|
||||
the v4→v5 step creates the `knowledge/` skeleton.
|
||||
|
||||
```bash
|
||||
wren context upgrade --dry-run # preview created/modified files
|
||||
wren context upgrade # apply
|
||||
wren context upgrade --to 5 # target a specific version
|
||||
```
|
||||
|
||||
To migrate `instructions.md` and the LanceDB memory into `knowledge/`, see
|
||||
[Migration](./migration.md).
|
||||
|
||||
---
|
||||
|
||||
## `wren docs` — Connection Info
|
||||
|
||||
### `wren docs connection-info <datasource>`
|
||||
@@ -114,7 +130,12 @@ Use this to check which fields are needed before creating a profile.
|
||||
|
||||
## `wren memory` — Schema & Query Memory
|
||||
|
||||
LanceDB-backed semantic memory for MDL schema search and NL-SQL retrieval. Install with the `memory` extra (separate from `main`):
|
||||
Schema and NL-SQL memory. NL→SQL pairs live in `knowledge/sql/*.md` (the source of truth);
|
||||
the LanceDB index is a derived artifact rebuilt from them.
|
||||
|
||||
`store`, `index`, and `recall` work **without** any extra — pairs are written to and
|
||||
searched over `knowledge/sql/` directly (token/substring matching). Install the `memory`
|
||||
extra only for **semantic** (embedding) recall and schema search (`wren memory fetch`):
|
||||
|
||||
```bash
|
||||
pip install 'wrenai[memory]'
|
||||
@@ -122,7 +143,10 @@ pip install 'wrenai[memory]'
|
||||
pip install 'wrenai[memory,main]'
|
||||
```
|
||||
|
||||
All `memory` subcommands accept `--path DIR` to override the default storage location (`~/.wren/memory/`).
|
||||
The backend is chosen automatically — LanceDB when the extra is installed, otherwise the
|
||||
dependency-free grep backend. Force one with `WREN_MEMORY_BACKEND=grep|lancedb`. All
|
||||
`memory` subcommands accept `--path DIR` to override the LanceDB storage location
|
||||
(`~/.wren/memory/`).
|
||||
|
||||
> **Note:** The `memory` extra bundles ~800MB of large unsigned native libraries (lancedb plus sentence-transformers/torch). On macOS, the first command that loads the memory stack can trigger a one-time XProtect/Gatekeeper scan and pause for up to about a minute before it finishes; this is normal macOS behavior, not a Wren error, and happens once per install or fresh virtual environment. With lazy memory loading, lightweight non-`memory` commands are unaffected — the scan is deferred to your first real memory use, not eliminated.
|
||||
|
||||
@@ -146,7 +170,10 @@ The default threshold (30,000 chars) can be overridden with `--threshold`.
|
||||
|
||||
### `wren memory index`
|
||||
|
||||
Parse the MDL manifest and index all schema items (models, columns, relationships, views) into LanceDB with local embeddings.
|
||||
Build the semantic index: schema items (models, columns, relationships, views) plus the
|
||||
NL→SQL pairs from `knowledge/sql/*.md` (re-running converges on the markdown). Requires the
|
||||
`memory` extra. Without it, the grep backend reads `knowledge/sql/` directly, so there is
|
||||
nothing to build and this command is a no-op.
|
||||
|
||||
```bash
|
||||
wren memory index # uses ~/.wren/mdl.json
|
||||
@@ -186,7 +213,8 @@ wren memory fetch -q "order date" --threshold 50000 --output json
|
||||
|
||||
### `wren memory store`
|
||||
|
||||
Store a natural-language-to-SQL pair for future few-shot retrieval.
|
||||
Store a natural-language-to-SQL pair. Writes `knowledge/sql/<slug>.md` (the source of
|
||||
truth, no extra required), then indexes it into LanceDB when the `memory` extra is present.
|
||||
|
||||
```bash
|
||||
wren memory store \
|
||||
@@ -197,7 +225,8 @@ wren memory store \
|
||||
|
||||
### `wren memory recall`
|
||||
|
||||
Search stored NL-SQL pairs by semantic similarity to a query.
|
||||
Search stored NL-SQL pairs — semantic similarity with the `memory` extra, token/substring
|
||||
matching (grep) without it. Each hit is annotated with its `knowledge/sql/*.md` path.
|
||||
|
||||
```bash
|
||||
wren memory recall -q "best customers"
|
||||
@@ -211,6 +240,26 @@ wren memory recall -q "monthly revenue" --datasource mysql --limit 5 --output js
|
||||
| `-d, --datasource` | Filter by data source |
|
||||
| `-o, --output` | Output format: `table` (default), `json` |
|
||||
|
||||
### `wren memory export`
|
||||
|
||||
One-time migration: export an existing LanceDB `query_history` into `knowledge/sql/*.md`
|
||||
(source, timestamp, and dedup preserved). Requires the `memory` extra to read LanceDB;
|
||||
leaves LanceDB intact. See [Migration](./migration.md).
|
||||
|
||||
```bash
|
||||
wren memory export # query_history → knowledge/sql/*.md
|
||||
wren memory export --include-seed # also export auto-generated seed pairs
|
||||
```
|
||||
|
||||
### `wren memory check`
|
||||
|
||||
Report drift between `knowledge/sql/*.md` and the derived index (which user pairs are not
|
||||
indexed, or indexed without a markdown source).
|
||||
|
||||
```bash
|
||||
wren memory check
|
||||
```
|
||||
|
||||
### `wren memory status`
|
||||
|
||||
Show index statistics: storage path, table names, and row counts.
|
||||
@@ -224,7 +273,8 @@ wren memory status
|
||||
|
||||
### `wren memory reset`
|
||||
|
||||
Drop all memory tables and start fresh.
|
||||
Drop the derived LanceDB index. Your `knowledge/sql/*.md` source files are **preserved** —
|
||||
rebuild the index any time with `wren memory index`.
|
||||
|
||||
```bash
|
||||
wren memory reset # prompts for confirmation
|
||||
|
||||
+40
-22
@@ -4,7 +4,7 @@ sidebar_label: MDL schema
|
||||
|
||||
# MDL schema reference
|
||||
|
||||
This page documents every YAML artifact in a Wren project — `wren_project.yml`, models, relationships, views, cubes, and `instructions.md` — with the full field surface for each.
|
||||
This page documents every YAML artifact in a Wren project — `wren_project.yml`, models, relationships, views, cubes, and the `knowledge/` files — with the full field surface for each.
|
||||
|
||||
> For the conceptual framing of MDL, see [What does MDL do for the agent?](/oss/concepts/what_is_mdl). For the project lifecycle commands, see [Manage project](/oss/guides/manage_project). For the canonical YAML compilation flow, run `wren context build` after editing.
|
||||
|
||||
@@ -31,20 +31,29 @@ my_project/
|
||||
│ └── revenue/
|
||||
│ └── metadata.yml
|
||||
├── relationships.yml # all relationships
|
||||
├── instructions.md # business and operational guidance for agents
|
||||
├── queries.yml # curated NL-SQL pairs (optional)
|
||||
├── knowledge/ # business context (schema_version 5+)
|
||||
│ ├── rules/ # business rules for agents (supersedes instructions.md)
|
||||
│ ├── glossary/ metrics/ caveats/
|
||||
│ ├── sql/ # NL→SQL pairs — source of truth for memory
|
||||
│ └── knowledge.yml # knowledge-axis schema_version (decoupled from MDL)
|
||||
├── instructions.md # deprecated — move into knowledge/rules/ (still read)
|
||||
├── queries.yml # legacy NL-SQL pairs — superseded by knowledge/sql/
|
||||
├── .wren/ # runtime state (gitignored)
|
||||
│ └── memory/ # LanceDB index files
|
||||
│ └── memory/ # derived LanceDB index (optional; rebuilt from knowledge/sql/)
|
||||
└── target/
|
||||
└── mdl.json # build output (gitignored)
|
||||
```
|
||||
|
||||
`wren_project.yml` carries a `schema_version`; **version 5** is the current layout. To
|
||||
upgrade an older project — and migrate `instructions.md` / memory into `knowledge/` — see
|
||||
[Migration](./migration.md).
|
||||
|
||||
YAML files use **snake_case** field names. The compiled `target/mdl.json` uses **camelCase** — the wire format expected by the engine.
|
||||
|
||||
## `wren_project.yml`
|
||||
|
||||
```yaml
|
||||
schema_version: 3
|
||||
schema_version: 5
|
||||
name: my_project
|
||||
version: "1.0"
|
||||
catalog: wren
|
||||
@@ -55,7 +64,7 @@ profile: my-pg
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `schema_version` | int | yes | Directory layout version. `2` = folder-per-entity, `3` = adds `dialect` field support (current). Owned by the CLI — bump with `wren context upgrade`. |
|
||||
| `schema_version` | int | yes | Project layout version (current: `5` — adds first-class `knowledge/`). `2` = folder-per-entity, `3` = `dialect` support, `4` = composite primary keys, `5` = `knowledge/`. Owned by the CLI — bump with `wren context upgrade` (see [Migration](./migration.md)). |
|
||||
| `name` | string | yes | Project identifier. |
|
||||
| `version` | string | no | User-defined project version (free-form, no parsing effect). |
|
||||
| `catalog` | string | no | **Wren AI namespace** — not your database catalog. Defaults to `wren`. |
|
||||
@@ -280,9 +289,11 @@ hierarchies:
|
||||
|
||||
Cubes are queried structurally via `wren cube query`, not by writing raw `GROUP BY` SQL. See [Pre-aggregate with cubes](/oss/guides/cubes) for the agent-facing recipe.
|
||||
|
||||
## Instructions (`instructions.md`)
|
||||
## Business rules (`knowledge/rules/`)
|
||||
|
||||
Free-form markdown with business and operational guidance for AI agents. Organized by topic with `##` headings — each heading and its body becomes a retrievable chunk in memory.
|
||||
Free-form markdown with business and operational guidance for AI agents — one file per
|
||||
topic under `knowledge/rules/`. Each file (and `##` heading within it) becomes a retrievable
|
||||
chunk in memory.
|
||||
|
||||
```markdown
|
||||
## Business rules
|
||||
@@ -297,28 +308,35 @@ Free-form markdown with business and operational guidance for AI agents. Organiz
|
||||
- Timestamps are stored in UTC.
|
||||
```
|
||||
|
||||
Instructions are consumed by agents, not by the engine. They are intentionally excluded from `target/mdl.json`. Agents access them via:
|
||||
Rules are consumed by agents, not by the engine — they are excluded from `target/mdl.json`.
|
||||
Agents access them via:
|
||||
|
||||
- `wren context instructions` — full text, run once at session start
|
||||
- `wren memory fetch -q "..."` — relevant chunks per query
|
||||
|
||||
## `queries.yml` (optional)
|
||||
> A top-level `instructions.md` is still read (alongside `knowledge/rules/`) but is
|
||||
> **deprecated** — move it into `knowledge/rules/`. See [Migration](./migration.md).
|
||||
|
||||
Curated natural-language-to-SQL pairs that seed memory. Same format as `wren memory dump` output:
|
||||
## NL→SQL pairs (`knowledge/sql/`)
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
pairs:
|
||||
- nl: "monthly revenue by product category"
|
||||
sql: |
|
||||
SELECT category, DATE_TRUNC('month', order_date) AS month, SUM(amount)
|
||||
FROM orders
|
||||
GROUP BY 1, 2
|
||||
source: user
|
||||
datasource: postgres-prod
|
||||
Confirmed natural-language-to-SQL pairs — one markdown file per pair under `knowledge/sql/`,
|
||||
the source of truth for memory recall. YAML frontmatter plus an optional body:
|
||||
|
||||
```markdown
|
||||
---
|
||||
nl: monthly revenue by product category
|
||||
sql: |
|
||||
SELECT category, DATE_TRUNC('month', order_date) AS month, SUM(amount)
|
||||
FROM orders
|
||||
GROUP BY 1, 2
|
||||
source: user
|
||||
datasource: postgres-prod
|
||||
---
|
||||
```
|
||||
|
||||
`wren memory index` auto-loads `queries.yml` after indexing the schema. Pairs added through `wren memory store` can be exported back to `queries.yml` with `wren memory dump`.
|
||||
`wren memory store` writes these files; `wren memory index` (re)builds the index from them.
|
||||
A legacy top-level `queries.yml` is still auto-loaded on `index` for the transition, but new
|
||||
pairs land in `knowledge/sql/`. See [Migration](./migration.md).
|
||||
|
||||
## Snake_case to camelCase mapping
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
---
|
||||
sidebar_label: Migration
|
||||
---
|
||||
|
||||
# Migration
|
||||
|
||||
How to move an existing project forward when the project layout `schema_version` changes.
|
||||
Each migration is forward-only, idempotent, and non-destructive — your existing files stay
|
||||
in place until you've verified the result. Run `wren context upgrade` to go to the latest
|
||||
version; the per-version notes below cover anything beyond the automatic restamp.
|
||||
|
||||
## Migrating to schema version 5
|
||||
|
||||
Version 5 keeps the per-folder MDL layout and adds **`knowledge/`** as a first-class home
|
||||
for business rules and NL→SQL pairs — the content that previously lived in
|
||||
`instructions.md`, `queries.yml`, and the LanceDB memory index.
|
||||
|
||||
> New projects from `wren context init` are already v5 — these steps are only for projects
|
||||
> created before v5.
|
||||
|
||||
### 1. Upgrade the layout
|
||||
|
||||
```bash
|
||||
wren context upgrade --dry-run # preview created/modified files
|
||||
wren context upgrade # restamp to schema_version 5 and create knowledge/
|
||||
```
|
||||
|
||||
For a v2–v4 project this bumps `schema_version` to 5 and creates the `knowledge/` skeleton
|
||||
(`rules/`, `glossary/`, `metrics/`, `caveats/`, `sql/`, and `knowledge.yml`); a v1 project
|
||||
is also restructured into the per-folder layout on the way through.
|
||||
|
||||
### 2. Move business rules into `knowledge/rules/`
|
||||
|
||||
`instructions.md` still works — `wren context build` and `wren memory index` read it
|
||||
alongside `knowledge/rules/*.md` — but it is **deprecated** and prints a notice. Move its
|
||||
content under `knowledge/rules/` (split by topic if you like):
|
||||
|
||||
```bash
|
||||
mv instructions.md knowledge/rules/general.md
|
||||
```
|
||||
|
||||
### 3. Migrate semantic memory to markdown
|
||||
|
||||
In v5 the markdown files under `knowledge/sql/` are the **source of truth** for NL→SQL
|
||||
pairs; the LanceDB index becomes a derived artifact rebuilt from them (like
|
||||
`target/mdl.json` is rebuilt from your YAML).
|
||||
|
||||
If you have an existing LanceDB memory at `~/.wren/memory`, export it (requires the
|
||||
`memory` extra to read LanceDB):
|
||||
|
||||
```bash
|
||||
wren memory export # query_history → knowledge/sql/*.md
|
||||
wren memory index # rebuild the derived index from knowledge/sql/
|
||||
wren memory recall -q "revenue" # verify recall still works
|
||||
wren memory reset # once verified — drops the derived index only
|
||||
```
|
||||
|
||||
`export` preserves each pair's source and timestamp, deduplicates by question, and **never
|
||||
deletes LanceDB** — you reset it yourself after verifying. Auto-generated seed pairs are
|
||||
skipped (they're regenerated on `index`); pass `--include-seed` to keep them. `queries.yml`
|
||||
is still loaded on `index` for the transition, but new pairs from `wren memory store` now
|
||||
land in `knowledge/sql/`.
|
||||
|
||||
`store`, `index`, and `recall` all work **without** `wren[memory]` — pairs are written to
|
||||
and searched over `knowledge/sql/` directly. Install `wren[memory]` only for semantic
|
||||
(embedding) recall and schema search. See the
|
||||
[CLI reference](./cli.md#wren-memory--schema--query-memory).
|
||||
|
||||
After migration the project is self-contained and git-friendly: MDL, `knowledge/`, and (on
|
||||
SaaS) `policy/` live together, and the memory index is reproducible from committed
|
||||
markdown. See the [MDL schema reference](./mdl.md) for the full layout.
|
||||
@@ -30,10 +30,13 @@ Override the entire global directory with `WREN_HOME`.
|
||||
| `views/<name>/sql.yml` | Optional separate `statement` file for views. | ✅ yes |
|
||||
| `cubes/<name>/metadata.yml` | Cube definitions. | ✅ yes |
|
||||
| `relationships.yml` | All relationships. | ✅ yes |
|
||||
| `instructions.md` | LLM-facing natural-language guidance. | ✅ yes |
|
||||
| `queries.yml` | Curated NL-SQL pairs (seed for memory). | ✅ yes |
|
||||
| `knowledge/rules/` | LLM-facing business rules (supersedes `instructions.md`). | ✅ yes |
|
||||
| `knowledge/sql/` | Confirmed NL→SQL pairs — source of truth for memory. | ✅ yes |
|
||||
| `knowledge/knowledge.yml` | Knowledge-axis `schema_version`. | ✅ yes |
|
||||
| `instructions.md` | Deprecated — move into `knowledge/rules/` (still read if present). | ⚠️ legacy |
|
||||
| `queries.yml` | Legacy NL-SQL pairs — superseded by `knowledge/sql/`. | ⚠️ legacy |
|
||||
| `.env` | Per-project `.env` for `${VAR}` interpolation. | ❌ gitignore |
|
||||
| `.wren/memory/` | LanceDB index files (schema + query history). | ❌ gitignore |
|
||||
| `.wren/memory/` | Derived LanceDB index (rebuilt from `knowledge/sql/`). | ❌ gitignore |
|
||||
| `target/mdl.json` | Compiled MDL manifest (rebuildable). | ❌ gitignore |
|
||||
|
||||
## Environment variables
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# Total revenue
|
||||
|
||||
**NL:** What is the total revenue across all orders?
|
||||
|
||||
```sql
|
||||
SELECT SUM(amount) AS total_revenue FROM orders
|
||||
```
|
||||
---
|
||||
nl: What is the total revenue across all orders?
|
||||
sql: |
|
||||
SELECT SUM(amount) AS total_revenue FROM orders
|
||||
source: user
|
||||
tags:
|
||||
- revenue
|
||||
---
|
||||
|
||||
Reference in New Issue
Block a user