feat(ai): search docs with bash (#9269)

* feat(ai): search docs with bash

* fix(ai): block broad docs searches with quoted patterns
This commit is contained in:
YANG QIA
2026-04-29 17:17:22 +08:00
committed by GitHub
parent b0b1a9c555
commit f7229ff266
21 changed files with 665 additions and 3416 deletions
+2
View File
@@ -58,6 +58,8 @@
"@types/react": "18.3.18",
"@types/react-dom": "^18.0.0",
"@typescript-eslint/parser": "^6.2.0",
"**/@mongodb-js/zstd": "file:./packages/plugins/@nocobase/plugin-ai/npm-shims/@mongodb-js/zstd",
"**/node-liblzma": "file:./packages/plugins/@nocobase/plugin-ai/npm-shims/node-liblzma",
"react-router-dom": "^6.30.1",
"react-router": "^6.30.1",
"react": "^18.0.0",
@@ -1,980 +0,0 @@
/**
* This file is part of the NocoBase (R) project.
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
* Authors: NocoBase Team.
*
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import fg from 'fast-glob';
import fs from 'fs-extra';
import path from 'path';
import { storagePathJoin } from '@nocobase/utils';
import { FlexSearchIndex } from '@nocobase/ai';
import type Application from '../application';
import { findAllPlugins } from '../plugin-manager/findPackageNames';
import { PluginManager } from '../plugin-manager';
export type DocsIndexOptions = {
pkg?: string | string[];
};
interface BuildResult {
created: boolean;
reason?: string;
conflicts?: string[];
}
type DirectoryChildren = Map<
string,
{
files: string[];
directories: string[];
}
>;
const DOCS_STORAGE_DIR = storagePathJoin('ai', 'docs');
const REFERENCE_START = '<!-- docs:references:start -->';
const REFERENCE_END = '<!-- docs:references:end -->';
const SPLIT_REFERENCE_START = '<!-- docs:splits:start -->';
const SPLIT_REFERENCE_END = '<!-- docs:splits:end -->';
const SPLIT_MAX_LENGTH = 5000;
const SPLIT_MAX_CODE_BLOCKS = 3;
const CREATE_INDEX_RETRY_MAX = 3;
const CREATE_INDEX_RETRY_DELAY_MS = 200;
interface DocEntryMeta {
absolutePath: string;
relativePath: string;
canonicalPath: string;
content: string;
title: string;
description: string;
hasFrontMatter: boolean;
isIndex: boolean;
moduleName: string;
moduleRoot: string;
packageName: string;
}
interface ModuleGroup {
moduleName: string;
description: string;
moduleRoot: string;
packageName: string;
entries: DocEntryMeta[];
directoryChildren: DirectoryChildren;
docMap: Map<string, DocEntryMeta>;
}
interface ModuleMeta {
module?: string;
description?: string;
source?: string;
}
type ModuleMetaEntry = ModuleMeta & {
module: string;
description?: string;
source?: string;
};
function normalizeMetaEntries(meta: ModuleMeta | ModuleMeta[] | null, fallbackModule: string) {
if (Array.isArray(meta)) {
return meta
.filter((item) => item && typeof item.module === 'string')
.map((item) => ({
module: item.module.trim(),
description: typeof item.description === 'string' ? item.description.trim() : '',
source: typeof item.source === 'string' ? item.source.trim() : '',
}))
.filter((item) => item.module);
}
if (meta && typeof meta.module === 'string') {
return [
{
module: meta.module.trim() || fallbackModule,
description: typeof meta.description === 'string' ? meta.description.trim() : '',
source: typeof meta.source === 'string' ? meta.source.trim() : '',
},
];
}
return [
{
module: fallbackModule,
description: '',
source: '',
},
];
}
async function resolvePkgs(pkg?: string | string[]) {
if (!pkg) {
return await findAllPlugins();
}
const scopes = (Array.isArray(pkg) ? pkg : pkg.split(',')).map((item) => item.trim()).filter(Boolean);
return Array.from(new Set(scopes));
}
function buildDirectoryChildren(files: string[], docsDir: string): DirectoryChildren {
const map: DirectoryChildren = new Map();
const ensureDir = (dir: string) => {
if (!map.has(dir)) {
map.set(dir, { files: [], directories: [] });
}
};
ensureDir(docsDir);
files.forEach((file) => {
const dir = path.dirname(file);
ensureDir(dir);
map.get(dir)?.files.push(file);
});
files.forEach((file) => {
let current = path.dirname(file);
while (current && current.startsWith(docsDir)) {
const parent = path.dirname(current);
if (parent && parent.startsWith(docsDir)) {
ensureDir(parent);
const list = map.get(parent);
if (list && !list.directories.includes(current)) {
list.directories.push(current);
}
}
if (current === docsDir) break;
current = path.dirname(current);
}
});
return map;
}
async function resolveSourcePath(source: string, metaFile: string) {
if (!source) return '';
if (path.isAbsolute(source)) {
return (await fs.pathExists(source)) ? source : '';
}
const fromRepo = path.resolve(process.cwd(), source);
if (await fs.pathExists(fromRepo)) {
return fromRepo;
}
const fromMeta = path.resolve(path.dirname(metaFile), source);
if (await fs.pathExists(fromMeta)) {
return fromMeta;
}
return '';
}
function errorToString(error: unknown) {
if (error instanceof Error) {
return error.stack || error.message;
}
return String(error);
}
async function collectModuleGroups(packageName: string): Promise<ModuleGroup[]> {
const packageJsonPath = require.resolve(`${packageName}/package.json`);
const packageDir = path.dirname(packageJsonPath);
const distDocsDir = path.join(packageDir, 'dist', 'ai', 'docs');
const srcDocsDir = path.join(packageDir, 'src', 'ai', 'docs');
const preferSrc = process.env.APP_ENV !== 'production';
const preferredDocsDir = preferSrc ? srcDocsDir : distDocsDir;
const fallbackDocsDir = preferSrc ? distDocsDir : srcDocsDir;
const docsDir = (await fs.pathExists(preferredDocsDir)) ? preferredDocsDir : fallbackDocsDir;
if (!(await fs.pathExists(docsDir))) {
return [];
}
const rootMetaPath = path.join(docsDir, 'meta.json');
let metaEntries: ModuleMetaEntry[] = [];
if (await fs.pathExists(rootMetaPath)) {
try {
const rootMeta = (await fs.readJson(rootMetaPath)) as ModuleMeta | ModuleMeta[];
metaEntries = normalizeMetaEntries(rootMeta, path.basename(docsDir));
} catch {
metaEntries = [];
}
}
const moduleMetaFiles =
metaEntries.length === 0
? await fg(['*/meta.json'], {
cwd: docsDir,
onlyFiles: true,
absolute: true,
})
: [];
if (!metaEntries.length && !moduleMetaFiles.length) {
return [];
}
moduleMetaFiles.sort();
const moduleRoots = moduleMetaFiles.map((metaPath) => path.dirname(metaPath));
const groups: ModuleGroup[] = [];
const entriesToProcess =
metaEntries.length > 0
? metaEntries.map((entry) => ({ metaFile: rootMetaPath, entry }))
: moduleMetaFiles.map((metaFile) => ({ metaFile, entry: undefined as ModuleMetaEntry | undefined }));
for (const item of entriesToProcess) {
const metaFile = item.metaFile;
const moduleRoot = path.dirname(metaFile);
const moduleDirName = path.basename(moduleRoot);
let perFileEntries: ModuleMetaEntry[] = [];
if (item.entry) {
perFileEntries = [item.entry];
} else {
try {
const meta = (await fs.readJson(metaFile)) as ModuleMeta | ModuleMeta[];
perFileEntries = normalizeMetaEntries(meta, moduleDirName);
} catch {
perFileEntries = normalizeMetaEntries(null, moduleDirName);
}
}
for (const metaEntry of perFileEntries) {
const moduleName = metaEntry.module || moduleDirName;
const description = metaEntry.description || '';
const source = metaEntry.source || '';
const defaultModuleRoot = path.join(docsDir, moduleName);
let effectiveModuleRoot = defaultModuleRoot;
if (source && preferSrc) {
const resolvedSource = await resolveSourcePath(source, metaFile);
if (resolvedSource) {
effectiveModuleRoot = resolvedSource;
}
}
if (!(await fs.pathExists(effectiveModuleRoot))) {
continue;
}
const ignore: string[] = [];
if (moduleRoot === docsDir && effectiveModuleRoot === moduleRoot) {
for (const otherRoot of moduleRoots) {
if (otherRoot === moduleRoot) continue;
const rel = path.relative(moduleRoot, otherRoot).split(path.sep).join('/');
if (rel && !rel.startsWith('..')) {
ignore.push(`${rel}/**`);
}
}
}
const files = await fg(['**/*.{md,mdx}'], {
cwd: effectiveModuleRoot,
onlyFiles: true,
absolute: true,
ignore,
});
if (!files.length) {
continue;
}
files.sort();
const directoryChildren = buildDirectoryChildren(files, effectiveModuleRoot);
const docEntries: DocEntryMeta[] = await Promise.all(
files.map(async (file) => {
const relativePath = path.relative(effectiveModuleRoot, file);
const normalizedRelativePath = relativePath.split(path.sep).join('/');
const canonicalPath = path.posix.join(moduleName, normalizedRelativePath);
const content = await fs.readFile(file, 'utf8');
const meta = extractDocMetadata(content, file);
return {
absolutePath: file,
relativePath,
canonicalPath,
content,
moduleName,
moduleRoot,
packageName,
...meta,
};
}),
);
const docMap = new Map<string, DocEntryMeta>(docEntries.map((entry) => [entry.absolutePath, entry]));
groups.push({
moduleName,
description,
moduleRoot,
packageName,
entries: docEntries,
directoryChildren,
docMap,
});
}
}
return groups;
}
async function buildDocsIndexForPackages(packageNames: string[]): Promise<Map<string, BuildResult>> {
const moduleGroups: Map<string, ModuleGroup[]> = new Map();
const moduleDescriptions: Map<string, string> = new Map();
const conflicts: Map<string, string[]> = new Map();
for (const packageName of packageNames) {
const groups = await collectModuleGroups(packageName);
for (const group of groups) {
const existing = moduleGroups.get(group.moduleName);
if (existing) {
existing.push(group);
} else {
moduleGroups.set(group.moduleName, [group]);
}
if (group.description) {
const existingDesc = moduleDescriptions.get(group.moduleName);
if (existingDesc && existingDesc !== group.description) {
const list = conflicts.get(group.moduleName) || [];
list.push(
`${group.packageName}: description mismatch (existing="${existingDesc}", new="${group.description}")`,
);
conflicts.set(group.moduleName, list);
} else if (!existingDesc) {
moduleDescriptions.set(group.moduleName, group.description);
}
}
}
}
const results = new Map<string, BuildResult>();
for (const [moduleName, groups] of moduleGroups.entries()) {
const outputDir = path.join(DOCS_STORAGE_DIR, moduleName);
if (!groups.length) {
results.set(moduleName, { created: false, reason: 'no doc files found' });
continue;
}
const docsOutputDir = outputDir;
const index = new FlexSearchIndex();
const fileMap: Record<number, string> = {};
let currentId = 1;
let indexedDocs = 0;
const markdownExt = new Set(['.md', '.mdx']);
const storagePathMap = new Map<string, DocEntryMeta>();
const moduleConflicts: string[] = [];
await fs.remove(outputDir);
await fs.ensureDir(docsOutputDir);
const sortedGroups = groups.slice().sort((a, b) => {
if (a.packageName !== b.packageName) {
return a.packageName.localeCompare(b.packageName);
}
return a.moduleRoot.localeCompare(b.moduleRoot);
});
for (const group of sortedGroups) {
for (const entry of group.entries) {
const storageDocPath = path.join(docsOutputDir, entry.relativePath);
const existing = storagePathMap.get(storageDocPath);
if (existing) {
moduleConflicts.push(
`Duplicate path "${entry.relativePath}" from ${entry.packageName} and ${existing.packageName}`,
);
continue;
}
storagePathMap.set(storageDocPath, entry);
await fs.ensureDir(path.dirname(storageDocPath));
let processedContent = rewriteRelativeLinks(entry.content, entry);
const splitResult = splitMarkdownIfNeeded(processedContent, entry);
const splitRefs = splitResult.splits.map((split, index) => {
const splitRelativePath = entry.relativePath.replace(/\.mdx?$/i, '') + split.suffix;
const splitCanonicalPath = path.posix.join(entry.moduleName, splitRelativePath.split(path.sep).join('/'));
return {
index,
splitRelativePath,
splitCanonicalPath,
...split,
};
});
processedContent = splitResult.content;
if (entry.isIndex) {
processedContent = applyReferencesToIndex(
processedContent,
entry.absolutePath,
group.directoryChildren,
group.docMap,
);
}
if (splitRefs.length) {
processedContent = applySplitReferences(
processedContent,
splitRefs.map((split) => ({
pathRef: split.splitCanonicalPath,
title: split.title,
description: split.description,
})),
);
}
if (!entry.hasFrontMatter) {
processedContent = injectFrontMatter(processedContent, entry);
}
await fs.writeFile(storageDocPath, processedContent, 'utf8');
for (const split of splitRefs) {
const splitStoragePath = path.join(docsOutputDir, split.splitRelativePath);
if (storagePathMap.has(splitStoragePath)) {
moduleConflicts.push(`Duplicate split path "${split.splitRelativePath}" in ${entry.packageName}`);
continue;
}
storagePathMap.set(splitStoragePath, entry);
await fs.ensureDir(path.dirname(splitStoragePath));
let splitContent = split.content;
splitContent = injectFrontMatter(splitContent, {
title: split.title,
description: split.description,
});
await fs.writeFile(splitStoragePath, splitContent, 'utf8');
}
const ext = path.extname(entry.absolutePath).toLowerCase();
if (!markdownExt.has(ext)) {
continue;
}
const content = processedContent.trim();
if (!content) {
continue;
}
const docId = currentId++;
await index.addAsync(docId, processedContent);
fileMap[docId] = entry.canonicalPath;
indexedDocs++;
}
}
if (!indexedDocs) {
await fs.remove(outputDir);
results.set(moduleName, { created: false, reason: 'no markdown doc content to index' });
continue;
}
const indexData: Record<string, string> = {};
index.export((key, data) => {
indexData[key] = data;
});
await fs.ensureDir(outputDir);
await fs.writeJSON(path.join(outputDir, 'index.json'), indexData, { spaces: 2 });
await fs.writeJSON(path.join(outputDir, 'files.json'), fileMap, { spaces: 2 });
const allConflicts = [...(conflicts.get(moduleName) || []), ...moduleConflicts];
if (allConflicts.length) {
results.set(moduleName, { created: true, conflicts: allConflicts });
} else {
results.set(moduleName, { created: true });
}
}
if (moduleGroups.size) {
const metaOutput: Record<string, { description: string }> = {};
for (const moduleName of moduleGroups.keys()) {
metaOutput[moduleName] = {
description: moduleDescriptions.get(moduleName) || '',
};
}
await fs.ensureDir(DOCS_STORAGE_DIR);
await fs.writeJSON(path.join(DOCS_STORAGE_DIR, 'meta.json'), metaOutput, { spaces: 2 });
}
return results;
}
function extractDocMetadata(content: string, filePath: string) {
const normalized = content.replace(/\r\n/g, '\n');
const hasFrontMatter = normalized.startsWith('---\n') && normalized.indexOf('\n---', 4) !== -1;
let searchStart = 0;
if (hasFrontMatter) {
const closingIndex = normalized.indexOf('\n---', 4);
if (closingIndex !== -1) {
const closingLineEnd = normalized.indexOf('\n', closingIndex + 4);
searchStart = closingLineEnd === -1 ? normalized.length : closingLineEnd + 1;
}
}
let title = '';
let description = '';
const lines = normalized.slice(searchStart).split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();
if (line.startsWith('# ')) {
title = line.replace(/^#\s+/, '').trim();
let j = i + 1;
const descParts: string[] = [];
while (j < lines.length) {
const descLine = lines[j].trim();
if (!descLine) {
if (descParts.length) break;
j++;
continue;
}
if (descLine.startsWith('#')) break;
descParts.push(descLine);
if (descParts.join(' ').length > 160) break;
j++;
}
description = descParts.join(' ');
break;
}
}
if (!title) {
const base = path.basename(filePath, path.extname(filePath));
title = base
.split(/[-_]/)
.map((segment) => segment.charAt(0).toUpperCase() + segment.slice(1))
.join(' ');
}
if (!description) {
description = `Reference snippet for ${title}`;
}
return {
title,
description,
hasFrontMatter,
isIndex: path.basename(filePath).toLowerCase() === 'index.md',
};
}
function injectFrontMatter(content: string, meta: { title: string; description: string }) {
const trimmed = content.replace(/^\uFEFF/, '');
const fmLines = [
'---',
`title: ${JSON.stringify(meta.title)}`,
`description: ${JSON.stringify(meta.description)}`,
'---',
];
const frontMatter = `${fmLines.join('\n')}\n`;
const separator = trimmed.startsWith('\n') ? '' : '\n';
return `${frontMatter}${separator}${trimmed}`;
}
function splitFrontMatter(content: string) {
const normalized = content.replace(/\r\n/g, '\n');
if (!normalized.startsWith('---\n')) {
return { frontMatter: '', body: normalized, hasFrontMatter: false };
}
const closingIndex = normalized.indexOf('\n---', 4);
if (closingIndex === -1) {
return { frontMatter: '', body: normalized, hasFrontMatter: false };
}
const closingLineEnd = normalized.indexOf('\n', closingIndex + 4);
const bodyStart = closingLineEnd === -1 ? normalized.length : closingLineEnd + 1;
return {
frontMatter: normalized.slice(0, bodyStart),
body: normalized.slice(bodyStart),
hasFrontMatter: true,
};
}
function getPrimaryHeading(body: string) {
const match = body.match(/^#\s+(.+)$/m);
return match?.[1]?.trim() || '';
}
function countCodeBlocks(content: string) {
return (content.match(/```[\s\S]*?```/g) || []).length;
}
function normalizeHeadingText(text: string) {
return text.replace(/^#+\s+/, '').trim();
}
function splitByHeadings(body: string) {
const lines = body.split('\n');
const sections: Array<{ level: number; heading: string; content: string[] }> = [];
let current: { level: number; heading: string; content: string[] } | null = null;
for (const line of lines) {
const headingMatch = line.match(/^(#{2,3})\s+(.+)$/);
if (headingMatch) {
if (current) {
sections.push(current);
}
current = {
level: headingMatch[1].length,
heading: normalizeHeadingText(headingMatch[0]),
content: [],
};
continue;
}
if (current) {
current.content.push(line);
}
}
if (current) {
sections.push(current);
}
return sections.filter((section) => section.content.join('\n').trim().length > 0);
}
function splitExamples(body: string) {
const codeRegex = /```[\s\S]*?```/g;
const headingRegex = /^(#{2,3})\s+(.+)$/gm;
const headings: Array<{ index: number; level: number; text: string; key: string }> = [];
for (const match of body.matchAll(headingRegex)) {
const raw = normalizeHeadingText(match[0]);
const key = raw.toLowerCase();
headings.push({
index: match.index ?? 0,
level: match[1].length,
text: raw,
key,
});
}
const examples: Array<{ heading: string; content: string }> = [];
const excludeKeys = new Set([
'type definition',
'type definition (simplified)',
'parameters',
'return value',
'类型定义',
'参数',
'返回值',
]);
const codeMatches = Array.from(body.matchAll(codeRegex));
const exampleCodeRanges: Array<{ start: number; end: number }> = [];
const exampleSectionRanges: Array<{ start: number; end: number }> = [];
for (const match of codeMatches) {
const code = match[0];
const index = match.index ?? 0;
const priorHeadings = headings.filter((item) => item.index < index);
const currentH2 = priorHeadings.filter((item) => item.level === 2).slice(-1)[0];
const currentH3 = priorHeadings.filter((item) => item.level === 3).slice(-1)[0];
const h2Key = currentH2?.key || '';
const h3Key = currentH3?.key || '';
if (excludeKeys.has(h2Key) || excludeKeys.has(h3Key)) {
continue;
}
const heading = currentH3?.text || currentH2?.text || '';
examples.push({ heading, content: code });
exampleCodeRanges.push({ start: index, end: index + code.length });
if (currentH3 || currentH2) {
const anchor = currentH3 || currentH2;
const nextHeadingIndex =
headings.filter((item) => item.index > anchor.index && item.level <= anchor.level).slice(0, 1)[0]?.index ??
body.length;
exampleSectionRanges.push({ start: anchor.index, end: nextHeadingIndex });
}
}
let cleanedBody = body;
if (exampleSectionRanges.length) {
const ranges = exampleSectionRanges
.sort((a, b) => a.start - b.start)
.filter((range, index, list) => index === 0 || range.start >= list[index - 1].end);
let result = '';
let last = 0;
for (const range of ranges) {
result += body.slice(last, range.start);
last = range.end;
}
result += body.slice(last);
cleanedBody = result;
} else if (exampleCodeRanges.length) {
const ranges = exampleCodeRanges
.sort((a, b) => a.start - b.start)
.filter((range, index, list) => index === 0 || range.start >= list[index - 1].end);
let result = '';
let last = 0;
for (const range of ranges) {
result += body.slice(last, range.start);
last = range.end;
}
result += body.slice(last);
cleanedBody = result;
}
return { examples, cleanedBody };
}
function splitMarkdownIfNeeded(content: string, entry: DocEntryMeta) {
const { body, frontMatter, hasFrontMatter } = splitFrontMatter(content);
const bodyLength = body.trim().length;
const codeBlocks = countCodeBlocks(body);
const shouldSplitByLength = bodyLength > SPLIT_MAX_LENGTH;
const shouldSplitByExamples = codeBlocks > SPLIT_MAX_CODE_BLOCKS;
if (!shouldSplitByLength && !shouldSplitByExamples) {
return {
splits: [] as Array<{ suffix: string; title: string; description: string; content: string }>,
content,
};
}
const h1 = getPrimaryHeading(body) || entry.title;
const splits: Array<{ suffix: string; title: string; description: string; content: string }> = [];
let nextBody = body;
let didSplitByExamples = false;
if (shouldSplitByExamples) {
const { examples, cleanedBody } = splitExamples(body);
if (examples.length) {
didSplitByExamples = true;
nextBody = cleanedBody;
examples.forEach((example, index) => {
const headingLine = example.heading ? `## ${example.heading}\n\n` : '';
const exampleContent = `# ${h1}\n\n${headingLine}${example.content}\n`;
const title = example.heading ? `${example.heading} Example` : `${h1} Example ${index + 1}`;
splits.push({
suffix: `__example-${index + 1}.md`,
title,
description: `Extracted example from ${entry.title}`,
content: exampleContent,
});
});
}
}
if (shouldSplitByLength && !didSplitByExamples) {
const sections = splitByHeadings(body);
sections.forEach((section, index) => {
const headingLine = `${'#'.repeat(section.level)} ${section.heading}`;
const sectionBody = section.content.join('\n').trim();
const chunkContent = `# ${h1}\n\n${headingLine}\n${sectionBody}\n`;
const title = section.heading || `${h1} Chunk ${index + 1}`;
splits.push({
suffix: `__chunk-${index + 1}.md`,
title,
description: `Extracted section from ${entry.title}`,
content: chunkContent,
});
});
}
const rebuilt = hasFrontMatter ? `${frontMatter}${nextBody}` : nextBody;
return { splits, content: rebuilt };
}
function applySplitReferences(content: string, splits: Array<{ pathRef: string; title: string; description: string }>) {
if (!splits.length) return content;
const refLines = splits.map((split) => referenceLine(split.title, split.description, split.pathRef));
const baseContent = stripSplitReferenceBlock(content).trimEnd();
const block = `${SPLIT_REFERENCE_START}\n\n## Extracted references\n\n${refLines.join(
'\n',
)}\n\n${SPLIT_REFERENCE_END}`;
if (!baseContent) {
return `${block}\n`;
}
return `${baseContent}\n\n${block}\n`;
}
function stripSplitReferenceBlock(content: string) {
const start = content.indexOf(SPLIT_REFERENCE_START);
if (start === -1) return content;
const end = content.indexOf(SPLIT_REFERENCE_END, start + SPLIT_REFERENCE_START.length);
if (end === -1) return content;
const before = content.slice(0, start).trimEnd();
const after = content.slice(end + SPLIT_REFERENCE_END.length).trimStart();
if (!before) return after;
if (!after) return `${before}\n`;
return `${before}\n\n${after}`;
}
function rewriteRelativeLinks(content: string, entry: DocEntryMeta) {
const { body, frontMatter, hasFrontMatter } = splitFrontMatter(content);
const segments: Array<{ type: 'code' | 'text'; content: string }> = [];
const codeRegex = /```[\s\S]*?```/g;
let lastIndex = 0;
for (const match of body.matchAll(codeRegex)) {
const index = match.index ?? 0;
if (index > lastIndex) {
segments.push({ type: 'text', content: body.slice(lastIndex, index) });
}
segments.push({ type: 'code', content: match[0] });
lastIndex = index + match[0].length;
}
if (lastIndex < body.length) {
segments.push({ type: 'text', content: body.slice(lastIndex) });
}
const dir = path.posix.dirname(entry.relativePath.split(path.sep).join('/'));
const rewritten = segments
.map((segment) => {
if (segment.type === 'code') return segment.content;
return segment.content.replace(/(!?)\[[^\]]+]\(([^)]+)\)/g, (match, bang, link) => {
if (bang) return match;
const trimmed = link.trim();
if (
trimmed.startsWith('http://') ||
trimmed.startsWith('https://') ||
trimmed.startsWith('/') ||
trimmed.startsWith('#') ||
trimmed.startsWith('mailto:') ||
trimmed.startsWith('tel:')
) {
return match;
}
const hashIndex = trimmed.indexOf('#');
const hash = hashIndex >= 0 ? trimmed.slice(hashIndex) : '';
const beforeHash = hashIndex >= 0 ? trimmed.slice(0, hashIndex) : trimmed;
const queryIndex = beforeHash.indexOf('?');
const query = queryIndex >= 0 ? beforeHash.slice(queryIndex) : '';
const pathPart = queryIndex >= 0 ? beforeHash.slice(0, queryIndex) : beforeHash;
const resolved = path.posix.normalize(path.posix.join(dir, pathPart));
const normalized = resolved.startsWith('.') ? resolved.replace(/^(\.\.\/?)+/, '') : resolved;
const absolutePath = path.posix.join('/', entry.moduleName, normalized);
const nextLink = `${absolutePath}${query}${hash}`;
return match.replace(link, nextLink);
});
})
.join('');
if (!hasFrontMatter) {
return rewritten;
}
return `${frontMatter}${rewritten}`;
}
function applyReferencesToIndex(
content: string,
filePath: string,
directoryChildren: DirectoryChildren,
docMap: Map<string, DocEntryMeta>,
) {
const dir = path.dirname(filePath);
const entry = directoryChildren.get(dir);
if (!entry) {
return stripExistingReferenceBlock(content);
}
const refLines: string[] = [];
const refs = new Set<string>();
const childFiles = entry.files.filter((file) => file !== path.join(dir, 'index.md'));
for (const child of childFiles.sort()) {
const meta = docMap.get(child);
if (!meta) continue;
const refPath = meta.canonicalPath;
if (refs.has(refPath)) continue;
refs.add(refPath);
refLines.push(referenceLine(meta.title, meta.description, refPath));
}
for (const subDir of entry.directories.sort()) {
const indexFile = path.join(subDir, 'index.md');
const meta = docMap.get(indexFile);
if (!meta) continue;
const refPath = meta.canonicalPath;
if (refs.has(refPath)) continue;
refs.add(refPath);
refLines.push(referenceLine(meta.title, meta.description, refPath));
}
if (!refLines.length) {
return stripExistingReferenceBlock(content);
}
const baseContent = stripExistingReferenceBlock(content).trimEnd();
const block = `${REFERENCE_START}\n\n## References\n\n${refLines.join('\n')}\n\n${REFERENCE_END}`;
if (!baseContent) {
return `${block}\n`;
}
return `${baseContent}\n\n${block}\n`;
}
function referenceLine(title: string, description: string, pathRef: string) {
const summary = description.length > 200 ? `${description.slice(0, 197)}...` : description;
const descText = summary ? ` - ${summary}` : '';
return `- **${title}** (\`${pathRef}\`)${descText}`;
}
function stripExistingReferenceBlock(content: string) {
const start = content.indexOf(REFERENCE_START);
if (start === -1) return content;
const end = content.indexOf(REFERENCE_END, start + REFERENCE_START.length);
if (end === -1) return content;
const before = content.slice(0, start).trimEnd();
const after = content.slice(end + REFERENCE_END.length).trimStart();
if (!before) return after;
if (!after) return `${before}\n`;
return `${before}\n\n${after}`;
}
export async function createDocsIndex(app: Application, options: DocsIndexOptions = {}) {
const pkgs = await resolvePkgs(options.pkg);
if (!pkgs.length) {
app.log.info('No plugin packages detected for docs index generation');
return;
}
const packageNames: string[] = [];
for (const pkg of pkgs) {
try {
const { packageName } = await PluginManager.parseName(pkg);
packageNames.push(packageName);
} catch (error) {
app.log.error(error, { pkg });
}
}
if (!packageNames.length) {
app.log.info('No plugin packages resolved for docs index generation');
return;
}
let results: Map<string, BuildResult> | null = null;
let lastError: unknown;
for (let attempt = 0; attempt < CREATE_INDEX_RETRY_MAX; attempt++) {
try {
results = await buildDocsIndexForPackages(packageNames);
break;
} catch (error) {
lastError = error;
if (attempt === CREATE_INDEX_RETRY_MAX - 1) {
break;
}
app.log.warn(`Docs index build failed, retrying (${attempt + 1}/${CREATE_INDEX_RETRY_MAX})`);
await new Promise((resolve) => setTimeout(resolve, CREATE_INDEX_RETRY_DELAY_MS * (attempt + 1)));
}
}
if (!results) {
const message = lastError ? errorToString(lastError) : 'Docs index build failed';
app.log.error(message);
return;
}
if (!results.size) {
app.log.info('No module docs found to index');
return;
}
for (const [moduleName, result] of results.entries()) {
if (result.created) {
app.log.info(`Docs index generated for module "${moduleName}"`);
if (result.conflicts?.length) {
app.log.warn(`Module "${moduleName}" has conflicts: ${result.conflicts.join('; ')}`);
}
} else {
app.log.info(`Skipped docs index for module "${moduleName}": ${result.reason}`);
}
}
}
+1 -9
View File
@@ -8,15 +8,7 @@
*/
import type Application from '../application';
import { createDocsIndex, DocsIndexOptions } from '../ai/create-docs-index';
export default (app: Application) => {
const ai = app.command('ai');
ai.command('create-docs-index')
.option('--pkg [pkg]', 'Generate docs index for the specified plugin package (comma separated).')
.action(async (...cliArgs) => {
const [opts] = cliArgs as [DocsIndexOptions?];
await createDocsIndex(app, opts);
});
app.command('ai');
};
@@ -10,7 +10,6 @@
/* istanbul ignore file -- @preserve */
import Application from '../application';
import { createDocsIndex } from '../ai/create-docs-index';
export default (app: Application) => {
app
@@ -24,9 +23,6 @@ export default (app: Application) => {
if (options.lang) {
process.env.INIT_APP_LANG = options.lang;
}
if (!process.env.VITEST) {
await createDocsIndex(app);
}
await app.install(options);
const reinstall = options.clean || options.force;
app.log.info(`app ${reinstall ? 'reinstalled' : 'installed'} successfully [v${app.getVersion()}]`);
@@ -12,7 +12,6 @@
import fs from 'fs-extra';
import { storagePathJoin } from '@nocobase/utils';
import Application from '../application';
import { createDocsIndex } from '../ai/create-docs-index';
import { ApplicationNotInstall } from '../errors/application-not-install';
export default (app: Application) => {
@@ -28,8 +27,6 @@ export default (app: Application) => {
if (upgrading) {
if (!process.env.VITEST) {
if (await app.isInstalled()) {
await createDocsIndex(app);
await app.upgrade();
}
}
@@ -39,8 +36,6 @@ export default (app: Application) => {
// skip
}
} else if (options.quickstart) {
await createDocsIndex(app);
if (await app.isInstalled()) {
await app.upgrade({ quickstart: true });
} else {
@@ -10,7 +10,6 @@
/* istanbul ignore file -- @preserve */
import Application from '../application';
import { createDocsIndex } from '../ai/create-docs-index';
/**
* TODO
@@ -21,10 +20,6 @@ export default (app: Application) => {
.ipc()
.auth()
.action(async (options) => {
if (!process.env.VITEST) {
await createDocsIndex(app);
}
await app.upgrade(options);
app.log.info(`✨ NocoBase has been upgraded to v${app.getVersion()}`);
});
@@ -3,6 +3,9 @@ import fg from 'fast-glob';
import fs from 'fs-extra';
import path from 'path';
const DOCS_SOURCE_DIR = path.resolve(__dirname, '../../../../docs/docs/en');
const DOCS_DIST_DIR = path.resolve(__dirname, 'dist/ai/docs/nocobase');
export default defineConfig({
beforeBuild: async () => {
const distPath = path.resolve(__dirname, 'dist');
@@ -24,6 +27,19 @@ export default defineConfig({
},
);
if (await fs.pathExists(DOCS_SOURCE_DIR)) {
log('copying NocoBase documentation files to dist/ai/docs/nocobase');
await fs.copy(DOCS_SOURCE_DIR, DOCS_DIST_DIR, {
overwrite: true,
filter: (src) => {
if (fs.lstatSync(src).isDirectory()) return true;
return /\.(md|mdx)$/i.test(src);
},
});
} else {
log(`skipping NocoBase documentation copy, source directory not found: ${DOCS_SOURCE_DIR}`);
}
log('remove zod src dir');
fg.sync('**/zod/src', {
cwd: path.resolve(__dirname, 'dist', 'node_modules'),
@@ -0,0 +1,2 @@
export function compress(): never;
export function decompress(): never;
@@ -0,0 +1,6 @@
function unsupported() {
throw new Error('@mongodb-js/zstd is not bundled with NocoBase.');
}
exports.compress = unsupported;
exports.decompress = unsupported;
@@ -0,0 +1,9 @@
{
"name": "@mongodb-js/zstd",
"version": "7.0.0",
"main": "./index.js",
"types": "./index.d.ts",
"exports": {
".": "./index.js"
}
}
@@ -0,0 +1,10 @@
function unsupported() {
throw new Error('node-liblzma is not bundled with NocoBase.');
}
export const xzSync = unsupported;
export const unxzSync = unsupported;
export default {
xzSync,
unxzSync,
};
@@ -0,0 +1,9 @@
{
"name": "node-liblzma",
"version": "2.2.0",
"type": "module",
"main": "./index.js",
"exports": {
".": "./index.js"
}
}
@@ -47,6 +47,7 @@
"echarts": "^5.5.0",
"echarts-for-react": "3.0.2",
"elkjs": "^0.10.0",
"just-bash": "^2.14.3",
"jsonrepair": "3.13.1",
"langchain": "^1.2.24",
"nodejs-snowflake": "^2.0.1",
@@ -1,69 +1,123 @@
---
scope: GENERAL
name: document-search
description: helps users search and read documentation using keyword-based indexing and file browsing capabilities.
description: helps users search and read NocoBase documentation using restricted bash commands.
introduction:
title: '{{t("ai.skills.documentSearch.title", { ns: "@nocobase/plugin-ai" })}}'
about: '{{t("ai.skills.documentSearch.about", { ns: "@nocobase/plugin-ai" })}}'
tools: ['searchDocs', 'readDocEntry']
tools: ['searchDocs']
---
You are a professional documentation assistant for NocoBase.
You help users find relevant documentation by searching indexed content and reading specific files or directories.
You help users find relevant documentation by running focused bash commands against the readonly documentation tree.
# Primary Workflows
# Documentation Root
## Search Documentation
- Commands run from `/docs/nocobase`.
- Use relative paths by default.
- Documentation files are Markdown or MDX files.
- The filesystem is readonly.
To find documentation on a specific topic:
# Available Tool
1. **Identify Search Keywords**
- Extract key terms, API names, or module identifiers from the user's query.
- Focus on specific identifiers rather than full sentences.
- `searchDocs`: Run a restricted bash script in `/docs/nocobase` to search or read documentation.
2. **Search Using Keywords**
- Use the `searchDocs` tool to find matching documents.
- Provide a module key (e.g., `runjs`, `workflow`) and relevant keywords.
- Review the returned matches to identify relevant content.
# Primary Workflow
3. **Read Matching Documents**
- Use the `readDocEntry` tool to read the content of matching files.
- Browse directories to explore the documentation structure if needed.
1. Identify specific search terms from the user's request, such as product names, feature names, API names, configuration keys, error messages, or command names.
2. Map the request to likely top-level directories before running commands.
- Installation / upgrade / deployment: `get-started`, `cluster-mode`
- UI builder / JS Block / RunJS / actions / fields: `interface-builder`
- Workflow: `workflow`, `flow-engine`
- Data modeling / collections / fields: `data-sources`, `database`
- Multi-app / multi-environment / App Supervisor: `multi-app`
- API / plugin development: `api`, `plugin-development`, `development`
3. Prefer answering from the first focused `searchDocs` result when it is enough. For follow-up questions, first reuse documentation already read in the conversation; call `searchDocs` again only if the answer still lacks evidence.
4. Search file paths and filenames first. Many documentation topics are reflected in paths such as `get-started/upgrading/docker.md`.
5. Read focused snippets from the best candidate files. Run content search only when likely file paths are unclear or the exact API/keyword must be verified.
6. When the snippets directly answer the question, respond to the user first instead of expanding the search. Offer to look deeper if the user needs implementation details or more sources.
7. Answer from the documentation content and include useful file paths when they help the user verify or continue reading.
## Browse Documentation Structure
# Command Guidance
To explore available documentation:
- Prefer compact, combined scripts over multiple small tool calls. Start narrow: a path search plus a few focused snippets is usually enough for an initial answer.
- Avoid broad full-tree scans. Do not use `find .`, `find /docs/nocobase`, or `rg ... .`.
- Do not pipe output into `rg`. In this docs shell, use `grep` for pipeline filtering.
- Prefer `rg --files <dir> | grep -Ei <pattern>` over `find` for file path discovery:
- `rg --files get-started cluster-mode | grep -Ei 'install|docker|upgrade|create-nocobase-app' | head -80`
- `rg --files interface-builder | grep -Ei 'runjs|js-' | head -80`
- `rg --files multi-app | head -80`
- Use `rg` for content search only after path search, and only when direct file reads are insufficient:
- `rg -n -i "upgrade|backup|docker compose" get-started cluster-mode | head -80`
- `rg -n -i "API key|token" integration security | head -80`
- Use `-g` filters with `rg`; do not use unsupported `--include` options:
- `rg -n "workflow" workflow flow-engine -g '*.md' -g '*.mdx' | head -80`
- Use `sed`, `head`, or `tail` to read focused snippets:
- `sed -n '1,160p' get-started/quickstart.md`
- `sed -n '40,120p' workflow/index.md`
- Keep output small. If a command may return many lines, pipe it through `head`, `sed`, or `tail`.
- Do not read large files fully unless the user explicitly needs the whole file.
- For concept explanations or comparisons, prefer the page section that directly defines or compares the concepts; avoid broad keyword searches across several large directories after a direct match is found unless the user asks for more depth.
- Do not write files or attempt to modify the documentation tree.
1. **List Available Modules**
- Call `searchDocs` with any module key to see which modules are available.
- The response will include `availableModules` listing all indexed documentation.
# Examples
2. **Explore Directory Contents**
- Use `readDocEntry` with a module path (e.g., `runjs` or `runjs/context`) to list directory contents.
- Navigate through the hierarchy by reading subdirectories.
Locate and read upgrade docs in one call:
# Available Tools
```bash
printf '## Candidate files\n'
rg --files get-started cluster-mode | grep -Ei 'upgrad|docker|git|create-nocobase-app' | head -40
- `searchDocs`: Search indexed documentation using FlexSearch-based keyword indexing. Provide a module key and keywords to find matching documents. Returns file paths that can be read with `readDocEntry`.
- `readDocEntry`: Read files or list directories inside the documentation storage. Supports reading up to 3 paths per call. Returns file content or directory listings.
printf '\n## create-nocobase-app\n'
sed -n '1,180p' get-started/upgrading/create-nocobase-app.md
# Path Format
printf '\n## Docker\n'
sed -n '1,220p' get-started/upgrading/docker.md
- Use canonical module-based paths: `moduleName/path/to/file.md`
- Examples: `runjs/context/router/index.md`, `workflow/triggers/manual.md`
- Leading slashes are optional: `/runjs/context` is equivalent to `runjs/context`
- Relative segments (`..`) and wildcards are not allowed for security reasons.
printf '\n## Git source\n'
sed -n '1,180p' get-started/upgrading/git.md
```
# Best Practices
Search for workflow trigger docs:
- Use specific keywords (API names, function names, module terms) rather than natural language queries.
- Start with a narrow search and broaden if needed.
- After finding relevant files, read them to provide detailed answers.
- Use directory browsing to explore the documentation structure when the exact file is unknown.
```bash
printf '## Candidate files\n'
rg --files workflow flow-engine | grep -Ei 'trigger|schedule|manual|workflow' | head -40
# Notes
printf '\n## Workflow docs\n'
sed -n '1,180p' workflow/index.md 2>/dev/null || true
```
- Document indexes must be created first using the `ai:create-docs-index` command.
- If no indexes are available, inform the user to run the indexing command.
- Search results include a relevance score based on keyword matching.
Read likely matches with labels:
```bash
printf '## Quick start\n'
sed -n '1,180p' get-started/quickstart.md
printf '\n## System requirements\n'
sed -n '1,160p' get-started/system-requirements.md
```
Find JS Block / RunJS docs:
```bash
printf '## Candidate files\n'
rg --files interface-builder | grep -Ei 'runjs|js-' | head -60
printf '\n## JS Block\n'
sed -n '1,180p' interface-builder/blocks/other-blocks/js-block.md
printf '\n## RunJS\n'
sed -n '1,120p' interface-builder/runjs.md
```
Find multi-app / multi-environment docs:
```bash
printf '## Candidate files\n'
rg --files multi-app | grep -Ei 'multi-app|remote|local|index' | head -40
printf '\n## Multi-environment mode\n'
sed -n '1,220p' multi-app/multi-app/remote.md
```
@@ -315,7 +315,7 @@
"ai.skills.frontendDeveloper.title": "Frontend developer",
"ai.skills.frontendDeveloper.about": "Assists with writing and testing JavaScript code for NocoBase workflows and frontend components.",
"ai.skills.documentSearch.title": "Document search",
"ai.skills.documentSearch.about": "Search and read indexed documentation using keyword-based search and file browsing.",
"ai.skills.documentSearch.about": "Search and read NocoBase documentation using restricted bash commands.",
"ai.skills.businessAnalysisReport.title": "Business analysis report",
"ai.skills.businessAnalysisReport.about": "Use the data-query workflow to analyze business data and produce stakeholder-facing reports with markdown and ECharts.",
"ai.tools.businessReportGenerator.title": "Business report generator",
@@ -318,7 +318,7 @@
"ai.skills.frontendDeveloper.title": "前端开发",
"ai.skills.frontendDeveloper.about": "协助编写和测试 nocobase 工作流和前端组件的 Javascript 代码。",
"ai.skills.documentSearch.title": "文档检索",
"ai.skills.documentSearch.about": "使用基于关键字的搜索和文件浏览搜索和读取索引文档。",
"ai.skills.documentSearch.about": "使用受限 Bash 命令搜索和读取 NocoBase 文档。",
"ai.skills.businessAnalysisReport.title": "业务分析报告",
"ai.skills.businessAnalysisReport.about": "使用 data-query 工作流分析业务数据,并输出面向业务干系人的 Markdown + ECharts 报告。",
"ai.tools.businessReportGenerator.title": "业务报告生成器",
@@ -26,7 +26,7 @@ import Snowflake from './snowflake';
import * as aiEmployeeActions from './resource/aiEmployees';
import { googleGenAIProviderOptions } from './llm-providers/google-genai';
import { AIEmployeeTrigger } from './workflow/triggers/ai-employee';
import { getWorkflowCallers, createDocsSearchTool, createReadDocEntryTool, loadDocsIndexes } from './tools';
import { getWorkflowCallers, createDocsSearchTool, type DocsFsCache } from './tools';
import { Model } from '@nocobase/database';
import { anthropicProviderOptions } from './llm-providers/anthropic';
import aiSettings from './resource/aiSettings';
@@ -63,6 +63,7 @@ export class PluginAIServer extends Plugin {
documentLoaders = new DocumentLoaders(this);
subAgentsDispatcher = new SubAgentsDispatcher(this);
knowledgeBaseManager = new KnowledgeBaseManager(this);
docsFsCache: DocsFsCache = null;
snowflake: Snowflake;
/**
@@ -104,7 +105,6 @@ export class PluginAIServer extends Plugin {
}
async load() {
await loadDocsIndexes();
this.registerLLMProviders();
this.registerTools();
this.defineResources();
@@ -129,7 +129,7 @@ export class PluginAIServer extends Plugin {
registerTools() {
const toolsManager = this.ai.toolsManager;
toolsManager.registerTools([createDocsSearchTool(), createReadDocEntryTool()]);
toolsManager.registerTools([createDocsSearchTool(this)]);
toolsManager.registerDynamicTools(getWorkflowCallers(this, 'workflowCaller'));
toolsManager.registerDynamicTools(getWorkflowTasks(this));
@@ -7,231 +7,57 @@
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import { z } from 'zod';
import path from 'path';
import { storagePathJoin } from '@nocobase/utils';
import fs from 'fs-extra';
import fg from 'fast-glob';
import { Index as FlexSearchIndex } from 'flexsearch';
import { z } from 'zod';
import { Bash, InMemoryFs, MountableFs, OverlayFs, type CommandName, type IFileSystem } from 'just-bash';
import { ToolsOptions } from '@nocobase/ai';
import type { PluginAIServer } from '../plugin';
const DEFAULT_DOCS_DIR = storagePathJoin('ai', 'docs');
const DOCS_MOUNT_POINT = '/docs/nocobase';
const DOCS_MAX_OUTPUT_LENGTH = 20000;
const DOCS_EXEC_TIMEOUT_MS = 10000;
const DOCS_MAX_FILE_READ_SIZE = 1024 * 1024;
type DocsIndexMeta = {
key: string;
description?: string;
docsDir: string;
indexFile: string;
filesMapFile: string;
};
const DOCS_COMMANDS: CommandName[] = [
'awk',
'basename',
'cat',
'comm',
'cut',
'dirname',
'du',
'echo',
'egrep',
'expr',
'false',
'fgrep',
'find',
'fold',
'grep',
'head',
'ls',
'nl',
'paste',
'printf',
'pwd',
'rg',
'sed',
'seq',
'sort',
'stat',
'tail',
'tr',
'true',
'uniq',
'wc',
'which',
'xargs',
];
export type DocEntryResult =
| {
type: 'file';
path: string;
content: string;
}
| {
type: 'directory';
path: string;
entries: Array<{ name: string; type: 'file' | 'directory'; path: string }>;
};
export type DocsFsCache = { docsDir: string; fs: IFileSystem } | null;
const docsModules = new Map<string, DocsIndexMeta>();
let docsBaseDir = DEFAULT_DOCS_DIR;
let docsMeta: Record<string, { description?: string }> = {};
const DOCS_INDEX_TTL_MS = 5 * 60 * 1000;
const DOCS_INDEX_MAX_CACHE = 5;
const DOCS_SEARCH_MAX_KEYWORDS = 5;
const DOCS_READ_MAX_PATHS = 3;
const docsIndexCache = new Map<
string,
{
index: FlexSearchIndex;
fileMap: Record<string, string>;
lastAccess: number;
}
>();
const docsIndexLoading = new Map<string, Promise<{ index: FlexSearchIndex; fileMap: Record<string, string> }>>();
export async function loadDocsIndexes(baseDir = DEFAULT_DOCS_DIR) {
docsBaseDir = baseDir;
docsModules.clear();
docsMeta = {};
docsIndexCache.clear();
docsIndexLoading.clear();
const exists = await fs.pathExists(docsBaseDir);
if (!exists) {
return;
}
const metaPath = path.join(docsBaseDir, 'meta.json');
if (await fs.pathExists(metaPath)) {
try {
const meta = await fs.readJson(metaPath);
if (meta && typeof meta === 'object') {
docsMeta = meta as Record<string, { description?: string }>;
}
} catch (error) {
// eslint-disable-next-line no-console
console.error('[docs-tools] Failed to read docs meta', { metaPath, error });
}
}
const indexFiles = await fg('**/index.json', { cwd: docsBaseDir, onlyFiles: true, absolute: true });
for (const indexFile of indexFiles) {
const pluginDir = path.dirname(indexFile);
const filesPath = path.join(pluginDir, 'files.json');
const docsDir = pluginDir;
try {
const [filesExists, docsExists] = await Promise.all([fs.pathExists(filesPath), fs.pathExists(docsDir)]);
if (!filesExists || !docsExists) {
continue;
}
const relativeKey = path.relative(docsBaseDir, pluginDir).split(path.sep).join('/');
if (!relativeKey) {
continue;
}
const description = docsMeta?.[relativeKey]?.description?.trim() || '';
docsModules.set(relativeKey, {
key: relativeKey,
description,
docsDir: path.resolve(docsDir),
indexFile,
filesMapFile: filesPath,
});
} catch (error) {
// eslint-disable-next-line no-console
console.error('[docs-tools] Failed to load docs index metadata', { indexFile, error });
}
}
}
export function getDocModuleKeys() {
return Array.from(docsModules.keys()).sort();
}
export function describeDocModules(emptyMessage = 'No document modules available.') {
const keys = getDocModuleKeys();
if (!keys.length) {
return emptyMessage;
}
const items = keys.map((key) => {
const desc = docsModules.get(key)?.description;
return desc ? `${key} (${desc})` : key;
});
return `Available modules: ${items.join(', ')}`;
}
export async function searchDocsModule(moduleKey: string, keywords: string[], limit = 5) {
const key = normalizeModuleKey(moduleKey);
const entry = docsModules.get(key);
if (!entry) {
throw new Error(`Document module "${key}" not found.`);
}
const { index: flexIndex, fileMap } = await getOrLoadDocsIndex(entry);
const searchTerms = buildSearchTerms(keywords);
if (!searchTerms.length) {
throw new Error('At least one valid keyword is required.');
}
const boundedLimit = Math.min(Math.max(limit ?? 5, 1), 20);
const perKeywordLimit = Math.min(Math.max(boundedLimit * 2, 4), 20);
const keywordHits = await Promise.all(
searchTerms.map(async (keyword) =>
(((await flexIndex.searchAsync(keyword, { limit: perKeywordLimit })) as (string | number)[]) || []).map((id) =>
String(id),
),
),
);
const aggregate = new Map<string, { score: number; bestRank: number }>();
for (const hits of keywordHits) {
for (let i = 0; i < hits.length; i++) {
const filePath = fileMap[hits[i]];
if (!filePath) {
continue;
}
const rankScore = perKeywordLimit - i;
const record = aggregate.get(filePath);
if (record) {
record.score += rankScore;
record.bestRank = Math.min(record.bestRank, i);
} else {
aggregate.set(filePath, { score: rankScore, bestRank: i });
}
}
}
const matches = Array.from(aggregate.entries())
.sort((a, b) => {
if (b[1].score !== a[1].score) {
return b[1].score - a[1].score;
}
if (a[1].bestRank !== b[1].bestRank) {
return a[1].bestRank - b[1].bestRank;
}
return a[0].localeCompare(b[0]);
})
.slice(0, boundedLimit)
.map(([filePath]) => filePath);
return {
key,
keywords: searchTerms,
matches,
};
}
export async function readDocEntry(docPath: string): Promise<DocEntryResult> {
const resolved = resolveDocPath(docPath);
if (!(await fs.pathExists(resolved.absolutePath))) {
throw new Error(`Path "${resolved.canonicalPath}" does not exist.`);
}
const stat = await fs.stat(resolved.absolutePath);
if (stat.isDirectory()) {
const names = await fs.readdir(resolved.absolutePath);
const entries = await Promise.all(
names.sort().map(async (name) => {
const absoluteChild = path.join(resolved.absolutePath, name);
const childStat = await fs.stat(absoluteChild);
const relativeChild = path.relative(resolved.meta.docsDir, absoluteChild).split(path.sep).join('/');
const childPath = relativeChild ? `${resolved.meta.key}/${relativeChild}` : resolved.meta.key;
const entryType: 'file' | 'directory' = childStat.isDirectory() ? 'directory' : 'file';
return {
name,
type: entryType,
path: childPath,
};
}),
);
return {
type: 'directory',
path: resolved.canonicalPath,
entries,
};
}
if (!stat.isFile()) {
throw new Error(`Path "${resolved.canonicalPath}" is not a regular file.`);
}
const content = await fs.readFile(resolved.absolutePath, 'utf8');
return {
type: 'file',
path: resolved.canonicalPath,
content,
};
}
export function createDocsSearchTool(): ToolsOptions {
const docsModulesDescription = describeDocModules('Docs modules unavailable. Run ai:create-docs-index first.');
export function createDocsSearchTool(plugin: PluginAIServer): ToolsOptions {
return {
scope: 'SPECIFIED',
defaultPermission: 'ALLOW',
@@ -241,50 +67,58 @@ export function createDocsSearchTool(): ToolsOptions {
},
definition: {
name: 'searchDocs',
description: `Search indexed documentation using a FlexSearch-based keyword index.
Provide a small keyword list (identifiers, API names, module terms). The tool will search them concurrently with limited parallelism.
Return matching document paths only. ${docsModulesDescription}`,
description: `Run a restricted bash script to search and read NocoBase documentation.
The documentation root is ${DOCS_MOUNT_POINT}, and commands run from that directory.
Use commands such as rg, find, grep, sed, awk, head, tail, cat, ls, and wc.
The filesystem is readonly; do not attempt to write files. Keep output focused and prefer reading specific snippets instead of whole large files.
Prefer "rg --files <dir> | grep -Ei <pattern>" for file path discovery, and "rg -n <pattern> <dir>" for content search.
Avoid broad full-tree scans such as "find ." or "rg ... .", and do not pipe output into rg. Use grep for pipeline filtering instead. First choose likely top-level directories (for example get-started, interface-builder, workflow, multi-app, data-sources), then search within those directories.
Start narrow with a path search and focused snippets. For follow-up questions, reuse docs already read when possible, and respond once the snippets directly answer the question instead of expanding the search for extra background.`,
schema: z.object({
module: z.string().min(1, 'module is required').describe('Module key, e.g. runjs'),
keywords: z
.array(z.string().min(1))
.min(1, 'keywords is required')
.max(DOCS_SEARCH_MAX_KEYWORDS, `keywords can contain up to ${DOCS_SEARCH_MAX_KEYWORDS} items`)
.describe('Keywords to search, e.g. ["router", "ctx.state", "runjs"]'),
limit: z.number().int().min(1).max(20).optional().describe('Maximum number of hits (default 5, max 20)'),
script: z
.string()
.min(1, 'script is required')
.max(4000, 'script is too long')
.describe('Bash script to search or read documentation under /docs/nocobase.'),
}),
},
invoke: async (ctx, args) => {
const available = getDocModuleKeys();
if (!available.length) {
const script = String(args?.script ?? '').trim();
if (!script) {
return {
status: 'error',
content: 'No document indexes available.',
content: 'script is required.',
};
}
const moduleKey = args?.module?.trim();
const keywords = Array.isArray(args?.keywords) ? args.keywords : [];
const limit = typeof args?.limit === 'number' ? args.limit : undefined;
if (!moduleKey || !keywords.length) {
const validationError = validateDocsScript(script);
if (validationError) {
return {
status: 'error',
content: `Both module and keywords are required. Available modules: ${available.join(', ')}`,
content: validationError,
};
}
try {
const result = await searchDocsModule(moduleKey, keywords, limit);
return {
status: 'success',
content: JSON.stringify({
module: result.key,
keywords: result.keywords,
matches: result.matches,
availableModules: available,
}),
};
const bash = await createBash(plugin);
const controller = new AbortController();
const timer: ReturnType<typeof setTimeout> = setTimeout(() => controller.abort(), DOCS_EXEC_TIMEOUT_MS);
try {
const result = await bash.exec(script, { cwd: DOCS_MOUNT_POINT, signal: controller.signal });
return {
status: 'success',
content: JSON.stringify({
exitCode: result.exitCode,
stdout: truncateOutput(result.stdout),
stderr: truncateOutput(result.stderr),
truncated: result.stdout.length > DOCS_MAX_OUTPUT_LENGTH || result.stderr.length > DOCS_MAX_OUTPUT_LENGTH,
...(result.exitCode === 124 ? { hint: getDocsSearchHint('The docs search command timed out.') } : {}),
}),
};
} finally {
clearTimeout(timer);
}
} catch (error) {
ctx.log?.error?.(error, {
module: 'ai',
@@ -293,249 +127,171 @@ Return matching document paths only. ${docsModulesDescription}`,
});
return {
status: 'error',
content: `Failed to search docs: ${error.message}`,
content: `Failed to search docs: ${error instanceof Error ? error.message : String(error)}`,
};
}
},
};
}
export function createReadDocEntryTool(): ToolsOptions {
return {
scope: 'SPECIFIED',
defaultPermission: 'ALLOW',
introduction: {
title: '{{t("Read documentation file")}}',
about: '{{t("Read docs tool description")}}',
},
definition: {
name: 'readDocEntry',
description:
'Read files or list directories inside storage/ai/docs using canonical module-based paths. Supports up to 3 paths per call.',
schema: z.object({
paths: z
.array(
z
.string()
.min(1, 'path is required')
.describe(
'Canonical path like runjs/context/router/index.md or /runjs/context/router/index.md. No wildcards, relative segments, or globbing.',
),
)
.min(1, 'paths is required')
.max(DOCS_READ_MAX_PATHS, `paths can contain up to ${DOCS_READ_MAX_PATHS} items`),
}),
},
invoke: async (ctx, args) => {
const available = getDocModuleKeys();
if (!available.length) {
return {
status: 'error',
content: 'No document indexes available.',
};
}
const rawPaths: unknown[] = Array.isArray(args?.paths) ? args.paths : [];
const paths: string[] = Array.from(
new Set(
rawPaths
.map((p) => String(p ?? '').trim())
.filter((p) => !!p)
.slice(0, DOCS_READ_MAX_PATHS),
),
);
if (!paths.length) {
return {
status: 'error',
content: `Paths are required. Available indexes: ${available.join(', ')}`,
};
}
async function createBash(plugin: PluginAIServer) {
const docsDir = await resolveBuiltinDocsDir();
const docsFs = await getDocsFs(plugin, docsDir);
const mountableFs = new MountableFs({ base: new InMemoryFs() });
mountableFs.mount(DOCS_MOUNT_POINT, docsFs);
try {
const entries = await Promise.all(paths.map((targetPath) => readDocEntry(targetPath)));
const blocks = entries.map((entry) => {
if (entry.type === 'directory') {
const listing = entry.entries
.map((item) => `${item.type === 'directory' ? 'DIR ' : 'FILE'} ${item.path}`)
.join('\n');
return `DIRECTORY ${entry.path}\n${listing}`;
}
const clean = entry.content.replace(/\r\n/g, '\n');
return `FILE ${entry.path}\n\`\`\`\n${clean}\n\`\`\``;
});
return {
status: 'success',
content: blocks.join('\n\n'),
};
} catch (error) {
ctx.log?.error?.(error, {
module: 'ai',
subModule: 'toolCalling',
toolName: 'readDocEntry',
});
return {
status: 'error',
content: `Failed to read docs entry: ${error.message}`,
};
}
return new Bash({
fs: mountableFs,
cwd: DOCS_MOUNT_POINT,
commands: DOCS_COMMANDS,
python: false,
javascript: false,
executionLimits: {
maxCommandCount: 1000,
maxLoopIterations: 2000,
maxAwkIterations: 2000,
maxSedIterations: 2000,
maxGlobOperations: 20000,
maxOutputSize: DOCS_MAX_OUTPUT_LENGTH + 4096,
maxStringLength: DOCS_MAX_OUTPUT_LENGTH + 4096,
},
};
defenseInDepth: false,
});
}
function normalizeModuleKey(input: string) {
const { pluginKey, relativePath } = parseDocPath(input);
if (relativePath) {
throw new Error('Index key should only contain the module key, e.g. runjs.');
async function getDocsFs(plugin: PluginAIServer, docsDir: string) {
if (plugin.docsFsCache?.docsDir === docsDir) {
return plugin.docsFsCache.fs;
}
return pluginKey;
const fs = new OverlayFs({
root: docsDir,
mountPoint: '/',
readOnly: true,
maxFileReadSize: DOCS_MAX_FILE_READ_SIZE,
});
plugin.docsFsCache = { docsDir, fs };
return fs;
}
function resolveDocPath(input: string) {
const { pluginKey, relativePath, canonicalPath } = parseDocPath(input);
const meta = docsModules.get(pluginKey);
if (!meta) {
throw new Error(`Document module "${pluginKey}" not found.`);
}
const safeRelative = relativePath || '';
const absolutePath = path.resolve(meta.docsDir, safeRelative);
if (!absolutePath.startsWith(meta.docsDir)) {
throw new Error('Access denied for the requested path.');
}
return {
meta,
absolutePath,
canonicalPath,
};
}
function parseDocPath(input: string) {
const segments = normalizeDocInput(input);
let pluginKey: string;
let relativeStart = 0;
if (segments[0].startsWith('@')) {
if (segments.length < 2) {
throw new Error('Missing package name segment in the path.');
async function resolveBuiltinDocsDir() {
const configured = process.env.NOCOBASE_AI_DOCS_DIR?.trim();
if (configured) {
const docsDir = path.resolve(configured);
if (await fs.pathExists(docsDir)) {
return docsDir;
}
pluginKey = `${segments[0]}/${segments[1]}`;
relativeStart = 2;
} else {
pluginKey = segments[0];
relativeStart = 1;
throw new Error(`NOCOBASE_AI_DOCS_DIR does not exist: ${docsDir}`);
}
const relativeSegments = segments.slice(relativeStart);
const relativePath = relativeSegments.join('/');
const canonicalPath = relativePath ? `${pluginKey}/${relativePath}` : pluginKey;
const sourceDocsDir = path.resolve(__dirname, '../../../../../../../docs/docs/en');
if (process.env.APP_ENV !== 'production' && (await fs.pathExists(sourceDocsDir))) {
return sourceDocsDir;
}
return {
pluginKey,
relativePath,
canonicalPath,
};
const packageDocsDir = path.resolve(__dirname, '../../ai/docs/nocobase');
if (await fs.pathExists(packageDocsDir)) {
return packageDocsDir;
}
if (await fs.pathExists(sourceDocsDir)) {
return sourceDocsDir;
}
throw new Error(`NocoBase documentation directory not found. Checked: ${packageDocsDir}, ${sourceDocsDir}`);
}
function normalizeDocInput(input: string) {
let value = (input ?? '').trim();
if (!value) {
throw new Error('Path is required.');
function truncateOutput(output: string) {
if (output.length <= DOCS_MAX_OUTPUT_LENGTH) {
return output;
}
return `${output.slice(0, DOCS_MAX_OUTPUT_LENGTH)}\n...[truncated]`;
}
function validateDocsScript(script: string) {
const commandText = maskQuotedText(script);
if (new RegExp(String.raw`(^|[\n;&|])\s*find\s+(?:\.\/?|/docs/nocobase/?)(?:\s|$)`).test(commandText)) {
return getDocsSearchHint('Broad full-tree scans with "find ." are not allowed.');
}
value = value.replace(/\\/g, '/');
value = value.replace(/^\/+/, '');
if (value.startsWith('storage/ai/docs/')) {
value = value.slice('storage/ai/docs/'.length);
}
if (!value) {
throw new Error('Path is required.');
if (
new RegExp(String.raw`(^|[\n;&|])\s*rg\b[^\n;&|]*\s(?:\.\/?|/docs/nocobase/?)\s*(?:[|;&\n]|$)`).test(commandText)
) {
return getDocsSearchHint('Broad full-tree content searches with "rg ... ." are not allowed.');
}
const rawSegments = value.split('/');
const segments: string[] = [];
for (const raw of rawSegments) {
const segment = raw.trim();
if (!segment || segment === '.') {
if (/[|]\s*rg\b/.test(commandText)) {
return getDocsSearchHint('Piping output into rg is not supported reliably by this docs shell.');
}
if (hasFindWithMultipleRoots(commandText)) {
return getDocsSearchHint('find with multiple starting directories is too slow for docs search.');
}
return null;
}
function maskQuotedText(script: string) {
let result = '';
let quote: '"' | "'" | null = null;
let escaped = false;
for (const char of script) {
if (quote) {
if (escaped) {
escaped = false;
} else if (char === '\\' && quote === '"') {
escaped = true;
} else if (char === quote) {
quote = null;
result += char;
continue;
}
result += ' ';
continue;
}
if (segment === '..') {
throw new Error('Relative segments are not allowed in document paths.');
}
if (segment.includes('*')) {
throw new Error('Wildcards are not allowed in document paths.');
}
segments.push(segment);
}
if (!segments.length) {
throw new Error('Path is required.');
}
return segments;
}
async function getOrLoadDocsIndex(entry: DocsIndexMeta) {
const now = Date.now();
const cached = docsIndexCache.get(entry.key);
if (cached && now - cached.lastAccess <= DOCS_INDEX_TTL_MS) {
cached.lastAccess = now;
return cached;
}
const loading = docsIndexLoading.get(entry.key);
if (loading) {
const result = await loading;
docsIndexCache.set(entry.key, { ...result, lastAccess: Date.now() });
return docsIndexCache.get(entry.key);
}
const loadPromise = (async () => {
const [indexData, fileMap] = await Promise.all([fs.readJSON(entry.indexFile), fs.readJSON(entry.filesMapFile)]);
const flexIndex = new FlexSearchIndex();
Object.entries(indexData).forEach(([importKey, data]) => {
if (typeof data === 'string') {
flexIndex.import(importKey, data);
}
});
return { index: flexIndex, fileMap };
})();
docsIndexLoading.set(entry.key, loadPromise);
try {
const result = await loadPromise;
docsIndexCache.set(entry.key, { ...result, lastAccess: Date.now() });
enforceDocsIndexCacheLimit();
return docsIndexCache.get(entry.key);
} finally {
docsIndexLoading.delete(entry.key);
}
}
function enforceDocsIndexCacheLimit() {
if (docsIndexCache.size <= DOCS_INDEX_MAX_CACHE) {
return;
}
const entries = Array.from(docsIndexCache.entries()).sort((a, b) => a[1].lastAccess - b[1].lastAccess);
const overflow = entries.length - DOCS_INDEX_MAX_CACHE;
for (let i = 0; i < overflow; i++) {
docsIndexCache.delete(entries[i][0]);
}
}
function buildSearchTerms(keywords: string[]) {
const seen = new Set<string>();
const terms: string[] = [];
for (const raw of keywords) {
const keyword = String(raw ?? '')
.trim()
.replace(/\s+/g, ' ');
if (!keyword || seen.has(keyword)) {
if (char === '"' || char === "'") {
quote = char;
result += char;
continue;
}
seen.add(keyword);
terms.push(keyword);
if (terms.length >= DOCS_SEARCH_MAX_KEYWORDS) {
break;
result += char;
}
return result;
}
function getDocsSearchHint(reason: string) {
return `${reason}
Use scoped searches instead:
- list top-level entries first: ls -d */
- find matching file paths: rg --files multi-app | grep -Ei 'remote|local|index' | head -80
- list files in one directory: rg --files multi-app | head -80
- search content inside likely directories: rg -n -i 'keyword|another keyword' multi-app -g '*.md' -g '*.mdx' | head -80
- read focused snippets from likely files: sed -n '1,180p' get-started/quickstart.md`;
}
function hasFindWithMultipleRoots(script: string) {
const findCommandPattern = /(^|[\n;&|])\s*find\s+([^\n;&|]+)/g;
let match: RegExpExecArray | null;
while ((match = findCommandPattern.exec(script))) {
const args = match[2].trim().split(/\s+/);
const roots = [];
for (const arg of args) {
if (arg.startsWith('-')) {
break;
}
if (arg === '2>/dev/null') {
continue;
}
roots.push(arg);
}
if (roots.length > 1) {
return true;
}
}
return terms;
return false;
}
@@ -50,18 +50,32 @@ Follow this exact order. Do NOT skip ahead to coding.
2. Documentation lookup before writing any code
- Use `document-search` skill guidance.
- You MUST call:
- `searchDocs`
- `readDocEntry`
- You MUST call `searchDocs`.
- Always search docs before coding when the task involves any of the following:
- RunJS / workflow / JS Block / JS Field / JS Item / JS Action / Event Flow / Linkage Rules
- `ctx` APIs, runtime constraints, rendering, routing, requests, imports, React, Antd
- any NocoBase-specific feature, component, schema, collection behavior, or API usage
- Do not rely on memory, prior experience, or "common NocoBase patterns" as a substitute for this step.
- Minimum requirement:
- search for the relevant module / keywords
- read the most relevant matching entry or entries
- use a compact Bash script that first searches file paths / filenames, then reads focused snippets from the best matches
- run broader content search only when path search is insufficient or ambiguous
- extract the concrete constraints or APIs you will rely on
- start narrow; prefer returning an initial answer once focused snippets confirm the needed constraints
- Prefer one combined `searchDocs` call over multiple small calls. Good pattern:
```bash
printf '## Candidate files\n'
rg --files interface-builder | grep -Ei 'runjs|js-|event|linkage' | head -50
printf '\n## JS Block\n'
sed -n '1,180p' interface-builder/blocks/other-blocks/js-block.md
printf '\n## RunJS\n'
sed -n '1,180p' interface-builder/runjs.md
```
- Avoid broad full-tree scans such as `find .` or `rg ... .`. Pick likely top-level directories first, then search inside them.
- Do not pipe output into `rg`; use `grep` for pipeline filtering, and call `rg --files <dir>` or `rg -n <pattern> <dir>` with explicit path arguments.
- Use `rg -g '*.md' -g '*.mdx'`; do not use unsupported `rg --include` options.
- Stop searching once the snippets directly confirm the API or constraint needed for the code. Do not read adjacent overview, quickstart, definition, lifecycle, or development pages just for extra background unless the user asks for more depth.
- Only after this step may you decide how to implement the solution.
3. Data inspection when data model is involved
@@ -84,7 +98,15 @@ Follow this exact order. Do NOT skip ahead to coding.
6. Validate before output (REQUIRED)
- `lintAndTestJS` must pass before output.
- If validation fails, fix the code and validate again.
- If validation fails, do not guess or repeatedly patch from memory.
- Treat every validation failure as new evidence and classify it before changing code:
- Unknown or missing `ctx` member / runtime API / library exposure → call `getContextApis`, `getContextVars`, or `getContextEnvs` again, then search docs for that exact API or error.
- Unsupported syntax, sandbox restriction, import/render/request error, React/Antd usage error → call `searchDocs` again with the exact diagnostic text and the related feature keywords.
- Collection, field, relation, filter, or record-shape error → call the data metadata tools again for the exact collection or field involved.
- Plain JavaScript syntax or type error that is fully explained by the diagnostic → fix directly, then validate again.
- After one failed direct fix, you MUST go back to runtime inspection, documentation lookup, or metadata lookup before another code change.
- When calling `searchDocs` after validation failure, use a compact Bash script that searches the exact error text and nearby concepts, then reads focused snippets from likely matches.
- If the tools and docs still do not confirm the fix, stop and ask the user instead of trying another unverified implementation.
# Coding Rules
@@ -27,7 +27,12 @@ When helping users with JavaScript code, follow this process:
3. **Validate the Code**
- Use `lintAndTestJS` to lint and test the code before final output
- Fix any errors reported by the linting tool
- Fix plain JavaScript syntax errors directly when the diagnostic fully explains the problem
- If the error involves NocoBase runtime APIs, `ctx`, sandbox restrictions, imports, rendering, React, Antd, requests, collections, fields, or record structure, do not guess. Go back to the relevant inspection or documentation tools before changing code:
- Runtime exposure errors: call `getContextEnvs`, `getContextVars`, or `getContextApis` again
- NocoBase API / runtime / sandbox / UI errors: use the documentation search skill again with the exact error and relevant feature keywords
- Data model errors: use data metadata tools again for the exact collection, field, or relation
- After one failed direct fix, you must gather new evidence from tools or docs before another code change
- Do not output final code unless it passes validation
4. **Submit the Code**
+257 -1898
View File
File diff suppressed because it is too large Load Diff