diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/README.md b/packages/@n8n/ai-workflow-builder.ee/evaluations/README.md index 73e7e0a1a19..5759f72a278 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/README.md +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/README.md @@ -319,6 +319,7 @@ Pairwise evaluation uses a dataset with custom do/don't criteria for each prompt | `--dos ` | Newline-separated "do" rules for local evaluation | - | | `--donts ` | Newline-separated "don't" rules for local evaluation | - | | `--notion-id ` | Filter to a single example by its `notion_id` metadata | (all examples) | +| `--technique ` | Filter examples by technique (must be in metadata `categories` field) | (all examples) | | `--max-examples ` | Limit number of examples to evaluate (useful for testing) | (no limit) | | `--repetitions ` | Number of times to repeat the entire evaluation | 1 | | `--generations ` | Number of workflow generations per prompt (for variance reduction) | 1 | @@ -366,6 +367,9 @@ pnpm eval:pairwise # Run a single example by notion_id pnpm eval:pairwise --notion-id 30d29454-b397-4a35-8e0b-74a2302fa81a +# Filter examples by technique (from metadata.categories field) +pnpm eval:pairwise --technique "1-technique-data" + # Run with 3 repetitions and 5 judges, custom experiment name pnpm eval:pairwise --repetitions 3 --judges 5 --name "my-experiment" diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/index.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/index.ts index a19070775bc..1a21523c113 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/index.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/index.ts @@ -21,6 +21,7 @@ const VALID_FLAGS = [ '--prompts-csv', '--repetitions', '--notion-id', + '--technique', '--judges', '--generations', '--concurrency', @@ -73,6 +74,7 @@ function parseCliArgs() { promptsCsvPath: getFlagValue('--prompts-csv') ?? process.env.PROMPTS_CSV_FILE, repetitions: getIntFlag('--repetitions', 1), notionId: getFlagValue('--notion-id'), + technique: getFlagValue('--technique'), numJudges: getIntFlag('--judges', 3), numGenerations: getIntFlag('--generations', 1, 10), concurrency: getIntFlag('--concurrency', 5), @@ -120,6 +122,7 @@ async function main(): Promise { await runPairwiseLangsmithEvaluation({ repetitions: args.repetitions, notionId: args.notionId, + technique: args.technique, numJudges: args.numJudges, numGenerations: args.numGenerations, verbose: args.verbose, diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/pairwise/judge-chain.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/pairwise/judge-chain.ts index 96547ed6786..8b70886acec 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/pairwise/judge-chain.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/pairwise/judge-chain.ts @@ -84,7 +84,12 @@ export async function evaluateWorkflowPairwise( ): Promise { const dos = input.evalCriteria?.dos ?? ''; const donts = input.evalCriteria?.donts ?? ''; - const criteriaList = `[DO]\n${dos}\n\n[DONT]\n${donts}`; + const formattedDonts = donts + .split('\n') + .filter((line) => line.trim().length > 0) + .map((line) => `DO NOT ${line}`) + .join('\n'); + const criteriaList = `[DO]\n${dos}\n\n[DON'T]\n${formattedDonts}`; const chain = createEvaluatorChain( llm, diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/pairwise/runner.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/pairwise/runner.ts index 96689f15291..9028b21f7c0 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/pairwise/runner.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/pairwise/runner.ts @@ -25,10 +25,22 @@ function getNotionId(metadata: unknown): string | undefined { return undefined; } -/** Filter examples by notion_id or limit count */ +/** Extract categories from metadata if present */ +function getCategories(metadata: unknown): string[] | undefined { + if (typeof metadata === 'object' && metadata !== null && 'categories' in metadata) { + const categories = (metadata as { categories: unknown }).categories; + return Array.isArray(categories) + ? categories.filter((c): c is string => typeof c === 'string') + : undefined; + } + return undefined; +} + +/** Filter examples by notion_id, technique, or limit count */ function filterExamples( allExamples: Example[], notionId: string | undefined, + technique: string | undefined, maxExamples: number | undefined, log: EvalLogger, ): Example[] { @@ -48,6 +60,33 @@ function filterExamples( return filtered; } + if (technique) { + log.warn(`🔍 Filtering by technique: ${technique}`); + const filtered = allExamples.filter((e) => { + const categories = getCategories(e.metadata); + return categories?.includes(technique); + }); + + if (filtered.length === 0) { + const availableTechniques = new Set(); + for (const example of allExamples) { + const categories = getCategories(example.metadata); + if (categories) { + for (const category of categories) { + availableTechniques.add(category); + } + } + } + throw new Error( + `No examples found with technique: ${technique}. Available techniques: ${Array.from(availableTechniques).sort().join(', ')}`, + ); + } + + log.success(`✅ Found ${filtered.length} example(s) with technique "${technique}"`); + log.verbose(`First example metadata: ${JSON.stringify(filtered[0].metadata, null, 2)}`); + return filtered; + } + if (maxExamples && maxExamples > 0) { log.warn(`➔ Limiting to ${maxExamples} example(s)`); return allExamples.slice(0, maxExamples); @@ -183,6 +222,7 @@ function displayLocalResults( export interface PairwiseEvaluationOptions { repetitions?: number; notionId?: string; + technique?: string; numJudges?: number; numGenerations?: number; verbose?: boolean; @@ -202,6 +242,7 @@ export async function runPairwiseLangsmithEvaluation( const { repetitions = DEFAULTS.REPETITIONS, notionId, + technique, numJudges = DEFAULTS.NUM_JUDGES, numGenerations = DEFAULTS.NUM_GENERATIONS, verbose = false, @@ -256,7 +297,7 @@ export async function runPairwiseLangsmithEvaluation( } log.verbose(`📊 Total examples in dataset: ${allExamples.length}`); - const data = filterExamples(allExamples, notionId, maxExamples, log); + const data = filterExamples(allExamples, notionId, technique, maxExamples, log); log.info(`➔ Running ${data.length} example(s) × ${repetitions} rep(s)`); // Create target (does all work) and evaluator (extracts pre-computed metrics) diff --git a/packages/@n8n/ai-workflow-builder.ee/src/prompts/agents/builder.prompt.ts b/packages/@n8n/ai-workflow-builder.ee/src/prompts/agents/builder.prompt.ts index 3321140c574..3c654d5c190 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/prompts/agents/builder.prompt.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/prompts/agents/builder.prompt.ts @@ -299,6 +299,12 @@ COMMON MISTAKES TO AVOID: ✅ Window Buffer Memory → AI Agent (CORRECT) `; +const DATA_TABLES = `**Data Tables VS 3rd party services** +- Data Tables provide built-in data storage within n8n, allowing you to persist and manage data directly in your workflows without external databases. +- Use the Data table node (n8n-nodes-base.dataTable) in workflows to retrieve, insert, update, or delete records. +- When storing, retrieving, or managing structured data within workflows, always prefer using n8n's native Data tables over external services like Google Sheets, Airtable, or other third-party databases if not specified otherwise. +`; + const RESTRICTIONS = `- Respond before calling validate_structure - Skip validation even if you think structure is correct - Add commentary between tool calls - execute tools silently @@ -331,6 +337,7 @@ export function buildBuilderPrompt(): string { .section('switch_node_pattern', SWITCH_NODE_PATTERN) .section('node_connection_examples', NODE_CONNECTION_EXAMPLES) .section('connection_type_examples', CONNECTION_TYPES) + .section('data_tables', DATA_TABLES) .section('do_not', RESTRICTIONS) .section('response_format', RESPONSE_FORMAT) .build(); diff --git a/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/chatbot.ts b/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/chatbot.ts index 7ad6e5f890f..00631a5d47a 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/chatbot.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/chatbot.ts @@ -79,6 +79,7 @@ Purpose: Fetches external data to enrich chatbot responses with real-time or org ### Database Nodes & Google Sheets +- Data Table (n8n-nodes-base.dataTable) - Postgres (n8n-nodes-base.postgres) - MySQL (n8n-nodes-base.mySql) - MongoDB (n8n-nodes-base.mongoDb) diff --git a/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/data-transformation.ts b/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/data-transformation.ts index 782bbfcd3d5..3047645e831 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/data-transformation.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/data-transformation.ts @@ -69,12 +69,17 @@ export class DataTransformationBestPractices implements BestPracticesDocument { #### Code Node (n8n-nodes-base.code) -**When NOT to Use**: Code node may be slower than core nodes (like Edit Fields, If, Switch, etc.) as Code nodes run in a sandboxed environment. Avoid the code node where possible — it should only be used for complex transformations that can't be done with other nodes. For example, DO NOT use it for: +**Built-in Nodes vs. Code Node** +- Prefer basic built-in nodes (Edit Fields, Filter, Split Out, Summarize, Aggregate, etc.) over Code node. Use Code only for complex logic that can't be achieved otherwise. +- Rule of thumb: if the goal can be achieved with fewer than 5 basic nodes, use basic nodes + +**When NOT to Use**: Code node may be slower than core nodes (like Edit Fields, If, Switch, Split Out, Aggregate, etc.) as Code nodes run in a sandboxed environment. Avoid the code node where possible — it should only be used for complex transformations that can't be done with other nodes. For example, DO NOT use it for: - Adding or removing fields from items (use the 'edit fields' node instead) - Single-line data transformations of item fields (use the 'edit fields' node instead) - Filtering items based on their fields (use the 'filter' node instead) -- Pivoting, aggregating or summarizing data across items (use the 'summarize' node instead) +- Pivoting or summarizing data across multiple items (use the 'summarize' node instead) - Splitting arrays inside items out into multiple items (use the 'split out' node instead) +- Aggregating multiple items into a single item (use the 'aggregate' node instead) - Sorting items in an array based on their fields (use the 'Sort' node instead) - Generating HTML from text or formatting text as HTML (use the 'HTML' node set to operation 'Generate HTML Template' or 'Convert to HTML Table' instead) @@ -118,15 +123,8 @@ return items; // or return [{ json: {...} }]; - **Purpose**: Process large datasets in chunks - **Use When**: Handling 100+ items with expensive operations (API calls, AI) -### Workflow Orchestration - -**Execute Workflow** (n8n-nodes-base.executeWorkflow): -- **Purpose**: Call sub-workflows for modular design -- **Best Practice**: Create reusable sub-workflows for common tasks like "Data Cleaning" or "Error Handler" - -**Error Trigger** (n8n-nodes-base.errorTrigger): -- **Purpose**: Create global error handling workflow -- **Best Practice**: Use as safety net to catch all workflow errors +## Input Data Validation +- Validate external data before processing: check for nulls, empty values, and edge cases (special chars, empty arrays) ## Common Pitfalls to Avoid @@ -145,28 +143,16 @@ return items; // or return [{ json: {...} }]; - **Fix**: Keep code nodes focused on single transformation aspect #### Merge Node Problems -- **Missing Keys**: Trying to merge on non-existent fields -- **Fix**: Validate both inputs have matching key fields - **Field Name Mismatch**: Different field names in sources -- **Fix**: Use Edit Fields node to normalize field names before merging - -### General Workflow Issues -- **No Error Handling**: Workflow crashes on unexpected data. **Fix**: Add IF nodes for validation, use error outputs -- **Hard-coded Values**: URLs, credentials, config in nodes. **Fix**: Use environment variables or config nodes -- **Missing Documentation**: No comments or descriptions -- **Fix**: Add sticky notes, node descriptions, code comments +- **Fix**: Normalize field names with Edit Fields before merging ### Performance Pitfalls - Processing large datasets without batching → timeouts - Not filtering early → unnecessary processing overhead - Excessive node chaining → visual clutter and slow execution -- Not using sub-workflows → unmaintainable monolithic workflows ### Data Validation Pitfalls -- Assuming input data is always perfect -- Not handling empty/null values -- Ignoring data type mismatches -- Missing edge case handling (special characters, empty arrays) +- Assuming input data is always perfect → runtime errors `; getDocumentation(): string { diff --git a/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/form-input.ts b/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/form-input.ts index 66ed2c6965d..2935e16b0fb 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/form-input.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/form-input.ts @@ -44,7 +44,7 @@ sequence. Use the n8n Form Trigger node to start the workflow and display the fi ## Data Collection & Aggregation -Collect and merge all user responses from each form step before writing to your destination (e.g., Google Sheets). Use +Collect and merge all user responses from each form step before writing to your destination (e.g., Data Table). Use Set or Merge nodes to combine data as needed. Make sure your JSON keys match the column names in your destination for automatic mapping. @@ -91,7 +91,7 @@ Purpose: Persist raw form data to a storage destination, preference should be fo but use the most applicable node depending on the user's request. Required nodes (use at least one): -- Data table (n8n-nodes-base.dataTable): Built-in n8n storage for quick setup +- Data table (n8n-nodes-base.dataTable): Built-in n8n storage for quick setup - preferred - Google Sheets (n8n-nodes-base.googleSheets): Best for simple spreadsheet storage - Airtable (n8n-nodes-base.airtable): Best for structured database with relationships - Postgres (n8n-nodes-base.postgres) / MySQL (n8n-nodes-base.mySql) / MongoDB (n8n-nodes-base.mongoDb): For production database storage diff --git a/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/scraping-and-research.ts b/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/scraping-and-research.ts index d815de8b172..af9733fda95 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/scraping-and-research.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/tools/best-practices/scraping-and-research.ts @@ -104,6 +104,10 @@ Pitfalls: Purpose: Introduces delays to respect rate limits and avoid overloading servers +### Data Tables (n8n-nodes-base.dataTable) + +Purpose: Stores scraped data in n8n's built-in persistent data storage + ### Google Sheets (n8n-nodes-base.googleSheets) Purpose: Stores scraped data in spreadsheets for easy access and sharing