mirror of
https://github.com/n8n-io/n8n.git
synced 2026-08-28 17:22:01 +08:00
feat(ai-builder): Improve generation across data_transformation category (#23609)
This commit is contained in:
Eugene
@@ -319,6 +319,7 @@ Pairwise evaluation uses a dataset with custom do/don't criteria for each prompt
|
||||
| `--dos <rules>` | Newline-separated "do" rules for local evaluation | - |
|
||||
| `--donts <rules>` | Newline-separated "don't" rules for local evaluation | - |
|
||||
| `--notion-id <id>` | Filter to a single example by its `notion_id` metadata | (all examples) |
|
||||
| `--technique <name>` | Filter examples by technique (must be in metadata `categories` field) | (all examples) |
|
||||
| `--max-examples <n>` | Limit number of examples to evaluate (useful for testing) | (no limit) |
|
||||
| `--repetitions <n>` | Number of times to repeat the entire evaluation | 1 |
|
||||
| `--generations <n>` | Number of workflow generations per prompt (for variance reduction) | 1 |
|
||||
@@ -366,6 +367,9 @@ pnpm eval:pairwise
|
||||
# Run a single example by notion_id
|
||||
pnpm eval:pairwise --notion-id 30d29454-b397-4a35-8e0b-74a2302fa81a
|
||||
|
||||
# Filter examples by technique (from metadata.categories field)
|
||||
pnpm eval:pairwise --technique "1-technique-data"
|
||||
|
||||
# Run with 3 repetitions and 5 judges, custom experiment name
|
||||
pnpm eval:pairwise --repetitions 3 --judges 5 --name "my-experiment"
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ const VALID_FLAGS = [
|
||||
'--prompts-csv',
|
||||
'--repetitions',
|
||||
'--notion-id',
|
||||
'--technique',
|
||||
'--judges',
|
||||
'--generations',
|
||||
'--concurrency',
|
||||
@@ -73,6 +74,7 @@ function parseCliArgs() {
|
||||
promptsCsvPath: getFlagValue('--prompts-csv') ?? process.env.PROMPTS_CSV_FILE,
|
||||
repetitions: getIntFlag('--repetitions', 1),
|
||||
notionId: getFlagValue('--notion-id'),
|
||||
technique: getFlagValue('--technique'),
|
||||
numJudges: getIntFlag('--judges', 3),
|
||||
numGenerations: getIntFlag('--generations', 1, 10),
|
||||
concurrency: getIntFlag('--concurrency', 5),
|
||||
@@ -120,6 +122,7 @@ async function main(): Promise<void> {
|
||||
await runPairwiseLangsmithEvaluation({
|
||||
repetitions: args.repetitions,
|
||||
notionId: args.notionId,
|
||||
technique: args.technique,
|
||||
numJudges: args.numJudges,
|
||||
numGenerations: args.numGenerations,
|
||||
verbose: args.verbose,
|
||||
|
||||
@@ -84,7 +84,12 @@ export async function evaluateWorkflowPairwise(
|
||||
): Promise<PairwiseEvaluationResult> {
|
||||
const dos = input.evalCriteria?.dos ?? '';
|
||||
const donts = input.evalCriteria?.donts ?? '';
|
||||
const criteriaList = `[DO]\n${dos}\n\n[DONT]\n${donts}`;
|
||||
const formattedDonts = donts
|
||||
.split('\n')
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line) => `DO NOT ${line}`)
|
||||
.join('\n');
|
||||
const criteriaList = `[DO]\n${dos}\n\n[DON'T]\n${formattedDonts}`;
|
||||
|
||||
const chain = createEvaluatorChain(
|
||||
llm,
|
||||
|
||||
@@ -25,10 +25,22 @@ function getNotionId(metadata: unknown): string | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Filter examples by notion_id or limit count */
|
||||
/** Extract categories from metadata if present */
|
||||
function getCategories(metadata: unknown): string[] | undefined {
|
||||
if (typeof metadata === 'object' && metadata !== null && 'categories' in metadata) {
|
||||
const categories = (metadata as { categories: unknown }).categories;
|
||||
return Array.isArray(categories)
|
||||
? categories.filter((c): c is string => typeof c === 'string')
|
||||
: undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Filter examples by notion_id, technique, or limit count */
|
||||
function filterExamples(
|
||||
allExamples: Example[],
|
||||
notionId: string | undefined,
|
||||
technique: string | undefined,
|
||||
maxExamples: number | undefined,
|
||||
log: EvalLogger,
|
||||
): Example[] {
|
||||
@@ -48,6 +60,33 @@ function filterExamples(
|
||||
return filtered;
|
||||
}
|
||||
|
||||
if (technique) {
|
||||
log.warn(`🔍 Filtering by technique: ${technique}`);
|
||||
const filtered = allExamples.filter((e) => {
|
||||
const categories = getCategories(e.metadata);
|
||||
return categories?.includes(technique);
|
||||
});
|
||||
|
||||
if (filtered.length === 0) {
|
||||
const availableTechniques = new Set<string>();
|
||||
for (const example of allExamples) {
|
||||
const categories = getCategories(example.metadata);
|
||||
if (categories) {
|
||||
for (const category of categories) {
|
||||
availableTechniques.add(category);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`No examples found with technique: ${technique}. Available techniques: ${Array.from(availableTechniques).sort().join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
log.success(`✅ Found ${filtered.length} example(s) with technique "${technique}"`);
|
||||
log.verbose(`First example metadata: ${JSON.stringify(filtered[0].metadata, null, 2)}`);
|
||||
return filtered;
|
||||
}
|
||||
|
||||
if (maxExamples && maxExamples > 0) {
|
||||
log.warn(`➔ Limiting to ${maxExamples} example(s)`);
|
||||
return allExamples.slice(0, maxExamples);
|
||||
@@ -183,6 +222,7 @@ function displayLocalResults(
|
||||
export interface PairwiseEvaluationOptions {
|
||||
repetitions?: number;
|
||||
notionId?: string;
|
||||
technique?: string;
|
||||
numJudges?: number;
|
||||
numGenerations?: number;
|
||||
verbose?: boolean;
|
||||
@@ -202,6 +242,7 @@ export async function runPairwiseLangsmithEvaluation(
|
||||
const {
|
||||
repetitions = DEFAULTS.REPETITIONS,
|
||||
notionId,
|
||||
technique,
|
||||
numJudges = DEFAULTS.NUM_JUDGES,
|
||||
numGenerations = DEFAULTS.NUM_GENERATIONS,
|
||||
verbose = false,
|
||||
@@ -256,7 +297,7 @@ export async function runPairwiseLangsmithEvaluation(
|
||||
}
|
||||
log.verbose(`📊 Total examples in dataset: ${allExamples.length}`);
|
||||
|
||||
const data = filterExamples(allExamples, notionId, maxExamples, log);
|
||||
const data = filterExamples(allExamples, notionId, technique, maxExamples, log);
|
||||
log.info(`➔ Running ${data.length} example(s) × ${repetitions} rep(s)`);
|
||||
|
||||
// Create target (does all work) and evaluator (extracts pre-computed metrics)
|
||||
|
||||
@@ -299,6 +299,12 @@ COMMON MISTAKES TO AVOID:
|
||||
✅ Window Buffer Memory → AI Agent (CORRECT)
|
||||
</connection_type_reference>`;
|
||||
|
||||
const DATA_TABLES = `**Data Tables VS 3rd party services**
|
||||
- Data Tables provide built-in data storage within n8n, allowing you to persist and manage data directly in your workflows without external databases.
|
||||
- Use the Data table node (n8n-nodes-base.dataTable) in workflows to retrieve, insert, update, or delete records.
|
||||
- When storing, retrieving, or managing structured data within workflows, always prefer using n8n's native Data tables over external services like Google Sheets, Airtable, or other third-party databases if not specified otherwise.
|
||||
`;
|
||||
|
||||
const RESTRICTIONS = `- Respond before calling validate_structure
|
||||
- Skip validation even if you think structure is correct
|
||||
- Add commentary between tool calls - execute tools silently
|
||||
@@ -331,6 +337,7 @@ export function buildBuilderPrompt(): string {
|
||||
.section('switch_node_pattern', SWITCH_NODE_PATTERN)
|
||||
.section('node_connection_examples', NODE_CONNECTION_EXAMPLES)
|
||||
.section('connection_type_examples', CONNECTION_TYPES)
|
||||
.section('data_tables', DATA_TABLES)
|
||||
.section('do_not', RESTRICTIONS)
|
||||
.section('response_format', RESPONSE_FORMAT)
|
||||
.build();
|
||||
|
||||
@@ -79,6 +79,7 @@ Purpose: Fetches external data to enrich chatbot responses with real-time or org
|
||||
|
||||
### Database Nodes & Google Sheets
|
||||
|
||||
- Data Table (n8n-nodes-base.dataTable)
|
||||
- Postgres (n8n-nodes-base.postgres)
|
||||
- MySQL (n8n-nodes-base.mySql)
|
||||
- MongoDB (n8n-nodes-base.mongoDb)
|
||||
|
||||
+11
-25
@@ -69,12 +69,17 @@ export class DataTransformationBestPractices implements BestPracticesDocument {
|
||||
|
||||
#### Code Node (n8n-nodes-base.code)
|
||||
|
||||
**When NOT to Use**: Code node may be slower than core nodes (like Edit Fields, If, Switch, etc.) as Code nodes run in a sandboxed environment. Avoid the code node where possible — it should only be used for complex transformations that can't be done with other nodes. For example, DO NOT use it for:
|
||||
**Built-in Nodes vs. Code Node**
|
||||
- Prefer basic built-in nodes (Edit Fields, Filter, Split Out, Summarize, Aggregate, etc.) over Code node. Use Code only for complex logic that can't be achieved otherwise.
|
||||
- Rule of thumb: if the goal can be achieved with fewer than 5 basic nodes, use basic nodes
|
||||
|
||||
**When NOT to Use**: Code node may be slower than core nodes (like Edit Fields, If, Switch, Split Out, Aggregate, etc.) as Code nodes run in a sandboxed environment. Avoid the code node where possible — it should only be used for complex transformations that can't be done with other nodes. For example, DO NOT use it for:
|
||||
- Adding or removing fields from items (use the 'edit fields' node instead)
|
||||
- Single-line data transformations of item fields (use the 'edit fields' node instead)
|
||||
- Filtering items based on their fields (use the 'filter' node instead)
|
||||
- Pivoting, aggregating or summarizing data across items (use the 'summarize' node instead)
|
||||
- Pivoting or summarizing data across multiple items (use the 'summarize' node instead)
|
||||
- Splitting arrays inside items out into multiple items (use the 'split out' node instead)
|
||||
- Aggregating multiple items into a single item (use the 'aggregate' node instead)
|
||||
- Sorting items in an array based on their fields (use the 'Sort' node instead)
|
||||
- Generating HTML from text or formatting text as HTML (use the 'HTML' node set to operation 'Generate HTML Template' or 'Convert to HTML Table' instead)
|
||||
|
||||
@@ -118,15 +123,8 @@ return items; // or return [{ json: {...} }];
|
||||
- **Purpose**: Process large datasets in chunks
|
||||
- **Use When**: Handling 100+ items with expensive operations (API calls, AI)
|
||||
|
||||
### Workflow Orchestration
|
||||
|
||||
**Execute Workflow** (n8n-nodes-base.executeWorkflow):
|
||||
- **Purpose**: Call sub-workflows for modular design
|
||||
- **Best Practice**: Create reusable sub-workflows for common tasks like "Data Cleaning" or "Error Handler"
|
||||
|
||||
**Error Trigger** (n8n-nodes-base.errorTrigger):
|
||||
- **Purpose**: Create global error handling workflow
|
||||
- **Best Practice**: Use as safety net to catch all workflow errors
|
||||
## Input Data Validation
|
||||
- Validate external data before processing: check for nulls, empty values, and edge cases (special chars, empty arrays)
|
||||
|
||||
## Common Pitfalls to Avoid
|
||||
|
||||
@@ -145,28 +143,16 @@ return items; // or return [{ json: {...} }];
|
||||
- **Fix**: Keep code nodes focused on single transformation aspect
|
||||
|
||||
#### Merge Node Problems
|
||||
- **Missing Keys**: Trying to merge on non-existent fields
|
||||
- **Fix**: Validate both inputs have matching key fields
|
||||
- **Field Name Mismatch**: Different field names in sources
|
||||
- **Fix**: Use Edit Fields node to normalize field names before merging
|
||||
|
||||
### General Workflow Issues
|
||||
- **No Error Handling**: Workflow crashes on unexpected data. **Fix**: Add IF nodes for validation, use error outputs
|
||||
- **Hard-coded Values**: URLs, credentials, config in nodes. **Fix**: Use environment variables or config nodes
|
||||
- **Missing Documentation**: No comments or descriptions
|
||||
- **Fix**: Add sticky notes, node descriptions, code comments
|
||||
- **Fix**: Normalize field names with Edit Fields before merging
|
||||
|
||||
### Performance Pitfalls
|
||||
- Processing large datasets without batching → timeouts
|
||||
- Not filtering early → unnecessary processing overhead
|
||||
- Excessive node chaining → visual clutter and slow execution
|
||||
- Not using sub-workflows → unmaintainable monolithic workflows
|
||||
|
||||
### Data Validation Pitfalls
|
||||
- Assuming input data is always perfect
|
||||
- Not handling empty/null values
|
||||
- Ignoring data type mismatches
|
||||
- Missing edge case handling (special characters, empty arrays)
|
||||
- Assuming input data is always perfect → runtime errors
|
||||
`;
|
||||
|
||||
getDocumentation(): string {
|
||||
|
||||
@@ -44,7 +44,7 @@ sequence. Use the n8n Form Trigger node to start the workflow and display the fi
|
||||
|
||||
## Data Collection & Aggregation
|
||||
|
||||
Collect and merge all user responses from each form step before writing to your destination (e.g., Google Sheets). Use
|
||||
Collect and merge all user responses from each form step before writing to your destination (e.g., Data Table). Use
|
||||
Set or Merge nodes to combine data as needed. Make sure your JSON keys match the column names in your destination for
|
||||
automatic mapping.
|
||||
|
||||
@@ -91,7 +91,7 @@ Purpose: Persist raw form data to a storage destination, preference should be fo
|
||||
but use the most applicable node depending on the user's request.
|
||||
|
||||
Required nodes (use at least one):
|
||||
- Data table (n8n-nodes-base.dataTable): Built-in n8n storage for quick setup
|
||||
- Data table (n8n-nodes-base.dataTable): Built-in n8n storage for quick setup - preferred
|
||||
- Google Sheets (n8n-nodes-base.googleSheets): Best for simple spreadsheet storage
|
||||
- Airtable (n8n-nodes-base.airtable): Best for structured database with relationships
|
||||
- Postgres (n8n-nodes-base.postgres) / MySQL (n8n-nodes-base.mySql) / MongoDB (n8n-nodes-base.mongoDb): For production database storage
|
||||
|
||||
@@ -104,6 +104,10 @@ Pitfalls:
|
||||
|
||||
Purpose: Introduces delays to respect rate limits and avoid overloading servers
|
||||
|
||||
### Data Tables (n8n-nodes-base.dataTable)
|
||||
|
||||
Purpose: Stores scraped data in n8n's built-in persistent data storage
|
||||
|
||||
### Google Sheets (n8n-nodes-base.googleSheets)
|
||||
|
||||
Purpose: Stores scraped data in spreadsheets for easy access and sharing
|
||||
|
||||
Reference in New Issue
Block a user