opensource/n8n
3
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-09-24 23:22:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(ai-builder): Filter LangSmith eval dataset by local file slugs (#29507)

Browse Source 
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
José Braulio González Valido
2026-04-29 13:30:29 +00:00
committed by GitHub
co-authored by Claude Opus 4.7
parent 794334cd79
commit 54d9286d92
1 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/@n8n/instance-ai/evaluations/cli/index.ts
+9 -6
View File
@@ -359,9 +359,11 @@ async function runWithLangSmith(config: RunConfig): Promise<MultiRunEvaluation>
`Starting evaluate() with concurrency=${String(args.concurrency)}, builds limited to ${String(MAX_CONCURRENT_BUILDS)}, iterations=${String(args.iterations)}`,
);
const sourceExamples = args.filter
? filteredExamplesIterable(lsClient, datasetName, args.filter, logger)
: lsClient.listExamples({ datasetName });
// Always filter the LangSmith dataset by the local file slugs. The local
// JSON files are the source of truth; the dataset accumulates orphans (the
// sync is additive — see langsmith/dataset-sync.ts) and we don't want to
// run scenarios whose JSON file no longer exists.
const sourceExamples = filteredExamplesIterable(lsClient, datasetName, args.filter, logger);
const evaluateData =
args.iterations > 1
? expandExamplesForIterations(sourceExamples, args.iterations)
@@ -454,15 +456,16 @@ async function* expandExamplesForIterations(
function filteredExamplesIterable(
lsClient: Client,
datasetName: string,
filter: string,
filter: string | undefined,
logger: EvalLogger,
): AsyncIterable<Example> {
const slugs = loadWorkflowTestCasesWithFiles(filter).map((tc) => tc.fileSlug);
const label = filter ? `Filter "${filter}"` : 'Local test cases';
if (slugs.length === 0) {
logger.info(`Filter "${filter}" matched no local test case files`);
logger.info(`${label} matched no local test case files`);
return (async function* () {})();
}
logger.info(`Filter "${filter}" matched ${String(slugs.length)} split(s): ${slugs.join(', ')}`);
logger.info(`${label} matched ${String(slugs.length)} split(s): ${slugs.join(', ')}`);
return lsClient.listExamples({ datasetName, splits: slugs });
}