From 943ee27686e44cd2335b8982357f4d8c9c1326be Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 2 Jul 2026 10:45:02 -0700 Subject: [PATCH] feat(langsmith): add run update, get run, and feedback tools (#5363) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * v0.6.29: login improvements, posthog telemetry (#4026) * feat(posthog): Add tracking on mothership abort (#4023) Co-authored-by: Theodore Li * fix(login): fix captcha headers for manual login (#4025) * fix(signup): fix turnstile key loading * fix(login): fix captcha header passing * Catch user already exists, remove login form captcha * feat(langsmith): add run update, get run, and feedback tools - add langsmith_update_run (PATCH /runs/{id}) to complete the create-then-patch tracing lifecycle - add langsmith_get_run (GET /runs/{id}) to read a run back - add langsmith_create_feedback (POST /feedback) to attach scores/corrections to runs - wire all three into the LangSmith block, reusing shared subBlocks across operations - fix feedback-capture template to use real feedback API instead of faking it via tagged runs - switch manual Object.fromEntries filtering to filterUndefined per repo convention * fix(langsmith): reject non-numeric feedback score instead of silently sending null Number() on an invalid score string produces NaN, which serializes to JSON null and would still be sent to LangSmith. Throw instead, matching the existing parseJsonValue error pattern. * fix(langsmith): harden error handling and validation on new run tools - update_run, get_run, create_feedback now check response.ok before parsing/returning, matching the cloudwatch/zoom convention instead of silently returning success on a 4xx/404 - block outputs schema now exposes inputs/outputs for Get Run - update_run requires at least one field to patch, matching the batch-ingest guard for post/patch * fix(langsmith): align get_run/update_run outputs and narrow feedback score type - get_run now also outputs runId (alias of id) so workflows can read the run identifier consistently across all operations on the block - update_run now parses and surfaces the response message instead of discarding the body entirely, matching create_run's pattern - narrow LangsmithCreateFeedbackParams.score to number — the tool param is declared as JSON-schema 'number' and the block's parseScore never produces a boolean, so the wider type was dead and misleading * fix(langsmith): reject empty-string patch fields and empty PATCH bodies - block mapper now normalizes blank name/end_time/status/error inputs to undefined instead of forwarding empty strings, which would clear those fields on the LangSmith run - update_run tool now throws if the filtered PATCH body is empty, guarding direct/programmatic tool calls that bypass the block's own "at least one field" check * fix(langsmith): normalize empty-string patch fields at the tool layer too Direct/agent tool calls bypass the block's own emptyToUndefined guard, so update_run now normalizes blank name/end_time/status/error itself before filtering, matching the block-level fix. --------- Co-authored-by: Theodore Li Co-authored-by: Siddharth Ganesan <33737564+Sg312@users.noreply.github.com> Co-authored-by: Vikhyath Mondreti Co-authored-by: Theodore Li --- apps/sim/blocks/blocks/langsmith.ts | 215 ++++++++++++++++++-- apps/sim/tools/langsmith/create_feedback.ts | 133 ++++++++++++ apps/sim/tools/langsmith/create_run.ts | 5 +- apps/sim/tools/langsmith/get_run.ts | 92 +++++++++ apps/sim/tools/langsmith/index.ts | 3 + apps/sim/tools/langsmith/types.ts | 79 ++++++- apps/sim/tools/langsmith/update_run.ts | 145 +++++++++++++ apps/sim/tools/registry.ts | 11 +- 8 files changed, 666 insertions(+), 17 deletions(-) create mode 100644 apps/sim/tools/langsmith/create_feedback.ts create mode 100644 apps/sim/tools/langsmith/get_run.ts create mode 100644 apps/sim/tools/langsmith/update_run.ts diff --git a/apps/sim/blocks/blocks/langsmith.ts b/apps/sim/blocks/blocks/langsmith.ts index cb8d20dff6..028fe76c32 100644 --- a/apps/sim/blocks/blocks/langsmith.ts +++ b/apps/sim/blocks/blocks/langsmith.ts @@ -23,6 +23,9 @@ export const LangsmithBlock: BlockConfig = { options: [ { label: 'Create Run', id: 'langsmith_create_run' }, { label: 'Create Runs Batch', id: 'langsmith_create_runs_batch' }, + { label: 'Update Run', id: 'langsmith_update_run' }, + { label: 'Get Run', id: 'langsmith_get_run' }, + { label: 'Create Feedback', id: 'langsmith_create_feedback' }, ], value: () => 'langsmith_create_run', }, @@ -41,13 +44,27 @@ export const LangsmithBlock: BlockConfig = { placeholder: 'Auto-generated if blank', condition: { field: 'operation', value: 'langsmith_create_run' }, }, + { + id: 'runId', + title: 'Run ID', + type: 'short-input', + placeholder: 'ID of the run to update, retrieve, or attach feedback to', + required: { + field: 'operation', + value: ['langsmith_update_run', 'langsmith_get_run', 'langsmith_create_feedback'], + }, + condition: { + field: 'operation', + value: ['langsmith_update_run', 'langsmith_get_run', 'langsmith_create_feedback'], + }, + }, { id: 'name', title: 'Name', type: 'short-input', placeholder: 'Run name', required: { field: 'operation', value: 'langsmith_create_run' }, - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, }, { id: 'run_type', @@ -78,7 +95,7 @@ export const LangsmithBlock: BlockConfig = { title: 'End Time', type: 'short-input', placeholder: '2025-01-01T12:00:30Z', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -94,7 +111,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Outputs', type: 'code', placeholder: '{"output":"value"}', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -102,7 +119,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Metadata', type: 'code', placeholder: '{"ls_model":"gpt-4"}', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -110,7 +127,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Tags', type: 'code', placeholder: '["production","workflow"]', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -150,7 +167,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Status', type: 'short-input', placeholder: 'success', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -158,7 +175,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Error', type: 'long-input', placeholder: 'Error message', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -174,7 +191,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Events', type: 'code', placeholder: '[{"event":"token","value":1}]', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -207,9 +224,66 @@ Required: id (existing run UUID), name, run_type ("tool"|"chain"|"llm"|"retrieve Common patch fields: outputs, end_time, status, error`, }, }, + { + id: 'key', + title: 'Feedback Key', + type: 'short-input', + placeholder: 'e.g. correctness, user_score', + required: { field: 'operation', value: 'langsmith_create_feedback' }, + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + }, + { + id: 'score', + title: 'Score', + type: 'short-input', + placeholder: 'e.g. 1, 0.5, 0', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + }, + { + id: 'value', + title: 'Value', + type: 'short-input', + placeholder: 'e.g. good, bad', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, + { + id: 'comment', + title: 'Comment', + type: 'long-input', + placeholder: 'Explanation for the feedback', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, + { + id: 'correction', + title: 'Correction', + type: 'code', + placeholder: '{"output":"the corrected value"}', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, + { + id: 'feedbackSourceType', + title: 'Feedback Source', + type: 'dropdown', + options: [ + { label: 'API', id: 'api' }, + { label: 'App', id: 'app' }, + { label: 'Model', id: 'model' }, + ], + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, ], tools: { - access: ['langsmith_create_run', 'langsmith_create_runs_batch'], + access: [ + 'langsmith_create_run', + 'langsmith_create_runs_batch', + 'langsmith_update_run', + 'langsmith_get_run', + 'langsmith_create_feedback', + ], config: { tool: (params) => params.operation, params: (params) => { @@ -227,6 +301,8 @@ Common patch fields: outputs, end_time, status, error`, return value } + const emptyToUndefined = (value: unknown) => (value === '' ? undefined : value) + if (params.operation === 'langsmith_create_runs_batch') { const post = parseJsonValue(params.post, 'post runs') const patch = parseJsonValue(params.patch, 'patch runs') @@ -242,6 +318,69 @@ Common patch fields: outputs, end_time, status, error`, } } + if (params.operation === 'langsmith_update_run') { + const name = emptyToUndefined(params.name) + const end_time = emptyToUndefined(params.end_time) + const outputs = parseJsonValue(params.outputs, 'outputs') + const extra = parseJsonValue(params.extra, 'metadata') + const tags = parseJsonValue(params.tags, 'tags') + const status = emptyToUndefined(params.status) + const error = emptyToUndefined(params.error) + const events = parseJsonValue(params.events, 'events') + + if ( + [name, end_time, outputs, extra, tags, status, error, events].every( + (value) => value === undefined + ) + ) { + throw new Error('Provide at least one field to update') + } + + return { + apiKey: params.apiKey, + runId: params.runId, + name, + end_time, + outputs, + extra, + tags, + status, + error, + events, + } + } + + if (params.operation === 'langsmith_get_run') { + return { + apiKey: params.apiKey, + runId: params.runId, + } + } + + if (params.operation === 'langsmith_create_feedback') { + const parseScore = (value: unknown) => { + if (value === undefined || value === null || value === '') { + return undefined + } + const parsed = Number(value) + if (Number.isNaN(parsed)) { + throw new Error(`Invalid score: "${value}" is not a number`) + } + return parsed + } + + return { + apiKey: params.apiKey, + runId: params.runId, + key: params.key, + score: parseScore(params.score), + value: params.value, + comment: params.comment, + correction: parseJsonValue(params.correction, 'correction'), + feedbackSourceType: params.feedbackSourceType || undefined, + } + } + return { apiKey: params.apiKey, id: params.id, @@ -269,6 +408,10 @@ Common patch fields: outputs, end_time, status, error`, operation: { type: 'string', description: 'Operation to perform' }, apiKey: { type: 'string', description: 'LangSmith API key' }, id: { type: 'string', description: 'Run identifier' }, + runId: { + type: 'string', + description: 'ID of the run to update, retrieve, or attach feedback to', + }, name: { type: 'string', description: 'Run name' }, run_type: { type: 'string', description: 'Run type' }, start_time: { type: 'string', description: 'Run start time (ISO)' }, @@ -287,13 +430,45 @@ Common patch fields: outputs, end_time, status, error`, events: { type: 'json', description: 'Events array' }, post: { type: 'json', description: 'Runs to ingest in batch' }, patch: { type: 'json', description: 'Runs to update in batch' }, + key: { type: 'string', description: 'Feedback metric name' }, + score: { type: 'string', description: 'Numeric score for the feedback metric' }, + value: { type: 'string', description: 'Categorical value for the feedback metric' }, + comment: { type: 'string', description: 'Comment explaining the feedback' }, + correction: { type: 'json', description: 'Corrected output for the run' }, + feedbackSourceType: { + type: 'string', + description: 'Origin of the feedback (api, app, or model)', + }, }, outputs: { - accepted: { type: 'boolean', description: 'Whether ingestion was accepted' }, - runId: { type: 'string', description: 'Run ID for single run' }, + accepted: { type: 'boolean', description: 'Whether ingestion or the update was accepted' }, + runId: { type: 'string', description: 'Run ID for single-run operations' }, runIds: { type: 'array', description: 'Run IDs for batch ingest' }, message: { type: 'string', description: 'LangSmith response message' }, messages: { type: 'array', description: 'Per-run response messages' }, + id: { type: 'string', description: 'Run ID (get run) or feedback ID (create feedback)' }, + name: { type: 'string', description: 'Run name (get run)' }, + runType: { type: 'string', description: 'Run type (get run)' }, + status: { type: 'string', description: 'Run status (get run)' }, + startTime: { type: 'string', description: 'Run start time (get run)' }, + endTime: { type: 'string', description: 'Run end time (get run)' }, + inputs: { type: 'json', description: 'Run inputs payload (get run)' }, + outputs: { type: 'json', description: 'Run outputs payload (get run)' }, + error: { type: 'string', description: 'Error details (get run)' }, + tags: { type: 'array', description: 'Tags attached to the run (get run)' }, + sessionId: { type: 'string', description: 'Project (session) ID the run belongs to (get run)' }, + traceId: { type: 'string', description: 'Trace ID (get run)' }, + parentRunId: { type: 'string', description: 'Parent run ID (get run)' }, + totalTokens: { type: 'number', description: 'Total tokens consumed by the run (get run)' }, + totalCost: { type: 'string', description: 'Total cost of the run (get run)' }, + key: { type: 'string', description: 'Feedback metric name (create feedback)' }, + score: { type: 'number', description: 'Score recorded for the feedback (create feedback)' }, + value: { + type: 'string', + description: 'Categorical value recorded for the feedback (create feedback)', + }, + comment: { type: 'string', description: 'Comment recorded for the feedback (create feedback)' }, + createdAt: { type: 'string', description: 'When the feedback was created (create feedback)' }, }, } @@ -324,11 +499,20 @@ export const LangsmithBlockMeta = { icon: LangsmithIcon, title: 'LangSmith feedback capture', prompt: - 'Build a workflow that collects user-reported agent failures from a table and forwards each as a tagged LangSmith run with the inputs and expected output for later review.', + 'Build a workflow that collects user-reported agent failures from a table and attaches each as scored LangSmith feedback on the originating run for later review.', modules: ['tables', 'agent', 'workflows'], category: 'engineering', tags: ['engineering', 'automation'], }, + { + icon: LangsmithIcon, + title: 'LangSmith run completion', + prompt: + 'Build a workflow that creates a LangSmith run when an agent step starts, then updates it with outputs, status, and end time once the step finishes so traces always show the full lifecycle.', + modules: ['agent', 'workflows'], + category: 'engineering', + tags: ['engineering', 'monitoring'], + }, { icon: LangsmithIcon, title: 'LangSmith batch run shipper', @@ -381,5 +565,12 @@ export const LangsmithBlockMeta = { content: '# Batch Export Runs\n\nShip multiple completed runs to LangSmith at once instead of one by one.\n\n## Steps\n1. Collect the runs to export, each with name, type, inputs, outputs, and timing.\n2. Assign a shared project so the runs land together.\n3. Submit them as a single batch.\n\n## Output\nReturn how many runs were exported, the project they landed in, and any runs that failed validation.', }, + { + name: 'attach-feedback-to-run', + description: + 'Attach a score, categorical value, or correction to an existing LangSmith run for evaluation.', + content: + '# Attach Feedback to a Run\n\nRecord a human or automated judgment on a run that already exists in LangSmith.\n\n## Steps\n1. Identify the run ID the feedback applies to.\n2. Choose a feedback key (e.g. "correctness", "user_score") and a score, value, or comment.\n3. Include a correction if the expected output is known.\n4. Submit the feedback.\n\n## Output\nConfirm the feedback ID and the run it was attached to.', + }, ], } as const satisfies BlockMeta diff --git a/apps/sim/tools/langsmith/create_feedback.ts b/apps/sim/tools/langsmith/create_feedback.ts new file mode 100644 index 0000000000..c1585bc0f4 --- /dev/null +++ b/apps/sim/tools/langsmith/create_feedback.ts @@ -0,0 +1,133 @@ +import { generateId } from '@sim/utils/id' +import { filterUndefined } from '@sim/utils/object' +import type { + LangsmithCreateFeedbackParams, + LangsmithCreateFeedbackResponse, +} from '@/tools/langsmith/types' +import type { ToolConfig } from '@/tools/types' + +export const langsmithCreateFeedbackTool: ToolConfig< + LangsmithCreateFeedbackParams, + LangsmithCreateFeedbackResponse +> = { + id: 'langsmith_create_feedback', + name: 'LangSmith Create Feedback', + description: 'Attach a score, correction, or comment to a LangSmith run.', + version: '1.0.0', + params: { + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'LangSmith API key', + }, + runId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'ID of the run to attach feedback to', + }, + key: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Feedback metric name (e.g. "correctness", "user_score")', + }, + score: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Numeric score for the feedback metric', + }, + value: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Categorical value for the feedback metric', + }, + comment: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Free-text comment explaining the feedback', + }, + correction: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Corrected output for the run', + }, + feedbackSourceType: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Origin of the feedback (api, app, or model)', + }, + }, + request: { + url: () => 'https://api.smith.langchain.com/feedback', + method: 'POST', + headers: (params) => ({ + 'X-Api-Key': params.apiKey, + 'Content-Type': 'application/json', + }), + body: (params) => { + const payload: Record = { + id: generateId(), + run_id: params.runId.trim(), + key: params.key, + score: params.score, + value: params.value, + comment: params.comment, + correction: params.correction, + feedback_source: params.feedbackSourceType + ? { type: params.feedbackSourceType } + : undefined, + } + + return filterUndefined(payload) + }, + }, + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + throw new Error(`LangSmith create feedback failed (${response.status}): ${errorText}`) + } + + const data = (await response.json()) as Record + + return { + success: true, + output: { + id: data.id as string, + key: data.key as string, + runId: (data.run_id as string) ?? null, + score: (data.score as number) ?? null, + value: (data.value as string | number | boolean) ?? null, + comment: (data.comment as string) ?? null, + createdAt: (data.created_at as string) ?? null, + }, + } + }, + outputs: { + id: { type: 'string', description: 'Feedback ID' }, + key: { type: 'string', description: 'Feedback metric name' }, + runId: { + type: 'string', + description: 'ID of the run the feedback was attached to', + optional: true, + }, + score: { type: 'number', description: 'Score recorded for the feedback', optional: true }, + value: { + type: 'string', + description: 'Categorical value recorded for the feedback', + optional: true, + }, + comment: { type: 'string', description: 'Comment recorded for the feedback', optional: true }, + createdAt: { + type: 'string', + description: 'When the feedback was created (ISO)', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/langsmith/create_run.ts b/apps/sim/tools/langsmith/create_run.ts index 510757ac23..0132dfdca2 100644 --- a/apps/sim/tools/langsmith/create_run.ts +++ b/apps/sim/tools/langsmith/create_run.ts @@ -1,3 +1,4 @@ +import { filterUndefined } from '@sim/utils/object' import type { LangsmithCreateRunParams, LangsmithCreateRunResponse } from '@/tools/langsmith/types' import { normalizeLangsmithRunPayload } from '@/tools/langsmith/utils' import type { ToolConfig } from '@/tools/types' @@ -141,9 +142,7 @@ export const langsmithCreateRunTool: ToolConfig< events: params.events, } - return Object.fromEntries( - Object.entries(normalizedPayload).filter(([, value]) => value !== undefined) - ) + return filterUndefined(normalizedPayload) }, }, transformResponse: async (response, params) => { diff --git a/apps/sim/tools/langsmith/get_run.ts b/apps/sim/tools/langsmith/get_run.ts new file mode 100644 index 0000000000..22582859ff --- /dev/null +++ b/apps/sim/tools/langsmith/get_run.ts @@ -0,0 +1,92 @@ +import type { LangsmithGetRunParams, LangsmithGetRunResponse } from '@/tools/langsmith/types' +import type { ToolConfig } from '@/tools/types' + +export const langsmithGetRunTool: ToolConfig = { + id: 'langsmith_get_run', + name: 'LangSmith Get Run', + description: 'Retrieve a single LangSmith run by ID.', + version: '1.0.0', + params: { + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'LangSmith API key', + }, + runId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'ID of the run to retrieve', + }, + }, + request: { + url: (params) => `https://api.smith.langchain.com/runs/${params.runId.trim()}`, + method: 'GET', + headers: (params) => ({ + 'X-Api-Key': params.apiKey, + }), + }, + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + throw new Error(`LangSmith get run failed (${response.status}): ${errorText}`) + } + + const data = (await response.json()) as Record + + return { + success: true, + output: { + id: data.id as string, + runId: data.id as string, + name: data.name as string, + runType: data.run_type as string, + status: (data.status as string) ?? null, + startTime: (data.start_time as string) ?? null, + endTime: (data.end_time as string) ?? null, + inputs: (data.inputs as Record) ?? null, + outputs: (data.outputs as Record) ?? null, + error: (data.error as string) ?? null, + tags: (data.tags as string[]) ?? [], + sessionId: (data.session_id as string) ?? null, + traceId: (data.trace_id as string) ?? null, + parentRunId: (data.parent_run_id as string) ?? null, + totalTokens: (data.total_tokens as number) ?? null, + totalCost: (data.total_cost as string) ?? null, + }, + } + }, + outputs: { + id: { type: 'string', description: 'Run ID' }, + runId: { + type: 'string', + description: 'Run ID (alias of id, for consistency with other operations)', + }, + name: { type: 'string', description: 'Run name' }, + runType: { + type: 'string', + description: 'Run type (tool, chain, llm, retriever, embedding, prompt, parser)', + }, + status: { type: 'string', description: 'Run status', optional: true }, + startTime: { type: 'string', description: 'Run start time (ISO)', optional: true }, + endTime: { type: 'string', description: 'Run end time (ISO)', optional: true }, + inputs: { type: 'json', description: 'Run inputs payload', optional: true }, + outputs: { type: 'json', description: 'Run outputs payload', optional: true }, + error: { type: 'string', description: 'Error details, if the run failed', optional: true }, + tags: { type: 'array', description: 'Tags attached to the run', items: { type: 'string' } }, + sessionId: { + type: 'string', + description: 'Project (session) ID the run belongs to', + optional: true, + }, + traceId: { type: 'string', description: 'Trace ID', optional: true }, + parentRunId: { type: 'string', description: 'Parent run ID', optional: true }, + totalTokens: { + type: 'number', + description: 'Total tokens consumed by the run', + optional: true, + }, + totalCost: { type: 'string', description: 'Total cost of the run', optional: true }, + }, +} diff --git a/apps/sim/tools/langsmith/index.ts b/apps/sim/tools/langsmith/index.ts index f173b10a21..5f7e6019a8 100644 --- a/apps/sim/tools/langsmith/index.ts +++ b/apps/sim/tools/langsmith/index.ts @@ -1,2 +1,5 @@ +export { langsmithCreateFeedbackTool } from '@/tools/langsmith/create_feedback' export { langsmithCreateRunTool } from '@/tools/langsmith/create_run' export { langsmithCreateRunsBatchTool } from '@/tools/langsmith/create_runs_batch' +export { langsmithGetRunTool } from '@/tools/langsmith/get_run' +export { langsmithUpdateRunTool } from '@/tools/langsmith/update_run' diff --git a/apps/sim/tools/langsmith/types.ts b/apps/sim/tools/langsmith/types.ts index 860a72e303..ff28916339 100644 --- a/apps/sim/tools/langsmith/types.ts +++ b/apps/sim/tools/langsmith/types.ts @@ -57,4 +57,81 @@ export interface LangsmithCreateRunsBatchResponse extends ToolResponse { } } -export type LangsmithResponse = LangsmithCreateRunResponse | LangsmithCreateRunsBatchResponse +export interface LangsmithUpdateRunParams { + apiKey: string + runId: string + name?: string + end_time?: string + outputs?: Record + extra?: Record + tags?: string[] + status?: string + error?: string + events?: Record[] +} + +export interface LangsmithUpdateRunResponse extends ToolResponse { + output: { + accepted: boolean + runId: string + message: string | null + } +} + +export interface LangsmithGetRunParams { + apiKey: string + runId: string +} + +export interface LangsmithGetRunResponse extends ToolResponse { + output: { + id: string + runId: string + name: string + runType: string + status: string | null + startTime: string | null + endTime: string | null + inputs: Record | null + outputs: Record | null + error: string | null + tags: string[] + sessionId: string | null + traceId: string | null + parentRunId: string | null + totalTokens: number | null + totalCost: string | null + } +} + +export type LangsmithFeedbackSourceType = 'api' | 'app' | 'model' + +export interface LangsmithCreateFeedbackParams { + apiKey: string + runId: string + key: string + score?: number + value?: string + comment?: string + correction?: Record + feedbackSourceType?: LangsmithFeedbackSourceType +} + +export interface LangsmithCreateFeedbackResponse extends ToolResponse { + output: { + id: string + key: string + runId: string | null + score: number | null + value: string | number | boolean | null + comment: string | null + createdAt: string | null + } +} + +export type LangsmithResponse = + | LangsmithCreateRunResponse + | LangsmithCreateRunsBatchResponse + | LangsmithUpdateRunResponse + | LangsmithGetRunResponse + | LangsmithCreateFeedbackResponse diff --git a/apps/sim/tools/langsmith/update_run.ts b/apps/sim/tools/langsmith/update_run.ts new file mode 100644 index 0000000000..28fa402860 --- /dev/null +++ b/apps/sim/tools/langsmith/update_run.ts @@ -0,0 +1,145 @@ +import { filterUndefined } from '@sim/utils/object' +import type { LangsmithUpdateRunParams, LangsmithUpdateRunResponse } from '@/tools/langsmith/types' +import type { ToolConfig } from '@/tools/types' + +export const langsmithUpdateRunTool: ToolConfig< + LangsmithUpdateRunParams, + LangsmithUpdateRunResponse +> = { + id: 'langsmith_update_run', + name: 'LangSmith Update Run', + description: 'Patch an existing LangSmith run with outputs, status, or timing once it completes.', + version: '1.0.0', + params: { + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'LangSmith API key', + }, + runId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'ID of the run to update', + }, + name: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Corrected run name', + }, + end_time: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Run end time in ISO-8601 format', + }, + outputs: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Outputs payload', + }, + extra: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Additional metadata (extra)', + }, + tags: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Array of tag strings', + }, + status: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Run status', + }, + error: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Error details', + }, + events: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Structured events array', + }, + }, + request: { + url: (params) => `https://api.smith.langchain.com/runs/${params.runId.trim()}`, + method: 'PATCH', + headers: (params) => ({ + 'X-Api-Key': params.apiKey, + 'Content-Type': 'application/json', + }), + body: (params) => { + const emptyToUndefined = (value?: string) => (value === '' ? undefined : value) + + const payload: Record = { + name: emptyToUndefined(params.name), + end_time: emptyToUndefined(params.end_time), + outputs: params.outputs, + extra: params.extra, + tags: params.tags, + status: emptyToUndefined(params.status), + error: emptyToUndefined(params.error), + events: params.events, + } + + const filtered = filterUndefined(payload) + if (Object.keys(filtered).length === 0) { + throw new Error('Provide at least one field to update') + } + + return filtered + }, + }, + transformResponse: async (response, params) => { + if (!response.ok) { + const errorText = await response.text() + throw new Error(`LangSmith update run failed (${response.status}): ${errorText}`) + } + + const responseText = await response.text() + let message: string | null = null + if (responseText) { + try { + const data = JSON.parse(responseText) as Record + message = typeof data.message === 'string' ? data.message : null + } catch { + // Response body isn't JSON (e.g. empty object or plain text) — no message to surface + } + } + + return { + success: true, + output: { + accepted: true, + runId: params?.runId.trim() ?? '', + message, + }, + } + }, + outputs: { + accepted: { + type: 'boolean', + description: 'Whether the run update was accepted', + }, + runId: { + type: 'string', + description: 'ID of the run that was updated', + }, + message: { + type: 'string', + description: 'Response message from LangSmith, if provided', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 4adbd38fb4..8b292a4a2e 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -1910,7 +1910,13 @@ import { knowledgeUploadChunkTool, knowledgeUpsertDocumentTool, } from '@/tools/knowledge' -import { langsmithCreateRunsBatchTool, langsmithCreateRunTool } from '@/tools/langsmith' +import { + langsmithCreateFeedbackTool, + langsmithCreateRunsBatchTool, + langsmithCreateRunTool, + langsmithGetRunTool, + langsmithUpdateRunTool, +} from '@/tools/langsmith' import { latexCompileTool, latexGetPackageTool, @@ -7250,6 +7256,9 @@ export const tools: Record = { linear_list_project_statuses: linearListProjectStatusesTool, langsmith_create_run: langsmithCreateRunTool, langsmith_create_runs_batch: langsmithCreateRunsBatchTool, + langsmith_update_run: langsmithUpdateRunTool, + langsmith_get_run: langsmithGetRunTool, + langsmith_create_feedback: langsmithCreateFeedbackTool, latex_compile: latexCompileTool, latex_get_package: latexGetPackageTool, latex_list_fonts: latexListFontsTool,