improvement(media-gen): retire vision block, add hosted key for fal ai for image/video gen, search visibility in cmd-k (#4684)

* improvement(media-gen): retire vision block, add hosted key for fal ai for image/video gen

* address comments
This commit is contained in:
Vikhyath Mondreti
2026-05-23 12:54:14 -07:00
committed by GitHub
parent 6b1210b8af
commit 4fab03c595
33 changed files with 934 additions and 111 deletions
@@ -280,7 +280,7 @@ Get the execution timeline for an Azure DevOps build — every stage, job, and t
| ↳ `warningCount` | number | Number of warnings |
| ↳ `startTime` | string | ISO 8601 start timestamp |
| ↳ `finishTime` | string | ISO 8601 finish timestamp |
| ↳ `failedRecords` | array | Subset of records where result === "failed" — use logId to fetch logs |
| ↳ `failedRecords` | array | Subset of records where result is failed, partiallySucceeded, or succeededWithIssues — use logId to fetch logs |
| ↳ `id` | string | Record GUID |
| ↳ `name` | string | Step name |
| ↳ `type` | string | Stage \| Phase \| Job \| Task |
@@ -333,7 +333,8 @@ Execute a WIQL query to search for work items in Azure DevOps and return full fi
| --------- | ---- | ----------- |
| `content` | string | Human-readable summary of matching work items |
| `metadata` | object | Work items metadata |
| ↳ `count` | number | Number of work items returned |
| ↳ `count` | number | Number of work items returned \(after hydration\) |
| ↳ `totalMatched` | number | Total number of work items matched by the WIQL query before hydration |
| ↳ `workItems` | array | Array of work item details |
| ↳ `id` | number | Work item ID |
| ↳ `title` | string | Work item title |
@@ -372,7 +373,7 @@ Fetch full details of a single work item by ID from Azure DevOps, including titl
### `azure_devops_get_work_items_batch`
Fetch full details for multiple work items by ID from Azure DevOps in a single call. Pass comma-separated IDs (e.g.
Fetch full details for multiple work items by ID from Azure DevOps. Pass comma-separated IDs (e.g.
#### Input
@@ -380,7 +381,7 @@ Fetch full details for multiple work items by ID from Azure DevOps in a single c
| --------- | ---- | -------- | ----------- |
| `organization` | string | Yes | Azure DevOps organization name |
| `project` | string | Yes | Azure DevOps project name |
| `ids` | string | Yes | Comma-separated work item IDs to fetch \(e.g. "123,456,789"\). Maximum 200 IDs. |
| `ids` | string | Yes | Comma-separated work item IDs to fetch \(e.g. "123,456,789"\). Lists longer than 200 IDs are chunked automatically. |
#### Output
@@ -389,6 +390,7 @@ Fetch full details for multiple work items by ID from Azure DevOps in a single c
| `content` | string | Human-readable summary of the fetched work items |
| `metadata` | object | Work items metadata |
| ↳ `count` | number | Number of work items returned |
| ↳ `totalRequested` | number | Total number of IDs requested \(across all chunks\) |
| ↳ `workItems` | array | Array of work item details |
| ↳ `id` | number | Work item ID |
| ↳ `title` | string | Work item title |
@@ -195,7 +195,6 @@
"upstash",
"vercel",
"video_generator",
"vision",
"wealthbox",
"webflow",
"whatsapp",
@@ -1,60 +0,0 @@
---
title: Vision
description: Analyze images with vision models
---
import { BlockInfoCard } from "@/components/ui/block-info-card"
<BlockInfoCard
type="vision_v2"
color="#4D5FFF"
/>
{/* MANUAL-CONTENT-START:intro */}
Vision is a tool that allows you to analyze images with vision models.
With Vision, you can:
- **Analyze images**: Analyze images with vision models
- **Extract text**: Extract text from images
- **Identify objects**: Identify objects in images
- **Describe images**: Describe images in detail
- **Generate images**: Generate images from text
In Sim, the Vision integration enables your agents to analyze images with vision models as part of their workflows. This allows for powerful automation scenarios that require analyzing images with vision models. Your agents can analyze images with vision models, extract text from images, identify objects in images, describe images in detail, and generate images from text. This integration bridges the gap between your AI workflows and your image analysis needs, enabling more sophisticated and image-centric automations. By connecting Sim with Vision, you can create agents that stay current with the latest information, provide more accurate responses, and deliver more value to users - all without requiring manual intervention or custom code.
{/* MANUAL-CONTENT-END */}
## Usage Instructions
Integrate Vision into the workflow. Can analyze images with vision models.
## Tools
### `vision_tool`
#### Input
| Parameter | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `apiKey` | string | Yes | API key for the selected model provider |
| `imageUrl` | string | No | Publicly accessible image URL |
| `imageFile` | file | No | Image file to analyze |
| `model` | string | No | Vision model to use \(gpt-4o, claude-3-opus-20240229, etc\) |
| `prompt` | string | No | Custom prompt for image analysis |
#### Output
| Parameter | Type | Description |
| --------- | ---- | ----------- |
| `content` | string | The analyzed content and description of the image |
| `model` | string | The vision model that was used for analysis |
| `tokens` | number | Total tokens used for the analysis |
| `usage` | object | Detailed token usage breakdown |
| ↳ `input_tokens` | number | Tokens used for input processing |
| ↳ `output_tokens` | number | Tokens used for response generation |
| ↳ `total_tokens` | number | Total tokens consumed |
@@ -31,7 +31,7 @@ Trigger workflow when an Azure DevOps build fails, is canceled, or partially suc
| `branch` | string | Source branch name \(refs/heads/ prefix stripped\) |
| `commitSha` | string | Source commit SHA |
| `triggeredBy` | string | Display name of the person who triggered the build |
| `triggeredByEmail` | string | Email/unique name of the person who triggered the build |
| `triggeredByEmail` | string | Email/unique name of the person who triggered the build, or null if not set |
| `startTime` | string | Build start time \(ISO 8601\) |
| `finishTime` | string | Build finish time \(ISO 8601\) |
| `buildUrl` | string | API URL for the build resource |
@@ -72,12 +72,12 @@ Trigger workflow when a work item is created in Azure DevOps
| `workItemType` | string | Work item type for Basic process \(e.g. Issue, Task, Epic\) |
| `title` | string | Work item title |
| `state` | string | Work item state for Basic process \(e.g. To Do, Doing, Done\) |
| `createdBy` | string | Display name of the creator |
| `assignedTo` | string | Assignee display name, or empty string if unassigned |
| `createdBy` | string | Display name of the creator, or null if not set |
| `assignedTo` | string | Assignee display name, or null if unassigned |
| `priority` | number | Priority \(1–4\), or 0 if not set |
| `areaPath` | string | Area path |
| `iterationPath` | string | Iteration path |
| `description` | string | Work item description \(HTML\), or empty string if not set |
| `description` | string | Work item description \(HTML\), or null if not set |
| `projectName` | string | Azure DevOps project name |
| `workItemUrl` | string | API URL for the work item resource |
@@ -57,7 +57,6 @@ import {
EvernoteIcon,
ExaAIIcon,
ExtendIcon,
EyeIcon,
FathomIcon,
FindymailIcon,
FirecrawlIcon,
@@ -405,7 +404,6 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
upstash: UpstashIcon,
vercel: VercelIcon,
video_generator_v3: VideoIcon,
vision_v2: EyeIcon,
wealthbox: WealthboxIcon,
webflow: WebflowIcon,
whatsapp: WhatsAppIcon,
@@ -1938,7 +1938,7 @@
},
{
"name": "Get Work Items Batch",
"description": "Fetch full details for multiple work items by ID from Azure DevOps in a single call. Pass comma-separated IDs (e.g. "
"description": "Fetch full details for multiple work items by ID from Azure DevOps. Pass comma-separated IDs (e.g. "
},
{
"name": "Create Work Item",
@@ -4156,10 +4156,6 @@
"name": "Fetch",
"description": "Fetch and parse a file from a URL with optional custom headers."
},
{
"name": "Get",
"description": "Get a workspace file object from a selected file or canonical workspace file ID."
},
{
"name": "Write",
"description": "Create a new workspace file. If a file with the same name already exists, a numeric suffix is added (e.g., "
@@ -4169,7 +4165,7 @@
"description": "Append content to an existing workspace file. The file must already exist. Content is added to the end of the file."
}
],
"operationCount": 5,
"operationCount": 4,
"triggers": [],
"triggerCount": 0,
"authType": "none",
@@ -14253,24 +14249,6 @@
"integrationTypes": ["ai", "design"],
"tags": ["video-generation", "llm"]
},
{
"type": "vision_v2",
"slug": "vision",
"name": "Vision",
"description": "Analyze images with vision models",
"longDescription": "Integrate Vision into the workflow. Can analyze images with vision models.",
"bgColor": "#4D5FFF",
"iconName": "EyeIcon",
"docsUrl": "https://docs.sim.ai/tools/vision",
"operations": [],
"operationCount": 0,
"triggers": [],
"triggerCount": 0,
"authType": "api-key",
"category": "tools",
"integrationTypes": ["ai", "documents"],
"tags": ["llm", "document-processing", "ocr"]
},
{
"type": "wealthbox",
"slug": "wealthbox",
+15
View File
@@ -32,6 +32,7 @@ import {
} from '@/lib/core/utils/stream-limits'
import { getBaseUrl } from '@/lib/core/utils/urls'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing'
const logger = createLogger('ImageProxyAPI')
const MAX_IMAGE_BYTES = 25 * 1024 * 1024
@@ -53,6 +54,7 @@ interface GeneratedImageResult {
revisedPrompt?: string
seed?: number
jobId?: string
falaiCost?: FalAICostMetadata
}
interface StoredImageResponse {
@@ -72,6 +74,8 @@ interface StoredImageResponse {
jobId?: string
contentType: string
}
__falaiCostDollars?: number
__falaiBilling?: FalAICostMetadata
}
export const POST = withRouteHandler(async (request: NextRequest) => {
@@ -935,6 +939,13 @@ async function generateWithFalAI(
revisedPrompt: getStringProperty(resultData, 'revised_prompt'),
seed: getNumberProperty(resultData, 'seed'),
jobId: falRequestId,
falaiCost: body.useHostedCostTracking
? await getFalAICostMetadata({
apiKey,
endpointId: modelConfig.endpoint,
requestId: falRequestId,
})
: undefined,
}
}
@@ -992,6 +1003,8 @@ async function storeGeneratedImage(
jobId: imageResult.jobId,
contentType: imageResult.contentType,
},
__falaiCostDollars: imageResult.falaiCost?.costDollars,
__falaiBilling: imageResult.falaiCost,
}
}
@@ -1024,5 +1037,7 @@ async function storeGeneratedImage(
jobId: imageResult.jobId,
contentType: imageResult.contentType,
},
__falaiCostDollars: imageResult.falaiCost?.costDollars,
__falaiBilling: imageResult.falaiCost,
}
}
+24 -1
View File
@@ -17,6 +17,7 @@ import {
readResponseToBufferWithLimit,
} from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import type { UserFile } from '@/executor/types'
@@ -138,6 +139,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let height: number | undefined
let jobId: string | undefined
let actualDuration: number | undefined
let falaiCost: FalAICostMetadata | undefined
if (body.visualReference) {
const denied = await assertToolFileAccess(
@@ -236,6 +238,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
resolution,
body.promptOptimizer,
body.generateAudio,
body.useHostedCostTracking === true,
requestId,
logger
)
@@ -244,6 +247,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
height = result.height
jobId = result.jobId
actualDuration = result.duration
falaiCost = result.falaiCost
} else {
return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 })
}
@@ -301,6 +305,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
provider,
model: model || 'default',
jobId,
__falaiCostDollars: falaiCost?.costDollars,
__falaiBilling: falaiCost,
})
}
@@ -333,6 +339,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
provider,
model: model || 'default',
jobId,
__falaiCostDollars: falaiCost?.costDollars,
__falaiBilling: falaiCost,
})
} catch (error) {
logger.error(`[${requestId}] Video proxy error:`, error)
@@ -1167,9 +1175,17 @@ async function generateWithFalAI(
resolution: string | undefined,
promptOptimizer: boolean | undefined,
generateAudio: boolean | undefined,
useHostedCostTracking: boolean,
requestId: string,
logger: ReturnType<typeof createLogger>
): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> {
): Promise<{
buffer: Buffer
width: number
height: number
jobId: string
duration: number
falaiCost?: FalAICostMetadata
}> {
logger.info(`[${requestId}] Starting Fal.ai generation with model: ${model}`)
const modelConfig = FALAI_MODEL_CONFIGS[model]
@@ -1312,6 +1328,13 @@ async function generateWithFalAI(
height,
jobId: requestIdFal,
duration: getNumberProperty(videoOutput, 'duration') || duration || 5,
falaiCost: useHostedCostTracking
? await getFalAICostMetadata({
apiKey,
endpointId: modelConfig.endpoint,
requestId: requestIdFal,
})
: undefined,
}
}
@@ -23,6 +23,7 @@ import {
FireworksIcon,
GeminiIcon,
GoogleIcon,
ImageIcon,
JinaAIIcon,
LinkupIcon,
MistralIcon,
@@ -85,6 +86,13 @@ const PROVIDERS: {
description: 'LLM calls',
placeholder: 'Enter your Fireworks API key',
},
{
id: 'falai',
name: 'Fal.ai',
icon: ImageIcon,
description: 'Image and video generation',
placeholder: 'Enter your Fal.ai API key',
},
{
id: 'firecrawl',
name: 'Firecrawl',
@@ -204,6 +204,12 @@ interface BlockData {
position: { x: number; y: number }
}
interface AddBlockFromToolbarDetail {
type?: unknown
enableTriggerMode?: unknown
presetOperation?: unknown
}
/**
* Main workflow canvas content component.
* Renders the ReactFlow canvas with blocks, edges, and all interactive features.
@@ -2004,7 +2010,7 @@ const WorkflowContent = React.memo(
/** Handles toolbar block click events to add blocks to the canvas. */
useEffect(() => {
const handleAddBlockFromToolbar = (event: CustomEvent) => {
const handleAddBlockFromToolbar = (event: CustomEvent<AddBlockFromToolbarDetail>) => {
// Check if user has permission to interact with blocks
if (!effectivePermissions.canEdit) {
return
@@ -2012,7 +2018,7 @@ const WorkflowContent = React.memo(
const { type, enableTriggerMode, presetOperation } = event.detail
if (!type) return
if (typeof type !== 'string' || !type) return
if (type === 'connectionBlock') return
const basePosition = getViewportCenter()
@@ -2070,8 +2076,10 @@ const WorkflowContent = React.memo(
undefined,
undefined,
autoConnectEdge,
enableTriggerMode,
presetOperation ? { operation: presetOperation } : undefined
enableTriggerMode === true,
typeof presetOperation === 'string' && presetOperation
? { operation: presetOperation }
: undefined
)
}
@@ -206,7 +206,10 @@ export function SearchModal({
type === 'trigger' && block.config ? hasTriggerCapability(block.config) : false
window.dispatchEvent(
new CustomEvent('add-block-from-toolbar', {
detail: { type: block.type, enableTriggerMode },
detail: {
type: block.type,
enableTriggerMode,
},
})
)
captureEvent(posthogRef.current, 'search_result_selected', {
@@ -376,12 +379,12 @@ export function SearchModal({
const filteredBlocks = useMemo(() => {
if (!isOnWorkflowPage) return []
return filterAndSort(blocks, (b) => `${b.name} block-${b.id}`, deferredSearch)
return filterAndSort(blocks, (b) => b.searchValue ?? `${b.name} block-${b.id}`, deferredSearch)
}, [isOnWorkflowPage, blocks, deferredSearch])
const filteredTools = useMemo(() => {
if (!isOnWorkflowPage) return []
return filterAndSort(tools, (t) => `${t.name} tool-${t.id}`, deferredSearch)
return filterAndSort(tools, (t) => t.searchValue ?? `${t.name} tool-${t.id}`, deferredSearch)
}, [isOnWorkflowPage, tools, deferredSearch])
const filteredTriggers = useMemo(() => {
@@ -104,6 +104,7 @@ import { SIDEBAR_WIDTH } from '@/stores/constants'
import { useFolderStore } from '@/stores/folders/store'
import { useSearchModalStore } from '@/stores/modals/search/store'
import { useMothershipDraftsStore } from '@/stores/mothership-drafts/store'
import { useProvidersStore } from '@/stores/providers'
import { useSidebarStore } from '@/stores/sidebar/store'
const logger = createLogger('Sidebar')
@@ -360,10 +361,18 @@ export const Sidebar = memo(function Sidebar() {
const { config: permissionConfig, filterBlocks } = usePermissionConfig()
const { navigateToSettings, getSettingsHref } = useSettingsNavigation()
const initializeSearchData = useSearchModalStore((state) => state.initializeData)
const providers = useProvidersStore((state) => state.providers)
const providerModelSignature = useMemo(
() =>
Object.values(providers)
.map((provider) => provider.models.join('\x00'))
.join('\x01'),
[providers]
)
useEffect(() => {
initializeSearchData(filterBlocks)
}, [initializeSearchData, filterBlocks])
}, [initializeSearchData, filterBlocks, providerModelSignature])
const setSidebarWidth = useSidebarStore((state) => state.setSidebarWidth)
const isCollapsed = useSidebarStore((state) => state.isCollapsed)
+59
View File
@@ -734,6 +734,65 @@ describe.concurrent('Blocks Module', () => {
expect(temperatureSubBlock?.min).toBe(0)
expect(temperatureSubBlock?.max).toBe(2)
})
it('should mark generator provider dropdowns as command-searchable', () => {
const imageGeneratorBlock = getBlock('image_generator_v2')
const videoGeneratorBlock = getBlock('video_generator_v3')
const imageProviderSubBlock = imageGeneratorBlock?.subBlocks.find(
(sb) => sb.id === 'provider'
)
const videoProviderSubBlock = videoGeneratorBlock?.subBlocks.find(
(sb) => sb.id === 'provider'
)
const imageProviderOptions = imageProviderSubBlock?.options
const videoProviderOptions = videoProviderSubBlock?.options
expect(imageGeneratorBlock?.hideFromToolbar).not.toBe(true)
expect(videoGeneratorBlock?.hideFromToolbar).not.toBe(true)
expect(imageProviderSubBlock?.commandSearchable).toBe(true)
expect(videoProviderSubBlock?.commandSearchable).toBe(true)
expect(imageProviderSubBlock?.value?.()).toBe('falai')
expect(videoProviderSubBlock?.value?.()).toBe('falai')
expect(
Array.isArray(imageProviderOptions) ? imageProviderOptions.map((option) => option.id) : []
).toContain('falai')
expect(
Array.isArray(videoProviderOptions) ? videoProviderOptions.map((option) => option.id) : []
).toContain('falai')
expect(getBlock('image_generator')?.hideFromToolbar).toBe(true)
expect(getBlock('video_generator_v2')?.hideFromToolbar).toBe(true)
})
it('should mark the agent model combobox as command-searchable', () => {
const agentBlock = getBlock('agent')
const modelSubBlock = agentBlock?.subBlocks.find((sb) => sb.id === 'model')
expect(agentBlock?.hideFromToolbar).not.toBe(true)
expect(modelSubBlock?.type).toBe('combobox')
expect(modelSubBlock?.commandSearchable).toBe(true)
})
it('should hide generator API keys on hosted only for Fal.ai providers', () => {
for (const blockType of ['image_generator_v2', 'video_generator_v3']) {
const block = getBlock(blockType)
const apiKeySubBlocks = block?.subBlocks.filter((sb) => sb.id === 'apiKey') ?? []
const falApiKeySubBlock = apiKeySubBlocks.find(
(sb) => sb.condition?.field === 'provider' && sb.condition.value === 'falai'
)
const nonFalApiKeySubBlock = apiKeySubBlocks.find(
(sb) =>
sb.condition?.field === 'provider' &&
sb.condition.value === 'falai' &&
sb.condition.not === true
)
expect(falApiKeySubBlock?.hideWhenHosted).toBe(true)
expect(nonFalApiKeySubBlock).toBeDefined()
expect(nonFalApiKeySubBlock?.hideWhenHosted).not.toBe(true)
}
})
})
describe('Block Consistency', () => {
+1
View File
@@ -133,6 +133,7 @@ Return ONLY the JSON array.`,
required: true,
defaultValue: 'claude-sonnet-4-6',
options: getModelOptions,
commandSearchable: true,
},
{
id: 'attachmentFiles',
+16 -4
View File
@@ -333,7 +333,8 @@ export const ImageGeneratorV2Block: BlockConfig<ImageGenerationResponse> = {
{ label: 'Google Gemini', id: 'gemini' },
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
],
value: () => 'openai',
commandSearchable: true,
value: () => 'falai',
},
{
id: 'model',
@@ -812,6 +813,18 @@ export const ImageGeneratorV2Block: BlockConfig<ImageGenerationResponse> = {
placeholder: 'Enter your provider API key',
password: true,
connectionDroppable: false,
hideWhenHosted: true,
condition: { field: 'provider', value: 'falai' },
},
{
id: 'apiKey',
title: 'API Key',
type: 'short-input',
required: true,
placeholder: 'Enter your provider API key',
password: true,
connectionDroppable: false,
condition: { field: 'provider', value: 'falai', not: true },
},
],
tools: {
@@ -819,14 +832,13 @@ export const ImageGeneratorV2Block: BlockConfig<ImageGenerationResponse> = {
config: {
tool: () => 'image_generate',
params: (params) => {
if (!params.apiKey) {
const provider = params.provider || 'openai'
if (provider !== 'falai' && !params.apiKey) {
throw new Error('API key is required')
}
if (!params.prompt) {
throw new Error('Prompt is required')
}
const provider = params.provider || 'openai'
const defaultModel =
provider === 'gemini'
? 'gemini-3.1-flash-image-preview'
+25 -2
View File
@@ -98,7 +98,7 @@ export const VideoGeneratorBlock: BlockConfig<VideoBlockResponse> = {
{ label: 'MiniMax Hailuo', id: 'minimax' },
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
],
value: () => 'runway',
value: () => 'falai',
required: true,
},
@@ -776,6 +776,17 @@ export const VideoGeneratorBlock: BlockConfig<VideoBlockResponse> = {
placeholder: 'Enter your provider API key',
password: true,
required: true,
hideWhenHosted: true,
condition: { field: 'provider', value: 'falai' },
},
{
id: 'apiKey',
title: 'API Key',
type: 'short-input',
placeholder: 'Enter your provider API key',
password: true,
required: true,
condition: { field: 'provider', value: 'falai', not: true },
},
],
@@ -894,7 +905,8 @@ export const VideoGeneratorV2Block: BlockConfig<VideoBlockResponse> = {
{ label: 'MiniMax Hailuo', id: 'minimax' },
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
],
value: () => 'runway',
commandSearchable: true,
value: () => 'falai',
required: true,
},
{
@@ -1537,6 +1549,17 @@ export const VideoGeneratorV2Block: BlockConfig<VideoBlockResponse> = {
placeholder: 'Enter your provider API key',
password: true,
required: true,
hideWhenHosted: true,
condition: { field: 'provider', value: 'falai' },
},
{
id: 'apiKey',
title: 'API Key',
type: 'short-input',
placeholder: 'Enter your provider API key',
password: true,
required: true,
condition: { field: 'provider', value: 'falai', not: true },
},
],
tools: {
+1 -1
View File
@@ -109,7 +109,7 @@ export const VisionV2Block: BlockConfig<VisionResponse> = {
type: 'vision_v2',
name: 'Vision',
description: 'Analyze images with vision models',
hideFromToolbar: false,
hideFromToolbar: true,
tools: {
access: ['vision_tool_v2'],
config: {
+2
View File
@@ -305,6 +305,8 @@ export interface SubBlockConfig {
multiSelect?: boolean
// Combobox specific: Enable search input in dropdown
searchable?: boolean
/** Dropdown-specific: include static options as Cmd K search entries that preset this subblock. */
commandSearchable?: boolean
// Wand configuration for AI assistance
wandConfig?: {
enabled: boolean
+1
View File
@@ -7,6 +7,7 @@ export const byokProviderIdSchema = z.enum([
'google',
'mistral',
'fireworks',
'falai',
'firecrawl',
'exa',
'serper',
@@ -37,6 +37,7 @@ export const imageToolBodySchema = z
workflowId: z.string().optional(),
executionId: z.string().optional(),
userId: z.string().optional(),
useHostedCostTracking: z.boolean().optional(),
})
.passthrough()
@@ -29,6 +29,7 @@ export const videoToolBodySchema = z
workflowId: z.string().optional(),
executionId: z.string().optional(),
userId: z.string().optional(),
useHostedCostTracking: z.boolean().optional(),
})
.passthrough()
+205
View File
@@ -0,0 +1,205 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { sleep } from '@sim/utils/helpers'
export const FALAI_HOSTED_KEY_MARKUP_MULTIPLIER = 1.5
export const FALAI_IMAGE_FALLBACK_PROVIDER_COST_DOLLARS = 0.05
export const FALAI_VIDEO_FALLBACK_PROVIDER_COST_DOLLARS = 0.25
const FALAI_BILLING_EVENT_ATTEMPTS = 2
const FALAI_BILLING_EVENT_RETRY_MS = 500
const logger = createLogger('FalAIPricing')
export interface FalAICostMetadata {
endpointId: string
requestId: string
costDollars: number
source: 'billing_events' | 'historical_estimate' | 'fallback_floor'
outputUnits?: number | null
unitPrice?: number | null
percentDiscount?: number | null
currency?: string
error?: string
}
interface FalAIBillingEvent {
request_id: string
endpoint_id: string
output_units: number | null
unit_price: number | null
percent_discount: number | null
cost_estimate_nano_usd: number
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
function getNumber(value: unknown): number | undefined {
return typeof value === 'number' && Number.isFinite(value) ? value : undefined
}
function getFalAIFallbackProviderCostDollars(endpointId: string): number {
const normalizedEndpointId = endpointId.toLowerCase()
const isImageEndpoint =
normalizedEndpointId.includes('image') ||
normalizedEndpointId.includes('nano-banana') ||
normalizedEndpointId.includes('seedream') ||
normalizedEndpointId.includes('flux') ||
normalizedEndpointId.includes('grok-imagine')
return isImageEndpoint
? FALAI_IMAGE_FALLBACK_PROVIDER_COST_DOLLARS
: FALAI_VIDEO_FALLBACK_PROVIDER_COST_DOLLARS
}
function parseBillingEvent(value: unknown): FalAIBillingEvent | undefined {
if (!isRecord(value)) return undefined
const requestId = value.request_id
const endpointId = value.endpoint_id
const costEstimateNanoUsd = getNumber(value.cost_estimate_nano_usd)
if (typeof requestId !== 'string' || typeof endpointId !== 'string') return undefined
if (costEstimateNanoUsd === undefined) return undefined
return {
request_id: requestId,
endpoint_id: endpointId,
output_units: getNumber(value.output_units) ?? null,
unit_price: getNumber(value.unit_price) ?? null,
percent_discount: getNumber(value.percent_discount) ?? null,
cost_estimate_nano_usd: costEstimateNanoUsd,
}
}
async function fetchFalAIBillingEvent(
apiKey: string,
requestId: string
): Promise<FalAIBillingEvent | undefined> {
const url = new URL('https://api.fal.ai/v1/models/billing-events')
url.searchParams.set('request_id', requestId)
url.searchParams.set('limit', '1')
let response: Response
try {
response = await fetch(url, {
headers: {
Authorization: `Key ${apiKey}`,
},
})
} catch (error) {
logger.warn('Failed to fetch Fal.ai billing event', {
requestId,
error: getErrorMessage(error, 'Unknown error'),
})
return undefined
}
if (!response.ok) return undefined
const data = await response.json().catch((error) => {
logger.warn('Failed to parse Fal.ai billing event response', {
requestId,
error: getErrorMessage(error, 'Unknown error'),
})
return undefined
})
if (!isRecord(data) || !Array.isArray(data.billing_events)) return undefined
return data.billing_events.map(parseBillingEvent).find(Boolean)
}
async function estimateFalAICallCost(
apiKey: string,
endpointId: string
): Promise<{ costDollars?: number; error?: string }> {
let response: Response
try {
response = await fetch('https://api.fal.ai/v1/models/pricing/estimate', {
method: 'POST',
headers: {
Authorization: `Key ${apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
estimate_type: 'historical_api_price',
endpoints: {
[endpointId]: {
call_quantity: 1,
},
},
}),
})
} catch (error) {
return { error: getErrorMessage(error, 'Unknown error') }
}
if (!response.ok) {
const error = await response.text().catch(() => '')
return { error: `Fal.ai pricing estimate failed: ${response.status} ${error}` }
}
const data = (await response.json()) as unknown
const totalCost = isRecord(data) ? getNumber(data.total_cost) : undefined
if (totalCost === undefined) {
return { error: 'Fal.ai pricing estimate missing total_cost' }
}
return { costDollars: totalCost }
}
export async function getFalAICostMetadata({
apiKey,
endpointId,
requestId,
}: {
apiKey: string
endpointId: string
requestId: string
}): Promise<FalAICostMetadata> {
for (let attempt = 0; attempt < FALAI_BILLING_EVENT_ATTEMPTS; attempt++) {
const event = await fetchFalAIBillingEvent(apiKey, requestId)
if (event) {
return {
endpointId: event.endpoint_id,
requestId: event.request_id,
costDollars: event.cost_estimate_nano_usd / 1_000_000_000,
source: 'billing_events',
outputUnits: event.output_units,
unitPrice: event.unit_price,
percentDiscount: event.percent_discount,
currency: 'USD',
}
}
if (attempt < FALAI_BILLING_EVENT_ATTEMPTS - 1) {
await sleep(FALAI_BILLING_EVENT_RETRY_MS)
}
}
const estimate = await estimateFalAICallCost(apiKey, endpointId)
if (estimate.costDollars !== undefined) {
return {
endpointId,
requestId,
costDollars: estimate.costDollars,
source: 'historical_estimate',
currency: 'USD',
}
}
logger.warn('Fal.ai cost metadata unavailable after generation completed', {
endpointId,
requestId,
error: estimate.error,
})
return {
endpointId,
requestId,
costDollars: getFalAIFallbackProviderCostDollars(endpointId),
source: 'fallback_floor',
currency: 'USD',
error: estimate.error,
}
}
+151
View File
@@ -0,0 +1,151 @@
import { createElement, type SVGProps } from 'react'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type { BlockConfig } from '@/blocks/types'
const { mockGetAllBlocks, mockGetToolOperationsIndex, mockGetTriggersForSidebar } = vi.hoisted(
() => ({
mockGetAllBlocks: vi.fn(),
mockGetToolOperationsIndex: vi.fn(() => []),
mockGetTriggersForSidebar: vi.fn(() => []),
})
)
vi.mock('@/blocks', () => ({
getAllBlocks: mockGetAllBlocks,
}))
vi.mock('@/lib/search/tool-operations', () => ({
getToolOperationsIndex: mockGetToolOperationsIndex,
}))
vi.mock('@/lib/workflows/triggers/trigger-utils', () => ({
getTriggersForSidebar: mockGetTriggersForSidebar,
}))
import {
buildCommandSearchableOptionSearchValue,
useSearchModalStore,
} from '@/stores/modals/search/store'
function TestIcon(props: SVGProps<SVGSVGElement>) {
return createElement('svg', props)
}
function createBlock(overrides: Partial<BlockConfig> = {}): BlockConfig {
return {
type: 'image_generator_v2',
name: 'Image Generator',
description: 'Generate images',
category: 'tools',
bgColor: '#4D5FFF',
icon: TestIcon,
subBlocks: [
{
id: 'provider',
title: 'Provider',
type: 'dropdown',
commandSearchable: true,
options: [
{ label: 'OpenAI', id: 'openai' },
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
{ label: 'Hidden Provider', id: 'hidden', hidden: true },
],
},
],
tools: { access: ['image_generate'] },
inputs: {},
outputs: {},
...overrides,
}
}
describe('search modal store', () => {
beforeEach(() => {
vi.clearAllMocks()
useSearchModalStore.setState({
isOpen: false,
data: {
blocks: [],
tools: [],
triggers: [],
toolOperations: [],
docs: [],
isInitialized: false,
},
})
})
describe('buildCommandSearchableOptionSearchValue', () => {
it('builds search terms for marked static dropdown options', () => {
const block = createBlock()
const searchValue = buildCommandSearchableOptionSearchValue(block)
expect(searchValue).toContain('Provider')
expect(searchValue).toContain('Fal.ai (Multi-Model)')
expect(searchValue).toContain('falai')
expect(searchValue).not.toContain('Hidden Provider')
expect(searchValue).not.toContain('hidden')
})
it('does not index dropdowns that only use in-dropdown search', () => {
const block = createBlock({
subBlocks: [
{
id: 'timezone',
title: 'Timezone',
type: 'dropdown',
searchable: true,
options: [{ label: 'UTC', id: 'utc' }],
},
],
})
expect(buildCommandSearchableOptionSearchValue(block)).toBe('')
})
it('builds search terms for marked combobox option functions', () => {
const block = createBlock({
subBlocks: [
{
id: 'model',
title: 'Model',
type: 'combobox',
commandSearchable: true,
options: () => [
{ label: 'claude-sonnet-4-6', id: 'claude-sonnet-4-6' },
{ label: 'Hidden Model', id: 'hidden-model', hidden: true },
],
},
],
})
const searchValue = buildCommandSearchableOptionSearchValue(block)
expect(searchValue).toContain('Model')
expect(searchValue).toContain('claude-sonnet-4-6')
expect(searchValue).not.toContain('Hidden Model')
expect(searchValue).not.toContain('hidden-model')
})
})
it('adds command-searchable options to visible block search values without extra rows', () => {
const visibleBlock = createBlock()
const hiddenBlock = createBlock({
type: 'hidden_generator',
hideFromToolbar: true,
})
mockGetAllBlocks.mockReturnValue([visibleBlock, hiddenBlock])
useSearchModalStore.getState().initializeData((blocks) => blocks)
const { tools } = useSearchModalStore.getState().data
expect(tools).toHaveLength(1)
expect(tools[0]).toEqual(
expect.objectContaining({
id: 'image_generator_v2',
searchValue: expect.stringContaining('Fal.ai (Multi-Model)'),
})
)
})
})
+43
View File
@@ -4,6 +4,7 @@ import { devtools } from 'zustand/middleware'
import { getToolOperationsIndex } from '@/lib/search/tool-operations'
import { getTriggersForSidebar } from '@/lib/workflows/triggers/trigger-utils'
import { getAllBlocks } from '@/blocks'
import type { BlockConfig, SubBlockConfig } from '@/blocks/types'
import type {
SearchBlockItem,
SearchData,
@@ -21,6 +22,47 @@ const initialData: SearchData = {
isInitialized: false,
}
type CommandSearchableOption = {
label: string
id: string
hidden?: boolean
}
function getCommandSearchableOptions(subBlock: SubBlockConfig): CommandSearchableOption[] {
if (!subBlock.options) return []
try {
const options = typeof subBlock.options === 'function' ? subBlock.options() : subBlock.options
return Array.isArray(options) ? options : []
} catch {
return []
}
}
export function buildCommandSearchableOptionSearchValue(block: BlockConfig): string {
const terms = new Set<string>()
for (const subBlock of block.subBlocks) {
if (
(subBlock.type !== 'dropdown' && subBlock.type !== 'combobox') ||
!subBlock.commandSearchable
) {
continue
}
for (const option of getCommandSearchableOptions(subBlock)) {
if (option.hidden) continue
const subBlockTitle = subBlock.title ?? subBlock.id
terms.add(subBlockTitle)
terms.add(option.label)
terms.add(option.id)
}
}
return Array.from(terms).join(' ')
}
export const useSearchModalStore = create<SearchModalState>()(
devtools(
(set, _) => ({
@@ -56,6 +98,7 @@ export const useSearchModalStore = create<SearchModalState>()(
icon: block.icon,
bgColor: block.bgColor || '#6B7280',
type: block.type,
searchValue: `${block.name} ${block.type} block-${block.type} ${buildCommandSearchableOptionSearchValue(block)}`,
}
if (block.category === 'blocks' && block.type !== 'starter') {
+1
View File
@@ -11,6 +11,7 @@ export interface SearchBlockItem {
bgColor: string
type: string
config?: BlockConfig
searchValue?: string
}
/**
+108
View File
@@ -0,0 +1,108 @@
/**
* @vitest-environment node
*/
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
FALAI_IMAGE_FALLBACK_PROVIDER_COST_DOLLARS,
getFalAICostMetadata,
} from '@/lib/tools/falai-pricing'
import { imageGenerateTool } from '@/tools/image/generate'
import { falaiVideoTool } from '@/tools/video/falai'
afterEach(() => {
vi.useRealTimers()
vi.unstubAllGlobals()
})
describe('Fal.ai hosted key pricing', () => {
it('applies hosted markup to image generation provider cost', () => {
const pricing = imageGenerateTool.hosting?.pricing
expect(pricing?.type).toBe('custom')
if (!pricing || pricing.type !== 'custom') throw new Error('Expected custom pricing')
const result = pricing.getCost(
{},
{
__falaiCostDollars: 0.1,
__falaiBilling: {
source: 'billing_events',
endpointId: 'fal-ai/nano-banana-2',
},
}
)
expect(typeof result).toBe('object')
if (typeof result === 'number') throw new Error('Expected structured pricing result')
expect(result.cost).toBeCloseTo(0.1 * FALAI_HOSTED_KEY_MARKUP_MULTIPLIER)
expect(result.metadata).toMatchObject({
providerCostDollars: 0.1,
markupMultiplier: FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
source: 'billing_events',
})
})
it('applies hosted markup to video generation provider cost', () => {
const pricing = falaiVideoTool.hosting?.pricing
expect(pricing?.type).toBe('custom')
if (!pricing || pricing.type !== 'custom') throw new Error('Expected custom pricing')
const result = pricing.getCost(
{},
{
__falaiCostDollars: 0.4,
__falaiBilling: {
source: 'billing_events',
endpointId: 'fal-ai/veo3.1',
},
}
)
expect(typeof result).toBe('object')
if (typeof result === 'number') throw new Error('Expected structured pricing result')
expect(result.cost).toBeCloseTo(0.4 * FALAI_HOSTED_KEY_MARKUP_MULTIPLIER)
expect(result.metadata).toMatchObject({
providerCostDollars: 0.4,
markupMultiplier: FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
source: 'billing_events',
})
})
it('returns fallback floor cost metadata instead of throwing when billing and estimate fail', async () => {
vi.useFakeTimers()
const mockFetch = vi
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ billing_events: [] }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
})
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ billing_events: [] }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
})
)
.mockResolvedValueOnce(new Response('pricing unavailable', { status: 500 }))
vi.stubGlobal('fetch', mockFetch)
const resultPromise = getFalAICostMetadata({
apiKey: 'fal-key',
endpointId: 'fal-ai/nano-banana-2',
requestId: 'request-1',
})
await vi.advanceTimersByTimeAsync(500)
const result = await resultPromise
expect(result).toMatchObject({
endpointId: 'fal-ai/nano-banana-2',
requestId: 'request-1',
costDollars: FALAI_IMAGE_FALLBACK_PROVIDER_COST_DOLLARS,
source: 'fallback_floor',
currency: 'USD',
})
})
})
+39
View File
@@ -1,3 +1,4 @@
import { FALAI_HOSTED_KEY_MARKUP_MULTIPLIER } from '@/lib/tools/falai-pricing'
import type { ImageGenerationParams, ImageGenerationResponse } from '@/tools/image/types'
import type { ToolConfig } from '@/tools/types'
@@ -113,6 +114,38 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
},
},
hosting: {
enabled: (params) => params.provider === 'falai',
envKeyPrefix: 'FALAI_API_KEY',
apiKeyParam: 'apiKey',
byokProviderId: 'falai',
pricing: {
type: 'custom',
getCost: (_params, output) => {
const providerCostDollars = output.__falaiCostDollars
if (typeof providerCostDollars !== 'number' || Number.isNaN(providerCostDollars)) {
throw new Error('Fal.ai image response missing cost data')
}
return {
cost: providerCostDollars * FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
metadata: {
...(typeof output.__falaiBilling === 'object' && output.__falaiBilling !== null
? (output.__falaiBilling as Record<string, unknown>)
: {}),
providerCostDollars,
markupMultiplier: FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
},
}
},
},
rateLimit: {
mode: 'per_request',
requestsPerMinute: 40,
burstMultiplier: 1,
},
},
request: {
url: '/api/tools/image',
method: 'POST',
@@ -122,6 +155,7 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
body: (
params: ImageGenerationParams & {
_context?: { workspaceId?: string; workflowId?: string; executionId?: string }
__usingHostedKey?: boolean
}
) => ({
provider: params.provider,
@@ -144,6 +178,7 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
workspaceId: params._context?.workspaceId,
workflowId: params._context?.workflowId,
executionId: params._context?.executionId,
useHostedCostTracking: params.__usingHostedKey === true,
}),
},
@@ -159,6 +194,8 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
provider?: string
model?: string
metadata?: ImageGenerationResponse['output']['metadata']
__falaiCostDollars?: number
__falaiBilling?: ImageGenerationResponse['output']['__falaiBilling']
}
if (!response.ok || data.error) {
@@ -203,6 +240,8 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
model: data.model || data.metadata?.model || '',
...data.metadata,
},
__falaiCostDollars: data.__falaiCostDollars,
__falaiBilling: data.__falaiBilling,
},
}
},
+11
View File
@@ -36,5 +36,16 @@ export interface ImageGenerationResponse extends ToolResponse {
jobId?: string
contentType?: string
}
__falaiCostDollars?: number
__falaiBilling?: {
endpointId: string
requestId: string
source: 'billing_events' | 'historical_estimate' | 'fallback_floor'
outputUnits?: number | null
unitPrice?: number | null
percentDiscount?: number | null
currency?: string
error?: string
}
}
}
+124
View File
@@ -2797,4 +2797,128 @@ describe('Cost Field Handling', () => {
Object.assign(tools, originalTools)
})
it('should skip hosted key injection when hosting predicate is false', async () => {
const mockTool = {
id: 'test_conditional_hosting',
name: 'Test Conditional Hosting',
description: 'A test tool with conditional hosted keys',
version: '1.0.0',
params: {
provider: { type: 'string', required: false },
apiKey: { type: 'string', required: false },
},
hosting: {
enabled: (params: { provider?: string }) => params.provider === 'hosted-provider',
envKeyPrefix: 'TEST_HOSTED_KEY',
apiKeyParam: 'apiKey',
pricing: {
type: 'per_request' as const,
cost: 0.005,
},
rateLimit: {
mode: 'per_request' as const,
requestsPerMinute: 100,
},
},
request: {
url: '/api/test/conditional-hosting',
method: 'POST' as const,
headers: () => ({ 'Content-Type': 'application/json' }),
},
transformResponse: vi.fn().mockResolvedValue({
success: true,
output: { result: 'success' },
}),
}
const originalTools = { ...tools }
;(tools as any).test_conditional_hosting = mockTool
global.fetch = Object.assign(
vi.fn().mockImplementation(async () => ({
ok: true,
status: 200,
headers: new Headers(),
json: () => Promise.resolve({ success: true }),
})),
{ preconnect: vi.fn() }
) as typeof fetch
const mockContext = createToolExecutionContext({
userId: 'user-123',
} as any)
const result = await executeTool(
'test_conditional_hosting',
{ provider: 'user-provider' },
{ executionContext: mockContext }
)
expect(result.success).toBe(true)
expect(mockRateLimiterFns.acquireKey).not.toHaveBeenCalled()
expect(result.output.cost).toBeUndefined()
Object.assign(tools, originalTools)
})
it('should skip hosted key injection when user provides an API key', async () => {
const mockTool = {
id: 'test_user_key_priority',
name: 'Test User Key Priority',
description: 'A test tool where user keys should win',
version: '1.0.0',
params: {
apiKey: { type: 'string', required: false },
},
hosting: {
envKeyPrefix: 'TEST_HOSTED_KEY',
apiKeyParam: 'apiKey',
pricing: {
type: 'per_request' as const,
cost: 0.005,
},
rateLimit: {
mode: 'per_request' as const,
requestsPerMinute: 100,
},
},
request: {
url: '/api/test/user-key-priority',
method: 'POST' as const,
headers: () => ({ 'Content-Type': 'application/json' }),
},
transformResponse: vi.fn().mockResolvedValue({
success: true,
output: { result: 'success' },
}),
}
const originalTools = { ...tools }
;(tools as any).test_user_key_priority = mockTool
global.fetch = Object.assign(
vi.fn().mockImplementation(async () => ({
ok: true,
status: 200,
headers: new Headers(),
json: () => Promise.resolve({ success: true }),
})),
{ preconnect: vi.fn() }
) as typeof fetch
const mockContext = createToolExecutionContext({
userId: 'user-123',
} as any)
const result = await executeTool(
'test_user_key_priority',
{ apiKey: 'user-api-key' },
{ executionContext: mockContext }
)
expect(result.success).toBe(true)
expect(mockRateLimiterFns.acquireKey).not.toHaveBeenCalled()
expect(result.output.cost).toBeUndefined()
Object.assign(tools, originalTools)
})
})
+8
View File
@@ -232,8 +232,15 @@ async function injectHostedKeyIfNeeded(
): Promise<HostedKeyInjectionResult> {
if (!tool.hosting) return { isUsingHostedKey: false }
if (!isHosted) return { isUsingHostedKey: false }
if (tool.hosting.enabled && !tool.hosting.enabled(params)) {
return { isUsingHostedKey: false }
}
const { envKeyPrefix, apiKeyParam, byokProviderId, rateLimit } = tool.hosting
const userProvidedKey = params[apiKeyParam]
if (typeof userProvidedKey === 'string' && userProvidedKey.trim().length > 0) {
return { isUsingHostedKey: false }
}
const { workspaceId, userId, workflowId } = resolveToolScope(params, executionContext)
@@ -299,6 +306,7 @@ async function injectHostedKeyIfNeeded(
}
params[apiKeyParam] = acquireResult.key
params.__usingHostedKey = true
logger.info(`[${requestId}] Using hosted key for ${tool.id} (${acquireResult.envVarName})`, {
keyIndex: acquireResult.keyIndex,
provider,
+3
View File
@@ -8,6 +8,7 @@ export type BYOKProviderId =
| 'google'
| 'mistral'
| 'fireworks'
| 'falai'
| 'firecrawl'
| 'exa'
| 'serper'
@@ -307,6 +308,8 @@ export type ToolHostingPricing<P = Record<string, unknown>> = PerRequestPricing
* no code changes needed.
*/
interface ToolHostingConfig<P = Record<string, unknown>> {
/** Optional predicate for tools where hosted keys only apply to some parameter combinations. */
enabled?: (params: P) => boolean
/**
* Env var name prefix for hosted keys.
* At runtime, `{envKeyPrefix}_COUNT` is read to determine how many keys exist,
+36
View File
@@ -1,3 +1,4 @@
import { FALAI_HOSTED_KEY_MARKUP_MULTIPLIER } from '@/lib/tools/falai-pricing'
import type { ToolConfig } from '@/tools/types'
import type { VideoParams, VideoResponse } from '@/tools/video/types'
import { parseBooleanParam, parseBooleanParamWithDefault } from '@/tools/video/utils'
@@ -68,6 +69,37 @@ export const falaiVideoTool: ToolConfig<VideoParams, VideoResponse> = {
},
},
hosting: {
envKeyPrefix: 'FALAI_API_KEY',
apiKeyParam: 'apiKey',
byokProviderId: 'falai',
pricing: {
type: 'custom',
getCost: (_params, output) => {
const providerCostDollars = output.__falaiCostDollars
if (typeof providerCostDollars !== 'number' || Number.isNaN(providerCostDollars)) {
throw new Error('Fal.ai video response missing cost data')
}
return {
cost: providerCostDollars * FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
metadata: {
...(typeof output.__falaiBilling === 'object' && output.__falaiBilling !== null
? (output.__falaiBilling as Record<string, unknown>)
: {}),
providerCostDollars,
markupMultiplier: FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
},
}
},
},
rateLimit: {
mode: 'per_request',
requestsPerMinute: 40,
burstMultiplier: 1,
},
},
request: {
url: '/api/tools/video',
method: 'POST',
@@ -77,6 +109,7 @@ export const falaiVideoTool: ToolConfig<VideoParams, VideoResponse> = {
body: (
params: VideoParams & {
_context?: { workspaceId?: string; workflowId?: string; executionId?: string }
__usingHostedKey?: boolean
}
) => ({
provider: 'falai',
@@ -91,6 +124,7 @@ export const falaiVideoTool: ToolConfig<VideoParams, VideoResponse> = {
workspaceId: params._context?.workspaceId,
workflowId: params._context?.workflowId,
executionId: params._context?.executionId,
useHostedCostTracking: params.__usingHostedKey === true,
}),
},
@@ -128,6 +162,8 @@ export const falaiVideoTool: ToolConfig<VideoParams, VideoResponse> = {
provider: 'falai',
model: data.model,
jobId: data.jobId,
__falaiCostDollars: data.__falaiCostDollars,
__falaiBilling: data.__falaiBilling,
},
}
},
+11
View File
@@ -33,6 +33,17 @@ export interface VideoResponse extends ToolResponse {
provider?: string
model?: string
jobId?: string
__falaiCostDollars?: number
__falaiBilling?: {
endpointId: string
requestId: string
source: 'billing_events' | 'historical_estimate' | 'fallback_floor'
outputUnits?: number | null
unitPrice?: number | null
percentDiscount?: number | null
currency?: string
error?: string
}
}
}