mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
improvement(media-gen): retire vision block, add hosted key for fal ai for image/video gen, search visibility in cmd-k (#4684)
* improvement(media-gen): retire vision block, add hosted key for fal ai for image/video gen * address comments
This commit is contained in:
@@ -280,7 +280,7 @@ Get the execution timeline for an Azure DevOps build — every stage, job, and t
|
||||
| ↳ `warningCount` | number | Number of warnings |
|
||||
| ↳ `startTime` | string | ISO 8601 start timestamp |
|
||||
| ↳ `finishTime` | string | ISO 8601 finish timestamp |
|
||||
| ↳ `failedRecords` | array | Subset of records where result === "failed" — use logId to fetch logs |
|
||||
| ↳ `failedRecords` | array | Subset of records where result is failed, partiallySucceeded, or succeededWithIssues — use logId to fetch logs |
|
||||
| ↳ `id` | string | Record GUID |
|
||||
| ↳ `name` | string | Step name |
|
||||
| ↳ `type` | string | Stage \| Phase \| Job \| Task |
|
||||
@@ -333,7 +333,8 @@ Execute a WIQL query to search for work items in Azure DevOps and return full fi
|
||||
| --------- | ---- | ----------- |
|
||||
| `content` | string | Human-readable summary of matching work items |
|
||||
| `metadata` | object | Work items metadata |
|
||||
| ↳ `count` | number | Number of work items returned |
|
||||
| ↳ `count` | number | Number of work items returned \(after hydration\) |
|
||||
| ↳ `totalMatched` | number | Total number of work items matched by the WIQL query before hydration |
|
||||
| ↳ `workItems` | array | Array of work item details |
|
||||
| ↳ `id` | number | Work item ID |
|
||||
| ↳ `title` | string | Work item title |
|
||||
@@ -372,7 +373,7 @@ Fetch full details of a single work item by ID from Azure DevOps, including titl
|
||||
|
||||
### `azure_devops_get_work_items_batch`
|
||||
|
||||
Fetch full details for multiple work items by ID from Azure DevOps in a single call. Pass comma-separated IDs (e.g.
|
||||
Fetch full details for multiple work items by ID from Azure DevOps. Pass comma-separated IDs (e.g.
|
||||
|
||||
#### Input
|
||||
|
||||
@@ -380,7 +381,7 @@ Fetch full details for multiple work items by ID from Azure DevOps in a single c
|
||||
| --------- | ---- | -------- | ----------- |
|
||||
| `organization` | string | Yes | Azure DevOps organization name |
|
||||
| `project` | string | Yes | Azure DevOps project name |
|
||||
| `ids` | string | Yes | Comma-separated work item IDs to fetch \(e.g. "123,456,789"\). Maximum 200 IDs. |
|
||||
| `ids` | string | Yes | Comma-separated work item IDs to fetch \(e.g. "123,456,789"\). Lists longer than 200 IDs are chunked automatically. |
|
||||
|
||||
#### Output
|
||||
|
||||
@@ -389,6 +390,7 @@ Fetch full details for multiple work items by ID from Azure DevOps in a single c
|
||||
| `content` | string | Human-readable summary of the fetched work items |
|
||||
| `metadata` | object | Work items metadata |
|
||||
| ↳ `count` | number | Number of work items returned |
|
||||
| ↳ `totalRequested` | number | Total number of IDs requested \(across all chunks\) |
|
||||
| ↳ `workItems` | array | Array of work item details |
|
||||
| ↳ `id` | number | Work item ID |
|
||||
| ↳ `title` | string | Work item title |
|
||||
|
||||
@@ -195,7 +195,6 @@
|
||||
"upstash",
|
||||
"vercel",
|
||||
"video_generator",
|
||||
"vision",
|
||||
"wealthbox",
|
||||
"webflow",
|
||||
"whatsapp",
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
---
|
||||
title: Vision
|
||||
description: Analyze images with vision models
|
||||
---
|
||||
|
||||
import { BlockInfoCard } from "@/components/ui/block-info-card"
|
||||
|
||||
<BlockInfoCard
|
||||
type="vision_v2"
|
||||
color="#4D5FFF"
|
||||
/>
|
||||
|
||||
{/* MANUAL-CONTENT-START:intro */}
|
||||
Vision is a tool that allows you to analyze images with vision models.
|
||||
|
||||
With Vision, you can:
|
||||
|
||||
- **Analyze images**: Analyze images with vision models
|
||||
- **Extract text**: Extract text from images
|
||||
- **Identify objects**: Identify objects in images
|
||||
- **Describe images**: Describe images in detail
|
||||
- **Generate images**: Generate images from text
|
||||
|
||||
In Sim, the Vision integration enables your agents to analyze images with vision models as part of their workflows. This allows for powerful automation scenarios that require analyzing images with vision models. Your agents can analyze images with vision models, extract text from images, identify objects in images, describe images in detail, and generate images from text. This integration bridges the gap between your AI workflows and your image analysis needs, enabling more sophisticated and image-centric automations. By connecting Sim with Vision, you can create agents that stay current with the latest information, provide more accurate responses, and deliver more value to users - all without requiring manual intervention or custom code.
|
||||
{/* MANUAL-CONTENT-END */}
|
||||
|
||||
|
||||
## Usage Instructions
|
||||
|
||||
Integrate Vision into the workflow. Can analyze images with vision models.
|
||||
|
||||
|
||||
|
||||
## Tools
|
||||
|
||||
### `vision_tool`
|
||||
|
||||
#### Input
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
| --------- | ---- | -------- | ----------- |
|
||||
| `apiKey` | string | Yes | API key for the selected model provider |
|
||||
| `imageUrl` | string | No | Publicly accessible image URL |
|
||||
| `imageFile` | file | No | Image file to analyze |
|
||||
| `model` | string | No | Vision model to use \(gpt-4o, claude-3-opus-20240229, etc\) |
|
||||
| `prompt` | string | No | Custom prompt for image analysis |
|
||||
|
||||
#### Output
|
||||
|
||||
| Parameter | Type | Description |
|
||||
| --------- | ---- | ----------- |
|
||||
| `content` | string | The analyzed content and description of the image |
|
||||
| `model` | string | The vision model that was used for analysis |
|
||||
| `tokens` | number | Total tokens used for the analysis |
|
||||
| `usage` | object | Detailed token usage breakdown |
|
||||
| ↳ `input_tokens` | number | Tokens used for input processing |
|
||||
| ↳ `output_tokens` | number | Tokens used for response generation |
|
||||
| ↳ `total_tokens` | number | Total tokens consumed |
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ Trigger workflow when an Azure DevOps build fails, is canceled, or partially suc
|
||||
| `branch` | string | Source branch name \(refs/heads/ prefix stripped\) |
|
||||
| `commitSha` | string | Source commit SHA |
|
||||
| `triggeredBy` | string | Display name of the person who triggered the build |
|
||||
| `triggeredByEmail` | string | Email/unique name of the person who triggered the build |
|
||||
| `triggeredByEmail` | string | Email/unique name of the person who triggered the build, or null if not set |
|
||||
| `startTime` | string | Build start time \(ISO 8601\) |
|
||||
| `finishTime` | string | Build finish time \(ISO 8601\) |
|
||||
| `buildUrl` | string | API URL for the build resource |
|
||||
@@ -72,12 +72,12 @@ Trigger workflow when a work item is created in Azure DevOps
|
||||
| `workItemType` | string | Work item type for Basic process \(e.g. Issue, Task, Epic\) |
|
||||
| `title` | string | Work item title |
|
||||
| `state` | string | Work item state for Basic process \(e.g. To Do, Doing, Done\) |
|
||||
| `createdBy` | string | Display name of the creator |
|
||||
| `assignedTo` | string | Assignee display name, or empty string if unassigned |
|
||||
| `createdBy` | string | Display name of the creator, or null if not set |
|
||||
| `assignedTo` | string | Assignee display name, or null if unassigned |
|
||||
| `priority` | number | Priority \(1–4\), or 0 if not set |
|
||||
| `areaPath` | string | Area path |
|
||||
| `iterationPath` | string | Iteration path |
|
||||
| `description` | string | Work item description \(HTML\), or empty string if not set |
|
||||
| `description` | string | Work item description \(HTML\), or null if not set |
|
||||
| `projectName` | string | Azure DevOps project name |
|
||||
| `workItemUrl` | string | API URL for the work item resource |
|
||||
|
||||
|
||||
@@ -57,7 +57,6 @@ import {
|
||||
EvernoteIcon,
|
||||
ExaAIIcon,
|
||||
ExtendIcon,
|
||||
EyeIcon,
|
||||
FathomIcon,
|
||||
FindymailIcon,
|
||||
FirecrawlIcon,
|
||||
@@ -405,7 +404,6 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
|
||||
upstash: UpstashIcon,
|
||||
vercel: VercelIcon,
|
||||
video_generator_v3: VideoIcon,
|
||||
vision_v2: EyeIcon,
|
||||
wealthbox: WealthboxIcon,
|
||||
webflow: WebflowIcon,
|
||||
whatsapp: WhatsAppIcon,
|
||||
|
||||
@@ -1938,7 +1938,7 @@
|
||||
},
|
||||
{
|
||||
"name": "Get Work Items Batch",
|
||||
"description": "Fetch full details for multiple work items by ID from Azure DevOps in a single call. Pass comma-separated IDs (e.g. "
|
||||
"description": "Fetch full details for multiple work items by ID from Azure DevOps. Pass comma-separated IDs (e.g. "
|
||||
},
|
||||
{
|
||||
"name": "Create Work Item",
|
||||
@@ -4156,10 +4156,6 @@
|
||||
"name": "Fetch",
|
||||
"description": "Fetch and parse a file from a URL with optional custom headers."
|
||||
},
|
||||
{
|
||||
"name": "Get",
|
||||
"description": "Get a workspace file object from a selected file or canonical workspace file ID."
|
||||
},
|
||||
{
|
||||
"name": "Write",
|
||||
"description": "Create a new workspace file. If a file with the same name already exists, a numeric suffix is added (e.g., "
|
||||
@@ -4169,7 +4165,7 @@
|
||||
"description": "Append content to an existing workspace file. The file must already exist. Content is added to the end of the file."
|
||||
}
|
||||
],
|
||||
"operationCount": 5,
|
||||
"operationCount": 4,
|
||||
"triggers": [],
|
||||
"triggerCount": 0,
|
||||
"authType": "none",
|
||||
@@ -14253,24 +14249,6 @@
|
||||
"integrationTypes": ["ai", "design"],
|
||||
"tags": ["video-generation", "llm"]
|
||||
},
|
||||
{
|
||||
"type": "vision_v2",
|
||||
"slug": "vision",
|
||||
"name": "Vision",
|
||||
"description": "Analyze images with vision models",
|
||||
"longDescription": "Integrate Vision into the workflow. Can analyze images with vision models.",
|
||||
"bgColor": "#4D5FFF",
|
||||
"iconName": "EyeIcon",
|
||||
"docsUrl": "https://docs.sim.ai/tools/vision",
|
||||
"operations": [],
|
||||
"operationCount": 0,
|
||||
"triggers": [],
|
||||
"triggerCount": 0,
|
||||
"authType": "api-key",
|
||||
"category": "tools",
|
||||
"integrationTypes": ["ai", "documents"],
|
||||
"tags": ["llm", "document-processing", "ocr"]
|
||||
},
|
||||
{
|
||||
"type": "wealthbox",
|
||||
"slug": "wealthbox",
|
||||
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
} from '@/lib/core/utils/stream-limits'
|
||||
import { getBaseUrl } from '@/lib/core/utils/urls'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing'
|
||||
|
||||
const logger = createLogger('ImageProxyAPI')
|
||||
const MAX_IMAGE_BYTES = 25 * 1024 * 1024
|
||||
@@ -53,6 +54,7 @@ interface GeneratedImageResult {
|
||||
revisedPrompt?: string
|
||||
seed?: number
|
||||
jobId?: string
|
||||
falaiCost?: FalAICostMetadata
|
||||
}
|
||||
|
||||
interface StoredImageResponse {
|
||||
@@ -72,6 +74,8 @@ interface StoredImageResponse {
|
||||
jobId?: string
|
||||
contentType: string
|
||||
}
|
||||
__falaiCostDollars?: number
|
||||
__falaiBilling?: FalAICostMetadata
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
@@ -935,6 +939,13 @@ async function generateWithFalAI(
|
||||
revisedPrompt: getStringProperty(resultData, 'revised_prompt'),
|
||||
seed: getNumberProperty(resultData, 'seed'),
|
||||
jobId: falRequestId,
|
||||
falaiCost: body.useHostedCostTracking
|
||||
? await getFalAICostMetadata({
|
||||
apiKey,
|
||||
endpointId: modelConfig.endpoint,
|
||||
requestId: falRequestId,
|
||||
})
|
||||
: undefined,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -992,6 +1003,8 @@ async function storeGeneratedImage(
|
||||
jobId: imageResult.jobId,
|
||||
contentType: imageResult.contentType,
|
||||
},
|
||||
__falaiCostDollars: imageResult.falaiCost?.costDollars,
|
||||
__falaiBilling: imageResult.falaiCost,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1024,5 +1037,7 @@ async function storeGeneratedImage(
|
||||
jobId: imageResult.jobId,
|
||||
contentType: imageResult.contentType,
|
||||
},
|
||||
__falaiCostDollars: imageResult.falaiCost?.costDollars,
|
||||
__falaiBilling: imageResult.falaiCost,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
readResponseToBufferWithLimit,
|
||||
} from '@/lib/core/utils/stream-limits'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing'
|
||||
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
|
||||
import { assertToolFileAccess } from '@/app/api/files/authorization'
|
||||
import type { UserFile } from '@/executor/types'
|
||||
@@ -138,6 +139,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
let height: number | undefined
|
||||
let jobId: string | undefined
|
||||
let actualDuration: number | undefined
|
||||
let falaiCost: FalAICostMetadata | undefined
|
||||
|
||||
if (body.visualReference) {
|
||||
const denied = await assertToolFileAccess(
|
||||
@@ -236,6 +238,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
resolution,
|
||||
body.promptOptimizer,
|
||||
body.generateAudio,
|
||||
body.useHostedCostTracking === true,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
@@ -244,6 +247,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
height = result.height
|
||||
jobId = result.jobId
|
||||
actualDuration = result.duration
|
||||
falaiCost = result.falaiCost
|
||||
} else {
|
||||
return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 })
|
||||
}
|
||||
@@ -301,6 +305,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
provider,
|
||||
model: model || 'default',
|
||||
jobId,
|
||||
__falaiCostDollars: falaiCost?.costDollars,
|
||||
__falaiBilling: falaiCost,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -333,6 +339,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
provider,
|
||||
model: model || 'default',
|
||||
jobId,
|
||||
__falaiCostDollars: falaiCost?.costDollars,
|
||||
__falaiBilling: falaiCost,
|
||||
})
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Video proxy error:`, error)
|
||||
@@ -1167,9 +1175,17 @@ async function generateWithFalAI(
|
||||
resolution: string | undefined,
|
||||
promptOptimizer: boolean | undefined,
|
||||
generateAudio: boolean | undefined,
|
||||
useHostedCostTracking: boolean,
|
||||
requestId: string,
|
||||
logger: ReturnType<typeof createLogger>
|
||||
): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> {
|
||||
): Promise<{
|
||||
buffer: Buffer
|
||||
width: number
|
||||
height: number
|
||||
jobId: string
|
||||
duration: number
|
||||
falaiCost?: FalAICostMetadata
|
||||
}> {
|
||||
logger.info(`[${requestId}] Starting Fal.ai generation with model: ${model}`)
|
||||
|
||||
const modelConfig = FALAI_MODEL_CONFIGS[model]
|
||||
@@ -1312,6 +1328,13 @@ async function generateWithFalAI(
|
||||
height,
|
||||
jobId: requestIdFal,
|
||||
duration: getNumberProperty(videoOutput, 'duration') || duration || 5,
|
||||
falaiCost: useHostedCostTracking
|
||||
? await getFalAICostMetadata({
|
||||
apiKey,
|
||||
endpointId: modelConfig.endpoint,
|
||||
requestId: requestIdFal,
|
||||
})
|
||||
: undefined,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
FireworksIcon,
|
||||
GeminiIcon,
|
||||
GoogleIcon,
|
||||
ImageIcon,
|
||||
JinaAIIcon,
|
||||
LinkupIcon,
|
||||
MistralIcon,
|
||||
@@ -85,6 +86,13 @@ const PROVIDERS: {
|
||||
description: 'LLM calls',
|
||||
placeholder: 'Enter your Fireworks API key',
|
||||
},
|
||||
{
|
||||
id: 'falai',
|
||||
name: 'Fal.ai',
|
||||
icon: ImageIcon,
|
||||
description: 'Image and video generation',
|
||||
placeholder: 'Enter your Fal.ai API key',
|
||||
},
|
||||
{
|
||||
id: 'firecrawl',
|
||||
name: 'Firecrawl',
|
||||
|
||||
@@ -204,6 +204,12 @@ interface BlockData {
|
||||
position: { x: number; y: number }
|
||||
}
|
||||
|
||||
interface AddBlockFromToolbarDetail {
|
||||
type?: unknown
|
||||
enableTriggerMode?: unknown
|
||||
presetOperation?: unknown
|
||||
}
|
||||
|
||||
/**
|
||||
* Main workflow canvas content component.
|
||||
* Renders the ReactFlow canvas with blocks, edges, and all interactive features.
|
||||
@@ -2004,7 +2010,7 @@ const WorkflowContent = React.memo(
|
||||
|
||||
/** Handles toolbar block click events to add blocks to the canvas. */
|
||||
useEffect(() => {
|
||||
const handleAddBlockFromToolbar = (event: CustomEvent) => {
|
||||
const handleAddBlockFromToolbar = (event: CustomEvent<AddBlockFromToolbarDetail>) => {
|
||||
// Check if user has permission to interact with blocks
|
||||
if (!effectivePermissions.canEdit) {
|
||||
return
|
||||
@@ -2012,7 +2018,7 @@ const WorkflowContent = React.memo(
|
||||
|
||||
const { type, enableTriggerMode, presetOperation } = event.detail
|
||||
|
||||
if (!type) return
|
||||
if (typeof type !== 'string' || !type) return
|
||||
if (type === 'connectionBlock') return
|
||||
|
||||
const basePosition = getViewportCenter()
|
||||
@@ -2070,8 +2076,10 @@ const WorkflowContent = React.memo(
|
||||
undefined,
|
||||
undefined,
|
||||
autoConnectEdge,
|
||||
enableTriggerMode,
|
||||
presetOperation ? { operation: presetOperation } : undefined
|
||||
enableTriggerMode === true,
|
||||
typeof presetOperation === 'string' && presetOperation
|
||||
? { operation: presetOperation }
|
||||
: undefined
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
+6
-3
@@ -206,7 +206,10 @@ export function SearchModal({
|
||||
type === 'trigger' && block.config ? hasTriggerCapability(block.config) : false
|
||||
window.dispatchEvent(
|
||||
new CustomEvent('add-block-from-toolbar', {
|
||||
detail: { type: block.type, enableTriggerMode },
|
||||
detail: {
|
||||
type: block.type,
|
||||
enableTriggerMode,
|
||||
},
|
||||
})
|
||||
)
|
||||
captureEvent(posthogRef.current, 'search_result_selected', {
|
||||
@@ -376,12 +379,12 @@ export function SearchModal({
|
||||
|
||||
const filteredBlocks = useMemo(() => {
|
||||
if (!isOnWorkflowPage) return []
|
||||
return filterAndSort(blocks, (b) => `${b.name} block-${b.id}`, deferredSearch)
|
||||
return filterAndSort(blocks, (b) => b.searchValue ?? `${b.name} block-${b.id}`, deferredSearch)
|
||||
}, [isOnWorkflowPage, blocks, deferredSearch])
|
||||
|
||||
const filteredTools = useMemo(() => {
|
||||
if (!isOnWorkflowPage) return []
|
||||
return filterAndSort(tools, (t) => `${t.name} tool-${t.id}`, deferredSearch)
|
||||
return filterAndSort(tools, (t) => t.searchValue ?? `${t.name} tool-${t.id}`, deferredSearch)
|
||||
}, [isOnWorkflowPage, tools, deferredSearch])
|
||||
|
||||
const filteredTriggers = useMemo(() => {
|
||||
|
||||
@@ -104,6 +104,7 @@ import { SIDEBAR_WIDTH } from '@/stores/constants'
|
||||
import { useFolderStore } from '@/stores/folders/store'
|
||||
import { useSearchModalStore } from '@/stores/modals/search/store'
|
||||
import { useMothershipDraftsStore } from '@/stores/mothership-drafts/store'
|
||||
import { useProvidersStore } from '@/stores/providers'
|
||||
import { useSidebarStore } from '@/stores/sidebar/store'
|
||||
|
||||
const logger = createLogger('Sidebar')
|
||||
@@ -360,10 +361,18 @@ export const Sidebar = memo(function Sidebar() {
|
||||
const { config: permissionConfig, filterBlocks } = usePermissionConfig()
|
||||
const { navigateToSettings, getSettingsHref } = useSettingsNavigation()
|
||||
const initializeSearchData = useSearchModalStore((state) => state.initializeData)
|
||||
const providers = useProvidersStore((state) => state.providers)
|
||||
const providerModelSignature = useMemo(
|
||||
() =>
|
||||
Object.values(providers)
|
||||
.map((provider) => provider.models.join('\x00'))
|
||||
.join('\x01'),
|
||||
[providers]
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
initializeSearchData(filterBlocks)
|
||||
}, [initializeSearchData, filterBlocks])
|
||||
}, [initializeSearchData, filterBlocks, providerModelSignature])
|
||||
|
||||
const setSidebarWidth = useSidebarStore((state) => state.setSidebarWidth)
|
||||
const isCollapsed = useSidebarStore((state) => state.isCollapsed)
|
||||
|
||||
@@ -734,6 +734,65 @@ describe.concurrent('Blocks Module', () => {
|
||||
expect(temperatureSubBlock?.min).toBe(0)
|
||||
expect(temperatureSubBlock?.max).toBe(2)
|
||||
})
|
||||
|
||||
it('should mark generator provider dropdowns as command-searchable', () => {
|
||||
const imageGeneratorBlock = getBlock('image_generator_v2')
|
||||
const videoGeneratorBlock = getBlock('video_generator_v3')
|
||||
|
||||
const imageProviderSubBlock = imageGeneratorBlock?.subBlocks.find(
|
||||
(sb) => sb.id === 'provider'
|
||||
)
|
||||
const videoProviderSubBlock = videoGeneratorBlock?.subBlocks.find(
|
||||
(sb) => sb.id === 'provider'
|
||||
)
|
||||
const imageProviderOptions = imageProviderSubBlock?.options
|
||||
const videoProviderOptions = videoProviderSubBlock?.options
|
||||
|
||||
expect(imageGeneratorBlock?.hideFromToolbar).not.toBe(true)
|
||||
expect(videoGeneratorBlock?.hideFromToolbar).not.toBe(true)
|
||||
expect(imageProviderSubBlock?.commandSearchable).toBe(true)
|
||||
expect(videoProviderSubBlock?.commandSearchable).toBe(true)
|
||||
expect(imageProviderSubBlock?.value?.()).toBe('falai')
|
||||
expect(videoProviderSubBlock?.value?.()).toBe('falai')
|
||||
expect(
|
||||
Array.isArray(imageProviderOptions) ? imageProviderOptions.map((option) => option.id) : []
|
||||
).toContain('falai')
|
||||
expect(
|
||||
Array.isArray(videoProviderOptions) ? videoProviderOptions.map((option) => option.id) : []
|
||||
).toContain('falai')
|
||||
expect(getBlock('image_generator')?.hideFromToolbar).toBe(true)
|
||||
expect(getBlock('video_generator_v2')?.hideFromToolbar).toBe(true)
|
||||
})
|
||||
|
||||
it('should mark the agent model combobox as command-searchable', () => {
|
||||
const agentBlock = getBlock('agent')
|
||||
const modelSubBlock = agentBlock?.subBlocks.find((sb) => sb.id === 'model')
|
||||
|
||||
expect(agentBlock?.hideFromToolbar).not.toBe(true)
|
||||
expect(modelSubBlock?.type).toBe('combobox')
|
||||
expect(modelSubBlock?.commandSearchable).toBe(true)
|
||||
})
|
||||
|
||||
it('should hide generator API keys on hosted only for Fal.ai providers', () => {
|
||||
for (const blockType of ['image_generator_v2', 'video_generator_v3']) {
|
||||
const block = getBlock(blockType)
|
||||
const apiKeySubBlocks = block?.subBlocks.filter((sb) => sb.id === 'apiKey') ?? []
|
||||
|
||||
const falApiKeySubBlock = apiKeySubBlocks.find(
|
||||
(sb) => sb.condition?.field === 'provider' && sb.condition.value === 'falai'
|
||||
)
|
||||
const nonFalApiKeySubBlock = apiKeySubBlocks.find(
|
||||
(sb) =>
|
||||
sb.condition?.field === 'provider' &&
|
||||
sb.condition.value === 'falai' &&
|
||||
sb.condition.not === true
|
||||
)
|
||||
|
||||
expect(falApiKeySubBlock?.hideWhenHosted).toBe(true)
|
||||
expect(nonFalApiKeySubBlock).toBeDefined()
|
||||
expect(nonFalApiKeySubBlock?.hideWhenHosted).not.toBe(true)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('Block Consistency', () => {
|
||||
|
||||
@@ -133,6 +133,7 @@ Return ONLY the JSON array.`,
|
||||
required: true,
|
||||
defaultValue: 'claude-sonnet-4-6',
|
||||
options: getModelOptions,
|
||||
commandSearchable: true,
|
||||
},
|
||||
{
|
||||
id: 'attachmentFiles',
|
||||
|
||||
@@ -333,7 +333,8 @@ export const ImageGeneratorV2Block: BlockConfig<ImageGenerationResponse> = {
|
||||
{ label: 'Google Gemini', id: 'gemini' },
|
||||
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
|
||||
],
|
||||
value: () => 'openai',
|
||||
commandSearchable: true,
|
||||
value: () => 'falai',
|
||||
},
|
||||
{
|
||||
id: 'model',
|
||||
@@ -812,6 +813,18 @@ export const ImageGeneratorV2Block: BlockConfig<ImageGenerationResponse> = {
|
||||
placeholder: 'Enter your provider API key',
|
||||
password: true,
|
||||
connectionDroppable: false,
|
||||
hideWhenHosted: true,
|
||||
condition: { field: 'provider', value: 'falai' },
|
||||
},
|
||||
{
|
||||
id: 'apiKey',
|
||||
title: 'API Key',
|
||||
type: 'short-input',
|
||||
required: true,
|
||||
placeholder: 'Enter your provider API key',
|
||||
password: true,
|
||||
connectionDroppable: false,
|
||||
condition: { field: 'provider', value: 'falai', not: true },
|
||||
},
|
||||
],
|
||||
tools: {
|
||||
@@ -819,14 +832,13 @@ export const ImageGeneratorV2Block: BlockConfig<ImageGenerationResponse> = {
|
||||
config: {
|
||||
tool: () => 'image_generate',
|
||||
params: (params) => {
|
||||
if (!params.apiKey) {
|
||||
const provider = params.provider || 'openai'
|
||||
if (provider !== 'falai' && !params.apiKey) {
|
||||
throw new Error('API key is required')
|
||||
}
|
||||
if (!params.prompt) {
|
||||
throw new Error('Prompt is required')
|
||||
}
|
||||
|
||||
const provider = params.provider || 'openai'
|
||||
const defaultModel =
|
||||
provider === 'gemini'
|
||||
? 'gemini-3.1-flash-image-preview'
|
||||
|
||||
@@ -98,7 +98,7 @@ export const VideoGeneratorBlock: BlockConfig<VideoBlockResponse> = {
|
||||
{ label: 'MiniMax Hailuo', id: 'minimax' },
|
||||
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
|
||||
],
|
||||
value: () => 'runway',
|
||||
value: () => 'falai',
|
||||
required: true,
|
||||
},
|
||||
|
||||
@@ -776,6 +776,17 @@ export const VideoGeneratorBlock: BlockConfig<VideoBlockResponse> = {
|
||||
placeholder: 'Enter your provider API key',
|
||||
password: true,
|
||||
required: true,
|
||||
hideWhenHosted: true,
|
||||
condition: { field: 'provider', value: 'falai' },
|
||||
},
|
||||
{
|
||||
id: 'apiKey',
|
||||
title: 'API Key',
|
||||
type: 'short-input',
|
||||
placeholder: 'Enter your provider API key',
|
||||
password: true,
|
||||
required: true,
|
||||
condition: { field: 'provider', value: 'falai', not: true },
|
||||
},
|
||||
],
|
||||
|
||||
@@ -894,7 +905,8 @@ export const VideoGeneratorV2Block: BlockConfig<VideoBlockResponse> = {
|
||||
{ label: 'MiniMax Hailuo', id: 'minimax' },
|
||||
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
|
||||
],
|
||||
value: () => 'runway',
|
||||
commandSearchable: true,
|
||||
value: () => 'falai',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
@@ -1537,6 +1549,17 @@ export const VideoGeneratorV2Block: BlockConfig<VideoBlockResponse> = {
|
||||
placeholder: 'Enter your provider API key',
|
||||
password: true,
|
||||
required: true,
|
||||
hideWhenHosted: true,
|
||||
condition: { field: 'provider', value: 'falai' },
|
||||
},
|
||||
{
|
||||
id: 'apiKey',
|
||||
title: 'API Key',
|
||||
type: 'short-input',
|
||||
placeholder: 'Enter your provider API key',
|
||||
password: true,
|
||||
required: true,
|
||||
condition: { field: 'provider', value: 'falai', not: true },
|
||||
},
|
||||
],
|
||||
tools: {
|
||||
|
||||
@@ -109,7 +109,7 @@ export const VisionV2Block: BlockConfig<VisionResponse> = {
|
||||
type: 'vision_v2',
|
||||
name: 'Vision',
|
||||
description: 'Analyze images with vision models',
|
||||
hideFromToolbar: false,
|
||||
hideFromToolbar: true,
|
||||
tools: {
|
||||
access: ['vision_tool_v2'],
|
||||
config: {
|
||||
|
||||
@@ -305,6 +305,8 @@ export interface SubBlockConfig {
|
||||
multiSelect?: boolean
|
||||
// Combobox specific: Enable search input in dropdown
|
||||
searchable?: boolean
|
||||
/** Dropdown-specific: include static options as Cmd K search entries that preset this subblock. */
|
||||
commandSearchable?: boolean
|
||||
// Wand configuration for AI assistance
|
||||
wandConfig?: {
|
||||
enabled: boolean
|
||||
|
||||
@@ -7,6 +7,7 @@ export const byokProviderIdSchema = z.enum([
|
||||
'google',
|
||||
'mistral',
|
||||
'fireworks',
|
||||
'falai',
|
||||
'firecrawl',
|
||||
'exa',
|
||||
'serper',
|
||||
|
||||
@@ -37,6 +37,7 @@ export const imageToolBodySchema = z
|
||||
workflowId: z.string().optional(),
|
||||
executionId: z.string().optional(),
|
||||
userId: z.string().optional(),
|
||||
useHostedCostTracking: z.boolean().optional(),
|
||||
})
|
||||
.passthrough()
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ export const videoToolBodySchema = z
|
||||
workflowId: z.string().optional(),
|
||||
executionId: z.string().optional(),
|
||||
userId: z.string().optional(),
|
||||
useHostedCostTracking: z.boolean().optional(),
|
||||
})
|
||||
.passthrough()
|
||||
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { sleep } from '@sim/utils/helpers'
|
||||
|
||||
export const FALAI_HOSTED_KEY_MARKUP_MULTIPLIER = 1.5
|
||||
export const FALAI_IMAGE_FALLBACK_PROVIDER_COST_DOLLARS = 0.05
|
||||
export const FALAI_VIDEO_FALLBACK_PROVIDER_COST_DOLLARS = 0.25
|
||||
const FALAI_BILLING_EVENT_ATTEMPTS = 2
|
||||
const FALAI_BILLING_EVENT_RETRY_MS = 500
|
||||
const logger = createLogger('FalAIPricing')
|
||||
|
||||
export interface FalAICostMetadata {
|
||||
endpointId: string
|
||||
requestId: string
|
||||
costDollars: number
|
||||
source: 'billing_events' | 'historical_estimate' | 'fallback_floor'
|
||||
outputUnits?: number | null
|
||||
unitPrice?: number | null
|
||||
percentDiscount?: number | null
|
||||
currency?: string
|
||||
error?: string
|
||||
}
|
||||
|
||||
interface FalAIBillingEvent {
|
||||
request_id: string
|
||||
endpoint_id: string
|
||||
output_units: number | null
|
||||
unit_price: number | null
|
||||
percent_discount: number | null
|
||||
cost_estimate_nano_usd: number
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function getNumber(value: unknown): number | undefined {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined
|
||||
}
|
||||
|
||||
function getFalAIFallbackProviderCostDollars(endpointId: string): number {
|
||||
const normalizedEndpointId = endpointId.toLowerCase()
|
||||
const isImageEndpoint =
|
||||
normalizedEndpointId.includes('image') ||
|
||||
normalizedEndpointId.includes('nano-banana') ||
|
||||
normalizedEndpointId.includes('seedream') ||
|
||||
normalizedEndpointId.includes('flux') ||
|
||||
normalizedEndpointId.includes('grok-imagine')
|
||||
|
||||
return isImageEndpoint
|
||||
? FALAI_IMAGE_FALLBACK_PROVIDER_COST_DOLLARS
|
||||
: FALAI_VIDEO_FALLBACK_PROVIDER_COST_DOLLARS
|
||||
}
|
||||
|
||||
function parseBillingEvent(value: unknown): FalAIBillingEvent | undefined {
|
||||
if (!isRecord(value)) return undefined
|
||||
|
||||
const requestId = value.request_id
|
||||
const endpointId = value.endpoint_id
|
||||
const costEstimateNanoUsd = getNumber(value.cost_estimate_nano_usd)
|
||||
|
||||
if (typeof requestId !== 'string' || typeof endpointId !== 'string') return undefined
|
||||
if (costEstimateNanoUsd === undefined) return undefined
|
||||
|
||||
return {
|
||||
request_id: requestId,
|
||||
endpoint_id: endpointId,
|
||||
output_units: getNumber(value.output_units) ?? null,
|
||||
unit_price: getNumber(value.unit_price) ?? null,
|
||||
percent_discount: getNumber(value.percent_discount) ?? null,
|
||||
cost_estimate_nano_usd: costEstimateNanoUsd,
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchFalAIBillingEvent(
|
||||
apiKey: string,
|
||||
requestId: string
|
||||
): Promise<FalAIBillingEvent | undefined> {
|
||||
const url = new URL('https://api.fal.ai/v1/models/billing-events')
|
||||
url.searchParams.set('request_id', requestId)
|
||||
url.searchParams.set('limit', '1')
|
||||
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Key ${apiKey}`,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
logger.warn('Failed to fetch Fal.ai billing event', {
|
||||
requestId,
|
||||
error: getErrorMessage(error, 'Unknown error'),
|
||||
})
|
||||
return undefined
|
||||
}
|
||||
|
||||
if (!response.ok) return undefined
|
||||
|
||||
const data = await response.json().catch((error) => {
|
||||
logger.warn('Failed to parse Fal.ai billing event response', {
|
||||
requestId,
|
||||
error: getErrorMessage(error, 'Unknown error'),
|
||||
})
|
||||
return undefined
|
||||
})
|
||||
if (!isRecord(data) || !Array.isArray(data.billing_events)) return undefined
|
||||
|
||||
return data.billing_events.map(parseBillingEvent).find(Boolean)
|
||||
}
|
||||
|
||||
async function estimateFalAICallCost(
|
||||
apiKey: string,
|
||||
endpointId: string
|
||||
): Promise<{ costDollars?: number; error?: string }> {
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetch('https://api.fal.ai/v1/models/pricing/estimate', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Key ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
estimate_type: 'historical_api_price',
|
||||
endpoints: {
|
||||
[endpointId]: {
|
||||
call_quantity: 1,
|
||||
},
|
||||
},
|
||||
}),
|
||||
})
|
||||
} catch (error) {
|
||||
return { error: getErrorMessage(error, 'Unknown error') }
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text().catch(() => '')
|
||||
return { error: `Fal.ai pricing estimate failed: ${response.status} ${error}` }
|
||||
}
|
||||
|
||||
const data = (await response.json()) as unknown
|
||||
const totalCost = isRecord(data) ? getNumber(data.total_cost) : undefined
|
||||
if (totalCost === undefined) {
|
||||
return { error: 'Fal.ai pricing estimate missing total_cost' }
|
||||
}
|
||||
|
||||
return { costDollars: totalCost }
|
||||
}
|
||||
|
||||
export async function getFalAICostMetadata({
|
||||
apiKey,
|
||||
endpointId,
|
||||
requestId,
|
||||
}: {
|
||||
apiKey: string
|
||||
endpointId: string
|
||||
requestId: string
|
||||
}): Promise<FalAICostMetadata> {
|
||||
for (let attempt = 0; attempt < FALAI_BILLING_EVENT_ATTEMPTS; attempt++) {
|
||||
const event = await fetchFalAIBillingEvent(apiKey, requestId)
|
||||
if (event) {
|
||||
return {
|
||||
endpointId: event.endpoint_id,
|
||||
requestId: event.request_id,
|
||||
costDollars: event.cost_estimate_nano_usd / 1_000_000_000,
|
||||
source: 'billing_events',
|
||||
outputUnits: event.output_units,
|
||||
unitPrice: event.unit_price,
|
||||
percentDiscount: event.percent_discount,
|
||||
currency: 'USD',
|
||||
}
|
||||
}
|
||||
|
||||
if (attempt < FALAI_BILLING_EVENT_ATTEMPTS - 1) {
|
||||
await sleep(FALAI_BILLING_EVENT_RETRY_MS)
|
||||
}
|
||||
}
|
||||
|
||||
const estimate = await estimateFalAICallCost(apiKey, endpointId)
|
||||
if (estimate.costDollars !== undefined) {
|
||||
return {
|
||||
endpointId,
|
||||
requestId,
|
||||
costDollars: estimate.costDollars,
|
||||
source: 'historical_estimate',
|
||||
currency: 'USD',
|
||||
}
|
||||
}
|
||||
|
||||
logger.warn('Fal.ai cost metadata unavailable after generation completed', {
|
||||
endpointId,
|
||||
requestId,
|
||||
error: estimate.error,
|
||||
})
|
||||
|
||||
return {
|
||||
endpointId,
|
||||
requestId,
|
||||
costDollars: getFalAIFallbackProviderCostDollars(endpointId),
|
||||
source: 'fallback_floor',
|
||||
currency: 'USD',
|
||||
error: estimate.error,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import { createElement, type SVGProps } from 'react'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { BlockConfig } from '@/blocks/types'
|
||||
|
||||
const { mockGetAllBlocks, mockGetToolOperationsIndex, mockGetTriggersForSidebar } = vi.hoisted(
|
||||
() => ({
|
||||
mockGetAllBlocks: vi.fn(),
|
||||
mockGetToolOperationsIndex: vi.fn(() => []),
|
||||
mockGetTriggersForSidebar: vi.fn(() => []),
|
||||
})
|
||||
)
|
||||
|
||||
vi.mock('@/blocks', () => ({
|
||||
getAllBlocks: mockGetAllBlocks,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/search/tool-operations', () => ({
|
||||
getToolOperationsIndex: mockGetToolOperationsIndex,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/workflows/triggers/trigger-utils', () => ({
|
||||
getTriggersForSidebar: mockGetTriggersForSidebar,
|
||||
}))
|
||||
|
||||
import {
|
||||
buildCommandSearchableOptionSearchValue,
|
||||
useSearchModalStore,
|
||||
} from '@/stores/modals/search/store'
|
||||
|
||||
function TestIcon(props: SVGProps<SVGSVGElement>) {
|
||||
return createElement('svg', props)
|
||||
}
|
||||
|
||||
function createBlock(overrides: Partial<BlockConfig> = {}): BlockConfig {
|
||||
return {
|
||||
type: 'image_generator_v2',
|
||||
name: 'Image Generator',
|
||||
description: 'Generate images',
|
||||
category: 'tools',
|
||||
bgColor: '#4D5FFF',
|
||||
icon: TestIcon,
|
||||
subBlocks: [
|
||||
{
|
||||
id: 'provider',
|
||||
title: 'Provider',
|
||||
type: 'dropdown',
|
||||
commandSearchable: true,
|
||||
options: [
|
||||
{ label: 'OpenAI', id: 'openai' },
|
||||
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
|
||||
{ label: 'Hidden Provider', id: 'hidden', hidden: true },
|
||||
],
|
||||
},
|
||||
],
|
||||
tools: { access: ['image_generate'] },
|
||||
inputs: {},
|
||||
outputs: {},
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
describe('search modal store', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
useSearchModalStore.setState({
|
||||
isOpen: false,
|
||||
data: {
|
||||
blocks: [],
|
||||
tools: [],
|
||||
triggers: [],
|
||||
toolOperations: [],
|
||||
docs: [],
|
||||
isInitialized: false,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildCommandSearchableOptionSearchValue', () => {
|
||||
it('builds search terms for marked static dropdown options', () => {
|
||||
const block = createBlock()
|
||||
const searchValue = buildCommandSearchableOptionSearchValue(block)
|
||||
|
||||
expect(searchValue).toContain('Provider')
|
||||
expect(searchValue).toContain('Fal.ai (Multi-Model)')
|
||||
expect(searchValue).toContain('falai')
|
||||
expect(searchValue).not.toContain('Hidden Provider')
|
||||
expect(searchValue).not.toContain('hidden')
|
||||
})
|
||||
|
||||
it('does not index dropdowns that only use in-dropdown search', () => {
|
||||
const block = createBlock({
|
||||
subBlocks: [
|
||||
{
|
||||
id: 'timezone',
|
||||
title: 'Timezone',
|
||||
type: 'dropdown',
|
||||
searchable: true,
|
||||
options: [{ label: 'UTC', id: 'utc' }],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
expect(buildCommandSearchableOptionSearchValue(block)).toBe('')
|
||||
})
|
||||
|
||||
it('builds search terms for marked combobox option functions', () => {
|
||||
const block = createBlock({
|
||||
subBlocks: [
|
||||
{
|
||||
id: 'model',
|
||||
title: 'Model',
|
||||
type: 'combobox',
|
||||
commandSearchable: true,
|
||||
options: () => [
|
||||
{ label: 'claude-sonnet-4-6', id: 'claude-sonnet-4-6' },
|
||||
{ label: 'Hidden Model', id: 'hidden-model', hidden: true },
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const searchValue = buildCommandSearchableOptionSearchValue(block)
|
||||
|
||||
expect(searchValue).toContain('Model')
|
||||
expect(searchValue).toContain('claude-sonnet-4-6')
|
||||
expect(searchValue).not.toContain('Hidden Model')
|
||||
expect(searchValue).not.toContain('hidden-model')
|
||||
})
|
||||
})
|
||||
|
||||
it('adds command-searchable options to visible block search values without extra rows', () => {
|
||||
const visibleBlock = createBlock()
|
||||
const hiddenBlock = createBlock({
|
||||
type: 'hidden_generator',
|
||||
hideFromToolbar: true,
|
||||
})
|
||||
|
||||
mockGetAllBlocks.mockReturnValue([visibleBlock, hiddenBlock])
|
||||
|
||||
useSearchModalStore.getState().initializeData((blocks) => blocks)
|
||||
|
||||
const { tools } = useSearchModalStore.getState().data
|
||||
expect(tools).toHaveLength(1)
|
||||
expect(tools[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 'image_generator_v2',
|
||||
searchValue: expect.stringContaining('Fal.ai (Multi-Model)'),
|
||||
})
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -4,6 +4,7 @@ import { devtools } from 'zustand/middleware'
|
||||
import { getToolOperationsIndex } from '@/lib/search/tool-operations'
|
||||
import { getTriggersForSidebar } from '@/lib/workflows/triggers/trigger-utils'
|
||||
import { getAllBlocks } from '@/blocks'
|
||||
import type { BlockConfig, SubBlockConfig } from '@/blocks/types'
|
||||
import type {
|
||||
SearchBlockItem,
|
||||
SearchData,
|
||||
@@ -21,6 +22,47 @@ const initialData: SearchData = {
|
||||
isInitialized: false,
|
||||
}
|
||||
|
||||
type CommandSearchableOption = {
|
||||
label: string
|
||||
id: string
|
||||
hidden?: boolean
|
||||
}
|
||||
|
||||
function getCommandSearchableOptions(subBlock: SubBlockConfig): CommandSearchableOption[] {
|
||||
if (!subBlock.options) return []
|
||||
|
||||
try {
|
||||
const options = typeof subBlock.options === 'function' ? subBlock.options() : subBlock.options
|
||||
return Array.isArray(options) ? options : []
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
export function buildCommandSearchableOptionSearchValue(block: BlockConfig): string {
|
||||
const terms = new Set<string>()
|
||||
|
||||
for (const subBlock of block.subBlocks) {
|
||||
if (
|
||||
(subBlock.type !== 'dropdown' && subBlock.type !== 'combobox') ||
|
||||
!subBlock.commandSearchable
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
for (const option of getCommandSearchableOptions(subBlock)) {
|
||||
if (option.hidden) continue
|
||||
|
||||
const subBlockTitle = subBlock.title ?? subBlock.id
|
||||
terms.add(subBlockTitle)
|
||||
terms.add(option.label)
|
||||
terms.add(option.id)
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(terms).join(' ')
|
||||
}
|
||||
|
||||
export const useSearchModalStore = create<SearchModalState>()(
|
||||
devtools(
|
||||
(set, _) => ({
|
||||
@@ -56,6 +98,7 @@ export const useSearchModalStore = create<SearchModalState>()(
|
||||
icon: block.icon,
|
||||
bgColor: block.bgColor || '#6B7280',
|
||||
type: block.type,
|
||||
searchValue: `${block.name} ${block.type} block-${block.type} ${buildCommandSearchableOptionSearchValue(block)}`,
|
||||
}
|
||||
|
||||
if (block.category === 'blocks' && block.type !== 'starter') {
|
||||
|
||||
@@ -11,6 +11,7 @@ export interface SearchBlockItem {
|
||||
bgColor: string
|
||||
type: string
|
||||
config?: BlockConfig
|
||||
searchValue?: string
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
|
||||
FALAI_IMAGE_FALLBACK_PROVIDER_COST_DOLLARS,
|
||||
getFalAICostMetadata,
|
||||
} from '@/lib/tools/falai-pricing'
|
||||
import { imageGenerateTool } from '@/tools/image/generate'
|
||||
import { falaiVideoTool } from '@/tools/video/falai'
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
describe('Fal.ai hosted key pricing', () => {
|
||||
it('applies hosted markup to image generation provider cost', () => {
|
||||
const pricing = imageGenerateTool.hosting?.pricing
|
||||
expect(pricing?.type).toBe('custom')
|
||||
if (!pricing || pricing.type !== 'custom') throw new Error('Expected custom pricing')
|
||||
|
||||
const result = pricing.getCost(
|
||||
{},
|
||||
{
|
||||
__falaiCostDollars: 0.1,
|
||||
__falaiBilling: {
|
||||
source: 'billing_events',
|
||||
endpointId: 'fal-ai/nano-banana-2',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
expect(typeof result).toBe('object')
|
||||
if (typeof result === 'number') throw new Error('Expected structured pricing result')
|
||||
expect(result.cost).toBeCloseTo(0.1 * FALAI_HOSTED_KEY_MARKUP_MULTIPLIER)
|
||||
expect(result.metadata).toMatchObject({
|
||||
providerCostDollars: 0.1,
|
||||
markupMultiplier: FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
|
||||
source: 'billing_events',
|
||||
})
|
||||
})
|
||||
|
||||
it('applies hosted markup to video generation provider cost', () => {
|
||||
const pricing = falaiVideoTool.hosting?.pricing
|
||||
expect(pricing?.type).toBe('custom')
|
||||
if (!pricing || pricing.type !== 'custom') throw new Error('Expected custom pricing')
|
||||
|
||||
const result = pricing.getCost(
|
||||
{},
|
||||
{
|
||||
__falaiCostDollars: 0.4,
|
||||
__falaiBilling: {
|
||||
source: 'billing_events',
|
||||
endpointId: 'fal-ai/veo3.1',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
expect(typeof result).toBe('object')
|
||||
if (typeof result === 'number') throw new Error('Expected structured pricing result')
|
||||
expect(result.cost).toBeCloseTo(0.4 * FALAI_HOSTED_KEY_MARKUP_MULTIPLIER)
|
||||
expect(result.metadata).toMatchObject({
|
||||
providerCostDollars: 0.4,
|
||||
markupMultiplier: FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
|
||||
source: 'billing_events',
|
||||
})
|
||||
})
|
||||
|
||||
it('returns fallback floor cost metadata instead of throwing when billing and estimate fail', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const mockFetch = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ billing_events: [] }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ billing_events: [] }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('pricing unavailable', { status: 500 }))
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const resultPromise = getFalAICostMetadata({
|
||||
apiKey: 'fal-key',
|
||||
endpointId: 'fal-ai/nano-banana-2',
|
||||
requestId: 'request-1',
|
||||
})
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
const result = await resultPromise
|
||||
|
||||
expect(result).toMatchObject({
|
||||
endpointId: 'fal-ai/nano-banana-2',
|
||||
requestId: 'request-1',
|
||||
costDollars: FALAI_IMAGE_FALLBACK_PROVIDER_COST_DOLLARS,
|
||||
source: 'fallback_floor',
|
||||
currency: 'USD',
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,3 +1,4 @@
|
||||
import { FALAI_HOSTED_KEY_MARKUP_MULTIPLIER } from '@/lib/tools/falai-pricing'
|
||||
import type { ImageGenerationParams, ImageGenerationResponse } from '@/tools/image/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
@@ -113,6 +114,38 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
enabled: (params) => params.provider === 'falai',
|
||||
envKeyPrefix: 'FALAI_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'falai',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
const providerCostDollars = output.__falaiCostDollars
|
||||
if (typeof providerCostDollars !== 'number' || Number.isNaN(providerCostDollars)) {
|
||||
throw new Error('Fal.ai image response missing cost data')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: providerCostDollars * FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
|
||||
metadata: {
|
||||
...(typeof output.__falaiBilling === 'object' && output.__falaiBilling !== null
|
||||
? (output.__falaiBilling as Record<string, unknown>)
|
||||
: {}),
|
||||
providerCostDollars,
|
||||
markupMultiplier: FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 40,
|
||||
burstMultiplier: 1,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: '/api/tools/image',
|
||||
method: 'POST',
|
||||
@@ -122,6 +155,7 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
|
||||
body: (
|
||||
params: ImageGenerationParams & {
|
||||
_context?: { workspaceId?: string; workflowId?: string; executionId?: string }
|
||||
__usingHostedKey?: boolean
|
||||
}
|
||||
) => ({
|
||||
provider: params.provider,
|
||||
@@ -144,6 +178,7 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
|
||||
workspaceId: params._context?.workspaceId,
|
||||
workflowId: params._context?.workflowId,
|
||||
executionId: params._context?.executionId,
|
||||
useHostedCostTracking: params.__usingHostedKey === true,
|
||||
}),
|
||||
},
|
||||
|
||||
@@ -159,6 +194,8 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
|
||||
provider?: string
|
||||
model?: string
|
||||
metadata?: ImageGenerationResponse['output']['metadata']
|
||||
__falaiCostDollars?: number
|
||||
__falaiBilling?: ImageGenerationResponse['output']['__falaiBilling']
|
||||
}
|
||||
|
||||
if (!response.ok || data.error) {
|
||||
@@ -203,6 +240,8 @@ export const imageGenerateTool: ToolConfig<ImageGenerationParams, ImageGeneratio
|
||||
model: data.model || data.metadata?.model || '',
|
||||
...data.metadata,
|
||||
},
|
||||
__falaiCostDollars: data.__falaiCostDollars,
|
||||
__falaiBilling: data.__falaiBilling,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
@@ -36,5 +36,16 @@ export interface ImageGenerationResponse extends ToolResponse {
|
||||
jobId?: string
|
||||
contentType?: string
|
||||
}
|
||||
__falaiCostDollars?: number
|
||||
__falaiBilling?: {
|
||||
endpointId: string
|
||||
requestId: string
|
||||
source: 'billing_events' | 'historical_estimate' | 'fallback_floor'
|
||||
outputUnits?: number | null
|
||||
unitPrice?: number | null
|
||||
percentDiscount?: number | null
|
||||
currency?: string
|
||||
error?: string
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2797,4 +2797,128 @@ describe('Cost Field Handling', () => {
|
||||
|
||||
Object.assign(tools, originalTools)
|
||||
})
|
||||
|
||||
it('should skip hosted key injection when hosting predicate is false', async () => {
|
||||
const mockTool = {
|
||||
id: 'test_conditional_hosting',
|
||||
name: 'Test Conditional Hosting',
|
||||
description: 'A test tool with conditional hosted keys',
|
||||
version: '1.0.0',
|
||||
params: {
|
||||
provider: { type: 'string', required: false },
|
||||
apiKey: { type: 'string', required: false },
|
||||
},
|
||||
hosting: {
|
||||
enabled: (params: { provider?: string }) => params.provider === 'hosted-provider',
|
||||
envKeyPrefix: 'TEST_HOSTED_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
pricing: {
|
||||
type: 'per_request' as const,
|
||||
cost: 0.005,
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request' as const,
|
||||
requestsPerMinute: 100,
|
||||
},
|
||||
},
|
||||
request: {
|
||||
url: '/api/test/conditional-hosting',
|
||||
method: 'POST' as const,
|
||||
headers: () => ({ 'Content-Type': 'application/json' }),
|
||||
},
|
||||
transformResponse: vi.fn().mockResolvedValue({
|
||||
success: true,
|
||||
output: { result: 'success' },
|
||||
}),
|
||||
}
|
||||
|
||||
const originalTools = { ...tools }
|
||||
;(tools as any).test_conditional_hosting = mockTool
|
||||
|
||||
global.fetch = Object.assign(
|
||||
vi.fn().mockImplementation(async () => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
headers: new Headers(),
|
||||
json: () => Promise.resolve({ success: true }),
|
||||
})),
|
||||
{ preconnect: vi.fn() }
|
||||
) as typeof fetch
|
||||
|
||||
const mockContext = createToolExecutionContext({
|
||||
userId: 'user-123',
|
||||
} as any)
|
||||
const result = await executeTool(
|
||||
'test_conditional_hosting',
|
||||
{ provider: 'user-provider' },
|
||||
{ executionContext: mockContext }
|
||||
)
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
expect(mockRateLimiterFns.acquireKey).not.toHaveBeenCalled()
|
||||
expect(result.output.cost).toBeUndefined()
|
||||
|
||||
Object.assign(tools, originalTools)
|
||||
})
|
||||
|
||||
it('should skip hosted key injection when user provides an API key', async () => {
|
||||
const mockTool = {
|
||||
id: 'test_user_key_priority',
|
||||
name: 'Test User Key Priority',
|
||||
description: 'A test tool where user keys should win',
|
||||
version: '1.0.0',
|
||||
params: {
|
||||
apiKey: { type: 'string', required: false },
|
||||
},
|
||||
hosting: {
|
||||
envKeyPrefix: 'TEST_HOSTED_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
pricing: {
|
||||
type: 'per_request' as const,
|
||||
cost: 0.005,
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request' as const,
|
||||
requestsPerMinute: 100,
|
||||
},
|
||||
},
|
||||
request: {
|
||||
url: '/api/test/user-key-priority',
|
||||
method: 'POST' as const,
|
||||
headers: () => ({ 'Content-Type': 'application/json' }),
|
||||
},
|
||||
transformResponse: vi.fn().mockResolvedValue({
|
||||
success: true,
|
||||
output: { result: 'success' },
|
||||
}),
|
||||
}
|
||||
|
||||
const originalTools = { ...tools }
|
||||
;(tools as any).test_user_key_priority = mockTool
|
||||
|
||||
global.fetch = Object.assign(
|
||||
vi.fn().mockImplementation(async () => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
headers: new Headers(),
|
||||
json: () => Promise.resolve({ success: true }),
|
||||
})),
|
||||
{ preconnect: vi.fn() }
|
||||
) as typeof fetch
|
||||
|
||||
const mockContext = createToolExecutionContext({
|
||||
userId: 'user-123',
|
||||
} as any)
|
||||
const result = await executeTool(
|
||||
'test_user_key_priority',
|
||||
{ apiKey: 'user-api-key' },
|
||||
{ executionContext: mockContext }
|
||||
)
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
expect(mockRateLimiterFns.acquireKey).not.toHaveBeenCalled()
|
||||
expect(result.output.cost).toBeUndefined()
|
||||
|
||||
Object.assign(tools, originalTools)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -232,8 +232,15 @@ async function injectHostedKeyIfNeeded(
|
||||
): Promise<HostedKeyInjectionResult> {
|
||||
if (!tool.hosting) return { isUsingHostedKey: false }
|
||||
if (!isHosted) return { isUsingHostedKey: false }
|
||||
if (tool.hosting.enabled && !tool.hosting.enabled(params)) {
|
||||
return { isUsingHostedKey: false }
|
||||
}
|
||||
|
||||
const { envKeyPrefix, apiKeyParam, byokProviderId, rateLimit } = tool.hosting
|
||||
const userProvidedKey = params[apiKeyParam]
|
||||
if (typeof userProvidedKey === 'string' && userProvidedKey.trim().length > 0) {
|
||||
return { isUsingHostedKey: false }
|
||||
}
|
||||
|
||||
const { workspaceId, userId, workflowId } = resolveToolScope(params, executionContext)
|
||||
|
||||
@@ -299,6 +306,7 @@ async function injectHostedKeyIfNeeded(
|
||||
}
|
||||
|
||||
params[apiKeyParam] = acquireResult.key
|
||||
params.__usingHostedKey = true
|
||||
logger.info(`[${requestId}] Using hosted key for ${tool.id} (${acquireResult.envVarName})`, {
|
||||
keyIndex: acquireResult.keyIndex,
|
||||
provider,
|
||||
|
||||
@@ -8,6 +8,7 @@ export type BYOKProviderId =
|
||||
| 'google'
|
||||
| 'mistral'
|
||||
| 'fireworks'
|
||||
| 'falai'
|
||||
| 'firecrawl'
|
||||
| 'exa'
|
||||
| 'serper'
|
||||
@@ -307,6 +308,8 @@ export type ToolHostingPricing<P = Record<string, unknown>> = PerRequestPricing
|
||||
* no code changes needed.
|
||||
*/
|
||||
interface ToolHostingConfig<P = Record<string, unknown>> {
|
||||
/** Optional predicate for tools where hosted keys only apply to some parameter combinations. */
|
||||
enabled?: (params: P) => boolean
|
||||
/**
|
||||
* Env var name prefix for hosted keys.
|
||||
* At runtime, `{envKeyPrefix}_COUNT` is read to determine how many keys exist,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { FALAI_HOSTED_KEY_MARKUP_MULTIPLIER } from '@/lib/tools/falai-pricing'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
import type { VideoParams, VideoResponse } from '@/tools/video/types'
|
||||
import { parseBooleanParam, parseBooleanParamWithDefault } from '@/tools/video/utils'
|
||||
@@ -68,6 +69,37 @@ export const falaiVideoTool: ToolConfig<VideoParams, VideoResponse> = {
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
envKeyPrefix: 'FALAI_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'falai',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
const providerCostDollars = output.__falaiCostDollars
|
||||
if (typeof providerCostDollars !== 'number' || Number.isNaN(providerCostDollars)) {
|
||||
throw new Error('Fal.ai video response missing cost data')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: providerCostDollars * FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
|
||||
metadata: {
|
||||
...(typeof output.__falaiBilling === 'object' && output.__falaiBilling !== null
|
||||
? (output.__falaiBilling as Record<string, unknown>)
|
||||
: {}),
|
||||
providerCostDollars,
|
||||
markupMultiplier: FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 40,
|
||||
burstMultiplier: 1,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: '/api/tools/video',
|
||||
method: 'POST',
|
||||
@@ -77,6 +109,7 @@ export const falaiVideoTool: ToolConfig<VideoParams, VideoResponse> = {
|
||||
body: (
|
||||
params: VideoParams & {
|
||||
_context?: { workspaceId?: string; workflowId?: string; executionId?: string }
|
||||
__usingHostedKey?: boolean
|
||||
}
|
||||
) => ({
|
||||
provider: 'falai',
|
||||
@@ -91,6 +124,7 @@ export const falaiVideoTool: ToolConfig<VideoParams, VideoResponse> = {
|
||||
workspaceId: params._context?.workspaceId,
|
||||
workflowId: params._context?.workflowId,
|
||||
executionId: params._context?.executionId,
|
||||
useHostedCostTracking: params.__usingHostedKey === true,
|
||||
}),
|
||||
},
|
||||
|
||||
@@ -128,6 +162,8 @@ export const falaiVideoTool: ToolConfig<VideoParams, VideoResponse> = {
|
||||
provider: 'falai',
|
||||
model: data.model,
|
||||
jobId: data.jobId,
|
||||
__falaiCostDollars: data.__falaiCostDollars,
|
||||
__falaiBilling: data.__falaiBilling,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
@@ -33,6 +33,17 @@ export interface VideoResponse extends ToolResponse {
|
||||
provider?: string
|
||||
model?: string
|
||||
jobId?: string
|
||||
__falaiCostDollars?: number
|
||||
__falaiBilling?: {
|
||||
endpointId: string
|
||||
requestId: string
|
||||
source: 'billing_events' | 'historical_estimate' | 'fallback_floor'
|
||||
outputUnits?: number | null
|
||||
unitPrice?: number | null
|
||||
percentDiscount?: number | null
|
||||
currency?: string
|
||||
error?: string
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user