diff --git a/apps/stage-pocket/package.json b/apps/stage-pocket/package.json index 5753f85186..b407576b63 100644 --- a/apps/stage-pocket/package.json +++ b/apps/stage-pocket/package.json @@ -119,6 +119,7 @@ "@iconify-json/simple-icons": "catalog:", "@iconify-json/solar": "catalog:", "@iconify-json/svg-spinners": "catalog:", + "@iconify-json/tabler": "catalog:", "@iconify-json/vscode-icons": "catalog:", "@intlify/unplugin-vue-i18n": "catalog:", "@proj-airi/cap-vite": "workspace:*", diff --git a/apps/stage-tamagotchi-kirie/package.json b/apps/stage-tamagotchi-kirie/package.json index e35719736d..c68d260a84 100644 --- a/apps/stage-tamagotchi-kirie/package.json +++ b/apps/stage-tamagotchi-kirie/package.json @@ -140,6 +140,7 @@ "@iconify-json/simple-icons": "catalog:", "@iconify-json/solar": "catalog:", "@iconify-json/svg-spinners": "catalog:", + "@iconify-json/tabler": "catalog:", "@iconify-json/vscode-icons": "catalog:", "@iconify/utils": "catalog:", "@intlify/unplugin-vue-i18n": "catalog:", diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index c3bca51522..b13c2a4d58 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -173,6 +173,7 @@ "@iconify-json/simple-icons": "catalog:", "@iconify-json/solar": "catalog:", "@iconify-json/svg-spinners": "catalog:", + "@iconify-json/tabler": "catalog:", "@iconify-json/vscode-icons": "catalog:", "@iconify/utils": "catalog:", "@intlify/unplugin-vue-i18n": "catalog:", diff --git a/packages/agents/README.md b/packages/agents/README.md index 6ddc3bbf6b..676ff53888 100644 --- a/packages/agents/README.md +++ b/packages/agents/README.md @@ -8,7 +8,7 @@ Each agent owns one prompt, one request shape, and one reply parser. An agent do | Agent | Entry | Use | | --- | --- | --- | -| Tool card | `@proj-airi/agents/tool-card` | Writes an OpenUI Lang card that shows the results of one tool. | +| Tool card | `@proj-airi/agents/tool-card` | Writes an OpenUI Lang card that shows the results of one tool. It can search icons with the tools that the caller passes. | | Voice end judge | `@proj-airi/agents/voice-end-judge` | Decides whether a paused spoken message is finished. | | Voice transcription rephrase | `@proj-airi/agents/voice-transcription-rephrase` | Cleans up the speech recognition text of one spoken message. | diff --git a/packages/agents/src/generate-text.ts b/packages/agents/src/generate-text.ts index d854b4f4b3..63e8744f00 100644 --- a/packages/agents/src/generate-text.ts +++ b/packages/agents/src/generate-text.ts @@ -1,4 +1,4 @@ -import type { Conversation } from '@proj-airi/core-agent' +import type { Conversation, StreamOptions } from '@proj-airi/core-agent' import type { GenerationProvider } from '@proj-airi/provider-inference' import { streamFrom } from '@proj-airi/core-agent' @@ -8,8 +8,11 @@ import { streamFrom } from '@proj-airi/core-agent' * * Agents receive this function instead of a provider, so each runtime keeps its own request policy. * For example, stage-ui passes its LLM store, which remembers provider compatibility. + * + * `tools` are the only tools of the request, for example an icon search. The provider adds none of its own, such as a + * web search. Text before a tool call only leads up to the call, so the reply is the text after the last call. */ -export type GenerateText = (conversation: Conversation, signal: AbortSignal) => Promise +export type GenerateText = (conversation: Conversation, signal: AbortSignal, tools?: Extract) => Promise /** * Creates a {@link GenerateText} that streams with the core-agent runtime and collects the text deltas. @@ -18,19 +21,23 @@ export type GenerateText = (conversation: Conversation, signal: AbortSignal) => * - A caller has no runtime of its own, for example an eval or a script. * * Returns: - * - A generator that requests no tools. + * - A generator that requests only the tools that a call passes. */ export function generateTextFrom(model: string, provider: GenerationProvider): GenerateText { - return async (conversation, signal) => { + return async (conversation, signal, tools) => { let reply = '' await streamFrom({ model, chatProvider: provider, conversation, options: { - supportsTools: false, + supportsTools: Boolean(tools), + tools, + providerTools: false, abortSignal: signal, onStreamEvent: (event) => { + if (event.type === 'tool-call') + reply = '' if (event.type === 'text-delta') reply += event.text }, diff --git a/packages/agents/src/tool-card/index.ts b/packages/agents/src/tool-card/index.ts index 46b7fbeaa6..8a3ae03639 100644 --- a/packages/agents/src/tool-card/index.ts +++ b/packages/agents/src/tool-card/index.ts @@ -11,7 +11,7 @@ import type { GenerateText } from '../generate-text' export const TOOL_CARD_INSTRUCTIONS = [ 'You design a card that shows the result of one tool call in a chat app.', 'The app saves the card and shows every later result of the same tool with it, so the card must work for other results of the same shape.', - 'You cannot call tools. The user message gives the result of one call as a sample. A result can be an object, a list, or text.', + 'You cannot call the tool of the card. The user message gives the result of one call as a sample. A result can be an object, a list, or text.', 'Read the result with exactly one Query, for example `data = Query("tool_name", {}, {items: []})`. The app answers that Query with the result of each call.', 'Read every value from the Query. Never copy values of the sample into the program.', 'Do not use Mutation, forms, or $variables.', @@ -96,6 +96,8 @@ export interface ToolCardAgentOptions { * The model reads each problem in the fix round, so a problem names the statement and the fix. */ validate: (program: string) => string[] | Promise + /** Tools that the model can use while it writes, for example an icon search. */ + tools?: Parameters[2] } /** @@ -112,7 +114,7 @@ export interface ToolCardAgentOptions { export function createToolCardAgent(options: ToolCardAgentOptions) { async function make(request: ToolCardRequest, signal: AbortSignal): Promise { const conversation = createToolCardConversation(request) - const reply = await options.generateText(conversation, signal) + const reply = await options.generateText(conversation, signal, options.tools) const card = readToolCardReply(reply) const problems = await options.validate(card.program) if (problems.length === 0) @@ -129,7 +131,7 @@ export function createToolCardAgent(options: ToolCardAgentOptions) { const fixed = readToolCardReply(await options.generateText({ turns: [ ...conversation.turns, { id: 'tool-card-fix', type: 'user', content: [{ type: 'text', text: fix }] }, - ] }, signal)) + ] }, signal, options.tools)) const remaining = await options.validate(fixed.program) if (remaining.length > 0) throw new Error(`The card program still has problems: ${remaining.join('; ')}`) diff --git a/packages/core-agent/src/runtime/llm-service.ts b/packages/core-agent/src/runtime/llm-service.ts index c20d0daa38..5fa02a1d2e 100644 --- a/packages/core-agent/src/runtime/llm-service.ts +++ b/packages/core-agent/src/runtime/llm-service.ts @@ -143,7 +143,7 @@ async function streamOnce({ try { const streamResult = request.protocol === 'responses' - ? streamResponses({ config: request.config, webSearch: supportedTools && request.webSearch, conversation, scope, options, tools, initialStep, onEvent }) + ? streamResponses({ config: request.config, webSearch: supportedTools && options?.providerTools !== false && request.webSearch, conversation, scope, options, tools, initialStep, onEvent }) : streamChatCompletions({ config: request.config, conversation, scope, options, tools, initialStep, onEvent, supportsContentArray: contentArraySupported }) // NOTICE: diff --git a/packages/core-agent/src/runtime/responses.ts b/packages/core-agent/src/runtime/responses.ts index b7a7225132..fb5e26d754 100644 --- a/packages/core-agent/src/runtime/responses.ts +++ b/packages/core-agent/src/runtime/responses.ts @@ -221,7 +221,7 @@ export function streamResponses(input: { maxOutputTokens: next.maxTokens ?? input.options?.maxTokens, headers: mergeRequestHeaders(nextRequest.config.headers, next.headers), tools: toolsSupported - ? nextRequest.webSearch ? [...(next.tools ?? []), { type: 'web_search' as const }] : next.tools + ? nextRequest.webSearch && input.options?.providerTools !== false ? [...(next.tools ?? []), { type: 'web_search' as const }] : next.tools : undefined, toolChoice: undefined, }) diff --git a/packages/core-agent/src/types/llm.ts b/packages/core-agent/src/types/llm.ts index 97c41c74ed..510dcb0be9 100644 --- a/packages/core-agent/src/types/llm.ts +++ b/packages/core-agent/src/types/llm.ts @@ -105,6 +105,12 @@ export interface StreamOptions { /** Provider tool-selection directive for one request. */ toolChoice?: ToolChoice tools?: Tool[] | (() => Promise) + /** + * Lets the provider add its own tools, for example the web search of OpenAI Responses that its settings turn on. + * A task beside the chat sets it to `false`, so its request carries only the tools that it passes. + * @default true + */ + providerTools?: boolean /** * Per-model runtime cache of whether the provider accepts content-part arrays * (e.g. `[{type:'text',...},{type:'image_url',...}]`) for `messages[].content`. diff --git a/packages/stage-ui/package.json b/packages/stage-ui/package.json index aac847a7ed..9b7b758cb2 100644 --- a/packages/stage-ui/package.json +++ b/packages/stage-ui/package.json @@ -80,6 +80,7 @@ "@formkit/auto-animate": "catalog:", "@huggingface/transformers": "catalog:", "@iconify-json/mingcute": "catalog:", + "@iconify-json/tabler": "catalog:", "@iconify/utils": "catalog:", "@moeru/eventa": "catalog:", "@moeru/std": "catalog:", diff --git a/packages/stage-ui/src/components/scenarios/chat/components/tool-call-group.vue b/packages/stage-ui/src/components/scenarios/chat/components/tool-call-group.vue index 020cd95b5e..3f5dc9a093 100644 --- a/packages/stage-ui/src/components/scenarios/chat/components/tool-call-group.vue +++ b/packages/stage-ui/src/components/scenarios/chat/components/tool-call-group.vue @@ -15,6 +15,7 @@ import CardView from '../../../../libs/cards/card-view.vue' import HitPill from './hit-pill.vue' import ToolCallPayload from './tool-call-payload.vue' +import { tablerIconMask } from '../../../../libs/tabler-icons' import { readCallSummary } from '../../../../stores/ai/chat-llm/tool-call-summary' import { useLlmToolsStore } from '../../../../stores/ai/chat-llm/tools' import { canRerunToolCall } from '../../../../stores/tool-call-rerun' @@ -51,8 +52,12 @@ const toolCards = useToolCardsStore() const calls = computed(() => { return props.items.map((item) => { - // The model says in the call what it does and why. The row shows it, and a new card of the tool learns from it. - const summary = readCallSummary(item.toolCall.args) + // The model says in the call what it does and why, with an icon for it. The row shows both, and a new card of the + // tool learns from the line. + const { text: summary, icons } = readCallSummary(item.toolCall.args) + // The icon draws in the element of the tool icon, so it keeps the size and the color of that icon. The tool icon + // shows when no candidate leads to an icon. + const iconMask = tablerIconMask(icons) const result = props.resultOf(item.sliceIndex) const widget = props.widgets[item.toolCall.toolName] const reading = widget?.read(result?.result, item.toolCall.args) @@ -69,6 +74,7 @@ const calls = computed(() => { // A widget title already says the state of the call. titled: Boolean(title), icon: toolIconOf(item.toolCall.toolName, llmTools.tools.findLast(tool => tool.function.name === item.toolCall.toolName)?.icon), + iconStyle: iconMask ? { '--un-icon': iconMask } : undefined, state, result: result?.result, widget: widget && reading ? { component: widget.component, ...reading } : undefined, @@ -128,7 +134,8 @@ function popLabelIn(el: Element, done: () => void) { function hitIcon(call: { state: ToolCallState, icon: string }) { if (call.state === 'executing') return 'i-svg-spinners:ring-resize' - return call.state === 'error' ? 'i-mingcute:close-circle-line' : call.icon + // The error icon takes the place of a tool icon, so it comes from the same set. + return call.state === 'error' ? 'i-tabler:circle-x' : call.icon } /** @@ -223,7 +230,7 @@ function rerun(item: ToolCallGroupItem) { - + @@ -253,7 +260,10 @@ function rerun(item: ToolCallGroupItem) { @click="toggleCall(call.key)" >