From bc41885f0bfb3483a93833f4bcfbc8ad3afe483d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=93=9D=E8=8E=93=F0=9F=AB=90?= Date: Tue, 20 Oct 2026 19:43:42 +0700 Subject: [PATCH] feat(stage-ui,agents,core-agent): let the model pick a Tabler icon for each tool call (#3051) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description A tool call row shows the summary of the call, but every call of a tool without its own icon shows the same magic wand. With this PR, the model picks an icon for each call, from the Tabler icon set. - Every tool gets a second argument `call_icon` after `call_summary`: up to three Tabler icon names for what the call does, best first. The tool never gets it, the same as `call_summary`. - The chat draws the first name that exists. A name that does not exist goes through a search of the icon names, so a close name still finds an icon. Without a match, the row keeps the default icon of the tool. - The picked icon replaces the `--un-icon` mask of the icon element that the row and the pill already have, so size and color stay. There is no extra request, and nothing is stored outside the recorded arguments. - Cards can use any Tabler icon. The card model and the chat model for reply cards get `builtIn_searchIcons`. It searches the words of the icon names, and a variant, for example `bookmark-off`, goes with its icon. The card validator rejects a name that does not exist. - The default tool icons, the skill icons, and the error icon of the pill are Tabler outline icons too, so one row of icons uses one set. - A task beside the chat, for example the card model, gets only its own tools. It does not get the tools of the chat, or the web search that OpenAI Responses adds from the provider settings. Why the model writes names: I compared names with local matching on two test sets. One set has 400 English tool call labels, in four kinds. The other has 104 labels in the 10 languages of AIRI. `gpt-6-sol` judged each icon. The first icon from names fit 89% and 91% of the labels. The best local method fit 74%. That method was a search of the Tabler tags with a trained ranker, or EmbeddingGemma on the label itself. Names cost a few tokens per call. In the 1437 labels, the names held a real icon name 1427 times, so the search that corrects a name runs rarely. ## Changes | Part | Change | | --- | --- | | `stores/ai/chat-llm/tool-call-summary.ts` | Adds `call_icon` after `call_summary`, and removes both before the tool gets its input. `readCallSummary` reads the line and the icon names, each with its own fallback. | | `libs/tabler-icons.ts` | New. Loads the Tabler icons on first use. Draws an icon by name for cards, searches icons by the words of their names, and turns `call_icon` candidates into the `--un-icon` mask of a row. | | `tools/search-icons.ts` | New. `builtIn_searchIcons` for the card model and for reply cards. | | `tool-call-group.vue` | The row and the pill draw the picked icon. The error icon of the pill is Tabler. | | `tools/icons.ts` | The default tool icons and the skill icons are Tabler outline icons. | | `libs/cards/catalog.ts`, `libs/cards/program.ts` | Cards draw Tabler icons, and the validator checks the names against the Tabler set. | | `stores/tool-cards.ts`, `stores/chat.ts` | The card model gets the icon search. A chat that draws reply cards gets it with the card tool. | | `composables/text-generation.ts`, `stores/ai/chat-llm/llm.ts`, `@proj-airi/agents` | `generateText` takes the tools of a task. A request with task tools gets no tools of the chat and no tools of the provider. | | `@proj-airi/core-agent` | New stream option `providerTools`. When it is `false`, OpenAI Responses adds no web search. | | `package.json` files | `@iconify-json/tabler` for stage-ui and for the UnoCSS classes of the desktop, mobile, and Kirie apps. It is the only source of the icons. | ## Verification - The pre-commit hook linted the staged files of each commit. I did not run the typecheck, the tests, or the full lint locally. CI runs them. - `.vishot/capture-tool-call-icons.mjs` (local and ignored) puts one reply with three calls into the stage-web chat. Each call has a summary and icon names: a web search and a weather call that finished, and a page read that failed. It opens the pill and captures at 1440×1600, light, English. The same script ran against `main` (`c8943f47c`) and against this branch. ## Visual changes | Before | After | |---|---| | ![](https://github.com/user-attachments/assets/800e4964-7bb4-485a-94e6-0eafd295b0c6) | ![](https://github.com/user-attachments/assets/5be33316-935d-4571-927d-5de193165c52) | | Chat / Tool call panel | Chat / Tool call panel | ## Additional Context - Each tool schema in a request grows by one more property, and each call carries up to three short names. - The icon search matches the words of the icon names only. A search for an idea, for example "save", finds fewer icons than a search of the Tabler tags. The model searches again with a thing that the icon draws, for example "device floppy". The names that a model writes in `call_icon` almost never need the search. - A tool card saved before this PR keeps the MingCute icon names that its model wrote. A name that Tabler does not have draws no icon. Making the card again fixes it. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- apps/stage-pocket/package.json | 1 + apps/stage-tamagotchi-kirie/package.json | 1 + apps/stage-tamagotchi/package.json | 1 + packages/agents/README.md | 2 +- packages/agents/src/generate-text.ts | 17 +- packages/agents/src/tool-card/index.ts | 8 +- .../core-agent/src/runtime/llm-service.ts | 2 +- packages/core-agent/src/runtime/responses.ts | 2 +- packages/core-agent/src/types/llm.ts | 6 + packages/stage-ui/package.json | 1 + .../chat/components/tool-call-group.vue | 20 +- .../src/composables/text-generation.ts | 13 +- packages/stage-ui/src/libs/cards/catalog.ts | 49 +---- packages/stage-ui/src/libs/cards/program.ts | 12 +- packages/stage-ui/src/libs/tabler-icons.ts | 180 ++++++++++++++++++ .../stage-ui/src/stores/ai/chat-llm/llm.ts | 14 +- .../stores/ai/chat-llm/tool-call-summary.ts | 74 ++++--- .../stage-ui/src/stores/chat.contract.test.ts | 6 +- packages/stage-ui/src/stores/chat.ts | 6 +- packages/stage-ui/src/stores/tool-cards.ts | 3 + packages/stage-ui/src/tools/icons.ts | 38 ++-- packages/stage-ui/src/tools/search-icons.ts | 44 +++++ pnpm-lock.yaml | 12 ++ 23 files changed, 396 insertions(+), 116 deletions(-) create mode 100644 packages/stage-ui/src/libs/tabler-icons.ts create mode 100644 packages/stage-ui/src/tools/search-icons.ts diff --git a/apps/stage-pocket/package.json b/apps/stage-pocket/package.json index 5753f85186..b407576b63 100644 --- a/apps/stage-pocket/package.json +++ b/apps/stage-pocket/package.json @@ -119,6 +119,7 @@ "@iconify-json/simple-icons": "catalog:", "@iconify-json/solar": "catalog:", "@iconify-json/svg-spinners": "catalog:", + "@iconify-json/tabler": "catalog:", "@iconify-json/vscode-icons": "catalog:", "@intlify/unplugin-vue-i18n": "catalog:", "@proj-airi/cap-vite": "workspace:*", diff --git a/apps/stage-tamagotchi-kirie/package.json b/apps/stage-tamagotchi-kirie/package.json index e35719736d..c68d260a84 100644 --- a/apps/stage-tamagotchi-kirie/package.json +++ b/apps/stage-tamagotchi-kirie/package.json @@ -140,6 +140,7 @@ "@iconify-json/simple-icons": "catalog:", "@iconify-json/solar": "catalog:", "@iconify-json/svg-spinners": "catalog:", + "@iconify-json/tabler": "catalog:", "@iconify-json/vscode-icons": "catalog:", "@iconify/utils": "catalog:", "@intlify/unplugin-vue-i18n": "catalog:", diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index c3bca51522..b13c2a4d58 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -173,6 +173,7 @@ "@iconify-json/simple-icons": "catalog:", "@iconify-json/solar": "catalog:", "@iconify-json/svg-spinners": "catalog:", + "@iconify-json/tabler": "catalog:", "@iconify-json/vscode-icons": "catalog:", "@iconify/utils": "catalog:", "@intlify/unplugin-vue-i18n": "catalog:", diff --git a/packages/agents/README.md b/packages/agents/README.md index 6ddc3bbf6b..676ff53888 100644 --- a/packages/agents/README.md +++ b/packages/agents/README.md @@ -8,7 +8,7 @@ Each agent owns one prompt, one request shape, and one reply parser. An agent do | Agent | Entry | Use | | --- | --- | --- | -| Tool card | `@proj-airi/agents/tool-card` | Writes an OpenUI Lang card that shows the results of one tool. | +| Tool card | `@proj-airi/agents/tool-card` | Writes an OpenUI Lang card that shows the results of one tool. It can search icons with the tools that the caller passes. | | Voice end judge | `@proj-airi/agents/voice-end-judge` | Decides whether a paused spoken message is finished. | | Voice transcription rephrase | `@proj-airi/agents/voice-transcription-rephrase` | Cleans up the speech recognition text of one spoken message. | diff --git a/packages/agents/src/generate-text.ts b/packages/agents/src/generate-text.ts index d854b4f4b3..63e8744f00 100644 --- a/packages/agents/src/generate-text.ts +++ b/packages/agents/src/generate-text.ts @@ -1,4 +1,4 @@ -import type { Conversation } from '@proj-airi/core-agent' +import type { Conversation, StreamOptions } from '@proj-airi/core-agent' import type { GenerationProvider } from '@proj-airi/provider-inference' import { streamFrom } from '@proj-airi/core-agent' @@ -8,8 +8,11 @@ import { streamFrom } from '@proj-airi/core-agent' * * Agents receive this function instead of a provider, so each runtime keeps its own request policy. * For example, stage-ui passes its LLM store, which remembers provider compatibility. + * + * `tools` are the only tools of the request, for example an icon search. The provider adds none of its own, such as a + * web search. Text before a tool call only leads up to the call, so the reply is the text after the last call. */ -export type GenerateText = (conversation: Conversation, signal: AbortSignal) => Promise +export type GenerateText = (conversation: Conversation, signal: AbortSignal, tools?: Extract) => Promise /** * Creates a {@link GenerateText} that streams with the core-agent runtime and collects the text deltas. @@ -18,19 +21,23 @@ export type GenerateText = (conversation: Conversation, signal: AbortSignal) => * - A caller has no runtime of its own, for example an eval or a script. * * Returns: - * - A generator that requests no tools. + * - A generator that requests only the tools that a call passes. */ export function generateTextFrom(model: string, provider: GenerationProvider): GenerateText { - return async (conversation, signal) => { + return async (conversation, signal, tools) => { let reply = '' await streamFrom({ model, chatProvider: provider, conversation, options: { - supportsTools: false, + supportsTools: Boolean(tools), + tools, + providerTools: false, abortSignal: signal, onStreamEvent: (event) => { + if (event.type === 'tool-call') + reply = '' if (event.type === 'text-delta') reply += event.text }, diff --git a/packages/agents/src/tool-card/index.ts b/packages/agents/src/tool-card/index.ts index 46b7fbeaa6..8a3ae03639 100644 --- a/packages/agents/src/tool-card/index.ts +++ b/packages/agents/src/tool-card/index.ts @@ -11,7 +11,7 @@ import type { GenerateText } from '../generate-text' export const TOOL_CARD_INSTRUCTIONS = [ 'You design a card that shows the result of one tool call in a chat app.', 'The app saves the card and shows every later result of the same tool with it, so the card must work for other results of the same shape.', - 'You cannot call tools. The user message gives the result of one call as a sample. A result can be an object, a list, or text.', + 'You cannot call the tool of the card. The user message gives the result of one call as a sample. A result can be an object, a list, or text.', 'Read the result with exactly one Query, for example `data = Query("tool_name", {}, {items: []})`. The app answers that Query with the result of each call.', 'Read every value from the Query. Never copy values of the sample into the program.', 'Do not use Mutation, forms, or $variables.', @@ -96,6 +96,8 @@ export interface ToolCardAgentOptions { * The model reads each problem in the fix round, so a problem names the statement and the fix. */ validate: (program: string) => string[] | Promise + /** Tools that the model can use while it writes, for example an icon search. */ + tools?: Parameters[2] } /** @@ -112,7 +114,7 @@ export interface ToolCardAgentOptions { export function createToolCardAgent(options: ToolCardAgentOptions) { async function make(request: ToolCardRequest, signal: AbortSignal): Promise { const conversation = createToolCardConversation(request) - const reply = await options.generateText(conversation, signal) + const reply = await options.generateText(conversation, signal, options.tools) const card = readToolCardReply(reply) const problems = await options.validate(card.program) if (problems.length === 0) @@ -129,7 +131,7 @@ export function createToolCardAgent(options: ToolCardAgentOptions) { const fixed = readToolCardReply(await options.generateText({ turns: [ ...conversation.turns, { id: 'tool-card-fix', type: 'user', content: [{ type: 'text', text: fix }] }, - ] }, signal)) + ] }, signal, options.tools)) const remaining = await options.validate(fixed.program) if (remaining.length > 0) throw new Error(`The card program still has problems: ${remaining.join('; ')}`) diff --git a/packages/core-agent/src/runtime/llm-service.ts b/packages/core-agent/src/runtime/llm-service.ts index c20d0daa38..5fa02a1d2e 100644 --- a/packages/core-agent/src/runtime/llm-service.ts +++ b/packages/core-agent/src/runtime/llm-service.ts @@ -143,7 +143,7 @@ async function streamOnce({ try { const streamResult = request.protocol === 'responses' - ? streamResponses({ config: request.config, webSearch: supportedTools && request.webSearch, conversation, scope, options, tools, initialStep, onEvent }) + ? streamResponses({ config: request.config, webSearch: supportedTools && options?.providerTools !== false && request.webSearch, conversation, scope, options, tools, initialStep, onEvent }) : streamChatCompletions({ config: request.config, conversation, scope, options, tools, initialStep, onEvent, supportsContentArray: contentArraySupported }) // NOTICE: diff --git a/packages/core-agent/src/runtime/responses.ts b/packages/core-agent/src/runtime/responses.ts index b7a7225132..fb5e26d754 100644 --- a/packages/core-agent/src/runtime/responses.ts +++ b/packages/core-agent/src/runtime/responses.ts @@ -221,7 +221,7 @@ export function streamResponses(input: { maxOutputTokens: next.maxTokens ?? input.options?.maxTokens, headers: mergeRequestHeaders(nextRequest.config.headers, next.headers), tools: toolsSupported - ? nextRequest.webSearch ? [...(next.tools ?? []), { type: 'web_search' as const }] : next.tools + ? nextRequest.webSearch && input.options?.providerTools !== false ? [...(next.tools ?? []), { type: 'web_search' as const }] : next.tools : undefined, toolChoice: undefined, }) diff --git a/packages/core-agent/src/types/llm.ts b/packages/core-agent/src/types/llm.ts index 97c41c74ed..510dcb0be9 100644 --- a/packages/core-agent/src/types/llm.ts +++ b/packages/core-agent/src/types/llm.ts @@ -105,6 +105,12 @@ export interface StreamOptions { /** Provider tool-selection directive for one request. */ toolChoice?: ToolChoice tools?: Tool[] | (() => Promise) + /** + * Lets the provider add its own tools, for example the web search of OpenAI Responses that its settings turn on. + * A task beside the chat sets it to `false`, so its request carries only the tools that it passes. + * @default true + */ + providerTools?: boolean /** * Per-model runtime cache of whether the provider accepts content-part arrays * (e.g. `[{type:'text',...},{type:'image_url',...}]`) for `messages[].content`. diff --git a/packages/stage-ui/package.json b/packages/stage-ui/package.json index aac847a7ed..9b7b758cb2 100644 --- a/packages/stage-ui/package.json +++ b/packages/stage-ui/package.json @@ -80,6 +80,7 @@ "@formkit/auto-animate": "catalog:", "@huggingface/transformers": "catalog:", "@iconify-json/mingcute": "catalog:", + "@iconify-json/tabler": "catalog:", "@iconify/utils": "catalog:", "@moeru/eventa": "catalog:", "@moeru/std": "catalog:", diff --git a/packages/stage-ui/src/components/scenarios/chat/components/tool-call-group.vue b/packages/stage-ui/src/components/scenarios/chat/components/tool-call-group.vue index 020cd95b5e..3f5dc9a093 100644 --- a/packages/stage-ui/src/components/scenarios/chat/components/tool-call-group.vue +++ b/packages/stage-ui/src/components/scenarios/chat/components/tool-call-group.vue @@ -15,6 +15,7 @@ import CardView from '../../../../libs/cards/card-view.vue' import HitPill from './hit-pill.vue' import ToolCallPayload from './tool-call-payload.vue' +import { tablerIconMask } from '../../../../libs/tabler-icons' import { readCallSummary } from '../../../../stores/ai/chat-llm/tool-call-summary' import { useLlmToolsStore } from '../../../../stores/ai/chat-llm/tools' import { canRerunToolCall } from '../../../../stores/tool-call-rerun' @@ -51,8 +52,12 @@ const toolCards = useToolCardsStore() const calls = computed(() => { return props.items.map((item) => { - // The model says in the call what it does and why. The row shows it, and a new card of the tool learns from it. - const summary = readCallSummary(item.toolCall.args) + // The model says in the call what it does and why, with an icon for it. The row shows both, and a new card of the + // tool learns from the line. + const { text: summary, icons } = readCallSummary(item.toolCall.args) + // The icon draws in the element of the tool icon, so it keeps the size and the color of that icon. The tool icon + // shows when no candidate leads to an icon. + const iconMask = tablerIconMask(icons) const result = props.resultOf(item.sliceIndex) const widget = props.widgets[item.toolCall.toolName] const reading = widget?.read(result?.result, item.toolCall.args) @@ -69,6 +74,7 @@ const calls = computed(() => { // A widget title already says the state of the call. titled: Boolean(title), icon: toolIconOf(item.toolCall.toolName, llmTools.tools.findLast(tool => tool.function.name === item.toolCall.toolName)?.icon), + iconStyle: iconMask ? { '--un-icon': iconMask } : undefined, state, result: result?.result, widget: widget && reading ? { component: widget.component, ...reading } : undefined, @@ -128,7 +134,8 @@ function popLabelIn(el: Element, done: () => void) { function hitIcon(call: { state: ToolCallState, icon: string }) { if (call.state === 'executing') return 'i-svg-spinners:ring-resize' - return call.state === 'error' ? 'i-mingcute:close-circle-line' : call.icon + // The error icon takes the place of a tool icon, so it comes from the same set. + return call.state === 'error' ? 'i-tabler:circle-x' : call.icon } /** @@ -223,7 +230,7 @@ function rerun(item: ToolCallGroupItem) { - + @@ -253,7 +260,10 @@ function rerun(item: ToolCallGroupItem) { @click="toggleCall(call.key)" >