From f5607681333a8a3ba61fe3c1b01940df038be103 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 15 Sep 2025 19:25:43 +0100 Subject: [PATCH] Feat/Gemini Built In Tools (#5215) * feat: add Gemini built-in tools URL Context and Google Search for enhanced functionality * add ui for gemini built in tools --- .../components/nodes/agentflow/Agent/Agent.ts | 153 +++++++++++++----- .../src/views/agentflowsv2/AgentFlowNode.jsx | 50 +++++- 2 files changed, 158 insertions(+), 45 deletions(-) diff --git a/packages/components/nodes/agentflow/Agent/Agent.ts b/packages/components/nodes/agentflow/Agent/Agent.ts index 39759f0e..ae494210 100644 --- a/packages/components/nodes/agentflow/Agent/Agent.ts +++ b/packages/components/nodes/agentflow/Agent/Agent.ts @@ -81,7 +81,7 @@ class Agent_Agentflow implements INode { constructor() { this.label = 'Agent' this.name = 'agentAgentflow' - this.version = 2.0 + this.version = 2.1 this.type = 'Agent' this.category = 'Agent Flows' this.description = 'Dynamically choose and utilize tools during runtime, enabling multi-step reasoning' @@ -161,6 +161,27 @@ class Agent_Agentflow implements INode { agentModel: 'chatOpenAI' } }, + { + label: 'Gemini Built-in Tools', + name: 'agentToolsBuiltInGemini', + type: 'multiOptions', + optional: true, + options: [ + { + label: 'URL Context', + name: 'urlContext', + description: 'Extract content from given URLs' + }, + { + label: 'Google Search', + name: 'googleSearch', + description: 'Search real-time web content' + } + ], + show: { + agentModel: 'chatGoogleGenerativeAI' + } + }, { label: 'Tools', name: 'agentTools', @@ -765,6 +786,23 @@ class Agent_Agentflow implements INode { } } + const agentToolsBuiltInGemini = convertMultiOptionsToStringArray(nodeData.inputs?.agentToolsBuiltInGemini) + if (agentToolsBuiltInGemini && agentToolsBuiltInGemini.length > 0) { + for (const tool of agentToolsBuiltInGemini) { + const builtInTool: ICommonObject = { + [tool]: {} + } + ;(toolsInstance as any).push(builtInTool) + ;(availableTools as any).push({ + name: tool, + toolNode: { + label: tool, + name: tool + } + }) + } + } + if (llmNodeInstance && toolsInstance.length > 0) { if (llmNodeInstance.bindTools === undefined) { throw new Error(`Agent needs to have a function calling capable models.`) @@ -1177,53 +1215,80 @@ class Agent_Agentflow implements INode { return builtInUsedTools } - const { output, tools } = response.response_metadata + const { output, tools, groundingMetadata, urlContextMetadata } = response.response_metadata - if (!output || !Array.isArray(output) || output.length === 0 || !tools || !Array.isArray(tools) || tools.length === 0) { - return builtInUsedTools + // Handle OpenAI built-in tools + if (output && Array.isArray(output) && output.length > 0 && tools && Array.isArray(tools) && tools.length > 0) { + for (const outputItem of output) { + if (outputItem.type && outputItem.type.endsWith('_call')) { + let toolInput = outputItem.action ?? outputItem.code + let toolOutput = outputItem.status === 'completed' ? 'Success' : outputItem.status + + // Handle image generation calls specially + if (outputItem.type === 'image_generation_call') { + // Create input summary for image generation + toolInput = { + prompt: outputItem.revised_prompt || 'Image generation request', + size: outputItem.size || '1024x1024', + quality: outputItem.quality || 'standard', + output_format: outputItem.output_format || 'png' + } + + // Check if image has been processed (base64 replaced with file path) + if (outputItem.result && !outputItem.result.startsWith('data:') && !outputItem.result.includes('base64')) { + toolOutput = `Image generated and saved` + } else { + toolOutput = `Image generated (base64)` + } + } + + // Remove "_call" suffix to get the base tool name + const baseToolName = outputItem.type.replace('_call', '') + + // Find matching tool that includes the base name in its type + const matchingTool = tools.find((tool) => tool.type && tool.type.includes(baseToolName)) + + if (matchingTool) { + // Check for duplicates + if (builtInUsedTools.find((tool) => tool.tool === matchingTool.type)) { + continue + } + + builtInUsedTools.push({ + tool: matchingTool.type, + toolInput, + toolOutput + }) + } + } + } } - for (const outputItem of output) { - if (outputItem.type && outputItem.type.endsWith('_call')) { - let toolInput = outputItem.action ?? outputItem.code - let toolOutput = outputItem.status === 'completed' ? 'Success' : outputItem.status + // Handle Gemini googleSearch tool + if (groundingMetadata && groundingMetadata.webSearchQueries && Array.isArray(groundingMetadata.webSearchQueries)) { + // Check for duplicates + if (!builtInUsedTools.find((tool) => tool.tool === 'googleSearch')) { + builtInUsedTools.push({ + tool: 'googleSearch', + toolInput: { + queries: groundingMetadata.webSearchQueries + }, + toolOutput: `Searched for: ${groundingMetadata.webSearchQueries.join(', ')}` + }) + } + } - // Handle image generation calls specially - if (outputItem.type === 'image_generation_call') { - // Create input summary for image generation - toolInput = { - prompt: outputItem.revised_prompt || 'Image generation request', - size: outputItem.size || '1024x1024', - quality: outputItem.quality || 'standard', - output_format: outputItem.output_format || 'png' - } - - // Check if image has been processed (base64 replaced with file path) - if (outputItem.result && !outputItem.result.startsWith('data:') && !outputItem.result.includes('base64')) { - toolOutput = `Image generated and saved` - } else { - toolOutput = `Image generated (base64)` - } - } - - // Remove "_call" suffix to get the base tool name - const baseToolName = outputItem.type.replace('_call', '') - - // Find matching tool that includes the base name in its type - const matchingTool = tools.find((tool) => tool.type && tool.type.includes(baseToolName)) - - if (matchingTool) { - // Check for duplicates - if (builtInUsedTools.find((tool) => tool.tool === matchingTool.type)) { - continue - } - - builtInUsedTools.push({ - tool: matchingTool.type, - toolInput, - toolOutput - }) - } + // Handle Gemini urlContext tool + if (urlContextMetadata && urlContextMetadata.urlMetadata && Array.isArray(urlContextMetadata.urlMetadata)) { + // Check for duplicates + if (!builtInUsedTools.find((tool) => tool.tool === 'urlContext')) { + builtInUsedTools.push({ + tool: 'urlContext', + toolInput: { + urlMetadata: urlContextMetadata.urlMetadata + }, + toolOutput: `Processed ${urlContextMetadata.urlMetadata.length} URL(s)` + }) } } diff --git a/packages/ui/src/views/agentflowsv2/AgentFlowNode.jsx b/packages/ui/src/views/agentflowsv2/AgentFlowNode.jsx index 3a6c689d..7941d69b 100644 --- a/packages/ui/src/views/agentflowsv2/AgentFlowNode.jsx +++ b/packages/ui/src/views/agentflowsv2/AgentFlowNode.jsx @@ -24,7 +24,8 @@ import { IconAlertCircleFilled, IconCode, IconWorldWww, - IconPhoto + IconPhoto, + IconBrandGoogle } from '@tabler/icons-react' import StopCircleIcon from '@mui/icons-material/StopCircle' import CancelIcon from '@mui/icons-material/Cancel' @@ -142,6 +143,17 @@ const AgentFlowNode = ({ data }) => { } } + const getBuiltInGeminiToolIcon = (toolName) => { + switch (toolName) { + case 'urlContext': + return + case 'googleSearch': + return + default: + return null + } + } + useEffect(() => { if (ref.current) { setTimeout(() => { @@ -433,6 +445,16 @@ const AgentFlowNode = ({ data }) => { : [], toolProperty: 'builtInTool', isBuiltInOpenAI: true + }, + { + tools: data.inputs?.agentToolsBuiltInGemini + ? (typeof data.inputs.agentToolsBuiltInGemini === 'string' + ? JSON.parse(data.inputs.agentToolsBuiltInGemini) + : data.inputs.agentToolsBuiltInGemini + ).map((tool) => ({ builtInTool: tool })) + : [], + toolProperty: 'builtInTool', + isBuiltInGemini: true } ] @@ -493,6 +515,32 @@ const AgentFlowNode = ({ data }) => { ] } + // Handle built-in Gemini tools with icons + if (config.isBuiltInGemini) { + const icon = getBuiltInGeminiToolIcon(toolName) + if (!icon) return [] + + return [ + + {icon} + + ] + } + return [