Chore/Update langchain version, openai, mistral, vertex, anthropic (#2180)

* update langchain version, openai, mistral, vertex, anthropic, introduced toolagent

* upgrade @google/generative-ai 0.7.0, replicate and faiss-node

* update cohere ver

* adding chatCohere to streaming

* update gemini to have image upload

* update google genai, remove aiplugin

packages/components/nodes/agents/MistralAIToolAgent/MistralAIToolAgent.ts

@@ -34,7 +34,7 @@ class MistralAIToolAgent_Agents implements INode {
         this.type = 'AgentExecutor'
         this.category = 'Agents'
         this.icon = 'MistralAI.svg'
-        this.badge = 'NEW'
+        this.badge = 'DEPRECATING'
         this.description = `Agent that uses MistralAI Function Calling to pick the tools and args to call`
         this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
         this.inputs = [

packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts

@@ -2,7 +2,7 @@ import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, IN
 import OpenAI from 'openai'
 import { DataSource } from 'typeorm'
 import { getCredentialData, getCredentialParam, getUserHome } from '../../../src/utils'
-import { MessageContentImageFile, MessageContentText } from 'openai/resources/beta/threads/messages/messages'
+import { ImageFileContentBlock, TextContentBlock } from 'openai/resources/beta/threads/messages/messages'
 import * as fsDefault from 'node:fs'
 import * as path from 'node:path'
 import fetch from 'node-fetch'
@@ -392,7 +392,7 @@ class OpenAIAssistant_Agents implements INode {
             const fileAnnotations = []
             for (let i = 0; i < assistantMessages[0].content.length; i += 1) {
                 if (assistantMessages[0].content[i].type === 'text') {
-                    const content = assistantMessages[0].content[i] as MessageContentText
+                    const content = assistantMessages[0].content[i] as TextContentBlock
 
                     if (content.text.annotations) {
                         const message_content = content.text
@@ -406,8 +406,7 @@ class OpenAIAssistant_Agents implements INode {
                             let filePath = ''
 
                             // Gather citations based on annotation attributes
-                            const file_citation = (annotation as OpenAI.Beta.Threads.Messages.MessageContentText.Text.FileCitation)
-                                .file_citation
+                            const file_citation = (annotation as OpenAI.Beta.Threads.Messages.FileCitationAnnotation).file_citation
                             if (file_citation) {
                                 const cited_file = await openai.files.retrieve(file_citation.file_id)
                                 // eslint-disable-next-line no-useless-escape
@@ -421,7 +420,7 @@ class OpenAIAssistant_Agents implements INode {
                                     })
                                 }
                             } else {
-                                const file_path = (annotation as OpenAI.Beta.Threads.Messages.MessageContentText.Text.FilePath).file_path
+                                const file_path = (annotation as OpenAI.Beta.Threads.Messages.FilePathAnnotation).file_path
                                 if (file_path) {
                                     const cited_file = await openai.files.retrieve(file_path.file_id)
                                     // eslint-disable-next-line no-useless-escape
@@ -452,7 +451,7 @@ class OpenAIAssistant_Agents implements INode {
                     const lenticularBracketRegex = /【[^】]*】/g
                     returnVal = returnVal.replace(lenticularBracketRegex, '')
                 } else {
-                    const content = assistantMessages[0].content[i] as MessageContentImageFile
+                    const content = assistantMessages[0].content[i] as ImageFileContentBlock
                     const fileId = content.image_file.file_id
                     const fileObj = await openai.files.retrieve(fileId)
                     const dirPath = path.join(getUserHome(), '.flowise', 'openai-assistant')
@@ -533,7 +532,7 @@ const downloadFile = async (fileObj: any, filePath: string, dirPath: string, ope
     }
 }
 
-const formatToOpenAIAssistantTool = (tool: any): OpenAI.Beta.AssistantCreateParams.AssistantToolsFunction => {
+const formatToOpenAIAssistantTool = (tool: any): OpenAI.Beta.FunctionTool => {
     return {
         type: 'function',
         function: {

packages/components/nodes/agents/OpenAIToolAgent/OpenAIToolAgent.ts

@@ -36,7 +36,7 @@ class OpenAIToolAgent_Agents implements INode {
         this.icon = 'function.svg'
         this.description = `Agent that uses OpenAI Function Calling to pick the tools and args to call`
         this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
-        this.badge = 'NEW'
+        this.badge = 'DEPRECATING'
         this.inputs = [
             {
                 label: 'Tools',

packages/components/nodes/agents/ReActAgentChat/ReActAgentChat.ts

@@ -13,7 +13,7 @@ import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalU
 import { checkInputs, Moderation } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'
 
-class MRKLAgentChat_Agents implements INode {
+class ReActAgentChat_Agents implements INode {
     label: string
     name: string
     version: number
@@ -27,7 +27,7 @@ class MRKLAgentChat_Agents implements INode {
 
     constructor(fields?: { sessionId?: string }) {
         this.label = 'ReAct Agent for Chat Models'
-        this.name = 'mrklAgentChat'
+        this.name = 'reactAgentChat'
         this.version = 4.0
         this.type = 'AgentExecutor'
         this.category = 'Agents'
@@ -157,4 +157,4 @@ class MRKLAgentChat_Agents implements INode {
     }
 }
 
-module.exports = { nodeClass: MRKLAgentChat_Agents }
+module.exports = { nodeClass: ReActAgentChat_Agents }

packages/components/nodes/agents/ReActAgentLLM/ReActAgentLLM.ts

@@ -11,7 +11,7 @@ import { createReactAgent } from '../../../src/agents'
 import { checkInputs, Moderation } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'
 
-class MRKLAgentLLM_Agents implements INode {
+class ReActAgentLLM_Agents implements INode {
     label: string
     name: string
     version: number
@@ -24,7 +24,7 @@ class MRKLAgentLLM_Agents implements INode {
 
     constructor() {
         this.label = 'ReAct Agent for LLMs'
-        this.name = 'mrklAgentLLM'
+        this.name = 'reactAgentLLM'
         this.version = 2.0
         this.type = 'AgentExecutor'
         this.category = 'Agents'
@@ -107,4 +107,4 @@ class MRKLAgentLLM_Agents implements INode {
     }
 }
 
-module.exports = { nodeClass: MRKLAgentLLM_Agents }
+module.exports = { nodeClass: ReActAgentLLM_Agents }

packages/components/nodes/agents/ToolAgent/ToolAgent.ts

@@ -0,0 +1,260 @@
+import { flatten } from 'lodash'
+import { BaseMessage } from '@langchain/core/messages'
+import { ChainValues } from '@langchain/core/utils/types'
+import { RunnableSequence } from '@langchain/core/runnables'
+import { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
+import { formatToOpenAIToolMessages } from 'langchain/agents/format_scratchpad/openai_tools'
+import { type ToolsAgentStep } from 'langchain/agents/openai/output_parser'
+import { getBaseClasses } from '../../../src/utils'
+import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams, IUsedTool, IVisionChatModal } from '../../../src/Interface'
+import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
+import { AgentExecutor, ToolCallingAgentOutputParser } from '../../../src/agents'
+import { Moderation, checkInputs, streamResponse } from '../../moderation/Moderation'
+import { formatResponse } from '../../outputparsers/OutputParserHelpers'
+import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
+
+class ToolAgent_Agents implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    sessionId?: string
+    badge?: string
+
+    constructor(fields?: { sessionId?: string }) {
+        this.label = 'Tool Agent'
+        this.name = 'toolAgent'
+        this.version = 1.0
+        this.type = 'AgentExecutor'
+        this.category = 'Agents'
+        this.icon = 'toolAgent.png'
+        this.description = `Agent that uses Function Calling to pick the tools and args to call`
+        this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
+        this.badge = 'NEW'
+        this.inputs = [
+            {
+                label: 'Tools',
+                name: 'tools',
+                type: 'Tool',
+                list: true
+            },
+            {
+                label: 'Memory',
+                name: 'memory',
+                type: 'BaseChatMemory'
+            },
+            {
+                label: 'Tool Calling Chat Model',
+                name: 'model',
+                type: 'BaseChatModel',
+                description:
+                    'Only compatible with models that are capable of function calling. ChatOpenAI, ChatMistral, ChatAnthropic, ChatVertexAI'
+            },
+            {
+                label: 'System Message',
+                name: 'systemMessage',
+                type: 'string',
+                default: `You are a helpful AI assistant.`,
+                rows: 4,
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Input Moderation',
+                description: 'Detect text that could generate harmful output and prevent it from being sent to the language model',
+                name: 'inputModeration',
+                type: 'Moderation',
+                optional: true,
+                list: true
+            },
+            {
+                label: 'Max Iterations',
+                name: 'maxIterations',
+                type: 'number',
+                optional: true,
+                additionalParams: true
+            }
+        ]
+        this.sessionId = fields?.sessionId
+    }
+
+    async init(nodeData: INodeData, input: string, options: ICommonObject): Promise<any> {
+        return prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
+    }
+
+    async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | ICommonObject> {
+        const memory = nodeData.inputs?.memory as FlowiseMemory
+        const moderations = nodeData.inputs?.inputModeration as Moderation[]
+
+        const isStreamable = options.socketIO && options.socketIOClientId
+
+        if (moderations && moderations.length > 0) {
+            try {
+                // Use the output of the moderation chain as input for the OpenAI Function Agent
+                input = await checkInputs(moderations, input)
+            } catch (e) {
+                await new Promise((resolve) => setTimeout(resolve, 500))
+                if (isStreamable)
+                    streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
+                return formatResponse(e.message)
+            }
+        }
+
+        const executor = prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
+
+        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const callbacks = await additionalCallbacks(nodeData, options)
+
+        let res: ChainValues = {}
+        let sourceDocuments: ICommonObject[] = []
+        let usedTools: IUsedTool[] = []
+
+        if (isStreamable) {
+            const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
+            res = await executor.invoke({ input }, { callbacks: [loggerHandler, handler, ...callbacks] })
+            if (res.sourceDocuments) {
+                options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', flatten(res.sourceDocuments))
+                sourceDocuments = res.sourceDocuments
+            }
+            if (res.usedTools) {
+                options.socketIO.to(options.socketIOClientId).emit('usedTools', res.usedTools)
+                usedTools = res.usedTools
+            }
+        } else {
+            res = await executor.invoke({ input }, { callbacks: [loggerHandler, ...callbacks] })
+            if (res.sourceDocuments) {
+                sourceDocuments = res.sourceDocuments
+            }
+            if (res.usedTools) {
+                usedTools = res.usedTools
+            }
+        }
+
+        let output = res?.output as string
+
+        // Claude 3 Opus tends to spit out <thinking>..</thinking> as well, discard that in final output
+        const regexPattern: RegExp = /<thinking>[\s\S]*?<\/thinking>/
+        const matches: RegExpMatchArray | null = output.match(regexPattern)
+        if (matches) {
+            for (const match of matches) {
+                output = output.replace(match, '')
+            }
+        }
+
+        await memory.addChatMessages(
+            [
+                {
+                    text: input,
+                    type: 'userMessage'
+                },
+                {
+                    text: output,
+                    type: 'apiMessage'
+                }
+            ],
+            this.sessionId
+        )
+
+        let finalRes = output
+
+        if (sourceDocuments.length || usedTools.length) {
+            const finalRes: ICommonObject = { text: output }
+            if (sourceDocuments.length) {
+                finalRes.sourceDocuments = flatten(sourceDocuments)
+            }
+            if (usedTools.length) {
+                finalRes.usedTools = usedTools
+            }
+            return finalRes
+        }
+
+        return finalRes
+    }
+}
+
+const prepareAgent = (nodeData: INodeData, options: ICommonObject, flowObj: { sessionId?: string; chatId?: string; input?: string }) => {
+    const model = nodeData.inputs?.model as BaseChatModel
+    const maxIterations = nodeData.inputs?.maxIterations as string
+    const memory = nodeData.inputs?.memory as FlowiseMemory
+    const systemMessage = nodeData.inputs?.systemMessage as string
+    let tools = nodeData.inputs?.tools
+    tools = flatten(tools)
+    const memoryKey = memory.memoryKey ? memory.memoryKey : 'chat_history'
+    const inputKey = memory.inputKey ? memory.inputKey : 'input'
+
+    const prompt = ChatPromptTemplate.fromMessages([
+        ['system', systemMessage],
+        new MessagesPlaceholder(memoryKey),
+        ['human', `{${inputKey}}`],
+        new MessagesPlaceholder('agent_scratchpad')
+    ])
+
+    if (llmSupportsVision(model)) {
+        const visionChatModel = model as IVisionChatModal
+        const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+        if (messageContent?.length) {
+            visionChatModel.setVisionModel()
+
+            // Pop the `agent_scratchpad` MessagePlaceHolder
+            let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder
+            if (prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) {
+                const lastMessage = prompt.promptMessages.pop() as HumanMessagePromptTemplate
+                const template = (lastMessage.prompt as PromptTemplate).template as string
+                const msg = HumanMessagePromptTemplate.fromTemplate([
+                    ...messageContent,
+                    {
+                        text: template
+                    }
+                ])
+                msg.inputVariables = lastMessage.inputVariables
+                prompt.promptMessages.push(msg)
+            }
+
+            // Add the `agent_scratchpad` MessagePlaceHolder back
+            prompt.promptMessages.push(messagePlaceholder)
+        } else {
+            visionChatModel.revertToOriginalModel()
+        }
+    }
+
+    if (model.bindTools === undefined) {
+        throw new Error(`This agent requires that the "bindTools()" method be implemented on the input model.`)
+    }
+
+    const modelWithTools = model.bindTools(tools)
+
+    const runnableAgent = RunnableSequence.from([
+        {
+            [inputKey]: (i: { input: string; steps: ToolsAgentStep[] }) => i.input,
+            agent_scratchpad: (i: { input: string; steps: ToolsAgentStep[] }) => formatToOpenAIToolMessages(i.steps),
+            [memoryKey]: async (_: { input: string; steps: ToolsAgentStep[] }) => {
+                const messages = (await memory.getChatMessages(flowObj?.sessionId, true)) as BaseMessage[]
+                return messages ?? []
+            }
+        },
+        prompt,
+        modelWithTools,
+        new ToolCallingAgentOutputParser()
+    ])
+
+    const executor = AgentExecutor.fromAgentAndTools({
+        agent: runnableAgent,
+        tools,
+        sessionId: flowObj?.sessionId,
+        chatId: flowObj?.chatId,
+        input: flowObj?.input,
+        verbose: process.env.DEBUG === 'true' ? true : false,
+        maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
+    })
+
+    return executor
+}
+
+module.exports = { nodeClass: ToolAgent_Agents }

packages/components/nodes/agents/XMLAgent/XMLAgent.ts

@@ -57,7 +57,6 @@ class XMLAgent_Agents implements INode {
         this.type = 'XMLAgent'
         this.category = 'Agents'
         this.icon = 'xmlagent.svg'
-        this.badge = 'NEW'
         this.description = `Agent that is designed for LLMs that are good for reasoning/writing XML (e.g: Anthropic Claude)`
         this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
         this.inputs = [