Chore/Update langchain version, openai, mistral, vertex, anthropic (#2180)

* update langchain version, openai, mistral, vertex, anthropic, introduced toolagent

* upgrade @google/generative-ai 0.7.0, replicate and faiss-node

* update cohere ver

* adding chatCohere to streaming

* update gemini to have image upload

* update google genai, remove aiplugin

packages/components/src/agents.ts

@@ -1,14 +1,16 @@
 import { flatten } from 'lodash'
 import { ChainValues } from '@langchain/core/utils/types'
 import { AgentStep, AgentAction } from '@langchain/core/agents'
-import { BaseMessage, FunctionMessage, AIMessage } from '@langchain/core/messages'
-import { OutputParserException } from '@langchain/core/output_parsers'
+import { BaseMessage, FunctionMessage, AIMessage, isBaseMessage } from '@langchain/core/messages'
+import { ToolCall } from '@langchain/core/messages/tool'
+import { OutputParserException, BaseOutputParser } from '@langchain/core/output_parsers'
 import { BaseLanguageModel } from '@langchain/core/language_models/base'
 import { CallbackManager, CallbackManagerForChainRun, Callbacks } from '@langchain/core/callbacks/manager'
 import { ToolInputParsingException, Tool, StructuredToolInterface } from '@langchain/core/tools'
 import { Runnable, RunnableSequence, RunnablePassthrough } from '@langchain/core/runnables'
 import { Serializable } from '@langchain/core/load/serializable'
 import { renderTemplate } from '@langchain/core/prompts'
+import { ChatGeneration } from '@langchain/core/outputs'
 import { BaseChain, SerializedLLMChain } from 'langchain/chains'
 import {
     CreateReactAgentParams,
@@ -824,3 +826,75 @@ export class XMLAgentOutputParser extends AgentActionOutputParser {
         throw new Error('getFormatInstructions not implemented inside XMLAgentOutputParser.')
     }
 }
+
+abstract class AgentMultiActionOutputParser extends BaseOutputParser<AgentAction[] | AgentFinish> {}
+
+type ToolsAgentAction = AgentAction & {
+    toolCallId: string
+    messageLog?: BaseMessage[]
+}
+
+export type ToolsAgentStep = AgentStep & {
+    action: ToolsAgentAction
+}
+
+function parseAIMessageToToolAction(message: AIMessage): ToolsAgentAction[] | AgentFinish {
+    const stringifiedMessageContent = typeof message.content === 'string' ? message.content : JSON.stringify(message.content)
+    let toolCalls: ToolCall[] = []
+    if (message.tool_calls !== undefined && message.tool_calls.length > 0) {
+        toolCalls = message.tool_calls
+    } else {
+        if (message.additional_kwargs.tool_calls === undefined || message.additional_kwargs.tool_calls.length === 0) {
+            return {
+                returnValues: { output: message.content },
+                log: stringifiedMessageContent
+            }
+        }
+        // Best effort parsing
+        for (const toolCall of message.additional_kwargs.tool_calls ?? []) {
+            const functionName = toolCall.function?.name
+            try {
+                const args = JSON.parse(toolCall.function.arguments)
+                toolCalls.push({ name: functionName, args, id: toolCall.id })
+            } catch (e: any) {
+                throw new OutputParserException(
+                    `Failed to parse tool arguments from chat model response. Text: "${JSON.stringify(toolCalls)}". ${e}`
+                )
+            }
+        }
+    }
+    return toolCalls.map((toolCall, i) => {
+        const messageLog = i === 0 ? [message] : []
+        const log = `Invoking "${toolCall.name}" with ${JSON.stringify(toolCall.args ?? {})}\n${stringifiedMessageContent}`
+        return {
+            tool: toolCall.name as string,
+            toolInput: toolCall.args,
+            toolCallId: toolCall.id ?? '',
+            log,
+            messageLog
+        }
+    })
+}
+
+export class ToolCallingAgentOutputParser extends AgentMultiActionOutputParser {
+    lc_namespace = ['langchain', 'agents', 'tool_calling']
+
+    static lc_name() {
+        return 'ToolCallingAgentOutputParser'
+    }
+
+    async parse(text: string): Promise<AgentAction[] | AgentFinish> {
+        throw new Error(`ToolCallingAgentOutputParser can only parse messages.\nPassed input: ${text}`)
+    }
+
+    async parseResult(generations: ChatGeneration[]) {
+        if ('message' in generations[0] && isBaseMessage(generations[0].message)) {
+            return parseAIMessageToToolAction(generations[0].message)
+        }
+        throw new Error('parseResult on ToolCallingAgentOutputParser only works on ChatGeneration output')
+    }
+
+    getFormatInstructions(): string {
+        throw new Error('getFormatInstructions not implemented inside ToolCallingAgentOutputParser.')
+    }
+}