Chore/Update langchain version, openai, mistral, vertex, anthropic (#2180)

* update langchain version, openai, mistral, vertex, anthropic, introduced toolagent * upgrade @google/generative-ai 0.7.0, replicate and faiss-node * update cohere ver * adding chatCohere to streaming * update gemini to have image upload * update google genai, remove aiplugin
2026-06-28 15:00:57 +03:00 · 2024-04-20 02:20:30 +01:00
parent f5be889ea8
commit 95beaba9d9
27 changed files with 1541 additions and 1008 deletions
@@ -34,7 +34,7 @@ class MistralAIToolAgent_Agents implements INode {
        this.type = 'AgentExecutor'
        this.category = 'Agents'
        this.icon = 'MistralAI.svg'
-        this.badge = 'NEW'
+        this.badge = 'DEPRECATING'
        this.description = `Agent that uses MistralAI Function Calling to pick the tools and args to call`
        this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
        this.inputs = [
@@ -2,7 +2,7 @@ import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, IN
 import OpenAI from 'openai'
 import { DataSource } from 'typeorm'
 import { getCredentialData, getCredentialParam, getUserHome } from '../../../src/utils'
-import { MessageContentImageFile, MessageContentText } from 'openai/resources/beta/threads/messages/messages'
+import { ImageFileContentBlock, TextContentBlock } from 'openai/resources/beta/threads/messages/messages'
 import * as fsDefault from 'node:fs'
 import * as path from 'node:path'
 import fetch from 'node-fetch'
@@ -392,7 +392,7 @@ class OpenAIAssistant_Agents implements INode {
            const fileAnnotations = []
            for (let i = 0; i < assistantMessages[0].content.length; i += 1) {
                if (assistantMessages[0].content[i].type === 'text') {
-                    const content = assistantMessages[0].content[i] as MessageContentText
+                    const content = assistantMessages[0].content[i] as TextContentBlock

                    if (content.text.annotations) {
                        const message_content = content.text
@@ -406,8 +406,7 @@ class OpenAIAssistant_Agents implements INode {
                            let filePath = ''

                            // Gather citations based on annotation attributes
-                            const file_citation = (annotation as OpenAI.Beta.Threads.Messages.MessageContentText.Text.FileCitation)
-                                .file_citation
+                            const file_citation = (annotation as OpenAI.Beta.Threads.Messages.FileCitationAnnotation).file_citation
                            if (file_citation) {
                                const cited_file = await openai.files.retrieve(file_citation.file_id)
                                // eslint-disable-next-line no-useless-escape
@@ -421,7 +420,7 @@ class OpenAIAssistant_Agents implements INode {
                                    })
                                }
                            } else {
-                                const file_path = (annotation as OpenAI.Beta.Threads.Messages.MessageContentText.Text.FilePath).file_path
+                                const file_path = (annotation as OpenAI.Beta.Threads.Messages.FilePathAnnotation).file_path
                                if (file_path) {
                                    const cited_file = await openai.files.retrieve(file_path.file_id)
                                    // eslint-disable-next-line no-useless-escape
@@ -452,7 +451,7 @@ class OpenAIAssistant_Agents implements INode {
                    const lenticularBracketRegex = /【[^】]*】/g
                    returnVal = returnVal.replace(lenticularBracketRegex, '')
                } else {
-                    const content = assistantMessages[0].content[i] as MessageContentImageFile
+                    const content = assistantMessages[0].content[i] as ImageFileContentBlock
                    const fileId = content.image_file.file_id
                    const fileObj = await openai.files.retrieve(fileId)
                    const dirPath = path.join(getUserHome(), '.flowise', 'openai-assistant')
@@ -533,7 +532,7 @@ const downloadFile = async (fileObj: any, filePath: string, dirPath: string, ope
    }
 }

-const formatToOpenAIAssistantTool = (tool: any): OpenAI.Beta.AssistantCreateParams.AssistantToolsFunction => {
+const formatToOpenAIAssistantTool = (tool: any): OpenAI.Beta.FunctionTool => {
    return {
        type: 'function',
        function: {
@@ -36,7 +36,7 @@ class OpenAIToolAgent_Agents implements INode {
        this.icon = 'function.svg'
        this.description = `Agent that uses OpenAI Function Calling to pick the tools and args to call`
        this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
-        this.badge = 'NEW'
+        this.badge = 'DEPRECATING'
        this.inputs = [
            {
                label: 'Tools',
@@ -13,7 +13,7 @@ import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalU
 import { checkInputs, Moderation } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'

-class MRKLAgentChat_Agents implements INode {
+class ReActAgentChat_Agents implements INode {
    label: string
    name: string
    version: number
@@ -27,7 +27,7 @@ class MRKLAgentChat_Agents implements INode {

    constructor(fields?: { sessionId?: string }) {
        this.label = 'ReAct Agent for Chat Models'
-        this.name = 'mrklAgentChat'
+        this.name = 'reactAgentChat'
        this.version = 4.0
        this.type = 'AgentExecutor'
        this.category = 'Agents'
@@ -157,4 +157,4 @@ class MRKLAgentChat_Agents implements INode {
    }
 }

-module.exports = { nodeClass: MRKLAgentChat_Agents }
+module.exports = { nodeClass: ReActAgentChat_Agents }
@@ -11,7 +11,7 @@ import { createReactAgent } from '../../../src/agents'
 import { checkInputs, Moderation } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'

-class MRKLAgentLLM_Agents implements INode {
+class ReActAgentLLM_Agents implements INode {
    label: string
    name: string
    version: number
@@ -24,7 +24,7 @@ class MRKLAgentLLM_Agents implements INode {

    constructor() {
        this.label = 'ReAct Agent for LLMs'
-        this.name = 'mrklAgentLLM'
+        this.name = 'reactAgentLLM'
        this.version = 2.0
        this.type = 'AgentExecutor'
        this.category = 'Agents'
@@ -107,4 +107,4 @@ class MRKLAgentLLM_Agents implements INode {
    }
 }

-module.exports = { nodeClass: MRKLAgentLLM_Agents }
+module.exports = { nodeClass: ReActAgentLLM_Agents }
@@ -0,0 +1,260 @@
+import { flatten } from 'lodash'
+import { BaseMessage } from '@langchain/core/messages'
+import { ChainValues } from '@langchain/core/utils/types'
+import { RunnableSequence } from '@langchain/core/runnables'
+import { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
+import { formatToOpenAIToolMessages } from 'langchain/agents/format_scratchpad/openai_tools'
+import { type ToolsAgentStep } from 'langchain/agents/openai/output_parser'
+import { getBaseClasses } from '../../../src/utils'
+import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams, IUsedTool, IVisionChatModal } from '../../../src/Interface'
+import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
+import { AgentExecutor, ToolCallingAgentOutputParser } from '../../../src/agents'
+import { Moderation, checkInputs, streamResponse } from '../../moderation/Moderation'
+import { formatResponse } from '../../outputparsers/OutputParserHelpers'
+import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
+
+class ToolAgent_Agents implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    sessionId?: string
+    badge?: string
+
+    constructor(fields?: { sessionId?: string }) {
+        this.label = 'Tool Agent'
+        this.name = 'toolAgent'
+        this.version = 1.0
+        this.type = 'AgentExecutor'
+        this.category = 'Agents'
+        this.icon = 'toolAgent.png'
+        this.description = `Agent that uses Function Calling to pick the tools and args to call`
+        this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
+        this.badge = 'NEW'
+        this.inputs = [
+            {
+                label: 'Tools',
+                name: 'tools',
+                type: 'Tool',
+                list: true
+            },
+            {
+                label: 'Memory',
+                name: 'memory',
+                type: 'BaseChatMemory'
+            },
+            {
+                label: 'Tool Calling Chat Model',
+                name: 'model',
+                type: 'BaseChatModel',
+                description:
+                    'Only compatible with models that are capable of function calling. ChatOpenAI, ChatMistral, ChatAnthropic, ChatVertexAI'
+            },
+            {
+                label: 'System Message',
+                name: 'systemMessage',
+                type: 'string',
+                default: `You are a helpful AI assistant.`,
+                rows: 4,
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Input Moderation',
+                description: 'Detect text that could generate harmful output and prevent it from being sent to the language model',
+                name: 'inputModeration',
+                type: 'Moderation',
+                optional: true,
+                list: true
+            },
+            {
+                label: 'Max Iterations',
+                name: 'maxIterations',
+                type: 'number',
+                optional: true,
+                additionalParams: true
+            }
+        ]
+        this.sessionId = fields?.sessionId
+    }
+
+    async init(nodeData: INodeData, input: string, options: ICommonObject): Promise<any> {
+        return prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
+    }
+
+    async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | ICommonObject> {
+        const memory = nodeData.inputs?.memory as FlowiseMemory
+        const moderations = nodeData.inputs?.inputModeration as Moderation[]
+
+        const isStreamable = options.socketIO && options.socketIOClientId
+
+        if (moderations && moderations.length > 0) {
+            try {
+                // Use the output of the moderation chain as input for the OpenAI Function Agent
+                input = await checkInputs(moderations, input)
+            } catch (e) {
+                await new Promise((resolve) => setTimeout(resolve, 500))
+                if (isStreamable)
+                    streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
+                return formatResponse(e.message)
+            }
+        }
+
+        const executor = prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
+
+        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const callbacks = await additionalCallbacks(nodeData, options)
+
+        let res: ChainValues = {}
+        let sourceDocuments: ICommonObject[] = []
+        let usedTools: IUsedTool[] = []
+
+        if (isStreamable) {
+            const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
+            res = await executor.invoke({ input }, { callbacks: [loggerHandler, handler, ...callbacks] })
+            if (res.sourceDocuments) {
+                options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', flatten(res.sourceDocuments))
+                sourceDocuments = res.sourceDocuments
+            }
+            if (res.usedTools) {
+                options.socketIO.to(options.socketIOClientId).emit('usedTools', res.usedTools)
+                usedTools = res.usedTools
+            }
+        } else {
+            res = await executor.invoke({ input }, { callbacks: [loggerHandler, ...callbacks] })
+            if (res.sourceDocuments) {
+                sourceDocuments = res.sourceDocuments
+            }
+            if (res.usedTools) {
+                usedTools = res.usedTools
+            }
+        }
+
+        let output = res?.output as string
+
+        // Claude 3 Opus tends to spit out <thinking>..</thinking> as well, discard that in final output
+        const regexPattern: RegExp = /<thinking>[\s\S]*?<\/thinking>/
+        const matches: RegExpMatchArray | null = output.match(regexPattern)
+        if (matches) {
+            for (const match of matches) {
+                output = output.replace(match, '')
+            }
+        }
+
+        await memory.addChatMessages(
+            [
+                {
+                    text: input,
+                    type: 'userMessage'
+                },
+                {
+                    text: output,
+                    type: 'apiMessage'
+                }
+            ],
+            this.sessionId
+        )
+
+        let finalRes = output
+
+        if (sourceDocuments.length || usedTools.length) {
+            const finalRes: ICommonObject = { text: output }
+            if (sourceDocuments.length) {
+                finalRes.sourceDocuments = flatten(sourceDocuments)
+            }
+            if (usedTools.length) {
+                finalRes.usedTools = usedTools
+            }
+            return finalRes
+        }
+
+        return finalRes
+    }
+}
+
+const prepareAgent = (nodeData: INodeData, options: ICommonObject, flowObj: { sessionId?: string; chatId?: string; input?: string }) => {
+    const model = nodeData.inputs?.model as BaseChatModel
+    const maxIterations = nodeData.inputs?.maxIterations as string
+    const memory = nodeData.inputs?.memory as FlowiseMemory
+    const systemMessage = nodeData.inputs?.systemMessage as string
+    let tools = nodeData.inputs?.tools
+    tools = flatten(tools)
+    const memoryKey = memory.memoryKey ? memory.memoryKey : 'chat_history'
+    const inputKey = memory.inputKey ? memory.inputKey : 'input'
+
+    const prompt = ChatPromptTemplate.fromMessages([
+        ['system', systemMessage],
+        new MessagesPlaceholder(memoryKey),
+        ['human', `{${inputKey}}`],
+        new MessagesPlaceholder('agent_scratchpad')
+    ])
+
+    if (llmSupportsVision(model)) {
+        const visionChatModel = model as IVisionChatModal
+        const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+        if (messageContent?.length) {
+            visionChatModel.setVisionModel()
+
+            // Pop the `agent_scratchpad` MessagePlaceHolder
+            let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder
+            if (prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) {
+                const lastMessage = prompt.promptMessages.pop() as HumanMessagePromptTemplate
+                const template = (lastMessage.prompt as PromptTemplate).template as string
+                const msg = HumanMessagePromptTemplate.fromTemplate([
+                    ...messageContent,
+                    {
+                        text: template
+                    }
+                ])
+                msg.inputVariables = lastMessage.inputVariables
+                prompt.promptMessages.push(msg)
+            }
+
+            // Add the `agent_scratchpad` MessagePlaceHolder back
+            prompt.promptMessages.push(messagePlaceholder)
+        } else {
+            visionChatModel.revertToOriginalModel()
+        }
+    }
+
+    if (model.bindTools === undefined) {
+        throw new Error(`This agent requires that the "bindTools()" method be implemented on the input model.`)
+    }
+
+    const modelWithTools = model.bindTools(tools)
+
+    const runnableAgent = RunnableSequence.from([
+        {
+            [inputKey]: (i: { input: string; steps: ToolsAgentStep[] }) => i.input,
+            agent_scratchpad: (i: { input: string; steps: ToolsAgentStep[] }) => formatToOpenAIToolMessages(i.steps),
+            [memoryKey]: async (_: { input: string; steps: ToolsAgentStep[] }) => {
+                const messages = (await memory.getChatMessages(flowObj?.sessionId, true)) as BaseMessage[]
+                return messages ?? []
+            }
+        },
+        prompt,
+        modelWithTools,
+        new ToolCallingAgentOutputParser()
+    ])
+
+    const executor = AgentExecutor.fromAgentAndTools({
+        agent: runnableAgent,
+        tools,
+        sessionId: flowObj?.sessionId,
+        chatId: flowObj?.chatId,
+        input: flowObj?.input,
+        verbose: process.env.DEBUG === 'true' ? true : false,
+        maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
+    })
+
+    return executor
+}
+
+module.exports = { nodeClass: ToolAgent_Agents }
@@ -57,7 +57,6 @@ class XMLAgent_Agents implements INode {
        this.type = 'XMLAgent'
        this.category = 'Agents'
        this.icon = 'xmlagent.svg'
-        this.badge = 'NEW'
        this.description = `Agent that is designed for LLMs that are good for reasoning/writing XML (e.g: Anthropic Claude)`
        this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
        this.inputs = [
@@ -1,10 +1,10 @@
 import { HarmBlockThreshold, HarmCategory } from '@google/generative-ai'
 import type { SafetySetting } from '@google/generative-ai'
-import { ChatGoogleGenerativeAI, GoogleGenerativeAIChatInput } from '@langchain/google-genai'
 import { BaseCache } from '@langchain/core/caches'
-import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, IMultiModalOption, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
 import { convertMultiOptionsToStringArray, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
+import { ChatGoogleGenerativeAI, GoogleGenerativeAIChatInput } from './FlowiseChatGoogleGenerativeAI'

 class GoogleGenerativeAI_ChatModels implements INode {
    label: string
@@ -139,6 +139,15 @@ class GoogleGenerativeAI_ChatModels implements INode {
                ],
                optional: true,
                additionalParams: true
+            },
+            {
+                label: 'Allow Image Uploads',
+                name: 'allowImageUploads',
+                type: 'boolean',
+                description:
+                    'Automatically uses vision model when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent',
+                default: false,
+                optional: true
            }
        ]
    }
@@ -162,20 +171,21 @@ class GoogleGenerativeAI_ChatModels implements INode {
        const harmCategory = nodeData.inputs?.harmCategory as string
        const harmBlockThreshold = nodeData.inputs?.harmBlockThreshold as string
        const cache = nodeData.inputs?.cache as BaseCache
+        const streaming = nodeData.inputs?.streaming as boolean
+
+        const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean

        const obj: Partial<GoogleGenerativeAIChatInput> = {
            apiKey: apiKey,
            modelName: modelName,
-            maxOutputTokens: 2048
+            streaming: streaming ?? true
        }

        if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10)
-
-        const model = new ChatGoogleGenerativeAI(obj)
-        if (topP) model.topP = parseFloat(topP)
-        if (topK) model.topK = parseFloat(topK)
-        if (cache) model.cache = cache
-        if (temperature) model.temperature = parseFloat(temperature)
+        if (topP) obj.topP = parseFloat(topP)
+        if (topK) obj.topK = parseFloat(topK)
+        if (cache) obj.cache = cache
+        if (temperature) obj.temperature = parseFloat(temperature)

        // Safety Settings
        let harmCategories: string[] = convertMultiOptionsToStringArray(harmCategory)
@@ -188,7 +198,16 @@ class GoogleGenerativeAI_ChatModels implements INode {
                threshold: harmBlockThresholds[index] as HarmBlockThreshold
            }
        })
-        if (safetySettings.length > 0) model.safetySettings = safetySettings
+        if (safetySettings.length > 0) obj.safetySettings = safetySettings
+
+        const multiModalOption: IMultiModalOption = {
+            image: {
+                allowImageUploads: allowImageUploads ?? false
+            }
+        }
+
+        const model = new ChatGoogleGenerativeAI(nodeData.id, obj)
+        model.setMultiModalOption(multiModalOption)

        return model
    }
@@ -0,0 +1,550 @@
+import { BaseMessage, AIMessage, AIMessageChunk, isBaseMessage, ChatMessage, MessageContent } from '@langchain/core/messages'
+import { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager'
+import { BaseChatModel, type BaseChatModelParams } from '@langchain/core/language_models/chat_models'
+import { ChatGeneration, ChatGenerationChunk, ChatResult } from '@langchain/core/outputs'
+import { ToolCall } from '@langchain/core/messages/tool'
+import { NewTokenIndices } from '@langchain/core/callbacks/base'
+import {
+    EnhancedGenerateContentResponse,
+    Content,
+    Part,
+    Tool,
+    GenerativeModel,
+    GoogleGenerativeAI as GenerativeAI
+} from '@google/generative-ai'
+import type { SafetySetting } from '@google/generative-ai'
+import { ICommonObject, IMultiModalOption, IVisionChatModal } from '../../../src'
+import { StructuredToolInterface } from '@langchain/core/tools'
+import { isStructuredTool } from '@langchain/core/utils/function_calling'
+import { zodToJsonSchema } from 'zod-to-json-schema'
+
+interface TokenUsage {
+    completionTokens?: number
+    promptTokens?: number
+    totalTokens?: number
+}
+
+export interface GoogleGenerativeAIChatInput extends BaseChatModelParams {
+    modelName?: string
+    model?: string
+    temperature?: number
+    maxOutputTokens?: number
+    topP?: number
+    topK?: number
+    stopSequences?: string[]
+    safetySettings?: SafetySetting[]
+    apiKey?: string
+    streaming?: boolean
+}
+
+class LangchainChatGoogleGenerativeAI extends BaseChatModel implements GoogleGenerativeAIChatInput {
+    modelName = 'gemini-pro'
+
+    temperature?: number
+
+    maxOutputTokens?: number
+
+    topP?: number
+
+    topK?: number
+
+    stopSequences: string[] = []
+
+    safetySettings?: SafetySetting[]
+
+    apiKey?: string
+
+    streaming = false
+
+    private client: GenerativeModel
+
+    get _isMultimodalModel() {
+        return this.modelName.includes('vision') || this.modelName.startsWith('gemini-1.5')
+    }
+
+    constructor(fields?: GoogleGenerativeAIChatInput) {
+        super(fields ?? {})
+
+        this.modelName = fields?.model?.replace(/^models\//, '') ?? fields?.modelName?.replace(/^models\//, '') ?? 'gemini-pro'
+
+        this.maxOutputTokens = fields?.maxOutputTokens ?? this.maxOutputTokens
+
+        if (this.maxOutputTokens && this.maxOutputTokens < 0) {
+            throw new Error('`maxOutputTokens` must be a positive integer')
+        }
+
+        this.temperature = fields?.temperature ?? this.temperature
+        if (this.temperature && (this.temperature < 0 || this.temperature > 1)) {
+            throw new Error('`temperature` must be in the range of [0.0,1.0]')
+        }
+
+        this.topP = fields?.topP ?? this.topP
+        if (this.topP && this.topP < 0) {
+            throw new Error('`topP` must be a positive integer')
+        }
+
+        if (this.topP && this.topP > 1) {
+            throw new Error('`topP` must be below 1.')
+        }
+
+        this.topK = fields?.topK ?? this.topK
+        if (this.topK && this.topK < 0) {
+            throw new Error('`topK` must be a positive integer')
+        }
+
+        this.stopSequences = fields?.stopSequences ?? this.stopSequences
+
+        this.apiKey = fields?.apiKey ?? process.env['GOOGLE_API_KEY']
+        if (!this.apiKey) {
+            throw new Error(
+                'Please set an API key for Google GenerativeAI ' +
+                    'in the environment variable GOOGLE_API_KEY ' +
+                    'or in the `apiKey` field of the ' +
+                    'ChatGoogleGenerativeAI constructor'
+            )
+        }
+
+        this.safetySettings = fields?.safetySettings ?? this.safetySettings
+        if (this.safetySettings && this.safetySettings.length > 0) {
+            const safetySettingsSet = new Set(this.safetySettings.map((s) => s.category))
+            if (safetySettingsSet.size !== this.safetySettings.length) {
+                throw new Error('The categories in `safetySettings` array must be unique')
+            }
+        }
+
+        this.streaming = fields?.streaming ?? this.streaming
+
+        this.getClient()
+    }
+
+    getClient(tools?: Tool[]) {
+        this.client = new GenerativeAI(this.apiKey ?? '').getGenerativeModel({
+            model: this.modelName,
+            tools,
+            safetySettings: this.safetySettings as SafetySetting[],
+            generationConfig: {
+                candidateCount: 1,
+                stopSequences: this.stopSequences,
+                maxOutputTokens: this.maxOutputTokens,
+                temperature: this.temperature,
+                topP: this.topP,
+                topK: this.topK
+            }
+        })
+    }
+
+    _combineLLMOutput() {
+        return []
+    }
+
+    _llmType() {
+        return 'googlegenerativeai'
+    }
+
+    override bindTools(tools: (StructuredToolInterface | Record<string, unknown>)[], kwargs?: Partial<ICommonObject>) {
+        //@ts-ignore
+        return this.bind({ tools: convertToGeminiTools(tools), ...kwargs })
+    }
+
+    convertFunctionResponse(prompts: Content[]) {
+        for (let i = 0; i < prompts.length; i += 1) {
+            if (prompts[i].role === 'function') {
+                if (prompts[i - 1].role === 'model') {
+                    const toolName = prompts[i - 1].parts[0].functionCall?.name ?? ''
+                    prompts[i].parts = [
+                        {
+                            functionResponse: {
+                                name: toolName,
+                                response: {
+                                    name: toolName,
+                                    content: prompts[i].parts[0].text
+                                }
+                            }
+                        }
+                    ]
+                }
+            }
+        }
+    }
+
+    async _generateNonStreaming(
+        prompt: Content[],
+        options: this['ParsedCallOptions'],
+        _runManager?: CallbackManagerForLLMRun
+    ): Promise<ChatResult> {
+        //@ts-ignore
+        const tools = options.tools ?? []
+
+        this.convertFunctionResponse(prompt)
+
+        if (tools.length > 0) {
+            this.getClient(tools)
+        } else {
+            this.getClient()
+        }
+        const res = await this.caller.callWithOptions({ signal: options?.signal }, async () => {
+            let output
+            try {
+                output = await this.client.generateContent({
+                    contents: prompt
+                })
+            } catch (e: any) {
+                if (e.message?.includes('400 Bad Request')) {
+                    e.status = 400
+                }
+                throw e
+            }
+            return output
+        })
+        const generationResult = mapGenerateContentResultToChatResult(res.response)
+        await _runManager?.handleLLMNewToken(generationResult.generations?.length ? generationResult.generations[0].text : '')
+        return generationResult
+    }
+
+    async _generate(
+        messages: BaseMessage[],
+        options: this['ParsedCallOptions'],
+        runManager?: CallbackManagerForLLMRun
+    ): Promise<ChatResult> {
+        const prompt = convertBaseMessagesToContent(messages, this._isMultimodalModel)
+
+        // Handle streaming
+        if (this.streaming) {
+            const tokenUsage: TokenUsage = {}
+            const stream = this._streamResponseChunks(messages, options, runManager)
+            const finalChunks: Record<number, ChatGenerationChunk> = {}
+            for await (const chunk of stream) {
+                const index = (chunk.generationInfo as NewTokenIndices)?.completion ?? 0
+                if (finalChunks[index] === undefined) {
+                    finalChunks[index] = chunk
+                } else {
+                    finalChunks[index] = finalChunks[index].concat(chunk)
+                }
+            }
+            const generations = Object.entries(finalChunks)
+                .sort(([aKey], [bKey]) => parseInt(aKey, 10) - parseInt(bKey, 10))
+                .map(([_, value]) => value)
+
+            return { generations, llmOutput: { estimatedTokenUsage: tokenUsage } }
+        }
+        return this._generateNonStreaming(prompt, options, runManager)
+    }
+
+    async *_streamResponseChunks(
+        messages: BaseMessage[],
+        options: this['ParsedCallOptions'],
+        runManager?: CallbackManagerForLLMRun
+    ): AsyncGenerator<ChatGenerationChunk> {
+        const prompt = convertBaseMessagesToContent(messages, this._isMultimodalModel)
+        //@ts-ignore
+        if (options.tools !== undefined && options.tools.length > 0) {
+            const result = await this._generateNonStreaming(prompt, options, runManager)
+            const generationMessage = result.generations[0].message as AIMessage
+            if (generationMessage === undefined) {
+                throw new Error('Could not parse Groq output.')
+            }
+            const toolCallChunks = generationMessage.tool_calls?.map((toolCall, i) => ({
+                name: toolCall.name,
+                args: JSON.stringify(toolCall.args),
+                id: toolCall.id,
+                index: i
+            }))
+            yield new ChatGenerationChunk({
+                message: new AIMessageChunk({
+                    content: generationMessage.content,
+                    additional_kwargs: generationMessage.additional_kwargs,
+                    tool_call_chunks: toolCallChunks
+                }),
+                text: generationMessage.tool_calls?.length ? '' : (generationMessage.content as string)
+            })
+        } else {
+            const stream = await this.caller.callWithOptions({ signal: options?.signal }, async () => {
+                this.getClient()
+                const { stream } = await this.client.generateContentStream({
+                    contents: prompt
+                })
+                return stream
+            })
+
+            for await (const response of stream) {
+                const chunk = convertResponseContentToChatGenerationChunk(response)
+                if (!chunk) {
+                    continue
+                }
+
+                yield chunk
+                await runManager?.handleLLMNewToken(chunk.text ?? '')
+            }
+        }
+    }
+}
+
+export class ChatGoogleGenerativeAI extends LangchainChatGoogleGenerativeAI implements IVisionChatModal {
+    configuredModel: string
+    configuredMaxToken?: number
+    multiModalOption: IMultiModalOption
+    id: string
+
+    constructor(id: string, fields?: GoogleGenerativeAIChatInput) {
+        super(fields)
+        this.id = id
+        this.configuredModel = fields?.modelName ?? ''
+        this.configuredMaxToken = fields?.maxOutputTokens
+    }
+
+    revertToOriginalModel(): void {
+        super.modelName = this.configuredModel
+        super.maxOutputTokens = this.configuredMaxToken
+    }
+
+    setMultiModalOption(multiModalOption: IMultiModalOption): void {
+        this.multiModalOption = multiModalOption
+    }
+
+    setVisionModel(): void {
+        if (this.modelName !== 'gemini-pro-vision' && this.modelName !== 'gemini-1.5-pro-latest') {
+            super.modelName = 'gemini-1.5-pro-latest'
+            super.maxOutputTokens = this.configuredMaxToken ? this.configuredMaxToken : 8192
+        }
+    }
+}
+
+function getMessageAuthor(message: BaseMessage) {
+    const type = message._getType()
+    if (ChatMessage.isInstance(message)) {
+        return message.role
+    }
+    return message.name ?? type
+}
+
+function convertAuthorToRole(author: string) {
+    switch (author) {
+        /**
+         *  Note: Gemini currently is not supporting system messages
+         *  we will convert them to human messages and merge with following
+         * */
+        case 'ai':
+        case 'model': // getMessageAuthor returns message.name. code ex.: return message.name ?? type;
+            return 'model'
+        case 'system':
+        case 'human':
+            return 'user'
+        case 'function':
+        case 'tool':
+            return 'function'
+        default:
+            throw new Error(`Unknown / unsupported author: ${author}`)
+    }
+}
+
+function convertMessageContentToParts(content: MessageContent, isMultimodalModel: boolean): Part[] {
+    if (typeof content === 'string') {
+        return [{ text: content }]
+    }
+
+    return content.map((c) => {
+        if (c.type === 'text') {
+            return {
+                text: c.text
+            }
+        }
+
+        if (c.type === 'tool_use') {
+            return {
+                functionCall: c.functionCall
+            }
+        }
+
+        /*if (c.type === "tool_use" || c.type === "tool_result") {
+            // TODO: Fix when SDK types are fixed
+            return {
+              ...contentPart,
+              // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            } as any;
+        }*/
+
+        if (c.type === 'image_url') {
+            if (!isMultimodalModel) {
+                throw new Error(`This model does not support images`)
+            }
+            let source
+            if (typeof c.image_url === 'string') {
+                source = c.image_url
+            } else if (typeof c.image_url === 'object' && 'url' in c.image_url) {
+                source = c.image_url.url
+            } else {
+                throw new Error('Please provide image as base64 encoded data URL')
+            }
+            const [dm, data] = source.split(',')
+            if (!dm.startsWith('data:')) {
+                throw new Error('Please provide image as base64 encoded data URL')
+            }
+
+            const [mimeType, encoding] = dm.replace(/^data:/, '').split(';')
+            if (encoding !== 'base64') {
+                throw new Error('Please provide image as base64 encoded data URL')
+            }
+
+            return {
+                inlineData: {
+                    data,
+                    mimeType
+                }
+            }
+        }
+        throw new Error(`Unknown content type ${(c as { type: string }).type}`)
+    })
+}
+
+function convertBaseMessagesToContent(messages: BaseMessage[], isMultimodalModel: boolean) {
+    return messages.reduce<{
+        content: Content[]
+        mergeWithPreviousContent: boolean
+    }>(
+        (acc, message, index) => {
+            if (!isBaseMessage(message)) {
+                throw new Error('Unsupported message input')
+            }
+            const author = getMessageAuthor(message)
+            if (author === 'system' && index !== 0) {
+                throw new Error('System message should be the first one')
+            }
+            const role = convertAuthorToRole(author)
+
+            const prevContent = acc.content[acc.content.length]
+            if (!acc.mergeWithPreviousContent && prevContent && prevContent.role === role) {
+                throw new Error('Google Generative AI requires alternate messages between authors')
+            }
+
+            const parts = convertMessageContentToParts(message.content, isMultimodalModel)
+
+            if (acc.mergeWithPreviousContent) {
+                const prevContent = acc.content[acc.content.length - 1]
+                if (!prevContent) {
+                    throw new Error('There was a problem parsing your system message. Please try a prompt without one.')
+                }
+                prevContent.parts.push(...parts)
+
+                return {
+                    mergeWithPreviousContent: false,
+                    content: acc.content
+                }
+            }
+            const content: Content = {
+                role,
+                parts
+            }
+            return {
+                mergeWithPreviousContent: author === 'system',
+                content: [...acc.content, content]
+            }
+        },
+        { content: [], mergeWithPreviousContent: false }
+    ).content
+}
+
+function mapGenerateContentResultToChatResult(response: EnhancedGenerateContentResponse): ChatResult {
+    // if rejected or error, return empty generations with reason in filters
+    if (!response.candidates || response.candidates.length === 0 || !response.candidates[0]) {
+        return {
+            generations: [],
+            llmOutput: {
+                filters: response?.promptFeedback
+            }
+        }
+    }
+
+    const [candidate] = response.candidates
+    const { content, ...generationInfo } = candidate
+    const text = content.parts.map(({ text }) => text).join('')
+
+    if (content.parts.some((part) => part.functionCall)) {
+        const toolCalls: ToolCall[] = []
+        for (const fcPart of content.parts) {
+            const fc = fcPart.functionCall
+            if (fc) {
+                const { name, args } = fc
+                toolCalls.push({ name, args })
+            }
+        }
+
+        const functionCalls = toolCalls.map((tool) => {
+            return { functionCall: { name: tool.name, args: tool.args }, type: 'tool_use' }
+        })
+        const generation: ChatGeneration = {
+            text,
+            message: new AIMessage({
+                content: functionCalls,
+                name: !content ? undefined : content.role,
+                additional_kwargs: generationInfo,
+                tool_calls: toolCalls
+            }),
+            generationInfo
+        }
+        return {
+            generations: [generation]
+        }
+    } else {
+        const generation: ChatGeneration = {
+            text,
+            message: new AIMessage({
+                content: text,
+                name: !content ? undefined : content.role,
+                additional_kwargs: generationInfo
+            }),
+            generationInfo
+        }
+
+        return {
+            generations: [generation]
+        }
+    }
+}
+
+function convertResponseContentToChatGenerationChunk(response: EnhancedGenerateContentResponse): ChatGenerationChunk | null {
+    if (!response.candidates || response.candidates.length === 0) {
+        return null
+    }
+    const [candidate] = response.candidates
+    const { content, ...generationInfo } = candidate
+    const text = content?.parts[0]?.text ?? ''
+
+    return new ChatGenerationChunk({
+        text,
+        message: new AIMessageChunk({
+            content: text,
+            name: !content ? undefined : content.role,
+            // Each chunk can have unique "generationInfo", and merging strategy is unclear,
+            // so leave blank for now.
+            additional_kwargs: {}
+        }),
+        generationInfo
+    })
+}
+
+function zodToGeminiParameters(zodObj: any) {
+    // Gemini doesn't accept either the $schema or additionalProperties
+    // attributes, so we need to explicitly remove them.
+    const jsonSchema: any = zodToJsonSchema(zodObj)
+    // eslint-disable-next-line unused-imports/no-unused-vars
+    const { $schema, additionalProperties, ...rest } = jsonSchema
+    return rest
+}
+
+function convertToGeminiTools(structuredTools: (StructuredToolInterface | Record<string, unknown>)[]) {
+    return [
+        {
+            functionDeclarations: structuredTools.map((structuredTool) => {
+                if (isStructuredTool(structuredTool)) {
+                    const jsonSchema = zodToGeminiParameters(structuredTool.schema)
+                    return {
+                        name: structuredTool.name,
+                        description: structuredTool.description,
+                        parameters: jsonSchema
+                    }
+                }
+                return structuredTool
+            })
+        }
+    ]
+}
@@ -1,6 +1,5 @@
-import { GoogleAuthOptions } from 'google-auth-library'
 import { BaseCache } from '@langchain/core/caches'
-import { ChatGoogleVertexAI, GoogleVertexAIChatInput } from '@langchain/community/chat_models/googlevertexai'
+import { ChatVertexAI, ChatVertexAIInput } from '@langchain/google-vertexai'
 import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
@@ -20,12 +19,12 @@ class GoogleVertexAI_ChatModels implements INode {
    constructor() {
        this.label = 'ChatGoogleVertexAI'
        this.name = 'chatGoogleVertexAI'
-        this.version = 3.0
+        this.version = 4.0
        this.type = 'ChatGoogleVertexAI'
        this.icon = 'GoogleVertex.svg'
        this.category = 'Chat Models'
        this.description = 'Wrapper around VertexAI large language models that use the Chat endpoint'
-        this.baseClasses = [this.type, ...getBaseClasses(ChatGoogleVertexAI)]
+        this.baseClasses = [this.type, ...getBaseClasses(ChatVertexAI)]
        this.credential = {
            label: 'Connect Credential',
            name: 'credential',
@@ -72,6 +71,15 @@ class GoogleVertexAI_ChatModels implements INode {
                step: 0.1,
                optional: true,
                additionalParams: true
+            },
+            {
+                label: 'Top Next Highest Probability Tokens',
+                name: 'topK',
+                type: 'number',
+                description: `Decode using top-k sampling: consider the set of top_k most probable tokens. Must be positive`,
+                step: 1,
+                optional: true,
+                additionalParams: true
            }
        ]
    }
@@ -89,7 +97,7 @@ class GoogleVertexAI_ChatModels implements INode {
        const googleApplicationCredential = getCredentialParam('googleApplicationCredential', credentialData, nodeData)
        const projectID = getCredentialParam('projectID', credentialData, nodeData)

-        const authOptions: GoogleAuthOptions = {}
+        const authOptions: ICommonObject = {}
        if (Object.keys(credentialData).length !== 0) {
            if (!googleApplicationCredentialFilePath && !googleApplicationCredential)
                throw new Error('Please specify your Google Application Credential')
@@ -111,8 +119,9 @@ class GoogleVertexAI_ChatModels implements INode {
        const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string
        const topP = nodeData.inputs?.topP as string
        const cache = nodeData.inputs?.cache as BaseCache
+        const topK = nodeData.inputs?.topK as string

-        const obj: GoogleVertexAIChatInput<GoogleAuthOptions> = {
+        const obj: ChatVertexAIInput = {
            temperature: parseFloat(temperature),
            model: modelName
        }
@@ -121,8 +130,9 @@ class GoogleVertexAI_ChatModels implements INode {
        if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10)
        if (topP) obj.topP = parseFloat(topP)
        if (cache) obj.cache = cache
+        if (topK) obj.topK = parseFloat(topK)

-        const model = new ChatGoogleVertexAI(obj)
+        const model = new ChatVertexAI(obj)
        return model
    }
 }
@@ -1,37 +1,9 @@
-import { ChatCompletionResponse, ToolCalls as MistralAIToolCalls } from '@mistralai/mistralai'
 import { BaseCache } from '@langchain/core/caches'
-import { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager'
-import { NewTokenIndices } from '@langchain/core/callbacks/base'
-import { ChatGeneration, ChatGenerationChunk, ChatResult } from '@langchain/core/outputs'
-import {
-    MessageType,
-    type BaseMessage,
-    MessageContent,
-    AIMessage,
-    HumanMessage,
-    HumanMessageChunk,
-    AIMessageChunk,
-    ToolMessageChunk,
-    ChatMessageChunk
-} from '@langchain/core/messages'
-import { ChatMistralAI as LangchainChatMistralAI, ChatMistralAIInput } from '@langchain/mistralai'
+import { ChatMistralAI, ChatMistralAIInput } from '@langchain/mistralai'
 import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { getModels, MODEL_TYPE } from '../../../src/modelLoader'

-interface TokenUsage {
-    completionTokens?: number
-    promptTokens?: number
-    totalTokens?: number
-}
-
-type MistralAIInputMessage = {
-    role: string
-    name?: string
-    content: string | string[]
-    tool_calls?: MistralAIToolCalls[] | any[]
-}
-
 class ChatMistral_ChatModels implements INode {
    label: string
    name: string
@@ -170,243 +142,4 @@ class ChatMistral_ChatModels implements INode {
    }
 }

-class ChatMistralAI extends LangchainChatMistralAI {
-    async _generate(
-        messages: BaseMessage[],
-        options?: this['ParsedCallOptions'],
-        runManager?: CallbackManagerForLLMRun
-    ): Promise<ChatResult> {
-        const tokenUsage: TokenUsage = {}
-        const params = this.invocationParams(options)
-        const mistralMessages = this.convertMessagesToMistralMessages(messages)
-        const input = {
-            ...params,
-            messages: mistralMessages
-        }
-
-        // Handle streaming
-        if (this.streaming) {
-            const stream = this._streamResponseChunks(messages, options, runManager)
-            const finalChunks: Record<number, ChatGenerationChunk> = {}
-            for await (const chunk of stream) {
-                const index = (chunk.generationInfo as NewTokenIndices)?.completion ?? 0
-                if (finalChunks[index] === undefined) {
-                    finalChunks[index] = chunk
-                } else {
-                    finalChunks[index] = finalChunks[index].concat(chunk)
-                }
-            }
-            const generations = Object.entries(finalChunks)
-                .sort(([aKey], [bKey]) => parseInt(aKey, 10) - parseInt(bKey, 10))
-                .map(([_, value]) => value)
-
-            return { generations, llmOutput: { estimatedTokenUsage: tokenUsage } }
-        }
-
-        // Not streaming, so we can just call the API once.
-        const response = await this.completionWithRetry(input, false)
-
-        const { completion_tokens: completionTokens, prompt_tokens: promptTokens, total_tokens: totalTokens } = response?.usage ?? {}
-
-        if (completionTokens) {
-            tokenUsage.completionTokens = (tokenUsage.completionTokens ?? 0) + completionTokens
-        }
-
-        if (promptTokens) {
-            tokenUsage.promptTokens = (tokenUsage.promptTokens ?? 0) + promptTokens
-        }
-
-        if (totalTokens) {
-            tokenUsage.totalTokens = (tokenUsage.totalTokens ?? 0) + totalTokens
-        }
-
-        const generations: ChatGeneration[] = []
-        for (const part of response?.choices ?? []) {
-            if ('delta' in part) {
-                throw new Error('Delta not supported in non-streaming mode.')
-            }
-            if (!('message' in part)) {
-                throw new Error('No message found in the choice.')
-            }
-            const text = part.message?.content ?? ''
-            const generation: ChatGeneration = {
-                text,
-                message: this.mistralAIResponseToChatMessage(part)
-            }
-            if (part.finish_reason) {
-                generation.generationInfo = { finish_reason: part.finish_reason }
-            }
-            generations.push(generation)
-        }
-        return {
-            generations,
-            llmOutput: { tokenUsage }
-        }
-    }
-
-    async *_streamResponseChunks(
-        messages: BaseMessage[],
-        options?: this['ParsedCallOptions'],
-        runManager?: CallbackManagerForLLMRun
-    ): AsyncGenerator<ChatGenerationChunk> {
-        const mistralMessages = this.convertMessagesToMistralMessages(messages)
-        const params = this.invocationParams(options)
-        const input = {
-            ...params,
-            messages: mistralMessages
-        }
-
-        const streamIterable = await this.completionWithRetry(input, true)
-        for await (const data of streamIterable) {
-            const choice = data?.choices[0]
-            if (!choice || !('delta' in choice)) {
-                continue
-            }
-
-            const { delta } = choice
-            if (!delta) {
-                continue
-            }
-            const newTokenIndices = {
-                prompt: 0,
-                completion: choice.index ?? 0
-            }
-            const message = this._convertDeltaToMessageChunk(delta)
-            if (message === null) {
-                // Do not yield a chunk if the message is empty
-                continue
-            }
-            const generationChunk = new ChatGenerationChunk({
-                message,
-                text: delta.content ?? '',
-                generationInfo: newTokenIndices
-            })
-            yield generationChunk
-
-            void runManager?.handleLLMNewToken(generationChunk.text ?? '', newTokenIndices, undefined, undefined, undefined, {
-                chunk: generationChunk
-            })
-        }
-        if (options?.signal?.aborted) {
-            throw new Error('AbortError')
-        }
-    }
-
-    _convertDeltaToMessageChunk(delta: {
-        role?: string | undefined
-        content?: string | undefined
-        tool_calls?: MistralAIToolCalls[] | undefined
-    }) {
-        if (!delta.content && !delta.tool_calls) {
-            return null
-        }
-        // Our merge additional kwargs util function will throw unless there
-        // is an index key in each tool object (as seen in OpenAI's) so we
-        // need to insert it here.
-        const toolCallsWithIndex = delta.tool_calls?.length
-            ? delta.tool_calls?.map((toolCall, index) => ({
-                  ...toolCall,
-                  index
-              }))
-            : undefined
-
-        let role = 'assistant'
-        if (delta.role) {
-            role = delta.role
-        } else if (toolCallsWithIndex) {
-            role = 'tool'
-        }
-        const content = delta.content ?? ''
-        let additional_kwargs
-        if (toolCallsWithIndex) {
-            additional_kwargs = {
-                tool_calls: toolCallsWithIndex
-            }
-        } else {
-            additional_kwargs = {}
-        }
-
-        if (role === 'user') {
-            return new HumanMessageChunk({ content })
-        } else if (role === 'assistant') {
-            return new AIMessageChunk({ content, additional_kwargs })
-        } else if (role === 'tool') {
-            return new ToolMessageChunk({
-                content,
-                additional_kwargs,
-                tool_call_id: toolCallsWithIndex?.[0].id ?? ''
-            })
-        } else {
-            return new ChatMessageChunk({ content, role })
-        }
-    }
-
-    convertMessagesToMistralMessages(messages: Array<BaseMessage>): Array<MistralAIInputMessage> {
-        const getRole = (role: MessageType) => {
-            switch (role) {
-                case 'human':
-                    return 'user'
-                case 'ai':
-                    return 'assistant'
-                case 'tool':
-                    return 'tool'
-                case 'function':
-                    return 'function'
-                case 'system':
-                    return 'system'
-                default:
-                    throw new Error(`Unknown message type: ${role}`)
-            }
-        }
-
-        const getContent = (content: MessageContent): string => {
-            if (typeof content === 'string') {
-                return content
-            }
-            throw new Error(`ChatMistralAI does not support non text message content. Received: ${JSON.stringify(content, null, 2)}`)
-        }
-
-        const mistralMessages = []
-        for (const msg of messages) {
-            const msgObj: MistralAIInputMessage = {
-                role: getRole(msg._getType()),
-                content: getContent(msg.content)
-            }
-            if (getRole(msg._getType()) === 'tool') {
-                msgObj.role = 'assistant'
-                msgObj.tool_calls = msg.additional_kwargs?.tool_calls ?? []
-            } else if (getRole(msg._getType()) === 'function') {
-                msgObj.role = 'tool'
-                msgObj.name = msg.name
-            }
-
-            mistralMessages.push(msgObj)
-        }
-
-        return mistralMessages
-    }
-
-    mistralAIResponseToChatMessage(choice: ChatCompletionResponse['choices'][0]): BaseMessage {
-        const { message } = choice
-        // MistralAI SDK does not include tool_calls in the non
-        // streaming return type, so we need to extract it like this
-        // to satisfy typescript.
-        let toolCalls: MistralAIToolCalls[] = []
-        if ('tool_calls' in message) {
-            toolCalls = message.tool_calls as MistralAIToolCalls[]
-        }
-        switch (message.role) {
-            case 'assistant':
-                return new AIMessage({
-                    content: message.content ?? '',
-                    additional_kwargs: {
-                        tool_calls: toolCalls
-                    }
-                })
-            default:
-                return new HumanMessage(message.content ?? '')
-        }
-    }
-}
-
 module.exports = { nodeClass: ChatMistral_ChatModels }
@@ -33,7 +33,9 @@ export class ChatOpenAI extends LangchainChatOpenAI implements IVisionChatModal
    }

    setVisionModel(): void {
-        super.modelName = 'gpt-4-vision-preview'
-        super.maxTokens = this.configuredMaxToken ? this.configuredMaxToken : 1024
+        if (this.modelName !== 'gpt-4-turbo' && !this.modelName.includes('vision')) {
+            super.modelName = 'gpt-4-turbo'
+            super.maxTokens = this.configuredMaxToken ? this.configuredMaxToken : 1024
+        }
    }
 }
@@ -1,45 +0,0 @@
-import { AIPluginTool } from '@langchain/community/tools/aiplugin'
-import { INode, INodeData, INodeParams } from '../../../src/Interface'
-import { getBaseClasses } from '../../../src/utils'
-
-class AIPlugin implements INode {
-    label: string
-    name: string
-    version: number
-    description: string
-    type: string
-    icon: string
-    category: string
-    baseClasses: string[]
-    inputs?: INodeParams[]
-    badge: string
-
-    constructor() {
-        this.label = 'AI Plugin'
-        this.name = 'aiPlugin'
-        this.version = 1.0
-        this.type = 'AIPlugin'
-        this.icon = 'aiplugin.svg'
-        this.category = 'Tools'
-        this.description = 'Execute actions using ChatGPT Plugin Url'
-        this.baseClasses = [this.type, ...getBaseClasses(AIPluginTool)]
-        this.badge = 'DEPRECATING'
-        this.inputs = [
-            {
-                label: 'Plugin Url',
-                name: 'pluginUrl',
-                type: 'string',
-                placeholder: 'https://www.klarna.com/.well-known/ai-plugin.json'
-            }
-        ]
-    }
-
-    async init(nodeData: INodeData): Promise<any> {
-        const pluginUrl = nodeData.inputs?.pluginUrl as string
-        const aiplugin = await AIPluginTool.fromPluginUrl(pluginUrl)
-
-        return aiplugin
-    }
-}
-
-module.exports = { nodeClass: AIPlugin }
@@ -1,6 +0,0 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M8.57535 17.5607C7.7943 16.7796 7.7943 15.5133 8.57535 14.7323L14.2322 9.07542C15.0132 8.29437 16.2796 8.29437 17.0606 9.07542L19.1819 11.1967C20.744 12.7588 20.744 15.2915 19.1819 16.8536L16.3535 19.682C14.7914 21.2441 12.2588 21.2441 10.6967 19.682L8.57535 17.5607Z" stroke="black" stroke-width="2"/>
-<path d="M11.404 7.66112L9.28271 5.53979" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M7.16136 11.9038L5.04004 9.78247" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M17.7676 18.2678L24.8386 25.3389" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
-</svg>