Merge pull request #1898 from vinodkiran/FEATURE/AzureOpenAI_vision

Claude 3 - Adding Support for Image Uploads / Multimodal Inputs

packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts

@@ -4,15 +4,14 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'
 import { ChainValues } from '@langchain/core/utils/types'
 import { AgentStep } from '@langchain/core/agents'
-import { renderTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
+import { renderTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
 import { RunnableSequence } from '@langchain/core/runnables'
 import { ChatConversationalAgent } from 'langchain/agents'
 import { getBaseClasses } from '../../../src/utils'
 import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
-import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { AgentExecutor } from '../../../src/agents'
-import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
-import { addImagesToMessages } from '../../../src/multiModalUtils'
 import { checkInputs, Moderation } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'
 
@@ -171,33 +170,32 @@ const prepareAgent = async (
         outputParser
     })
 
-    if (model instanceof ChatOpenAI) {
+    if (llmSupportsVision(model)) {
-        let humanImageMessages: HumanMessage[] = []
+        const visionChatModel = model as IVisionChatModal
         const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
 
         if (messageContent?.length) {
-            // Change model to gpt-4-vision
+            visionChatModel.setVisionModel()
-            model.modelName = 'gpt-4-vision-preview'
-
-            // Change default max token to higher when using gpt-4-vision
-            model.maxTokens = 1024
-
-            for (const msg of messageContent) {
-                humanImageMessages.push(new HumanMessage({ content: [msg] }))
-            }
 
             // Pop the `agent_scratchpad` MessagePlaceHolder
             let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder
-
+            if (prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) {
-            // Add the HumanMessage for images
+                const lastMessage = prompt.promptMessages.pop() as HumanMessagePromptTemplate
-            prompt.promptMessages.push(...humanImageMessages)
+                const template = (lastMessage.prompt as PromptTemplate).template as string
+                const msg = HumanMessagePromptTemplate.fromTemplate([
+                    ...messageContent,
+                    {
+                        text: template
+                    }
+                ])
+                msg.inputVariables = lastMessage.inputVariables
+                prompt.promptMessages.push(msg)
+            }
 
             // Add the `agent_scratchpad` MessagePlaceHolder back
             prompt.promptMessages.push(messagePlaceholder)
         } else {
-            // revert to previous values if image upload is empty
+            visionChatModel.revertToOriginalModel()
-            model.modelName = model.configuredModel
-            model.maxTokens = model.configuredMaxToken
         }
     }
 

packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts

@@ -1,17 +1,15 @@
 import { flatten } from 'lodash'
 import { AgentExecutor } from 'langchain/agents'
-import { HumanMessage } from '@langchain/core/messages'
 import { ChatPromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts'
 import { Tool } from '@langchain/core/tools'
 import type { PromptTemplate } from '@langchain/core/prompts'
 import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { pull } from 'langchain/hub'
 import { additionalCallbacks } from '../../../src/handler'
-import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
 import { createReactAgent } from '../../../src/agents'
-import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
-import { addImagesToMessages } from '../../../src/multiModalUtils'
 import { checkInputs, Moderation } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'
 
@@ -90,23 +88,26 @@ class MRKLAgentChat_Agents implements INode {
         const prompt = await pull<PromptTemplate>('hwchase17/react-chat')
         let chatPromptTemplate = undefined
 
-        if (model instanceof ChatOpenAI) {
+        if (llmSupportsVision(model)) {
+            const visionChatModel = model as IVisionChatModal
             const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
 
             if (messageContent?.length) {
-                // Change model to gpt-4-vision
+                // Change model to vision supported
-                model.modelName = 'gpt-4-vision-preview'
+                visionChatModel.setVisionModel()
-
-                // Change default max token to higher when using gpt-4-vision
-                model.maxTokens = 1024
-
                 const oldTemplate = prompt.template as string
-                chatPromptTemplate = ChatPromptTemplate.fromMessages([HumanMessagePromptTemplate.fromTemplate(oldTemplate)])
+
-                chatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
+                const msg = HumanMessagePromptTemplate.fromTemplate([
+                    ...messageContent,
+                    {
+                        text: oldTemplate
+                    }
+                ])
+                msg.inputVariables = prompt.inputVariables
+                chatPromptTemplate = ChatPromptTemplate.fromMessages([msg])
             } else {
                 // revert to previous values if image upload is empty
-                model.modelName = model.configuredModel
+                visionChatModel.revertToOriginalModel()
-                model.maxTokens = model.configuredMaxToken
             }
         }
 

packages/components/nodes/agents/XMLAgent/XMLAgent.ts

@@ -1,8 +1,8 @@
 import { flatten } from 'lodash'
 import { ChainValues } from '@langchain/core/utils/types'
 import { AgentStep } from '@langchain/core/agents'
+import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { RunnableSequence } from '@langchain/core/runnables'
-import { ChatOpenAI } from '@langchain/openai'
 import { Tool } from '@langchain/core/tools'
 import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
 import { XMLAgentOutputParser } from 'langchain/agents/xml/output_parser'
@@ -160,7 +160,7 @@ const prepareAgent = async (
     flowObj: { sessionId?: string; chatId?: string; input?: string },
     chatHistory: IMessage[] = []
 ) => {
-    const model = nodeData.inputs?.model as ChatOpenAI
+    const model = nodeData.inputs?.model as BaseChatModel
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const systemMessage = nodeData.inputs?.systemMessage as string
     let tools = nodeData.inputs?.tools

packages/components/nodes/chains/ConversationChain/ConversationChain.ts

@@ -1,14 +1,30 @@
 import { ConversationChain } from 'langchain/chains'
-import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts'
+import {
+    ChatPromptTemplate,
+    HumanMessagePromptTemplate,
+    MessagesPlaceholder,
+    SystemMessagePromptTemplate,
+    BaseMessagePromptTemplateLike,
+    PromptTemplate
+} from '@langchain/core/prompts'
 import { RunnableSequence } from '@langchain/core/runnables'
 import { StringOutputParser } from '@langchain/core/output_parsers'
+import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { HumanMessage } from '@langchain/core/messages'
 import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
 import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'
-import { addImagesToMessages } from '../../../src/multiModalUtils'
+import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
 import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
-import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import {
+    IVisionChatModal,
+    FlowiseMemory,
+    ICommonObject,
+    INode,
+    INodeData,
+    INodeParams,
+    MessageContentImageUrl
+} from '../../../src/Interface'
 import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
 import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
 
@@ -145,16 +161,32 @@ class ConversationChain_Chains implements INode {
     }
 }
 
-const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => {
+const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageContentImageUrl[]) => {
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const prompt = nodeData.inputs?.systemMessagePrompt as string
     const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate
+    let model = nodeData.inputs?.model as BaseChatModel
 
     if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) {
         const sysPrompt = chatPromptTemplate.promptMessages[0]
         const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1]
         const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt]
-        if (humanImageMessages.length) messages.push(...humanImageMessages)
+
+        // OpenAI works better when separate images into standalone human messages
+        if (model instanceof ChatOpenAI && humanImageMessages.length) {
+            messages.push(new HumanMessage({ content: [...humanImageMessages] }))
+        } else if (humanImageMessages.length) {
+            const lastMessage = messages.pop() as HumanMessagePromptTemplate
+            const template = (lastMessage.prompt as PromptTemplate).template as string
+            const msg = HumanMessagePromptTemplate.fromTemplate([
+                ...humanImageMessages,
+                {
+                    text: template
+                }
+            ])
+            msg.inputVariables = lastMessage.inputVariables
+            messages.push(msg)
+        }
 
         const chatPrompt = ChatPromptTemplate.fromMessages(messages)
         if ((chatPromptTemplate as any).promptValues) {
@@ -165,12 +197,18 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage
         return chatPrompt
     }
 
-    const messages = [
+    const messages: BaseMessagePromptTemplateLike[] = [
         SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage),
-        new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
+        new MessagesPlaceholder(memory.memoryKey ?? 'chat_history')
-        HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)
     ]
-    if (humanImageMessages.length) messages.push(...(humanImageMessages as any[]))
+
+    // OpenAI works better when separate images into standalone human messages
+    if (model instanceof ChatOpenAI && humanImageMessages.length) {
+        messages.push(HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`))
+        messages.push(new HumanMessage({ content: [...humanImageMessages] }))
+    } else if (humanImageMessages.length) {
+        messages.push(HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages]))
+    }
 
     const chatPrompt = ChatPromptTemplate.fromMessages(messages)
 
@@ -179,32 +217,23 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage
 
 const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
     const chatHistory = options.chatHistory
-    let model = nodeData.inputs?.model as ChatOpenAI
+    let model = nodeData.inputs?.model as BaseChatModel
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const memoryKey = memory.memoryKey ?? 'chat_history'
 
-    let humanImageMessages: HumanMessage[] = []
+    let messageContent: MessageContentImageUrl[] = []
-    if (model instanceof ChatOpenAI) {
+    if (llmSupportsVision(model)) {
-        const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+        messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
-
+        const visionChatModel = model as IVisionChatModal
         if (messageContent?.length) {
-            // Change model to gpt-4-vision
+            visionChatModel.setVisionModel()
-            model.modelName = 'gpt-4-vision-preview'
-
-            // Change default max token to higher when using gpt-4-vision
-            model.maxTokens = 1024
-
-            for (const msg of messageContent) {
-                humanImageMessages.push(new HumanMessage({ content: [msg] }))
-            }
         } else {
             // revert to previous values if image upload is empty
-            model.modelName = model.configuredModel
+            visionChatModel.revertToOriginalModel()
-            model.maxTokens = model.configuredMaxToken
         }
     }
 
-    const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages)
+    const chatPrompt = prepareChatPrompt(nodeData, messageContent)
     let promptVariables = {}
     const promptValuesRaw = (chatPrompt as any).promptValues
     if (promptValuesRaw) {
@@ -228,7 +257,7 @@ const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: s
             },
             ...promptVariables
         },
-        prepareChatPrompt(nodeData, humanImageMessages),
+        prepareChatPrompt(nodeData, messageContent),
         model,
         new StringOutputParser()
     ])

packages/components/nodes/chains/LLMChain/LLMChain.ts

@@ -1,16 +1,15 @@
 import { BaseLanguageModel, BaseLanguageModelCallOptions } from '@langchain/core/language_models/base'
 import { BaseLLMOutputParser, BaseOutputParser } from '@langchain/core/output_parsers'
 import { HumanMessage } from '@langchain/core/messages'
-import { ChatPromptTemplate, FewShotPromptTemplate, PromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts'
+import { ChatPromptTemplate, FewShotPromptTemplate, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
 import { OutputFixingParser } from 'langchain/output_parsers'
 import { LLMChain } from 'langchain/chains'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { IVisionChatModal, ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
-import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
+import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
 import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
 import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
 import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers'
-import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
-import { addImagesToMessages } from '../../../src/multiModalUtils'
 
 class LLMChain_Chains implements INode {
     label: string
@@ -164,7 +163,6 @@ const runPrediction = async (
     const socketIO = isStreaming ? options.socketIO : undefined
     const socketIOClientId = isStreaming ? options.socketIOClientId : ''
     const moderations = nodeData.inputs?.inputModeration as Moderation[]
-    let model = nodeData.inputs?.model as ChatOpenAI
 
     if (moderations && moderations.length > 0) {
         try {
@@ -183,24 +181,39 @@ const runPrediction = async (
      * TO: { "value": "hello i am ben\n\n\thow are you?" }
      */
     const promptValues = handleEscapeCharacters(promptValuesRaw, true)
-    const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
 
-    if (chain.llm instanceof ChatOpenAI) {
+    if (llmSupportsVision(chain.llm)) {
-        const chatOpenAI = chain.llm as ChatOpenAI
+        const visionChatModel = chain.llm as IVisionChatModal
+        const messageContent = addImagesToMessages(nodeData, options, visionChatModel.multiModalOption)
         if (messageContent?.length) {
             // Change model to gpt-4-vision && max token to higher when using gpt-4-vision
-            chatOpenAI.modelName = 'gpt-4-vision-preview'
+            visionChatModel.setVisionModel()
-            chatOpenAI.maxTokens = 1024
             // Add image to the message
             if (chain.prompt instanceof PromptTemplate) {
                 const existingPromptTemplate = chain.prompt.template as string
-                let newChatPromptTemplate = ChatPromptTemplate.fromMessages([
+                const msg = HumanMessagePromptTemplate.fromTemplate([
-                    HumanMessagePromptTemplate.fromTemplate(existingPromptTemplate)
+                    ...messageContent,
+                    {
+                        text: existingPromptTemplate
+                    }
                 ])
-                newChatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
+                msg.inputVariables = chain.prompt.inputVariables
-                chain.prompt = newChatPromptTemplate
+                chain.prompt = ChatPromptTemplate.fromMessages([msg])
             } else if (chain.prompt instanceof ChatPromptTemplate) {
+                if (chain.prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) {
+                    const lastMessage = chain.prompt.promptMessages.pop() as HumanMessagePromptTemplate
+                    const template = (lastMessage.prompt as PromptTemplate).template as string
+                    const msg = HumanMessagePromptTemplate.fromTemplate([
+                        ...messageContent,
+                        {
+                            text: template
+                        }
+                    ])
+                    msg.inputVariables = lastMessage.inputVariables
+                    chain.prompt.promptMessages.push(msg)
+                } else {
                     chain.prompt.promptMessages.push(new HumanMessage({ content: messageContent }))
+                }
             } else if (chain.prompt instanceof FewShotPromptTemplate) {
                 let existingFewShotPromptTemplate = chain.prompt.examplePrompt.template as string
                 let newFewShotPromptTemplate = ChatPromptTemplate.fromMessages([
@@ -212,8 +225,7 @@ const runPrediction = async (
             }
         } else {
             // revert to previous values if image upload is empty
-            chatOpenAI.modelName = model.configuredModel
+            visionChatModel.revertToOriginalModel()
-            chatOpenAI.maxTokens = model.configuredMaxToken
         }
     }
 

packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts

@@ -1,8 +1,9 @@
-import { AnthropicInput, ChatAnthropic } from '@langchain/anthropic'
+import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic'
 import { BaseCache } from '@langchain/core/caches'
 import { BaseLLMParams } from '@langchain/core/language_models/llms'
-import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { ChatAnthropic } from './FlowiseChatAntrhopic'
 
 class ChatAnthropic_ChatModels implements INode {
     label: string
@@ -19,12 +20,12 @@ class ChatAnthropic_ChatModels implements INode {
     constructor() {
         this.label = 'ChatAnthropic'
         this.name = 'chatAnthropic'
-        this.version = 3.0
+        this.version = 4.0
         this.type = 'ChatAnthropic'
         this.icon = 'Anthropic.svg'
         this.category = 'Chat Models'
         this.description = 'Wrapper around ChatAnthropic large language models that use the Chat endpoint'
-        this.baseClasses = [this.type, ...getBaseClasses(ChatAnthropic)]
+        this.baseClasses = [this.type, ...getBaseClasses(LangchainChatAnthropic)]
         this.credential = {
             label: 'Connect Credential',
             name: 'credential',
@@ -147,6 +148,15 @@ class ChatAnthropic_ChatModels implements INode {
                 step: 0.1,
                 optional: true,
                 additionalParams: true
+            },
+            {
+                label: 'Allow Image Uploads',
+                name: 'allowImageUploads',
+                type: 'boolean',
+                description:
+                    'Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent',
+                default: false,
+                optional: true
             }
         ]
     }
@@ -163,6 +173,8 @@ class ChatAnthropic_ChatModels implements INode {
         const credentialData = await getCredentialData(nodeData.credential ?? '', options)
         const anthropicApiKey = getCredentialParam('anthropicApiKey', credentialData, nodeData)
 
+        const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
+
         const obj: Partial<AnthropicInput> & BaseLLMParams & { anthropicApiKey?: string } = {
             temperature: parseFloat(temperature),
             modelName,
@@ -175,7 +187,14 @@ class ChatAnthropic_ChatModels implements INode {
         if (topK) obj.topK = parseFloat(topK)
         if (cache) obj.cache = cache
 
-        const model = new ChatAnthropic(obj)
+        const multiModalOption: IMultiModalOption = {
+            image: {
+                allowImageUploads: allowImageUploads ?? false
+            }
+        }
+
+        const model = new ChatAnthropic(nodeData.id, obj)
+        model.setMultiModalOption(multiModalOption)
         return model
     }
 }

packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts

@@ -0,0 +1,33 @@
+import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic'
+import { IVisionChatModal, IMultiModalOption } from '../../../src'
+import { BaseLLMParams } from '@langchain/core/language_models/llms'
+
+export class ChatAnthropic extends LangchainChatAnthropic implements IVisionChatModal {
+    configuredModel: string
+    configuredMaxToken: number
+    multiModalOption: IMultiModalOption
+    id: string
+
+    constructor(id: string, fields: Partial<AnthropicInput> & BaseLLMParams & { anthropicApiKey?: string }) {
+        super(fields)
+        this.id = id
+        this.configuredModel = fields?.modelName || 'claude-3-opus-20240229'
+        this.configuredMaxToken = fields?.maxTokens ?? 256
+    }
+
+    revertToOriginalModel(): void {
+        super.modelName = this.configuredModel
+        super.maxTokens = this.configuredMaxToken
+    }
+
+    setMultiModalOption(multiModalOption: IMultiModalOption): void {
+        this.multiModalOption = multiModalOption
+    }
+
+    setVisionModel(): void {
+        if (!this.modelName.startsWith('claude-3')) {
+            super.modelName = 'claude-3-opus-20240229'
+            super.maxTokens = 1024
+        }
+    }
+}

packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts

@@ -228,7 +228,7 @@ class ChatOpenAI_ChatModels implements INode {
 
         const obj: Partial<OpenAIChatInput> &
             Partial<AzureOpenAIInput> &
-            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption } = {
+            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput } = {
             temperature: parseFloat(temperature),
             modelName,
             openAIApiKey,
@@ -265,10 +265,9 @@ class ChatOpenAI_ChatModels implements INode {
                 imageResolution
             }
         }
-        obj.multiModalOption = multiModalOption
 
         const model = new ChatOpenAI(nodeData.id, obj)
-
+        model.setMultiModalOption(multiModalOption)
         return model
     }
 }

packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts

@@ -1,39 +1,39 @@
 import type { ClientOptions } from 'openai'
-import {
+import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, LegacyOpenAIInput, AzureOpenAIInput } from '@langchain/openai'
-    ChatOpenAI as LangchainChatOpenAI,
-    OpenAIChatInput,
-    LegacyOpenAIInput,
-    AzureOpenAIInput,
-    ChatOpenAICallOptions
-} from '@langchain/openai'
 import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
-import { BaseMessageLike } from '@langchain/core/messages'
+import { IMultiModalOption, IVisionChatModal } from '../../../src'
-import { Callbacks } from '@langchain/core/callbacks/manager'
-import { LLMResult } from '@langchain/core/outputs'
-import { IMultiModalOption } from '../../../src'
 
-export class ChatOpenAI extends LangchainChatOpenAI {
+export class ChatOpenAI extends LangchainChatOpenAI implements IVisionChatModal {
     configuredModel: string
-    configuredMaxToken?: number
+    configuredMaxToken: number
-    multiModalOption?: IMultiModalOption
+    multiModalOption: IMultiModalOption
     id: string
 
     constructor(
         id: string,
         fields?: Partial<OpenAIChatInput> &
             Partial<AzureOpenAIInput> &
-            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption },
+            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput },
         /** @deprecated */
         configuration?: ClientOptions & LegacyOpenAIInput
     ) {
         super(fields, configuration)
         this.id = id
-        this.multiModalOption = fields?.multiModalOption
         this.configuredModel = fields?.modelName ?? 'gpt-3.5-turbo'
-        this.configuredMaxToken = fields?.maxTokens
+        this.configuredMaxToken = fields?.maxTokens ?? 256
     }
 
-    async generate(messages: BaseMessageLike[][], options?: string[] | ChatOpenAICallOptions, callbacks?: Callbacks): Promise<LLMResult> {
+    revertToOriginalModel(): void {
-        return super.generate(messages, options, callbacks)
+        super.modelName = this.configuredModel
+        super.maxTokens = this.configuredMaxToken
+    }
+
+    setMultiModalOption(multiModalOption: IMultiModalOption): void {
+        this.multiModalOption = multiModalOption
+    }
+
+    setVisionModel(): void {
+        super.modelName = 'gpt-4-vision-preview'
+        super.maxTokens = 1024
     }
 }

packages/components/package.json

@@ -29,7 +29,7 @@
         "@google-ai/generativelanguage": "^0.2.1",
         "@google/generative-ai": "^0.1.3",
         "@huggingface/inference": "^2.6.1",
-        "@langchain/anthropic": "^0.0.10",
+        "@langchain/anthropic": "^0.1.4",
         "@langchain/cohere": "^0.0.5",
         "@langchain/community": "^0.0.30",
         "@langchain/google-genai": "^0.0.10",

packages/components/src/Interface.ts

@@ -270,3 +270,14 @@ export abstract class FlowiseSummaryMemory extends ConversationSummaryMemory imp
     abstract addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId?: string): Promise<void>
     abstract clearChatMessages(overrideSessionId?: string): Promise<void>
 }
+
+export interface IVisionChatModal {
+    id: string
+    configuredModel: string
+    configuredMaxToken: number
+    multiModalOption: IMultiModalOption
+
+    setVisionModel(): void
+    revertToOriginalModel(): void
+    setMultiModalOption(multiModalOption: IMultiModalOption): void
+}

packages/components/src/multiModalUtils.ts

@@ -1,5 +1,4 @@
-import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface'
+import { IVisionChatModal, ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface'
-import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI'
 import path from 'path'
 import { getStoragePath } from './utils'
 import fs from 'fs'
@@ -12,7 +11,7 @@ export const addImagesToMessages = (
     const imageContent: MessageContentImageUrl[] = []
     let model = nodeData.inputs?.model
 
-    if (model instanceof ChatOpenAI && multiModalOption) {
+    if (llmSupportsVision(model) && multiModalOption) {
         // Image Uploaded
         if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) {
             const imageUploads = getImageUploads(options.uploads)
@@ -46,3 +45,5 @@ export const getAudioUploads = (uploads: IFileUpload[]) => {
 export const getImageUploads = (uploads: IFileUpload[]) => {
     return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('image/'))
 }
+
+export const llmSupportsVision = (value: any): value is IVisionChatModal => !!value?.multiModalOption

packages/server/marketplaces/chatflows/Claude LLM.json

@@ -160,7 +160,7 @@
             "data": {
                 "id": "chatAnthropic_0",
                 "label": "ChatAnthropic",
-                "version": 3,
+                "version": 4,
                 "name": "chatAnthropic",
                 "type": "ChatAnthropic",
                 "baseClasses": ["ChatAnthropic", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -288,6 +288,15 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatAnthropic_0-input-topK-number"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatAnthropic_0-input-allowImageUploads-boolean"
                     }
                 ],
                 "inputAnchors": [
@@ -305,7 +314,8 @@
                     "temperature": 0.9,
                     "maxTokensToSample": "",
                     "topP": "",
-                    "topK": ""
+                    "topK": "",
+                    "allowImageUploads": true
                 },
                 "outputAnchors": [
                     {

packages/server/src/index.ts

@@ -1533,7 +1533,7 @@ export class App {
         if (!chatflow) return `Chatflow ${chatflowid} not found`
 
         const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent']
-        const uploadProcessingNodes = ['chatOpenAI']
+        const uploadProcessingNodes = ['chatOpenAI', 'chatAnthropic']
 
         const flowObj = JSON.parse(chatflow.flowData)
         const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = []