diff --git a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts index 802ee6f8..fe1af11c 100644 --- a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts +++ b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts @@ -4,15 +4,14 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages' import { ChainValues } from '@langchain/core/utils/types' import { AgentStep } from '@langchain/core/agents' -import { renderTemplate, MessagesPlaceholder } from '@langchain/core/prompts' +import { renderTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts' import { RunnableSequence } from '@langchain/core/runnables' import { ChatConversationalAgent } from 'langchain/agents' import { getBaseClasses } from '../../../src/utils' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' -import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' +import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { AgentExecutor } from '../../../src/agents' -import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { addImagesToMessages } from '../../../src/multiModalUtils' +import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' import { checkInputs, Moderation } from '../../moderation/Moderation' import { formatResponse } from '../../outputparsers/OutputParserHelpers' @@ -171,33 +170,32 @@ const prepareAgent = async ( outputParser }) - if (model instanceof ChatOpenAI) { - let humanImageMessages: HumanMessage[] = [] + if (llmSupportsVision(model)) { + const visionChatModel = model as IVisionChatModal const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) if (messageContent?.length) { - // Change model to gpt-4-vision - model.modelName = 'gpt-4-vision-preview' - - // Change default max token to higher when using gpt-4-vision - model.maxTokens = 1024 - - for (const msg of messageContent) { - humanImageMessages.push(new HumanMessage({ content: [msg] })) - } + visionChatModel.setVisionModel() // Pop the `agent_scratchpad` MessagePlaceHolder let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder - - // Add the HumanMessage for images - prompt.promptMessages.push(...humanImageMessages) + if (prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) { + const lastMessage = prompt.promptMessages.pop() as HumanMessagePromptTemplate + const template = (lastMessage.prompt as PromptTemplate).template as string + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...messageContent, + { + text: template + } + ]) + msg.inputVariables = lastMessage.inputVariables + prompt.promptMessages.push(msg) + } // Add the `agent_scratchpad` MessagePlaceHolder back prompt.promptMessages.push(messagePlaceholder) } else { - // revert to previous values if image upload is empty - model.modelName = model.configuredModel - model.maxTokens = model.configuredMaxToken + visionChatModel.revertToOriginalModel() } } diff --git a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts index a2d0f259..81efe43b 100644 --- a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts +++ b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts @@ -1,17 +1,15 @@ import { flatten } from 'lodash' import { AgentExecutor } from 'langchain/agents' -import { HumanMessage } from '@langchain/core/messages' import { ChatPromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts' import { Tool } from '@langchain/core/tools' import type { PromptTemplate } from '@langchain/core/prompts' import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { pull } from 'langchain/hub' import { additionalCallbacks } from '../../../src/handler' -import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' +import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { createReactAgent } from '../../../src/agents' -import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { addImagesToMessages } from '../../../src/multiModalUtils' +import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' import { checkInputs, Moderation } from '../../moderation/Moderation' import { formatResponse } from '../../outputparsers/OutputParserHelpers' @@ -90,23 +88,26 @@ class MRKLAgentChat_Agents implements INode { const prompt = await pull('hwchase17/react-chat') let chatPromptTemplate = undefined - if (model instanceof ChatOpenAI) { + if (llmSupportsVision(model)) { + const visionChatModel = model as IVisionChatModal const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) if (messageContent?.length) { - // Change model to gpt-4-vision - model.modelName = 'gpt-4-vision-preview' - - // Change default max token to higher when using gpt-4-vision - model.maxTokens = 1024 - + // Change model to vision supported + visionChatModel.setVisionModel() const oldTemplate = prompt.template as string - chatPromptTemplate = ChatPromptTemplate.fromMessages([HumanMessagePromptTemplate.fromTemplate(oldTemplate)]) - chatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent })) + + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...messageContent, + { + text: oldTemplate + } + ]) + msg.inputVariables = prompt.inputVariables + chatPromptTemplate = ChatPromptTemplate.fromMessages([msg]) } else { // revert to previous values if image upload is empty - model.modelName = model.configuredModel - model.maxTokens = model.configuredMaxToken + visionChatModel.revertToOriginalModel() } } diff --git a/packages/components/nodes/agents/XMLAgent/XMLAgent.ts b/packages/components/nodes/agents/XMLAgent/XMLAgent.ts index 167214f8..175dcefd 100644 --- a/packages/components/nodes/agents/XMLAgent/XMLAgent.ts +++ b/packages/components/nodes/agents/XMLAgent/XMLAgent.ts @@ -1,8 +1,8 @@ import { flatten } from 'lodash' import { ChainValues } from '@langchain/core/utils/types' import { AgentStep } from '@langchain/core/agents' +import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { RunnableSequence } from '@langchain/core/runnables' -import { ChatOpenAI } from '@langchain/openai' import { Tool } from '@langchain/core/tools' import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder } from '@langchain/core/prompts' import { XMLAgentOutputParser } from 'langchain/agents/xml/output_parser' @@ -160,7 +160,7 @@ const prepareAgent = async ( flowObj: { sessionId?: string; chatId?: string; input?: string }, chatHistory: IMessage[] = [] ) => { - const model = nodeData.inputs?.model as ChatOpenAI + const model = nodeData.inputs?.model as BaseChatModel const memory = nodeData.inputs?.memory as FlowiseMemory const systemMessage = nodeData.inputs?.systemMessage as string let tools = nodeData.inputs?.tools diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index 25d80bee..16493adc 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -1,14 +1,30 @@ import { ConversationChain } from 'langchain/chains' -import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts' +import { + ChatPromptTemplate, + HumanMessagePromptTemplate, + MessagesPlaceholder, + SystemMessagePromptTemplate, + BaseMessagePromptTemplateLike, + PromptTemplate +} from '@langchain/core/prompts' import { RunnableSequence } from '@langchain/core/runnables' import { StringOutputParser } from '@langchain/core/output_parsers' +import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { HumanMessage } from '@langchain/core/messages' import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse } from '../../outputparsers/OutputParserHelpers' -import { addImagesToMessages } from '../../../src/multiModalUtils' +import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { + IVisionChatModal, + FlowiseMemory, + ICommonObject, + INode, + INodeData, + INodeParams, + MessageContentImageUrl +} from '../../../src/Interface' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' @@ -145,16 +161,32 @@ class ConversationChain_Chains implements INode { } } -const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => { +const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageContentImageUrl[]) => { const memory = nodeData.inputs?.memory as FlowiseMemory const prompt = nodeData.inputs?.systemMessagePrompt as string const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate + let model = nodeData.inputs?.model as BaseChatModel if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) { const sysPrompt = chatPromptTemplate.promptMessages[0] const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1] const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt] - if (humanImageMessages.length) messages.push(...humanImageMessages) + + // OpenAI works better when separate images into standalone human messages + if (model instanceof ChatOpenAI && humanImageMessages.length) { + messages.push(new HumanMessage({ content: [...humanImageMessages] })) + } else if (humanImageMessages.length) { + const lastMessage = messages.pop() as HumanMessagePromptTemplate + const template = (lastMessage.prompt as PromptTemplate).template as string + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...humanImageMessages, + { + text: template + } + ]) + msg.inputVariables = lastMessage.inputVariables + messages.push(msg) + } const chatPrompt = ChatPromptTemplate.fromMessages(messages) if ((chatPromptTemplate as any).promptValues) { @@ -165,12 +197,18 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage return chatPrompt } - const messages = [ + const messages: BaseMessagePromptTemplateLike[] = [ SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage), - new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), - HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`) + new MessagesPlaceholder(memory.memoryKey ?? 'chat_history') ] - if (humanImageMessages.length) messages.push(...(humanImageMessages as any[])) + + // OpenAI works better when separate images into standalone human messages + if (model instanceof ChatOpenAI && humanImageMessages.length) { + messages.push(HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)) + messages.push(new HumanMessage({ content: [...humanImageMessages] })) + } else if (humanImageMessages.length) { + messages.push(HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages])) + } const chatPrompt = ChatPromptTemplate.fromMessages(messages) @@ -179,32 +217,23 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => { const chatHistory = options.chatHistory - let model = nodeData.inputs?.model as ChatOpenAI + let model = nodeData.inputs?.model as BaseChatModel const memory = nodeData.inputs?.memory as FlowiseMemory const memoryKey = memory.memoryKey ?? 'chat_history' - let humanImageMessages: HumanMessage[] = [] - if (model instanceof ChatOpenAI) { - const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) - + let messageContent: MessageContentImageUrl[] = [] + if (llmSupportsVision(model)) { + messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) + const visionChatModel = model as IVisionChatModal if (messageContent?.length) { - // Change model to gpt-4-vision - model.modelName = 'gpt-4-vision-preview' - - // Change default max token to higher when using gpt-4-vision - model.maxTokens = 1024 - - for (const msg of messageContent) { - humanImageMessages.push(new HumanMessage({ content: [msg] })) - } + visionChatModel.setVisionModel() } else { // revert to previous values if image upload is empty - model.modelName = model.configuredModel - model.maxTokens = model.configuredMaxToken + visionChatModel.revertToOriginalModel() } } - const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages) + const chatPrompt = prepareChatPrompt(nodeData, messageContent) let promptVariables = {} const promptValuesRaw = (chatPrompt as any).promptValues if (promptValuesRaw) { @@ -228,7 +257,7 @@ const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: s }, ...promptVariables }, - prepareChatPrompt(nodeData, humanImageMessages), + prepareChatPrompt(nodeData, messageContent), model, new StringOutputParser() ]) diff --git a/packages/components/nodes/chains/LLMChain/LLMChain.ts b/packages/components/nodes/chains/LLMChain/LLMChain.ts index 6adee1e1..fa0fd61c 100644 --- a/packages/components/nodes/chains/LLMChain/LLMChain.ts +++ b/packages/components/nodes/chains/LLMChain/LLMChain.ts @@ -1,16 +1,15 @@ import { BaseLanguageModel, BaseLanguageModelCallOptions } from '@langchain/core/language_models/base' import { BaseLLMOutputParser, BaseOutputParser } from '@langchain/core/output_parsers' import { HumanMessage } from '@langchain/core/messages' -import { ChatPromptTemplate, FewShotPromptTemplate, PromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts' +import { ChatPromptTemplate, FewShotPromptTemplate, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts' import { OutputFixingParser } from 'langchain/output_parsers' import { LLMChain } from 'langchain/chains' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' -import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' +import { IVisionChatModal, ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler' import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers' -import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { addImagesToMessages } from '../../../src/multiModalUtils' +import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' class LLMChain_Chains implements INode { label: string @@ -164,7 +163,6 @@ const runPrediction = async ( const socketIO = isStreaming ? options.socketIO : undefined const socketIOClientId = isStreaming ? options.socketIOClientId : '' const moderations = nodeData.inputs?.inputModeration as Moderation[] - let model = nodeData.inputs?.model as ChatOpenAI if (moderations && moderations.length > 0) { try { @@ -183,24 +181,39 @@ const runPrediction = async ( * TO: { "value": "hello i am ben\n\n\thow are you?" } */ const promptValues = handleEscapeCharacters(promptValuesRaw, true) - const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) - if (chain.llm instanceof ChatOpenAI) { - const chatOpenAI = chain.llm as ChatOpenAI + if (llmSupportsVision(chain.llm)) { + const visionChatModel = chain.llm as IVisionChatModal + const messageContent = addImagesToMessages(nodeData, options, visionChatModel.multiModalOption) if (messageContent?.length) { // Change model to gpt-4-vision && max token to higher when using gpt-4-vision - chatOpenAI.modelName = 'gpt-4-vision-preview' - chatOpenAI.maxTokens = 1024 + visionChatModel.setVisionModel() // Add image to the message if (chain.prompt instanceof PromptTemplate) { const existingPromptTemplate = chain.prompt.template as string - let newChatPromptTemplate = ChatPromptTemplate.fromMessages([ - HumanMessagePromptTemplate.fromTemplate(existingPromptTemplate) + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...messageContent, + { + text: existingPromptTemplate + } ]) - newChatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent })) - chain.prompt = newChatPromptTemplate + msg.inputVariables = chain.prompt.inputVariables + chain.prompt = ChatPromptTemplate.fromMessages([msg]) } else if (chain.prompt instanceof ChatPromptTemplate) { - chain.prompt.promptMessages.push(new HumanMessage({ content: messageContent })) + if (chain.prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) { + const lastMessage = chain.prompt.promptMessages.pop() as HumanMessagePromptTemplate + const template = (lastMessage.prompt as PromptTemplate).template as string + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...messageContent, + { + text: template + } + ]) + msg.inputVariables = lastMessage.inputVariables + chain.prompt.promptMessages.push(msg) + } else { + chain.prompt.promptMessages.push(new HumanMessage({ content: messageContent })) + } } else if (chain.prompt instanceof FewShotPromptTemplate) { let existingFewShotPromptTemplate = chain.prompt.examplePrompt.template as string let newFewShotPromptTemplate = ChatPromptTemplate.fromMessages([ @@ -212,8 +225,7 @@ const runPrediction = async ( } } else { // revert to previous values if image upload is empty - chatOpenAI.modelName = model.configuredModel - chatOpenAI.maxTokens = model.configuredMaxToken + visionChatModel.revertToOriginalModel() } } diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts index 844e7d25..392107a2 100644 --- a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts +++ b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts @@ -1,8 +1,9 @@ -import { AnthropicInput, ChatAnthropic } from '@langchain/anthropic' +import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic' import { BaseCache } from '@langchain/core/caches' import { BaseLLMParams } from '@langchain/core/language_models/llms' -import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { ChatAnthropic } from './FlowiseChatAntrhopic' class ChatAnthropic_ChatModels implements INode { label: string @@ -19,12 +20,12 @@ class ChatAnthropic_ChatModels implements INode { constructor() { this.label = 'ChatAnthropic' this.name = 'chatAnthropic' - this.version = 3.0 + this.version = 4.0 this.type = 'ChatAnthropic' this.icon = 'Anthropic.svg' this.category = 'Chat Models' this.description = 'Wrapper around ChatAnthropic large language models that use the Chat endpoint' - this.baseClasses = [this.type, ...getBaseClasses(ChatAnthropic)] + this.baseClasses = [this.type, ...getBaseClasses(LangchainChatAnthropic)] this.credential = { label: 'Connect Credential', name: 'credential', @@ -147,6 +148,15 @@ class ChatAnthropic_ChatModels implements INode { step: 0.1, optional: true, additionalParams: true + }, + { + label: 'Allow Image Uploads', + name: 'allowImageUploads', + type: 'boolean', + description: + 'Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + default: false, + optional: true } ] } @@ -163,6 +173,8 @@ class ChatAnthropic_ChatModels implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const anthropicApiKey = getCredentialParam('anthropicApiKey', credentialData, nodeData) + const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean + const obj: Partial & BaseLLMParams & { anthropicApiKey?: string } = { temperature: parseFloat(temperature), modelName, @@ -175,7 +187,14 @@ class ChatAnthropic_ChatModels implements INode { if (topK) obj.topK = parseFloat(topK) if (cache) obj.cache = cache - const model = new ChatAnthropic(obj) + const multiModalOption: IMultiModalOption = { + image: { + allowImageUploads: allowImageUploads ?? false + } + } + + const model = new ChatAnthropic(nodeData.id, obj) + model.setMultiModalOption(multiModalOption) return model } } diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts new file mode 100644 index 00000000..05665d1f --- /dev/null +++ b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts @@ -0,0 +1,33 @@ +import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic' +import { IVisionChatModal, IMultiModalOption } from '../../../src' +import { BaseLLMParams } from '@langchain/core/language_models/llms' + +export class ChatAnthropic extends LangchainChatAnthropic implements IVisionChatModal { + configuredModel: string + configuredMaxToken: number + multiModalOption: IMultiModalOption + id: string + + constructor(id: string, fields: Partial & BaseLLMParams & { anthropicApiKey?: string }) { + super(fields) + this.id = id + this.configuredModel = fields?.modelName || 'claude-3-opus-20240229' + this.configuredMaxToken = fields?.maxTokens ?? 256 + } + + revertToOriginalModel(): void { + super.modelName = this.configuredModel + super.maxTokens = this.configuredMaxToken + } + + setMultiModalOption(multiModalOption: IMultiModalOption): void { + this.multiModalOption = multiModalOption + } + + setVisionModel(): void { + if (!this.modelName.startsWith('claude-3')) { + super.modelName = 'claude-3-opus-20240229' + super.maxTokens = 1024 + } + } +} diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts index cc0b0efa..09b8c5b3 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts @@ -228,7 +228,7 @@ class ChatOpenAI_ChatModels implements INode { const obj: Partial & Partial & - BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption } = { + BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput } = { temperature: parseFloat(temperature), modelName, openAIApiKey, @@ -265,10 +265,9 @@ class ChatOpenAI_ChatModels implements INode { imageResolution } } - obj.multiModalOption = multiModalOption const model = new ChatOpenAI(nodeData.id, obj) - + model.setMultiModalOption(multiModalOption) return model } } diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts index 9049bb79..0227362c 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts @@ -1,39 +1,39 @@ import type { ClientOptions } from 'openai' -import { - ChatOpenAI as LangchainChatOpenAI, - OpenAIChatInput, - LegacyOpenAIInput, - AzureOpenAIInput, - ChatOpenAICallOptions -} from '@langchain/openai' +import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, LegacyOpenAIInput, AzureOpenAIInput } from '@langchain/openai' import { BaseChatModelParams } from '@langchain/core/language_models/chat_models' -import { BaseMessageLike } from '@langchain/core/messages' -import { Callbacks } from '@langchain/core/callbacks/manager' -import { LLMResult } from '@langchain/core/outputs' -import { IMultiModalOption } from '../../../src' +import { IMultiModalOption, IVisionChatModal } from '../../../src' -export class ChatOpenAI extends LangchainChatOpenAI { +export class ChatOpenAI extends LangchainChatOpenAI implements IVisionChatModal { configuredModel: string - configuredMaxToken?: number - multiModalOption?: IMultiModalOption + configuredMaxToken: number + multiModalOption: IMultiModalOption id: string constructor( id: string, fields?: Partial & Partial & - BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption }, + BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput }, /** @deprecated */ configuration?: ClientOptions & LegacyOpenAIInput ) { super(fields, configuration) this.id = id - this.multiModalOption = fields?.multiModalOption this.configuredModel = fields?.modelName ?? 'gpt-3.5-turbo' - this.configuredMaxToken = fields?.maxTokens + this.configuredMaxToken = fields?.maxTokens ?? 256 } - async generate(messages: BaseMessageLike[][], options?: string[] | ChatOpenAICallOptions, callbacks?: Callbacks): Promise { - return super.generate(messages, options, callbacks) + revertToOriginalModel(): void { + super.modelName = this.configuredModel + super.maxTokens = this.configuredMaxToken + } + + setMultiModalOption(multiModalOption: IMultiModalOption): void { + this.multiModalOption = multiModalOption + } + + setVisionModel(): void { + super.modelName = 'gpt-4-vision-preview' + super.maxTokens = 1024 } } diff --git a/packages/components/package.json b/packages/components/package.json index 6e2a977a..40a0f16e 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -29,7 +29,7 @@ "@google-ai/generativelanguage": "^0.2.1", "@google/generative-ai": "^0.1.3", "@huggingface/inference": "^2.6.1", - "@langchain/anthropic": "^0.0.10", + "@langchain/anthropic": "^0.1.4", "@langchain/cohere": "^0.0.5", "@langchain/community": "^0.0.30", "@langchain/google-genai": "^0.0.10", diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 0e280dea..798e91a2 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -270,3 +270,14 @@ export abstract class FlowiseSummaryMemory extends ConversationSummaryMemory imp abstract addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId?: string): Promise abstract clearChatMessages(overrideSessionId?: string): Promise } + +export interface IVisionChatModal { + id: string + configuredModel: string + configuredMaxToken: number + multiModalOption: IMultiModalOption + + setVisionModel(): void + revertToOriginalModel(): void + setMultiModalOption(multiModalOption: IMultiModalOption): void +} diff --git a/packages/components/src/multiModalUtils.ts b/packages/components/src/multiModalUtils.ts index 00cc5bf3..b3a51210 100644 --- a/packages/components/src/multiModalUtils.ts +++ b/packages/components/src/multiModalUtils.ts @@ -1,5 +1,4 @@ -import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' -import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI' +import { IVisionChatModal, ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' import path from 'path' import { getStoragePath } from './utils' import fs from 'fs' @@ -12,7 +11,7 @@ export const addImagesToMessages = ( const imageContent: MessageContentImageUrl[] = [] let model = nodeData.inputs?.model - if (model instanceof ChatOpenAI && multiModalOption) { + if (llmSupportsVision(model) && multiModalOption) { // Image Uploaded if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) { const imageUploads = getImageUploads(options.uploads) @@ -46,3 +45,5 @@ export const getAudioUploads = (uploads: IFileUpload[]) => { export const getImageUploads = (uploads: IFileUpload[]) => { return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('image/')) } + +export const llmSupportsVision = (value: any): value is IVisionChatModal => !!value?.multiModalOption diff --git a/packages/server/marketplaces/chatflows/Claude LLM.json b/packages/server/marketplaces/chatflows/Claude LLM.json index fdce533e..dc5f3254 100644 --- a/packages/server/marketplaces/chatflows/Claude LLM.json +++ b/packages/server/marketplaces/chatflows/Claude LLM.json @@ -160,7 +160,7 @@ "data": { "id": "chatAnthropic_0", "label": "ChatAnthropic", - "version": 3, + "version": 4, "name": "chatAnthropic", "type": "ChatAnthropic", "baseClasses": ["ChatAnthropic", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -288,6 +288,15 @@ "optional": true, "additionalParams": true, "id": "chatAnthropic_0-input-topK-number" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatAnthropic_0-input-allowImageUploads-boolean" } ], "inputAnchors": [ @@ -305,7 +314,8 @@ "temperature": 0.9, "maxTokensToSample": "", "topP": "", - "topK": "" + "topK": "", + "allowImageUploads": true }, "outputAnchors": [ { diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 094e3812..3b390a3f 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1533,7 +1533,7 @@ export class App { if (!chatflow) return `Chatflow ${chatflowid} not found` const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent'] - const uploadProcessingNodes = ['chatOpenAI'] + const uploadProcessingNodes = ['chatOpenAI', 'chatAnthropic'] const flowObj = JSON.parse(chatflow.flowData) const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = []