From da43fb7f44c205d92683623e2f21118043cf013a Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Wed, 6 Mar 2024 12:30:39 +0530 Subject: [PATCH 01/10] Base changes to enable AzureChatOpenAI for image upload --- .../AzureChatOpenAI/AzureChatOpenAI.ts | 56 +++++++++++++++++-- packages/server/src/index.ts | 2 +- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index ea924fd0..da1b69ef 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -1,8 +1,9 @@ -import { AzureOpenAIInput, ChatOpenAI, OpenAIChatInput } from '@langchain/openai' +import { AzureOpenAIInput, ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from '@langchain/openai' import { BaseCache } from '@langchain/core/caches' import { BaseLLMParams } from '@langchain/core/language_models/llms' -import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from "../../../src/Interface"; import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { ChatOpenAI } from '../ChatOpenAI/FlowiseChatOpenAI' class AzureChatOpenAI_ChatModels implements INode { label: string @@ -19,12 +20,12 @@ class AzureChatOpenAI_ChatModels implements INode { constructor() { this.label = 'Azure ChatOpenAI' this.name = 'azureChatOpenAI' - this.version = 2.0 + this.version = 3.0 this.type = 'AzureChatOpenAI' this.icon = 'Azure.svg' this.category = 'Chat Models' this.description = 'Wrapper around Azure OpenAI large language models that use the Chat endpoint' - this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)] + this.baseClasses = [this.type, ...getBaseClasses(LangchainChatOpenAI)] this.credential = { label: 'Connect Credential', name: 'credential', @@ -102,6 +103,38 @@ class AzureChatOpenAI_ChatModels implements INode { step: 1, optional: true, additionalParams: true + }, + { + label: 'Allow Image Uploads', + name: 'allowImageUploads', + type: 'boolean', + description: + 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + default: false, + optional: true + }, + { + label: 'Image Resolution', + description: 'This parameter controls the resolution in which the model views the image.', + name: 'imageResolution', + type: 'options', + options: [ + { + label: 'Low', + name: 'low' + }, + { + label: 'High', + name: 'high' + }, + { + label: 'Auto', + name: 'auto' + } + ], + default: 'low', + optional: false, + additionalParams: true } ] } @@ -122,7 +155,10 @@ class AzureChatOpenAI_ChatModels implements INode { const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) - const obj: Partial & BaseLLMParams & Partial = { + const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean + const imageResolution = nodeData.inputs?.imageResolution as string + + const obj: Partial & BaseLLMParams & Partial & { multiModalOption?: IMultiModalOption } = { temperature: parseFloat(temperature), modelName, azureOpenAIApiKey, @@ -138,7 +174,15 @@ class AzureChatOpenAI_ChatModels implements INode { if (timeout) obj.timeout = parseInt(timeout, 10) if (cache) obj.cache = cache - const model = new ChatOpenAI(obj) + const multiModalOption: IMultiModalOption = { + image: { + allowImageUploads: allowImageUploads ?? false, + imageResolution + } + } + obj.multiModalOption = multiModalOption + + const model = new ChatOpenAI(nodeData.id, obj) return model } } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index ab405e35..68e1709d 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1534,7 +1534,7 @@ export class App { if (!chatflow) return `Chatflow ${chatflowid} not found` const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent'] - const uploadProcessingNodes = ['chatOpenAI'] + const uploadProcessingNodes = ['chatOpenAI', 'azureChatOpenAI'] const flowObj = JSON.parse(chatflow.flowData) const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = [] From a2caf3e265e2bf6ddabf45374a07e9fac90673dd Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Thu, 7 Mar 2024 18:54:36 +0530 Subject: [PATCH 02/10] Structural Changes to support expansion to other chat models. --- packages/components/src/IVisionChatModal.ts | 12 ++++++++++++ packages/components/src/multiModalUtils.ts | 6 ++++-- 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 packages/components/src/IVisionChatModal.ts diff --git a/packages/components/src/IVisionChatModal.ts b/packages/components/src/IVisionChatModal.ts new file mode 100644 index 00000000..482ff70b --- /dev/null +++ b/packages/components/src/IVisionChatModal.ts @@ -0,0 +1,12 @@ +import { IMultiModalOption } from './Interface' + +export interface IVisionChatModal { + id: string + configuredModel: string + configuredMaxToken: number + multiModalOption: IMultiModalOption + + setVisionModel(): void + revertToOriginalModel(): void + setMultiModalOption(multiModalOption: IMultiModalOption): void +} diff --git a/packages/components/src/multiModalUtils.ts b/packages/components/src/multiModalUtils.ts index 00cc5bf3..186e85ca 100644 --- a/packages/components/src/multiModalUtils.ts +++ b/packages/components/src/multiModalUtils.ts @@ -1,8 +1,8 @@ import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' -import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI' import path from 'path' import { getStoragePath } from './utils' import fs from 'fs' +import { IVisionChatModal } from './IVisionChatModal' export const addImagesToMessages = ( nodeData: INodeData, @@ -12,7 +12,7 @@ export const addImagesToMessages = ( const imageContent: MessageContentImageUrl[] = [] let model = nodeData.inputs?.model - if (model instanceof ChatOpenAI && multiModalOption) { + if (llmSupportsVision(model) && multiModalOption) { // Image Uploaded if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) { const imageUploads = getImageUploads(options.uploads) @@ -46,3 +46,5 @@ export const getAudioUploads = (uploads: IFileUpload[]) => { export const getImageUploads = (uploads: IFileUpload[]) => { return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('image/')) } + +export const llmSupportsVision = (value: any): value is IVisionChatModal => !!value?.multiModalOption From 63b8c23072094c871b097e80606efa04db6c1932 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Thu, 7 Mar 2024 18:55:24 +0530 Subject: [PATCH 03/10] Addition of Claude for Image uploads --- .../ConversationalAgent.ts | 46 +++++++----- .../agents/MRKLAgentChat/MRKLAgentChat.ts | 30 ++++---- .../ConversationChain/ConversationChain.ts | 38 ++++------ .../nodes/chains/LLMChain/LLMChain.ts | 70 ++++++++++++------- .../chatmodels/ChatAnthropic/ChatAnthropic.ts | 29 ++++++-- .../ChatAnthropic/FlowiseChatAntrhopic.ts | 34 +++++++++ .../nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts | 5 +- .../ChatOpenAI/FlowiseChatOpenAI.ts | 37 +++++----- packages/components/package.json | 2 +- packages/server/src/index.ts | 2 +- 10 files changed, 185 insertions(+), 108 deletions(-) create mode 100644 packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts diff --git a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts index db6b37c6..62c46878 100644 --- a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts +++ b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts @@ -4,7 +4,12 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages' import { ChainValues } from '@langchain/core/utils/types' import { AgentStep } from '@langchain/core/agents' -import { renderTemplate, MessagesPlaceholder } from '@langchain/core/prompts' +import { + renderTemplate, + MessagesPlaceholder, + HumanMessagePromptTemplate, + PromptTemplate +} from "@langchain/core/prompts"; import { RunnableSequence } from '@langchain/core/runnables' import { ChatConversationalAgent } from 'langchain/agents' import { getBaseClasses } from '../../../src/utils' @@ -12,7 +17,8 @@ import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { AgentExecutor } from '../../../src/agents' import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { addImagesToMessages } from '../../../src/multiModalUtils' +import { addImagesToMessages, llmSupportsVision } from "../../../src/multiModalUtils"; +import { IVisionChatModal } from "../../../src/IVisionChatModal"; const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI. @@ -150,33 +156,39 @@ const prepareAgent = async ( outputParser }) - if (model instanceof ChatOpenAI) { - let humanImageMessages: HumanMessage[] = [] + if (llmSupportsVision(model)) { + const visionChatModel = model as IVisionChatModal + // let humanImageMessages: HumanMessage[] = [] const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) if (messageContent?.length) { - // Change model to gpt-4-vision - model.modelName = 'gpt-4-vision-preview' + visionChatModel.setVisionModel() - // Change default max token to higher when using gpt-4-vision - model.maxTokens = 1024 - - for (const msg of messageContent) { - humanImageMessages.push(new HumanMessage({ content: [msg] })) - } + // for (const msg of messageContent) { + // humanImageMessages.push(new HumanMessage({ content: [msg] })) + // } // Pop the `agent_scratchpad` MessagePlaceHolder let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder - + if (prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) { + const lastMessage = prompt.promptMessages.pop() as HumanMessagePromptTemplate + const template = (lastMessage.prompt as PromptTemplate).template as string + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...messageContent, + { + text: template + } + ]) + msg.inputVariables = lastMessage.inputVariables + prompt.promptMessages.push(msg) + } // Add the HumanMessage for images - prompt.promptMessages.push(...humanImageMessages) + //prompt.promptMessages.push(...humanImageMessages) // Add the `agent_scratchpad` MessagePlaceHolder back prompt.promptMessages.push(messagePlaceholder) } else { - // revert to previous values if image upload is empty - model.modelName = model.configuredModel - model.maxTokens = model.configuredMaxToken + visionChatModel.revertToOriginalModel() } } diff --git a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts index 5923d77e..e1f16fd8 100644 --- a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts +++ b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts @@ -1,6 +1,5 @@ import { flatten } from 'lodash' import { AgentExecutor } from 'langchain/agents' -import { HumanMessage } from '@langchain/core/messages' import { ChatPromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts' import { Tool } from '@langchain/core/tools' import type { PromptTemplate } from '@langchain/core/prompts' @@ -10,8 +9,8 @@ import { additionalCallbacks } from '../../../src/handler' import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { createReactAgent } from '../../../src/agents' -import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { addImagesToMessages } from '../../../src/multiModalUtils' +import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' +import { IVisionChatModal } from '../../../src/IVisionChatModal' class MRKLAgentChat_Agents implements INode { label: string @@ -68,23 +67,26 @@ class MRKLAgentChat_Agents implements INode { const prompt = await pull('hwchase17/react-chat') let chatPromptTemplate = undefined - if (model instanceof ChatOpenAI) { + if (llmSupportsVision(model)) { + const visionChatModel = model as IVisionChatModal const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) if (messageContent?.length) { - // Change model to gpt-4-vision - model.modelName = 'gpt-4-vision-preview' - - // Change default max token to higher when using gpt-4-vision - model.maxTokens = 1024 - + // Change model to vision supported + visionChatModel.setVisionModel() const oldTemplate = prompt.template as string - chatPromptTemplate = ChatPromptTemplate.fromMessages([HumanMessagePromptTemplate.fromTemplate(oldTemplate)]) - chatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent })) + + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...messageContent, + { + text: oldTemplate + } + ]) + msg.inputVariables = prompt.inputVariables + chatPromptTemplate = ChatPromptTemplate.fromMessages([msg]) } else { // revert to previous values if image upload is empty - model.modelName = model.configuredModel - model.maxTokens = model.configuredMaxToken + visionChatModel.revertToOriginalModel() } } diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index 25d80bee..0d572eca 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -2,15 +2,16 @@ import { ConversationChain } from 'langchain/chains' import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts' import { RunnableSequence } from '@langchain/core/runnables' import { StringOutputParser } from '@langchain/core/output_parsers' -import { HumanMessage } from '@langchain/core/messages' import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse } from '../../outputparsers/OutputParserHelpers' -import { addImagesToMessages } from '../../../src/multiModalUtils' +import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams, MessageContentImageUrl } from '../../../src/Interface' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' +import { IVisionChatModal } from '../../../src/IVisionChatModal' +import { MessageContent } from 'llamaindex' let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.` const inputKey = 'input' @@ -145,7 +146,7 @@ class ConversationChain_Chains implements INode { } } -const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => { +const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageContentImageUrl[]) => { const memory = nodeData.inputs?.memory as FlowiseMemory const prompt = nodeData.inputs?.systemMessagePrompt as string const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate @@ -154,7 +155,6 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage const sysPrompt = chatPromptTemplate.promptMessages[0] const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1] const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt] - if (humanImageMessages.length) messages.push(...humanImageMessages) const chatPrompt = ChatPromptTemplate.fromMessages(messages) if ((chatPromptTemplate as any).promptValues) { @@ -168,9 +168,8 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage const messages = [ SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage), new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), - HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`) + HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages]) ] - if (humanImageMessages.length) messages.push(...(humanImageMessages as any[])) const chatPrompt = ChatPromptTemplate.fromMessages(messages) @@ -183,28 +182,19 @@ const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: s const memory = nodeData.inputs?.memory as FlowiseMemory const memoryKey = memory.memoryKey ?? 'chat_history' - let humanImageMessages: HumanMessage[] = [] - if (model instanceof ChatOpenAI) { - const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) - + let messageContent: MessageContentImageUrl[] = [] + if (llmSupportsVision(model)) { + messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) + const visionChatModel = model as IVisionChatModal if (messageContent?.length) { - // Change model to gpt-4-vision - model.modelName = 'gpt-4-vision-preview' - - // Change default max token to higher when using gpt-4-vision - model.maxTokens = 1024 - - for (const msg of messageContent) { - humanImageMessages.push(new HumanMessage({ content: [msg] })) - } + visionChatModel.setVisionModel() } else { // revert to previous values if image upload is empty - model.modelName = model.configuredModel - model.maxTokens = model.configuredMaxToken + visionChatModel.revertToOriginalModel() } } - const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages) + const chatPrompt = prepareChatPrompt(nodeData, messageContent) let promptVariables = {} const promptValuesRaw = (chatPrompt as any).promptValues if (promptValuesRaw) { @@ -228,7 +218,7 @@ const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: s }, ...promptVariables }, - prepareChatPrompt(nodeData, humanImageMessages), + prepareChatPrompt(nodeData, messageContent), model, new StringOutputParser() ]) diff --git a/packages/components/nodes/chains/LLMChain/LLMChain.ts b/packages/components/nodes/chains/LLMChain/LLMChain.ts index 6adee1e1..a70a308e 100644 --- a/packages/components/nodes/chains/LLMChain/LLMChain.ts +++ b/packages/components/nodes/chains/LLMChain/LLMChain.ts @@ -1,16 +1,22 @@ -import { BaseLanguageModel, BaseLanguageModelCallOptions } from '@langchain/core/language_models/base' -import { BaseLLMOutputParser, BaseOutputParser } from '@langchain/core/output_parsers' -import { HumanMessage } from '@langchain/core/messages' -import { ChatPromptTemplate, FewShotPromptTemplate, PromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts' -import { OutputFixingParser } from 'langchain/output_parsers' -import { LLMChain } from 'langchain/chains' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' -import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' -import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' -import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' -import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers' -import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { addImagesToMessages } from '../../../src/multiModalUtils' +import { BaseLanguageModel, BaseLanguageModelCallOptions } from "@langchain/core/language_models/base"; +import { BaseLLMOutputParser, BaseOutputParser } from "@langchain/core/output_parsers"; +import { HumanMessage } from "@langchain/core/messages"; +import { + ChatPromptTemplate, + FewShotPromptTemplate, + HumanMessagePromptTemplate, + PromptTemplate +} from "@langchain/core/prompts"; +import { OutputFixingParser } from "langchain/output_parsers"; +import { LLMChain } from "langchain/chains"; +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from "../../../src/Interface"; +import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from "../../../src/handler"; +import { getBaseClasses, handleEscapeCharacters } from "../../../src/utils"; +import { checkInputs, Moderation, streamResponse } from "../../moderation/Moderation"; +import { formatResponse, injectOutputParser } from "../../outputparsers/OutputParserHelpers"; +import { ChatOpenAI } from "../../chatmodels/ChatOpenAI/FlowiseChatOpenAI"; +import { addImagesToMessages, llmSupportsVision } from "../../../src/multiModalUtils"; +import { IVisionChatModal } from "../../../src/IVisionChatModal"; class LLMChain_Chains implements INode { label: string @@ -183,24 +189,39 @@ const runPrediction = async ( * TO: { "value": "hello i am ben\n\n\thow are you?" } */ const promptValues = handleEscapeCharacters(promptValuesRaw, true) - const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) - if (chain.llm instanceof ChatOpenAI) { - const chatOpenAI = chain.llm as ChatOpenAI + if (llmSupportsVision(chain.llm)) { + const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) + const visionChatModel = chain.llm as IVisionChatModal if (messageContent?.length) { // Change model to gpt-4-vision && max token to higher when using gpt-4-vision - chatOpenAI.modelName = 'gpt-4-vision-preview' - chatOpenAI.maxTokens = 1024 + visionChatModel.setVisionModel() // Add image to the message if (chain.prompt instanceof PromptTemplate) { const existingPromptTemplate = chain.prompt.template as string - let newChatPromptTemplate = ChatPromptTemplate.fromMessages([ - HumanMessagePromptTemplate.fromTemplate(existingPromptTemplate) + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...messageContent, + { + text: existingPromptTemplate + } ]) - newChatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent })) - chain.prompt = newChatPromptTemplate + msg.inputVariables = chain.prompt.inputVariables + chain.prompt = ChatPromptTemplate.fromMessages([msg]) } else if (chain.prompt instanceof ChatPromptTemplate) { - chain.prompt.promptMessages.push(new HumanMessage({ content: messageContent })) + if (chain.prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) { + const lastMessage = chain.prompt.promptMessages.pop() as HumanMessagePromptTemplate + const template = (lastMessage.prompt as PromptTemplate).template as string + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...messageContent, + { + text: template + } + ]) + msg.inputVariables = lastMessage.inputVariables + chain.prompt.promptMessages.push(msg) + } else { + chain.prompt.promptMessages.push(new HumanMessage({ content: messageContent })) + } } else if (chain.prompt instanceof FewShotPromptTemplate) { let existingFewShotPromptTemplate = chain.prompt.examplePrompt.template as string let newFewShotPromptTemplate = ChatPromptTemplate.fromMessages([ @@ -212,8 +233,7 @@ const runPrediction = async ( } } else { // revert to previous values if image upload is empty - chatOpenAI.modelName = model.configuredModel - chatOpenAI.maxTokens = model.configuredMaxToken + visionChatModel.revertToOriginalModel() } } diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts index 844e7d25..392107a2 100644 --- a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts +++ b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts @@ -1,8 +1,9 @@ -import { AnthropicInput, ChatAnthropic } from '@langchain/anthropic' +import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic' import { BaseCache } from '@langchain/core/caches' import { BaseLLMParams } from '@langchain/core/language_models/llms' -import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { ChatAnthropic } from './FlowiseChatAntrhopic' class ChatAnthropic_ChatModels implements INode { label: string @@ -19,12 +20,12 @@ class ChatAnthropic_ChatModels implements INode { constructor() { this.label = 'ChatAnthropic' this.name = 'chatAnthropic' - this.version = 3.0 + this.version = 4.0 this.type = 'ChatAnthropic' this.icon = 'Anthropic.svg' this.category = 'Chat Models' this.description = 'Wrapper around ChatAnthropic large language models that use the Chat endpoint' - this.baseClasses = [this.type, ...getBaseClasses(ChatAnthropic)] + this.baseClasses = [this.type, ...getBaseClasses(LangchainChatAnthropic)] this.credential = { label: 'Connect Credential', name: 'credential', @@ -147,6 +148,15 @@ class ChatAnthropic_ChatModels implements INode { step: 0.1, optional: true, additionalParams: true + }, + { + label: 'Allow Image Uploads', + name: 'allowImageUploads', + type: 'boolean', + description: + 'Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + default: false, + optional: true } ] } @@ -163,6 +173,8 @@ class ChatAnthropic_ChatModels implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const anthropicApiKey = getCredentialParam('anthropicApiKey', credentialData, nodeData) + const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean + const obj: Partial & BaseLLMParams & { anthropicApiKey?: string } = { temperature: parseFloat(temperature), modelName, @@ -175,7 +187,14 @@ class ChatAnthropic_ChatModels implements INode { if (topK) obj.topK = parseFloat(topK) if (cache) obj.cache = cache - const model = new ChatAnthropic(obj) + const multiModalOption: IMultiModalOption = { + image: { + allowImageUploads: allowImageUploads ?? false + } + } + + const model = new ChatAnthropic(nodeData.id, obj) + model.setMultiModalOption(multiModalOption) return model } } diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts new file mode 100644 index 00000000..d52e9900 --- /dev/null +++ b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts @@ -0,0 +1,34 @@ +import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic' +import { IMultiModalOption } from '../../../src' +import { IVisionChatModal } from '../../../src/IVisionChatModal' +import { BaseLLMParams } from '@langchain/core/language_models/llms' + +export class ChatAnthropic extends LangchainChatAnthropic implements IVisionChatModal { + configuredModel: string + configuredMaxToken: number + multiModalOption: IMultiModalOption + id: string + + constructor(id: string, fields: Partial & BaseLLMParams & { anthropicApiKey?: string }) { + super(fields) + this.id = id + this.configuredModel = fields?.modelName || 'claude-3-opus-20240229' + this.configuredMaxToken = fields?.maxTokens ?? 256 + } + + revertToOriginalModel(): void { + super.modelName = this.configuredModel + super.maxTokens = this.configuredMaxToken + } + + setMultiModalOption(multiModalOption: IMultiModalOption): void { + this.multiModalOption = multiModalOption + } + + setVisionModel(): void { + if (!this.modelName.startsWith('claude-3')) { + super.modelName = 'claude-3-opus-20240229' + super.maxTokens = 1024 + } + } +} diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts index cc0b0efa..09b8c5b3 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts @@ -228,7 +228,7 @@ class ChatOpenAI_ChatModels implements INode { const obj: Partial & Partial & - BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption } = { + BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput } = { temperature: parseFloat(temperature), modelName, openAIApiKey, @@ -265,10 +265,9 @@ class ChatOpenAI_ChatModels implements INode { imageResolution } } - obj.multiModalOption = multiModalOption const model = new ChatOpenAI(nodeData.id, obj) - + model.setMultiModalOption(multiModalOption) return model } } diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts index 9049bb79..b00811d5 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts @@ -1,39 +1,40 @@ import type { ClientOptions } from 'openai' -import { - ChatOpenAI as LangchainChatOpenAI, - OpenAIChatInput, - LegacyOpenAIInput, - AzureOpenAIInput, - ChatOpenAICallOptions -} from '@langchain/openai' +import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, LegacyOpenAIInput, AzureOpenAIInput } from '@langchain/openai' import { BaseChatModelParams } from '@langchain/core/language_models/chat_models' -import { BaseMessageLike } from '@langchain/core/messages' -import { Callbacks } from '@langchain/core/callbacks/manager' -import { LLMResult } from '@langchain/core/outputs' import { IMultiModalOption } from '../../../src' +import { IVisionChatModal } from '../../../src/IVisionChatModal' -export class ChatOpenAI extends LangchainChatOpenAI { +export class ChatOpenAI extends LangchainChatOpenAI implements IVisionChatModal { configuredModel: string - configuredMaxToken?: number - multiModalOption?: IMultiModalOption + configuredMaxToken: number + multiModalOption: IMultiModalOption id: string constructor( id: string, fields?: Partial & Partial & - BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption }, + BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput }, /** @deprecated */ configuration?: ClientOptions & LegacyOpenAIInput ) { super(fields, configuration) this.id = id - this.multiModalOption = fields?.multiModalOption this.configuredModel = fields?.modelName ?? 'gpt-3.5-turbo' - this.configuredMaxToken = fields?.maxTokens + this.configuredMaxToken = fields?.maxTokens ?? 256 } - async generate(messages: BaseMessageLike[][], options?: string[] | ChatOpenAICallOptions, callbacks?: Callbacks): Promise { - return super.generate(messages, options, callbacks) + revertToOriginalModel(): void { + super.modelName = this.configuredModel + super.maxTokens = this.configuredMaxToken + } + + setMultiModalOption(multiModalOption: IMultiModalOption): void { + this.multiModalOption = multiModalOption + } + + setVisionModel(): void { + super.modelName = 'gpt-4-vision-preview' + super.maxTokens = 1024 } } diff --git a/packages/components/package.json b/packages/components/package.json index 69ca3b5f..30fe1c8c 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -29,7 +29,7 @@ "@google-ai/generativelanguage": "^0.2.1", "@google/generative-ai": "^0.1.3", "@huggingface/inference": "^2.6.1", - "@langchain/anthropic": "^0.0.10", + "@langchain/anthropic": "^0.1.4", "@langchain/cohere": "^0.0.5", "@langchain/community": "^0.0.30", "@langchain/google-genai": "^0.0.10", diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 68e1709d..8fd5f451 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1534,7 +1534,7 @@ export class App { if (!chatflow) return `Chatflow ${chatflowid} not found` const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent'] - const uploadProcessingNodes = ['chatOpenAI', 'azureChatOpenAI'] + const uploadProcessingNodes = ['chatOpenAI', 'chatAnthropic'] const flowObj = JSON.parse(chatflow.flowData) const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = [] From 7ab96a4c394e1d972092534bed9f56b9e8c46a3e Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Thu, 7 Mar 2024 18:55:47 +0530 Subject: [PATCH 04/10] start of changes to AzureOpenAI to support Image uploads --- .../AzureChatOpenAI/AzureChatOpenAI.ts | 77 +++++++++---------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index da1b69ef..155469ef 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -1,7 +1,7 @@ import { AzureOpenAIInput, ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from '@langchain/openai' import { BaseCache } from '@langchain/core/caches' import { BaseLLMParams } from '@langchain/core/language_models/llms' -import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from "../../../src/Interface"; +import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatOpenAI } from '../ChatOpenAI/FlowiseChatOpenAI' @@ -103,38 +103,38 @@ class AzureChatOpenAI_ChatModels implements INode { step: 1, optional: true, additionalParams: true - }, - { - label: 'Allow Image Uploads', - name: 'allowImageUploads', - type: 'boolean', - description: - 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', - default: false, - optional: true - }, - { - label: 'Image Resolution', - description: 'This parameter controls the resolution in which the model views the image.', - name: 'imageResolution', - type: 'options', - options: [ - { - label: 'Low', - name: 'low' - }, - { - label: 'High', - name: 'high' - }, - { - label: 'Auto', - name: 'auto' - } - ], - default: 'low', - optional: false, - additionalParams: true + // }, + // { + // label: 'Allow Image Uploads', + // name: 'allowImageUploads', + // type: 'boolean', + // description: + // 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + // default: false, + // optional: true + // }, + // { + // label: 'Image Resolution', + // description: 'This parameter controls the resolution in which the model views the image.', + // name: 'imageResolution', + // type: 'options', + // options: [ + // { + // label: 'Low', + // name: 'low' + // }, + // { + // label: 'High', + // name: 'high' + // }, + // { + // label: 'Auto', + // name: 'auto' + // } + // ], + // default: 'low', + // optional: false, + // additionalParams: true } ] } @@ -155,10 +155,10 @@ class AzureChatOpenAI_ChatModels implements INode { const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) - const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean - const imageResolution = nodeData.inputs?.imageResolution as string + // const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean + // const imageResolution = nodeData.inputs?.imageResolution as string - const obj: Partial & BaseLLMParams & Partial & { multiModalOption?: IMultiModalOption } = { + const obj: Partial & BaseLLMParams & Partial = { temperature: parseFloat(temperature), modelName, azureOpenAIApiKey, @@ -176,13 +176,12 @@ class AzureChatOpenAI_ChatModels implements INode { const multiModalOption: IMultiModalOption = { image: { - allowImageUploads: allowImageUploads ?? false, - imageResolution + allowImageUploads: false, } } - obj.multiModalOption = multiModalOption const model = new ChatOpenAI(nodeData.id, obj) + model.setMultiModalOption(multiModalOption) return model } } From 2b0ca6068676fe4c2657844d7ef28a670509f89d Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Thu, 7 Mar 2024 20:09:05 +0530 Subject: [PATCH 05/10] lint fixes --- .../ConversationalAgent.ts | 12 ++---- .../ConversationChain/ConversationChain.ts | 1 - .../nodes/chains/LLMChain/LLMChain.ts | 33 +++++++---------- .../AzureChatOpenAI/AzureChatOpenAI.ts | 37 +------------------ 4 files changed, 18 insertions(+), 65 deletions(-) diff --git a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts index 62c46878..6bb042b6 100644 --- a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts +++ b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts @@ -4,21 +4,15 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages' import { ChainValues } from '@langchain/core/utils/types' import { AgentStep } from '@langchain/core/agents' -import { - renderTemplate, - MessagesPlaceholder, - HumanMessagePromptTemplate, - PromptTemplate -} from "@langchain/core/prompts"; +import { renderTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts' import { RunnableSequence } from '@langchain/core/runnables' import { ChatConversationalAgent } from 'langchain/agents' import { getBaseClasses } from '../../../src/utils' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { AgentExecutor } from '../../../src/agents' -import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { addImagesToMessages, llmSupportsVision } from "../../../src/multiModalUtils"; -import { IVisionChatModal } from "../../../src/IVisionChatModal"; +import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' +import { IVisionChatModal } from '../../../src/IVisionChatModal' const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI. diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index 0d572eca..814fc1cc 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -11,7 +11,6 @@ import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams, MessageCon import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' import { IVisionChatModal } from '../../../src/IVisionChatModal' -import { MessageContent } from 'llamaindex' let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.` const inputKey = 'input' diff --git a/packages/components/nodes/chains/LLMChain/LLMChain.ts b/packages/components/nodes/chains/LLMChain/LLMChain.ts index a70a308e..738a8c4b 100644 --- a/packages/components/nodes/chains/LLMChain/LLMChain.ts +++ b/packages/components/nodes/chains/LLMChain/LLMChain.ts @@ -1,22 +1,17 @@ -import { BaseLanguageModel, BaseLanguageModelCallOptions } from "@langchain/core/language_models/base"; -import { BaseLLMOutputParser, BaseOutputParser } from "@langchain/core/output_parsers"; -import { HumanMessage } from "@langchain/core/messages"; -import { - ChatPromptTemplate, - FewShotPromptTemplate, - HumanMessagePromptTemplate, - PromptTemplate -} from "@langchain/core/prompts"; -import { OutputFixingParser } from "langchain/output_parsers"; -import { LLMChain } from "langchain/chains"; -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from "../../../src/Interface"; -import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from "../../../src/handler"; -import { getBaseClasses, handleEscapeCharacters } from "../../../src/utils"; -import { checkInputs, Moderation, streamResponse } from "../../moderation/Moderation"; -import { formatResponse, injectOutputParser } from "../../outputparsers/OutputParserHelpers"; -import { ChatOpenAI } from "../../chatmodels/ChatOpenAI/FlowiseChatOpenAI"; -import { addImagesToMessages, llmSupportsVision } from "../../../src/multiModalUtils"; -import { IVisionChatModal } from "../../../src/IVisionChatModal"; +import { BaseLanguageModel, BaseLanguageModelCallOptions } from '@langchain/core/language_models/base' +import { BaseLLMOutputParser, BaseOutputParser } from '@langchain/core/output_parsers' +import { HumanMessage } from '@langchain/core/messages' +import { ChatPromptTemplate, FewShotPromptTemplate, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts' +import { OutputFixingParser } from 'langchain/output_parsers' +import { LLMChain } from 'langchain/chains' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler' +import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' +import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' +import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers' +import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' +import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' +import { IVisionChatModal } from '../../../src/IVisionChatModal' class LLMChain_Chains implements INode { label: string diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index 155469ef..785bd3c5 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -103,38 +103,6 @@ class AzureChatOpenAI_ChatModels implements INode { step: 1, optional: true, additionalParams: true - // }, - // { - // label: 'Allow Image Uploads', - // name: 'allowImageUploads', - // type: 'boolean', - // description: - // 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', - // default: false, - // optional: true - // }, - // { - // label: 'Image Resolution', - // description: 'This parameter controls the resolution in which the model views the image.', - // name: 'imageResolution', - // type: 'options', - // options: [ - // { - // label: 'Low', - // name: 'low' - // }, - // { - // label: 'High', - // name: 'high' - // }, - // { - // label: 'Auto', - // name: 'auto' - // } - // ], - // default: 'low', - // optional: false, - // additionalParams: true } ] } @@ -155,9 +123,6 @@ class AzureChatOpenAI_ChatModels implements INode { const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) - // const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean - // const imageResolution = nodeData.inputs?.imageResolution as string - const obj: Partial & BaseLLMParams & Partial = { temperature: parseFloat(temperature), modelName, @@ -176,7 +141,7 @@ class AzureChatOpenAI_ChatModels implements INode { const multiModalOption: IMultiModalOption = { image: { - allowImageUploads: false, + allowImageUploads: false } } From bce7ff9ada6db56bfda729fdbc446d335d6d8537 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Fri, 8 Mar 2024 17:59:54 +0530 Subject: [PATCH 06/10] refactoring of interface into the common interface.ts file and misc changes --- .../ConversationalAgent/ConversationalAgent.ts | 9 +-------- .../nodes/agents/MRKLAgentChat/MRKLAgentChat.ts | 3 +-- .../chains/ConversationChain/ConversationChain.ts | 11 +++++++++-- .../components/nodes/chains/LLMChain/LLMChain.ts | 3 +-- .../chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts | 3 +-- .../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts | 3 +-- packages/components/src/IVisionChatModal.ts | 12 ------------ packages/components/src/Interface.ts | 11 +++++++++++ packages/components/src/multiModalUtils.ts | 3 +-- 9 files changed, 26 insertions(+), 32 deletions(-) delete mode 100644 packages/components/src/IVisionChatModal.ts diff --git a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts index 6bb042b6..14361e43 100644 --- a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts +++ b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts @@ -9,10 +9,9 @@ import { RunnableSequence } from '@langchain/core/runnables' import { ChatConversationalAgent } from 'langchain/agents' import { getBaseClasses } from '../../../src/utils' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' -import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' +import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { AgentExecutor } from '../../../src/agents' import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' -import { IVisionChatModal } from '../../../src/IVisionChatModal' const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI. @@ -158,10 +157,6 @@ const prepareAgent = async ( if (messageContent?.length) { visionChatModel.setVisionModel() - // for (const msg of messageContent) { - // humanImageMessages.push(new HumanMessage({ content: [msg] })) - // } - // Pop the `agent_scratchpad` MessagePlaceHolder let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder if (prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) { @@ -176,8 +171,6 @@ const prepareAgent = async ( msg.inputVariables = lastMessage.inputVariables prompt.promptMessages.push(msg) } - // Add the HumanMessage for images - //prompt.promptMessages.push(...humanImageMessages) // Add the `agent_scratchpad` MessagePlaceHolder back prompt.promptMessages.push(messagePlaceholder) diff --git a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts index e1f16fd8..f0466e08 100644 --- a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts +++ b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts @@ -6,11 +6,10 @@ import type { PromptTemplate } from '@langchain/core/prompts' import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { pull } from 'langchain/hub' import { additionalCallbacks } from '../../../src/handler' -import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' +import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { createReactAgent } from '../../../src/agents' import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' -import { IVisionChatModal } from '../../../src/IVisionChatModal' class MRKLAgentChat_Agents implements INode { label: string diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index 814fc1cc..ffea16f5 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -7,10 +7,17 @@ import { checkInputs, Moderation, streamResponse } from '../../moderation/Modera import { formatResponse } from '../../outputparsers/OutputParserHelpers' import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams, MessageContentImageUrl } from '../../../src/Interface' +import { + IVisionChatModal, + FlowiseMemory, + ICommonObject, + INode, + INodeData, + INodeParams, + MessageContentImageUrl +} from '../../../src/Interface' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' -import { IVisionChatModal } from '../../../src/IVisionChatModal' let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.` const inputKey = 'input' diff --git a/packages/components/nodes/chains/LLMChain/LLMChain.ts b/packages/components/nodes/chains/LLMChain/LLMChain.ts index 738a8c4b..dc59760c 100644 --- a/packages/components/nodes/chains/LLMChain/LLMChain.ts +++ b/packages/components/nodes/chains/LLMChain/LLMChain.ts @@ -4,14 +4,13 @@ import { HumanMessage } from '@langchain/core/messages' import { ChatPromptTemplate, FewShotPromptTemplate, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts' import { OutputFixingParser } from 'langchain/output_parsers' import { LLMChain } from 'langchain/chains' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { IVisionChatModal, ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler' import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers' import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' -import { IVisionChatModal } from '../../../src/IVisionChatModal' class LLMChain_Chains implements INode { label: string diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts index d52e9900..05665d1f 100644 --- a/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts +++ b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts @@ -1,6 +1,5 @@ import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic' -import { IMultiModalOption } from '../../../src' -import { IVisionChatModal } from '../../../src/IVisionChatModal' +import { IVisionChatModal, IMultiModalOption } from '../../../src' import { BaseLLMParams } from '@langchain/core/language_models/llms' export class ChatAnthropic extends LangchainChatAnthropic implements IVisionChatModal { diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts index b00811d5..0227362c 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts @@ -1,8 +1,7 @@ import type { ClientOptions } from 'openai' import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, LegacyOpenAIInput, AzureOpenAIInput } from '@langchain/openai' import { BaseChatModelParams } from '@langchain/core/language_models/chat_models' -import { IMultiModalOption } from '../../../src' -import { IVisionChatModal } from '../../../src/IVisionChatModal' +import { IMultiModalOption, IVisionChatModal } from '../../../src' export class ChatOpenAI extends LangchainChatOpenAI implements IVisionChatModal { configuredModel: string diff --git a/packages/components/src/IVisionChatModal.ts b/packages/components/src/IVisionChatModal.ts deleted file mode 100644 index 482ff70b..00000000 --- a/packages/components/src/IVisionChatModal.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { IMultiModalOption } from './Interface' - -export interface IVisionChatModal { - id: string - configuredModel: string - configuredMaxToken: number - multiModalOption: IMultiModalOption - - setVisionModel(): void - revertToOriginalModel(): void - setMultiModalOption(multiModalOption: IMultiModalOption): void -} diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 0e280dea..798e91a2 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -270,3 +270,14 @@ export abstract class FlowiseSummaryMemory extends ConversationSummaryMemory imp abstract addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId?: string): Promise abstract clearChatMessages(overrideSessionId?: string): Promise } + +export interface IVisionChatModal { + id: string + configuredModel: string + configuredMaxToken: number + multiModalOption: IMultiModalOption + + setVisionModel(): void + revertToOriginalModel(): void + setMultiModalOption(multiModalOption: IMultiModalOption): void +} diff --git a/packages/components/src/multiModalUtils.ts b/packages/components/src/multiModalUtils.ts index 186e85ca..b3a51210 100644 --- a/packages/components/src/multiModalUtils.ts +++ b/packages/components/src/multiModalUtils.ts @@ -1,8 +1,7 @@ -import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' +import { IVisionChatModal, ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' import path from 'path' import { getStoragePath } from './utils' import fs from 'fs' -import { IVisionChatModal } from './IVisionChatModal' export const addImagesToMessages = ( nodeData: INodeData, From c35eb0b7e55dfd66fcc012a31708a98a2bb9d0fd Mon Sep 17 00:00:00 2001 From: Henry Date: Sat, 9 Mar 2024 17:01:22 +0800 Subject: [PATCH 07/10] add separate logic for conversation chain for openai vision --- .../ConversationalAgent.ts | 1 - .../ConversationChain/ConversationChain.ts | 43 ++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts index 14361e43..404f4db4 100644 --- a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts +++ b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts @@ -151,7 +151,6 @@ const prepareAgent = async ( if (llmSupportsVision(model)) { const visionChatModel = model as IVisionChatModal - // let humanImageMessages: HumanMessage[] = [] const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) if (messageContent?.length) { diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index ffea16f5..16493adc 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -1,7 +1,16 @@ import { ConversationChain } from 'langchain/chains' -import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts' +import { + ChatPromptTemplate, + HumanMessagePromptTemplate, + MessagesPlaceholder, + SystemMessagePromptTemplate, + BaseMessagePromptTemplateLike, + PromptTemplate +} from '@langchain/core/prompts' import { RunnableSequence } from '@langchain/core/runnables' import { StringOutputParser } from '@langchain/core/output_parsers' +import { BaseChatModel } from '@langchain/core/language_models/chat_models' +import { HumanMessage } from '@langchain/core/messages' import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse } from '../../outputparsers/OutputParserHelpers' @@ -156,12 +165,29 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte const memory = nodeData.inputs?.memory as FlowiseMemory const prompt = nodeData.inputs?.systemMessagePrompt as string const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate + let model = nodeData.inputs?.model as BaseChatModel if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) { const sysPrompt = chatPromptTemplate.promptMessages[0] const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1] const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt] + // OpenAI works better when separate images into standalone human messages + if (model instanceof ChatOpenAI && humanImageMessages.length) { + messages.push(new HumanMessage({ content: [...humanImageMessages] })) + } else if (humanImageMessages.length) { + const lastMessage = messages.pop() as HumanMessagePromptTemplate + const template = (lastMessage.prompt as PromptTemplate).template as string + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...humanImageMessages, + { + text: template + } + ]) + msg.inputVariables = lastMessage.inputVariables + messages.push(msg) + } + const chatPrompt = ChatPromptTemplate.fromMessages(messages) if ((chatPromptTemplate as any).promptValues) { // @ts-ignore @@ -171,12 +197,19 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte return chatPrompt } - const messages = [ + const messages: BaseMessagePromptTemplateLike[] = [ SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage), - new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), - HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages]) + new MessagesPlaceholder(memory.memoryKey ?? 'chat_history') ] + // OpenAI works better when separate images into standalone human messages + if (model instanceof ChatOpenAI && humanImageMessages.length) { + messages.push(HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)) + messages.push(new HumanMessage({ content: [...humanImageMessages] })) + } else if (humanImageMessages.length) { + messages.push(HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages])) + } + const chatPrompt = ChatPromptTemplate.fromMessages(messages) return chatPrompt @@ -184,7 +217,7 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => { const chatHistory = options.chatHistory - let model = nodeData.inputs?.model as ChatOpenAI + let model = nodeData.inputs?.model as BaseChatModel const memory = nodeData.inputs?.memory as FlowiseMemory const memoryKey = memory.memoryKey ?? 'chat_history' From 66a83f886a441931a1cc25d6a5a3970d7dbb17cc Mon Sep 17 00:00:00 2001 From: Henry Date: Sat, 9 Mar 2024 17:25:56 +0800 Subject: [PATCH 08/10] update model typecast --- packages/components/nodes/agents/XMLAgent/XMLAgent.ts | 4 ++-- packages/components/nodes/chains/LLMChain/LLMChain.ts | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/components/nodes/agents/XMLAgent/XMLAgent.ts b/packages/components/nodes/agents/XMLAgent/XMLAgent.ts index 49109947..476a40da 100644 --- a/packages/components/nodes/agents/XMLAgent/XMLAgent.ts +++ b/packages/components/nodes/agents/XMLAgent/XMLAgent.ts @@ -1,8 +1,8 @@ import { flatten } from 'lodash' import { ChainValues } from '@langchain/core/utils/types' import { AgentStep } from '@langchain/core/agents' +import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { RunnableSequence } from '@langchain/core/runnables' -import { ChatOpenAI } from '@langchain/openai' import { Tool } from '@langchain/core/tools' import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder } from '@langchain/core/prompts' import { XMLAgentOutputParser } from 'langchain/agents/xml/output_parser' @@ -139,7 +139,7 @@ const prepareAgent = async ( flowObj: { sessionId?: string; chatId?: string; input?: string }, chatHistory: IMessage[] = [] ) => { - const model = nodeData.inputs?.model as ChatOpenAI + const model = nodeData.inputs?.model as BaseChatModel const memory = nodeData.inputs?.memory as FlowiseMemory const systemMessage = nodeData.inputs?.systemMessage as string let tools = nodeData.inputs?.tools diff --git a/packages/components/nodes/chains/LLMChain/LLMChain.ts b/packages/components/nodes/chains/LLMChain/LLMChain.ts index dc59760c..fa0fd61c 100644 --- a/packages/components/nodes/chains/LLMChain/LLMChain.ts +++ b/packages/components/nodes/chains/LLMChain/LLMChain.ts @@ -9,7 +9,6 @@ import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers' -import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' class LLMChain_Chains implements INode { @@ -164,7 +163,6 @@ const runPrediction = async ( const socketIO = isStreaming ? options.socketIO : undefined const socketIOClientId = isStreaming ? options.socketIOClientId : '' const moderations = nodeData.inputs?.inputModeration as Moderation[] - let model = nodeData.inputs?.model as ChatOpenAI if (moderations && moderations.length > 0) { try { @@ -185,8 +183,8 @@ const runPrediction = async ( const promptValues = handleEscapeCharacters(promptValuesRaw, true) if (llmSupportsVision(chain.llm)) { - const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) const visionChatModel = chain.llm as IVisionChatModal + const messageContent = addImagesToMessages(nodeData, options, visionChatModel.multiModalOption) if (messageContent?.length) { // Change model to gpt-4-vision && max token to higher when using gpt-4-vision visionChatModel.setVisionModel() From 4b9a7c9b9b7d03868a08a673f955201822149e62 Mon Sep 17 00:00:00 2001 From: Henry Date: Sat, 9 Mar 2024 17:28:31 +0800 Subject: [PATCH 09/10] revert azure chat openai --- .../AzureChatOpenAI/AzureChatOpenAI.ts | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index 785bd3c5..ea924fd0 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -1,9 +1,8 @@ -import { AzureOpenAIInput, ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from '@langchain/openai' +import { AzureOpenAIInput, ChatOpenAI, OpenAIChatInput } from '@langchain/openai' import { BaseCache } from '@langchain/core/caches' import { BaseLLMParams } from '@langchain/core/language_models/llms' -import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' -import { ChatOpenAI } from '../ChatOpenAI/FlowiseChatOpenAI' class AzureChatOpenAI_ChatModels implements INode { label: string @@ -20,12 +19,12 @@ class AzureChatOpenAI_ChatModels implements INode { constructor() { this.label = 'Azure ChatOpenAI' this.name = 'azureChatOpenAI' - this.version = 3.0 + this.version = 2.0 this.type = 'AzureChatOpenAI' this.icon = 'Azure.svg' this.category = 'Chat Models' this.description = 'Wrapper around Azure OpenAI large language models that use the Chat endpoint' - this.baseClasses = [this.type, ...getBaseClasses(LangchainChatOpenAI)] + this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)] this.credential = { label: 'Connect Credential', name: 'credential', @@ -139,14 +138,7 @@ class AzureChatOpenAI_ChatModels implements INode { if (timeout) obj.timeout = parseInt(timeout, 10) if (cache) obj.cache = cache - const multiModalOption: IMultiModalOption = { - image: { - allowImageUploads: false - } - } - - const model = new ChatOpenAI(nodeData.id, obj) - model.setMultiModalOption(multiModalOption) + const model = new ChatOpenAI(obj) return model } } From e04cfba7a53675e04845e0358e014daeac107b46 Mon Sep 17 00:00:00 2001 From: Henry Date: Sat, 9 Mar 2024 19:45:29 +0800 Subject: [PATCH 10/10] update marketplace template --- .../server/marketplaces/chatflows/Claude LLM.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/server/marketplaces/chatflows/Claude LLM.json b/packages/server/marketplaces/chatflows/Claude LLM.json index fdce533e..dc5f3254 100644 --- a/packages/server/marketplaces/chatflows/Claude LLM.json +++ b/packages/server/marketplaces/chatflows/Claude LLM.json @@ -160,7 +160,7 @@ "data": { "id": "chatAnthropic_0", "label": "ChatAnthropic", - "version": 3, + "version": 4, "name": "chatAnthropic", "type": "ChatAnthropic", "baseClasses": ["ChatAnthropic", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -288,6 +288,15 @@ "optional": true, "additionalParams": true, "id": "chatAnthropic_0-input-topK-number" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatAnthropic_0-input-allowImageUploads-boolean" } ], "inputAnchors": [ @@ -305,7 +314,8 @@ "temperature": 0.9, "maxTokensToSample": "", "topP": "", - "topK": "" + "topK": "", + "allowImageUploads": true }, "outputAnchors": [ {