From 85809a9ecc82486a90a1f4cc39515dd28b796c04 Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 15 Feb 2024 04:03:10 +0800 Subject: [PATCH] fix for concurrency --- .../ConversationChain/ConversationChain.ts | 49 +++++++++++++------ packages/components/src/Interface.ts | 17 +++++++ packages/components/src/multiModalUtils.ts | 11 +++-- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index ad49d829..22a32c7b 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -2,14 +2,15 @@ import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../ import { ConversationChain } from 'langchain/chains' import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from 'langchain/prompts' -import { BaseChatModel } from 'langchain/chat_models/base' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' import { RunnableSequence } from 'langchain/schema/runnable' import { StringOutputParser } from 'langchain/schema/output_parser' +import { HumanMessage } from 'langchain/schema' import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse } from '../../outputparsers/OutputParserHelpers' -import { injectChainNodeData } from '../../../src/multiModalUtils' +import { addImagesToMessages } from '../../../src/multiModalUtils' +import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.` const inputKey = 'input' @@ -93,7 +94,7 @@ class ConversationChain_Chains implements INode { async run(nodeData: INodeData, input: string, options: ICommonObject): Promise { const memory = nodeData.inputs?.memory - injectChainNodeData(nodeData, options) + // injectChainNodeData(nodeData, options) const chain = prepareChain(nodeData, options, this.sessionId) const moderations = nodeData.inputs?.inputModeration as Moderation[] @@ -145,7 +146,7 @@ class ConversationChain_Chains implements INode { } } -const prepareChatPrompt = (nodeData: INodeData) => { +const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => { const memory = nodeData.inputs?.memory as FlowiseMemory const prompt = nodeData.inputs?.systemMessagePrompt as string const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate @@ -153,12 +154,10 @@ const prepareChatPrompt = (nodeData: INodeData) => { if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) { const sysPrompt = chatPromptTemplate.promptMessages[0] const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1] - const chatPrompt = ChatPromptTemplate.fromMessages([ - sysPrompt, - new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), - humanPrompt - ]) + const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt] + if (humanImageMessages.length) messages.push(...humanImageMessages) + const chatPrompt = ChatPromptTemplate.fromMessages(messages) if ((chatPromptTemplate as any).promptValues) { // @ts-ignore chatPrompt.promptValues = (chatPromptTemplate as any).promptValues @@ -167,22 +166,44 @@ const prepareChatPrompt = (nodeData: INodeData) => { return chatPrompt } - const chatPrompt = ChatPromptTemplate.fromMessages([ + const messages = [ SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage), new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`) - ]) + ] + if (humanImageMessages.length) messages.push(...(humanImageMessages as any[])) + + const chatPrompt = ChatPromptTemplate.fromMessages(messages) return chatPrompt } const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => { const chatHistory = options.chatHistory - const model = nodeData.inputs?.model as BaseChatModel + let model = nodeData.inputs?.model as ChatOpenAI const memory = nodeData.inputs?.memory as FlowiseMemory const memoryKey = memory.memoryKey ?? 'chat_history' - const chatPrompt = prepareChatPrompt(nodeData) + const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) + let humanImageMessages: HumanMessage[] = [] + + if (messageContent?.length) { + // Change model to gpt-4-vision + model.modelName = 'gpt-4-vision-preview' + + // Change default max token to higher when using gpt-4-vision + model.maxTokens = 1024 + + for (const msg of messageContent) { + humanImageMessages.push(new HumanMessage({ content: [msg] })) + } + } else { + // revert to previous values if image upload is empty + model.modelName = model.configuredModel + model.maxTokens = model.configuredMaxToken + } + + const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages) let promptVariables = {} const promptValuesRaw = (chatPrompt as any).promptValues if (promptValuesRaw) { @@ -206,7 +227,7 @@ const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: s }, ...promptVariables }, - prepareChatPrompt(nodeData), + prepareChatPrompt(nodeData, humanImageMessages), model, new StringOutputParser() ]) diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 44818733..62cd3ba9 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -21,6 +21,8 @@ export type CommonType = string | number | boolean | undefined | null export type MessageType = 'apiMessage' | 'userMessage' +export type ImageDetail = 'auto' | 'low' | 'high' + /** * Others */ @@ -158,6 +160,21 @@ export interface IMultiModalOption { audio?: Record } +export type MessageContentText = { + type: 'text' + text: string +} + +export type MessageContentImageUrl = { + type: 'image_url' + image_url: + | string + | { + url: string + detail?: ImageDetail + } +} + /** * Classes */ diff --git a/packages/components/src/multiModalUtils.ts b/packages/components/src/multiModalUtils.ts index c321f088..246821db 100644 --- a/packages/components/src/multiModalUtils.ts +++ b/packages/components/src/multiModalUtils.ts @@ -1,10 +1,9 @@ -import { ICommonObject, IFileUpload, IMultiModalOption, INodeData } from './Interface' +import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' import { BaseChatModel } from 'langchain/chat_models/base' import { ChatOpenAI as LangchainChatOpenAI } from 'langchain/chat_models/openai' import path from 'path' import { getStoragePath } from './utils' import fs from 'fs' -import { MessageContent } from '@langchain/core/dist/messages' import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI' export const injectChainNodeData = (nodeData: INodeData, options: ICommonObject) => { @@ -16,8 +15,12 @@ export const injectChainNodeData = (nodeData: INodeData, options: ICommonObject) } } -export const addImagesToMessages = (nodeData: INodeData, options: ICommonObject, multiModalOption?: IMultiModalOption): MessageContent => { - const imageContent: MessageContent = [] +export const addImagesToMessages = ( + nodeData: INodeData, + options: ICommonObject, + multiModalOption?: IMultiModalOption +): MessageContentImageUrl[] => { + const imageContent: MessageContentImageUrl[] = [] let model = nodeData.inputs?.model if (model instanceof LangchainChatOpenAI && multiModalOption) {