fix for concurrency

packages/components/nodes/chains/ConversationChain/ConversationChain.ts

@@ -2,14 +2,15 @@ import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../
 import { ConversationChain } from 'langchain/chains'
 import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
 import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from 'langchain/prompts'
-import { BaseChatModel } from 'langchain/chat_models/base'
 import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
 import { RunnableSequence } from 'langchain/schema/runnable'
 import { StringOutputParser } from 'langchain/schema/output_parser'
+import { HumanMessage } from 'langchain/schema'
 import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
 import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'
-import { injectChainNodeData } from '../../../src/multiModalUtils'
+import { addImagesToMessages } from '../../../src/multiModalUtils'
+import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
 
 let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.`
 const inputKey = 'input'
@@ -93,7 +94,7 @@ class ConversationChain_Chains implements INode {
 
     async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
         const memory = nodeData.inputs?.memory
-        injectChainNodeData(nodeData, options)
+        // injectChainNodeData(nodeData, options)
 
         const chain = prepareChain(nodeData, options, this.sessionId)
         const moderations = nodeData.inputs?.inputModeration as Moderation[]
@@ -145,7 +146,7 @@ class ConversationChain_Chains implements INode {
     }
 }
 
-const prepareChatPrompt = (nodeData: INodeData) => {
+const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => {
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const prompt = nodeData.inputs?.systemMessagePrompt as string
     const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate
@@ -153,12 +154,10 @@ const prepareChatPrompt = (nodeData: INodeData) => {
     if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) {
         const sysPrompt = chatPromptTemplate.promptMessages[0]
         const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1]
-        const chatPrompt = ChatPromptTemplate.fromMessages([
+        const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt]
-            sysPrompt,
+        if (humanImageMessages.length) messages.push(...humanImageMessages)
-            new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
-            humanPrompt
-        ])
 
+        const chatPrompt = ChatPromptTemplate.fromMessages(messages)
         if ((chatPromptTemplate as any).promptValues) {
             // @ts-ignore
             chatPrompt.promptValues = (chatPromptTemplate as any).promptValues
@@ -167,22 +166,44 @@ const prepareChatPrompt = (nodeData: INodeData) => {
         return chatPrompt
     }
 
-    const chatPrompt = ChatPromptTemplate.fromMessages([
+    const messages = [
         SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage),
         new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
         HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)
-    ])
+    ]
+    if (humanImageMessages.length) messages.push(...(humanImageMessages as any[]))
+
+    const chatPrompt = ChatPromptTemplate.fromMessages(messages)
 
     return chatPrompt
 }
 
 const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
     const chatHistory = options.chatHistory
-    const model = nodeData.inputs?.model as BaseChatModel
+    let model = nodeData.inputs?.model as ChatOpenAI
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const memoryKey = memory.memoryKey ?? 'chat_history'
 
-    const chatPrompt = prepareChatPrompt(nodeData)
+    const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+    let humanImageMessages: HumanMessage[] = []
+
+    if (messageContent?.length) {
+        // Change model to gpt-4-vision
+        model.modelName = 'gpt-4-vision-preview'
+
+        // Change default max token to higher when using gpt-4-vision
+        model.maxTokens = 1024
+
+        for (const msg of messageContent) {
+            humanImageMessages.push(new HumanMessage({ content: [msg] }))
+        }
+    } else {
+        // revert to previous values if image upload is empty
+        model.modelName = model.configuredModel
+        model.maxTokens = model.configuredMaxToken
+    }
+
+    const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages)
     let promptVariables = {}
     const promptValuesRaw = (chatPrompt as any).promptValues
     if (promptValuesRaw) {
@@ -206,7 +227,7 @@ const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: s
             },
             ...promptVariables
         },
-        prepareChatPrompt(nodeData),
+        prepareChatPrompt(nodeData, humanImageMessages),
         model,
         new StringOutputParser()
     ])

packages/components/src/Interface.ts

@@ -21,6 +21,8 @@ export type CommonType = string | number | boolean | undefined | null
 
 export type MessageType = 'apiMessage' | 'userMessage'
 
+export type ImageDetail = 'auto' | 'low' | 'high'
+
 /**
  * Others
  */
@@ -158,6 +160,21 @@ export interface IMultiModalOption {
     audio?: Record<string, any>
 }
 
+export type MessageContentText = {
+    type: 'text'
+    text: string
+}
+
+export type MessageContentImageUrl = {
+    type: 'image_url'
+    image_url:
+        | string
+        | {
+              url: string
+              detail?: ImageDetail
+          }
+}
+
 /**
  * Classes
  */

packages/components/src/multiModalUtils.ts

@@ -1,10 +1,9 @@
-import { ICommonObject, IFileUpload, IMultiModalOption, INodeData } from './Interface'
+import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface'
 import { BaseChatModel } from 'langchain/chat_models/base'
 import { ChatOpenAI as LangchainChatOpenAI } from 'langchain/chat_models/openai'
 import path from 'path'
 import { getStoragePath } from './utils'
 import fs from 'fs'
-import { MessageContent } from '@langchain/core/dist/messages'
 import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI'
 
 export const injectChainNodeData = (nodeData: INodeData, options: ICommonObject) => {
@@ -16,8 +15,12 @@ export const injectChainNodeData = (nodeData: INodeData, options: ICommonObject)
     }
 }
 
-export const addImagesToMessages = (nodeData: INodeData, options: ICommonObject, multiModalOption?: IMultiModalOption): MessageContent => {
+export const addImagesToMessages = (
-    const imageContent: MessageContent = []
+    nodeData: INodeData,
+    options: ICommonObject,
+    multiModalOption?: IMultiModalOption
+): MessageContentImageUrl[] => {
+    const imageContent: MessageContentImageUrl[] = []
     let model = nodeData.inputs?.model
 
     if (model instanceof LangchainChatOpenAI && multiModalOption) {