reverting all image upload logic to individual chains/agents

packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts

@@ -11,7 +11,8 @@ import { getBaseClasses } from '../../../src/utils'
 import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
 import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { AgentExecutor } from '../../../src/agents'
-import { injectAgentExecutorNodeData } from '../../../src/multiModalUtils'
+import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { addImagesToMessages } from '../../../src/multiModalUtils'
 
 const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI.
 
@@ -82,14 +83,19 @@ class ConversationalAgent_Agents implements INode {
     }
 
     async init(nodeData: INodeData, input: string, options: ICommonObject): Promise<any> {
-        return prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
+        return prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
     }
 
     async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
         const memory = nodeData.inputs?.memory as FlowiseMemory
 
-        const executor = await prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
-        injectAgentExecutorNodeData(executor, nodeData, options)
+        const executor = await prepareAgent(
+            nodeData,
+            options,
+            { sessionId: this.sessionId, chatId: options.chatId, input },
+            options.chatHistory
+        )
+        // injectAgentExecutorNodeData(executor, nodeData, options)
 
         const loggerHandler = new ConsoleCallbackHandler(options.logger)
         const callbacks = await additionalCallbacks(nodeData, options)
@@ -123,6 +129,7 @@ class ConversationalAgent_Agents implements INode {
 
 const prepareAgent = async (
     nodeData: INodeData,
+    options: ICommonObject,
     flowObj: { sessionId?: string; chatId?: string; input?: string },
     chatHistory: IMessage[] = []
 ) => {
@@ -149,6 +156,32 @@ const prepareAgent = async (
         outputParser
     })
 
+    if (model instanceof ChatOpenAI) {
+        let humanImageMessages: HumanMessage[] = []
+        const chatModel = model as ChatOpenAI
+        const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+        if (messageContent?.length) {
+            // Change model to gpt-4-vision
+            chatModel.modelName = 'gpt-4-vision-preview'
+
+            // Change default max token to higher when using gpt-4-vision
+            chatModel.maxTokens = 1024
+
+            for (const msg of messageContent) {
+                humanImageMessages.push(new HumanMessage({ content: [msg] }))
+            }
+            let messagePlaceholder = prompt.promptMessages.pop()
+            prompt.promptMessages.push(...humanImageMessages)
+            // @ts-ignore
+            prompt.promptMessages.push(messagePlaceholder)
+        } else {
+            // revert to previous values if image upload is empty
+            chatModel.modelName = chatModel.configuredModel
+            chatModel.maxTokens = chatModel.configuredMaxToken
+        }
+    }
+
     const runnableAgent = RunnableSequence.from([
         {
             [inputKey]: (i: { input: string; steps: AgentStep[] }) => i.input,
@@ -169,7 +202,7 @@ const prepareAgent = async (
         sessionId: flowObj?.sessionId,
         chatId: flowObj?.chatId,
         input: flowObj?.input,
-        verbose: process.env.DEBUG === 'true' ? true : false
+        verbose: process.env.DEBUG === 'true'
     })
 
     return executor