touchup fixes

packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts

@@ -4,7 +4,7 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'
 import { ChainValues } from '@langchain/core/utils/types'
 import { AgentStep } from '@langchain/core/agents'
-import { renderTemplate } from '@langchain/core/prompts'
+import { renderTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
 import { RunnableSequence } from '@langchain/core/runnables'
 import { ChatConversationalAgent } from 'langchain/agents'
 import { getBaseClasses } from '../../../src/utils'
@@ -141,11 +141,6 @@ const prepareAgent = async (
     const memoryKey = memory.memoryKey ? memory.memoryKey : 'chat_history'
     const inputKey = memory.inputKey ? memory.inputKey : 'input'
 
-    /** Bind a stop token to the model */
-    const modelWithStop = model.bind({
-        stop: ['\nObservation']
-    })
-
     const outputParser = ChatConversationalAgent.getDefaultOutputParser({
         llm: model,
         toolNames: tools.map((tool) => tool.name)
@@ -158,30 +153,39 @@ const prepareAgent = async (
 
     if (model instanceof ChatOpenAI) {
         let humanImageMessages: HumanMessage[] = []
-        const chatModel = model as ChatOpenAI
         const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
 
         if (messageContent?.length) {
             // Change model to gpt-4-vision
-            chatModel.modelName = 'gpt-4-vision-preview'
+            model.modelName = 'gpt-4-vision-preview'
 
             // Change default max token to higher when using gpt-4-vision
-            chatModel.maxTokens = 1024
+            model.maxTokens = 1024
 
             for (const msg of messageContent) {
                 humanImageMessages.push(new HumanMessage({ content: [msg] }))
             }
-            let messagePlaceholder = prompt.promptMessages.pop()
+
+            // Pop the `agent_scratchpad` MessagePlaceHolder
+            let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder
+
+            // Add the HumanMessage for images
             prompt.promptMessages.push(...humanImageMessages)
-            // @ts-ignore
+
+            // Add the `agent_scratchpad` MessagePlaceHolder back
             prompt.promptMessages.push(messagePlaceholder)
         } else {
             // revert to previous values if image upload is empty
-            chatModel.modelName = chatModel.configuredModel
-            chatModel.maxTokens = chatModel.configuredMaxToken
+            model.modelName = model.configuredModel
+            model.maxTokens = model.configuredMaxToken
         }
     }
 
+    /** Bind a stop token to the model */
+    const modelWithStop = model.bind({
+        stop: ['\nObservation']
+    })
+
     const runnableAgent = RunnableSequence.from([
         {
             [inputKey]: (i: { input: string; steps: AgentStep[] }) => i.input,

packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts

@@ -11,8 +11,7 @@ import { createReactAgent } from '../../../src/agents'
 import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
 import { HumanMessage } from '@langchain/core/messages'
 import { addImagesToMessages } from '../../../src/multiModalUtils'
-import { ChatPromptTemplate, SystemMessagePromptTemplate } from 'langchain/prompts'
-// import { injectLcAgentExecutorNodeData } from '../../../src/multiModalUtils'
+import { ChatPromptTemplate, HumanMessagePromptTemplate } from 'langchain/prompts'
 
 class MRKLAgentChat_Agents implements INode {
     label: string
@@ -66,32 +65,33 @@ class MRKLAgentChat_Agents implements INode {
         let tools = nodeData.inputs?.tools as Tool[]
         tools = flatten(tools)
 
-        const promptWithChat = await pull<PromptTemplate>('hwchase17/react-chat')
+        const prompt = await pull<PromptTemplate>('hwchase17/react-chat')
         let chatPromptTemplate = undefined
+
         if (model instanceof ChatOpenAI) {
-            const chatModel = model as ChatOpenAI
             const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
 
             if (messageContent?.length) {
                 // Change model to gpt-4-vision
-                chatModel.modelName = 'gpt-4-vision-preview'
+                model.modelName = 'gpt-4-vision-preview'
 
                 // Change default max token to higher when using gpt-4-vision
-                chatModel.maxTokens = 1024
-                const oldTemplate = promptWithChat.template as string
-                chatPromptTemplate = ChatPromptTemplate.fromMessages([SystemMessagePromptTemplate.fromTemplate(oldTemplate)])
+                model.maxTokens = 1024
+
+                const oldTemplate = prompt.template as string
+                chatPromptTemplate = ChatPromptTemplate.fromMessages([HumanMessagePromptTemplate.fromTemplate(oldTemplate)])
                 chatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
             } else {
                 // revert to previous values if image upload is empty
-                chatModel.modelName = chatModel.configuredModel
-                chatModel.maxTokens = chatModel.configuredMaxToken
+                model.modelName = model.configuredModel
+                model.maxTokens = model.configuredMaxToken
             }
         }
 
         const agent = await createReactAgent({
             llm: model,
             tools,
-            prompt: chatPromptTemplate ?? promptWithChat
+            prompt: chatPromptTemplate ?? prompt
         })
 
         const executor = new AgentExecutor({