farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 23:01:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

reverting all image upload logic to individual chains/agents

Browse Source
This commit is contained in:
vinodkiran
2024-02-19 15:27:19 -08:00
parent 8bad360796
commit b31e8715f4
6 changed files with 177 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts
+38 -5
View File
@@ -11,7 +11,8 @@ import { getBaseClasses } from '../../../src/utils'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
import { AgentExecutor } from '../../../src/agents'
import { injectAgentExecutorNodeData } from '../../../src/multiModalUtils'
import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
import { addImagesToMessages } from '../../../src/multiModalUtils'
const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI.
@@ -82,14 +83,19 @@ class ConversationalAgent_Agents implements INode {
}
async init(nodeData: INodeData, input: string, options: ICommonObject): Promise<any> {
return prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
return prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
}
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
const memory = nodeData.inputs?.memory as FlowiseMemory
const executor = await prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
injectAgentExecutorNodeData(executor, nodeData, options)
const executor = await prepareAgent(
nodeData,
options,
{ sessionId: this.sessionId, chatId: options.chatId, input },
options.chatHistory
)
// injectAgentExecutorNodeData(executor, nodeData, options)
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
@@ -123,6 +129,7 @@ class ConversationalAgent_Agents implements INode {
const prepareAgent = async (
nodeData: INodeData,
options: ICommonObject,
flowObj: { sessionId?: string; chatId?: string; input?: string },
chatHistory: IMessage[] = []
) => {
@@ -149,6 +156,32 @@ const prepareAgent = async (
outputParser
})
if (model instanceof ChatOpenAI) {
let humanImageMessages: HumanMessage[] = []
const chatModel = model as ChatOpenAI
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
// Change model to gpt-4-vision
chatModel.modelName = 'gpt-4-vision-preview'
// Change default max token to higher when using gpt-4-vision
chatModel.maxTokens = 1024
for (const msg of messageContent) {
humanImageMessages.push(new HumanMessage({ content: [msg] }))
}
let messagePlaceholder = prompt.promptMessages.pop()
prompt.promptMessages.push(...humanImageMessages)
// @ts-ignore
prompt.promptMessages.push(messagePlaceholder)
} else {
// revert to previous values if image upload is empty
chatModel.modelName = chatModel.configuredModel
chatModel.maxTokens = chatModel.configuredMaxToken
}
}
const runnableAgent = RunnableSequence.from([
{
[inputKey]: (i: { input: string; steps: AgentStep[] }) => i.input,
@@ -169,7 +202,7 @@ const prepareAgent = async (
sessionId: flowObj?.sessionId,
chatId: flowObj?.chatId,
input: flowObj?.input,
verbose: process.env.DEBUG === 'true' ? true : false
verbose: process.env.DEBUG === 'true'
})
return executor
packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts
+28 -4
View File
@@ -7,7 +7,11 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { additionalCallbacks } from '../../../src/handler'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { injectLcAgentExecutorNodeData } from '../../../src/multiModalUtils'
import { ChatOpenAI } from "../../chatmodels/ChatOpenAI/FlowiseChatOpenAI";
import { HumanMessage } from "@langchain/core/messages";
import { addImagesToMessages } from "../../../src/multiModalUtils";
import { ChatPromptTemplate, SystemMessagePromptTemplate } from "langchain/prompts";
// import { injectLcAgentExecutorNodeData } from '../../../src/multiModalUtils'
class MRKLAgentChat_Agents implements INode {
label: string
@@ -54,19 +58,39 @@ class MRKLAgentChat_Agents implements INode {
tools = flatten(tools)
const promptWithChat = await pull<PromptTemplate>('hwchase17/react-chat')
let chatPromptTemplate = undefined
if (model instanceof ChatOpenAI) {
const chatModel = model as ChatOpenAI
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
// Change model to gpt-4-vision
chatModel.modelName = 'gpt-4-vision-preview'
// Change default max token to higher when using gpt-4-vision
chatModel.maxTokens = 1024
const oldTemplate = promptWithChat.template as string
let chatPromptTemplate = ChatPromptTemplate.fromMessages([SystemMessagePromptTemplate.fromTemplate(oldTemplate)])
chatPromptTemplate.promptMessages = [new HumanMessage({ content: messageContent })]
} else {
// revert to previous values if image upload is empty
chatModel.modelName = chatModel.configuredModel
chatModel.maxTokens = chatModel.configuredMaxToken
}
}
const agent = await createReactAgent({
llm: model,
tools,
prompt: promptWithChat
prompt: chatPromptTemplate ?? promptWithChat
})
const executor = new AgentExecutor({
agent,
tools,
verbose: process.env.DEBUG === 'true' ? true : false
verbose: process.env.DEBUG === 'true'
})
injectLcAgentExecutorNodeData(executor, nodeData, options)
// injectLcAgentExecutorNodeData(executor, nodeData, options)
const callbacks = await additionalCallbacks(nodeData, options)