farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 17:01:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

reverting all image upload logic to individual chains/agents

Browse Source
This commit is contained in:
vinodkiran
2024-02-19 15:27:19 -08:00
parent 8bad360796
commit b31e8715f4
6 changed files with 177 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/chains/ConversationChain/ConversationChain.ts
+42 -20
View File
@@ -1,15 +1,16 @@
import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts'
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { RunnableSequence } from '@langchain/core/runnables'
import { StringOutputParser } from '@langchain/core/output_parsers'
import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
import { ConversationChain } from 'langchain/chains'
import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from 'langchain/prompts'
import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
import { RunnableSequence } from 'langchain/schema/runnable'
import { StringOutputParser } from 'langchain/schema/output_parser'
import { HumanMessage } from 'langchain/schema'
import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
import { injectRunnableNodeData } from '../../../src/multiModalUtils'
import { addImagesToMessages } from '../../../src/multiModalUtils'
import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.`
const inputKey = 'input'
@@ -95,8 +96,6 @@ class ConversationChain_Chains implements INode {
const memory = nodeData.inputs?.memory
const chain = prepareChain(nodeData, options, this.sessionId)
injectRunnableNodeData(chain, nodeData, options)
const moderations = nodeData.inputs?.inputModeration as Moderation[]
if (moderations && moderations.length > 0) {
@@ -146,7 +145,7 @@ class ConversationChain_Chains implements INode {
}
}
const prepareChatPrompt = (nodeData: INodeData) => {
const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => {
const memory = nodeData.inputs?.memory as FlowiseMemory
const prompt = nodeData.inputs?.systemMessagePrompt as string
const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate
@@ -154,12 +153,10 @@ const prepareChatPrompt = (nodeData: INodeData) => {
if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) {
const sysPrompt = chatPromptTemplate.promptMessages[0]
const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1]
const chatPrompt = ChatPromptTemplate.fromMessages([
sysPrompt,
new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
humanPrompt
])
const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt]
if (humanImageMessages.length) messages.push(...humanImageMessages)
const chatPrompt = ChatPromptTemplate.fromMessages(messages)
if ((chatPromptTemplate as any).promptValues) {
// @ts-ignore
chatPrompt.promptValues = (chatPromptTemplate as any).promptValues
@@ -168,22 +165,47 @@ const prepareChatPrompt = (nodeData: INodeData) => {
return chatPrompt
}
const chatPrompt = ChatPromptTemplate.fromMessages([
const messages = [
SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage),
new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)
])
]
if (humanImageMessages.length) messages.push(...(humanImageMessages as any[]))
const chatPrompt = ChatPromptTemplate.fromMessages(messages)
return chatPrompt
}
const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
const chatHistory = options.chatHistory
const model = nodeData.inputs?.model as BaseChatModel
let model = nodeData.inputs?.model
const memory = nodeData.inputs?.memory as FlowiseMemory
const memoryKey = memory.memoryKey ?? 'chat_history'
const chatPrompt = prepareChatPrompt(nodeData)
let humanImageMessages: HumanMessage[] = []
if (model instanceof ChatOpenAI) {
const chatModel = model as ChatOpenAI
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
// Change model to gpt-4-vision
chatModel.modelName = 'gpt-4-vision-preview'
// Change default max token to higher when using gpt-4-vision
chatModel.maxTokens = 1024
for (const msg of messageContent) {
humanImageMessages.push(new HumanMessage({ content: [msg] }))
}
} else {
// revert to previous values if image upload is empty
chatModel.modelName = chatModel.configuredModel
chatModel.maxTokens = chatModel.configuredMaxToken
}
}
const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages)
let promptVariables = {}
const promptValuesRaw = (chatPrompt as any).promptValues
if (promptValuesRaw) {
@@ -207,7 +229,7 @@ const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: s
},
...promptVariables
},
prepareChatPrompt(nodeData),
prepareChatPrompt(nodeData, humanImageMessages),
model,
new StringOutputParser()
])