farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 13:00:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Addition of Claude for Image uploads

Browse Source
This commit is contained in:
vinodkiran
2024-03-07 18:55:24 +05:30
parent a2caf3e265
commit 63b8c23072
10 changed files with 185 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts
+29 -17
View File
@@ -4,7 +4,12 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'
import { ChainValues } from '@langchain/core/utils/types'
import { AgentStep } from '@langchain/core/agents'
import { renderTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
import {
renderTemplate,
MessagesPlaceholder,
HumanMessagePromptTemplate,
PromptTemplate
} from "@langchain/core/prompts";
import { RunnableSequence } from '@langchain/core/runnables'
import { ChatConversationalAgent } from 'langchain/agents'
import { getBaseClasses } from '../../../src/utils'
@@ -12,7 +17,8 @@ import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
import { AgentExecutor } from '../../../src/agents'
import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
import { addImagesToMessages } from '../../../src/multiModalUtils'
import { addImagesToMessages, llmSupportsVision } from "../../../src/multiModalUtils";
import { IVisionChatModal } from "../../../src/IVisionChatModal";
const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI.
@@ -150,33 +156,39 @@ const prepareAgent = async (
outputParser
})
if (model instanceof ChatOpenAI) {
let humanImageMessages: HumanMessage[] = []
if (llmSupportsVision(model)) {
const visionChatModel = model as IVisionChatModal
// let humanImageMessages: HumanMessage[] = []
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
// Change model to gpt-4-vision
model.modelName = 'gpt-4-vision-preview'
visionChatModel.setVisionModel()
// Change default max token to higher when using gpt-4-vision
model.maxTokens = 1024
for (const msg of messageContent) {
humanImageMessages.push(new HumanMessage({ content: [msg] }))
}
// for (const msg of messageContent) {
// humanImageMessages.push(new HumanMessage({ content: [msg] }))
// }
// Pop the `agent_scratchpad` MessagePlaceHolder
let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder
if (prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) {
const lastMessage = prompt.promptMessages.pop() as HumanMessagePromptTemplate
const template = (lastMessage.prompt as PromptTemplate).template as string
const msg = HumanMessagePromptTemplate.fromTemplate([
...messageContent,
{
text: template
}
])
msg.inputVariables = lastMessage.inputVariables
prompt.promptMessages.push(msg)
}
// Add the HumanMessage for images
prompt.promptMessages.push(...humanImageMessages)
//prompt.promptMessages.push(...humanImageMessages)
// Add the `agent_scratchpad` MessagePlaceHolder back
prompt.promptMessages.push(messagePlaceholder)
} else {
// revert to previous values if image upload is empty
model.modelName = model.configuredModel
model.maxTokens = model.configuredMaxToken
visionChatModel.revertToOriginalModel()
}
}
packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts
+16 -14
View File
@@ -1,6 +1,5 @@
import { flatten } from 'lodash'
import { AgentExecutor } from 'langchain/agents'
import { HumanMessage } from '@langchain/core/messages'
import { ChatPromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts'
import { Tool } from '@langchain/core/tools'
import type { PromptTemplate } from '@langchain/core/prompts'
@@ -10,8 +9,8 @@ import { additionalCallbacks } from '../../../src/handler'
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { createReactAgent } from '../../../src/agents'
import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
import { addImagesToMessages } from '../../../src/multiModalUtils'
import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
import { IVisionChatModal } from '../../../src/IVisionChatModal'
class MRKLAgentChat_Agents implements INode {
label: string
@@ -68,23 +67,26 @@ class MRKLAgentChat_Agents implements INode {
const prompt = await pull<PromptTemplate>('hwchase17/react-chat')
let chatPromptTemplate = undefined
if (model instanceof ChatOpenAI) {
if (llmSupportsVision(model)) {
const visionChatModel = model as IVisionChatModal
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
// Change model to gpt-4-vision
model.modelName = 'gpt-4-vision-preview'
// Change default max token to higher when using gpt-4-vision
model.maxTokens = 1024
// Change model to vision supported
visionChatModel.setVisionModel()
const oldTemplate = prompt.template as string
chatPromptTemplate = ChatPromptTemplate.fromMessages([HumanMessagePromptTemplate.fromTemplate(oldTemplate)])
chatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
const msg = HumanMessagePromptTemplate.fromTemplate([
...messageContent,
{
text: oldTemplate
}
])
msg.inputVariables = prompt.inputVariables
chatPromptTemplate = ChatPromptTemplate.fromMessages([msg])
} else {
// revert to previous values if image upload is empty
model.modelName = model.configuredModel
model.maxTokens = model.configuredMaxToken
visionChatModel.revertToOriginalModel()
}
}