farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 09:00:52 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1898 from vinodkiran/FEATURE/AzureOpenAI_vision

Browse Source 
Claude 3 - Adding Support for Image Uploads / Multimodal Inputs
This commit is contained in:
Henry Heng
2024-03-11 15:18:14 +08:00
committed by GitHub
parent 88dd9cc6b4 2d208e1f99
commit 5b94974d80
14 changed files with 230 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts
+19 -21
View File
@@ -4,15 +4,14 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'
import { ChainValues } from '@langchain/core/utils/types'
import { AgentStep } from '@langchain/core/agents'
import { renderTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
import { renderTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
import { RunnableSequence } from '@langchain/core/runnables'
import { ChatConversationalAgent } from 'langchain/agents'
import { getBaseClasses } from '../../../src/utils'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
import { AgentExecutor } from '../../../src/agents'
import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
import { addImagesToMessages } from '../../../src/multiModalUtils'
import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
import { checkInputs, Moderation } from '../../moderation/Moderation'
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
@@ -171,33 +170,32 @@ const prepareAgent = async (
outputParser
})
if (model instanceof ChatOpenAI) {
let humanImageMessages: HumanMessage[] = []
if (llmSupportsVision(model)) {
const visionChatModel = model as IVisionChatModal
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
// Change model to gpt-4-vision
model.modelName = 'gpt-4-vision-preview'
// Change default max token to higher when using gpt-4-vision
model.maxTokens = 1024
for (const msg of messageContent) {
humanImageMessages.push(new HumanMessage({ content: [msg] }))
}
visionChatModel.setVisionModel()
// Pop the `agent_scratchpad` MessagePlaceHolder
let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder
// Add the HumanMessage for images
prompt.promptMessages.push(...humanImageMessages)
if (prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) {
const lastMessage = prompt.promptMessages.pop() as HumanMessagePromptTemplate
const template = (lastMessage.prompt as PromptTemplate).template as string
const msg = HumanMessagePromptTemplate.fromTemplate([
...messageContent,
{
text: template
}
])
msg.inputVariables = lastMessage.inputVariables
prompt.promptMessages.push(msg)
}
// Add the `agent_scratchpad` MessagePlaceHolder back
prompt.promptMessages.push(messagePlaceholder)
} else {
// revert to previous values if image upload is empty
model.modelName = model.configuredModel
model.maxTokens = model.configuredMaxToken
visionChatModel.revertToOriginalModel()
}
}
packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts
+16 -15
View File
@@ -1,17 +1,15 @@
import { flatten } from 'lodash'
import { AgentExecutor } from 'langchain/agents'
import { HumanMessage } from '@langchain/core/messages'
import { ChatPromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts'
import { Tool } from '@langchain/core/tools'
import type { PromptTemplate } from '@langchain/core/prompts'
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { pull } from 'langchain/hub'
import { additionalCallbacks } from '../../../src/handler'
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { createReactAgent } from '../../../src/agents'
import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
import { addImagesToMessages } from '../../../src/multiModalUtils'
import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
import { checkInputs, Moderation } from '../../moderation/Moderation'
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
@@ -90,23 +88,26 @@ class MRKLAgentChat_Agents implements INode {
const prompt = await pull<PromptTemplate>('hwchase17/react-chat')
let chatPromptTemplate = undefined
if (model instanceof ChatOpenAI) {
if (llmSupportsVision(model)) {
const visionChatModel = model as IVisionChatModal
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
// Change model to gpt-4-vision
model.modelName = 'gpt-4-vision-preview'
// Change default max token to higher when using gpt-4-vision
model.maxTokens = 1024
// Change model to vision supported
visionChatModel.setVisionModel()
const oldTemplate = prompt.template as string
chatPromptTemplate = ChatPromptTemplate.fromMessages([HumanMessagePromptTemplate.fromTemplate(oldTemplate)])
chatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
const msg = HumanMessagePromptTemplate.fromTemplate([
...messageContent,
{
text: oldTemplate
}
])
msg.inputVariables = prompt.inputVariables
chatPromptTemplate = ChatPromptTemplate.fromMessages([msg])
} else {
// revert to previous values if image upload is empty
model.modelName = model.configuredModel
model.maxTokens = model.configuredMaxToken
visionChatModel.revertToOriginalModel()
}
}
packages/components/nodes/agents/XMLAgent/XMLAgent.ts
+2 -2
View File
@@ -1,8 +1,8 @@
import { flatten } from 'lodash'
import { ChainValues } from '@langchain/core/utils/types'
import { AgentStep } from '@langchain/core/agents'
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { RunnableSequence } from '@langchain/core/runnables'
import { ChatOpenAI } from '@langchain/openai'
import { Tool } from '@langchain/core/tools'
import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
import { XMLAgentOutputParser } from 'langchain/agents/xml/output_parser'
@@ -160,7 +160,7 @@ const prepareAgent = async (
flowObj: { sessionId?: string; chatId?: string; input?: string },
chatHistory: IMessage[] = []
) => {
const model = nodeData.inputs?.model as ChatOpenAI
const model = nodeData.inputs?.model as BaseChatModel
const memory = nodeData.inputs?.memory as FlowiseMemory
const systemMessage = nodeData.inputs?.systemMessage as string
let tools = nodeData.inputs?.tools
packages/components/nodes/chains/ConversationChain/ConversationChain.ts
+56 -27
View File
@@ -1,14 +1,30 @@
import { ConversationChain } from 'langchain/chains'
import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts'
import {
ChatPromptTemplate,
HumanMessagePromptTemplate,
MessagesPlaceholder,
SystemMessagePromptTemplate,
BaseMessagePromptTemplateLike,
PromptTemplate
} from '@langchain/core/prompts'
import { RunnableSequence } from '@langchain/core/runnables'
import { StringOutputParser } from '@langchain/core/output_parsers'
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { HumanMessage } from '@langchain/core/messages'
import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
import { addImagesToMessages } from '../../../src/multiModalUtils'
import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import {
IVisionChatModal,
FlowiseMemory,
ICommonObject,
INode,
INodeData,
INodeParams,
MessageContentImageUrl
} from '../../../src/Interface'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
@@ -145,16 +161,32 @@ class ConversationChain_Chains implements INode {
}
}
const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => {
const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageContentImageUrl[]) => {
const memory = nodeData.inputs?.memory as FlowiseMemory
const prompt = nodeData.inputs?.systemMessagePrompt as string
const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate
let model = nodeData.inputs?.model as BaseChatModel
if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) {
const sysPrompt = chatPromptTemplate.promptMessages[0]
const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1]
const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt]
if (humanImageMessages.length) messages.push(...humanImageMessages)
// OpenAI works better when separate images into standalone human messages
if (model instanceof ChatOpenAI && humanImageMessages.length) {
messages.push(new HumanMessage({ content: [...humanImageMessages] }))
} else if (humanImageMessages.length) {
const lastMessage = messages.pop() as HumanMessagePromptTemplate
const template = (lastMessage.prompt as PromptTemplate).template as string
const msg = HumanMessagePromptTemplate.fromTemplate([
...humanImageMessages,
{
text: template
}
])
msg.inputVariables = lastMessage.inputVariables
messages.push(msg)
}
const chatPrompt = ChatPromptTemplate.fromMessages(messages)
if ((chatPromptTemplate as any).promptValues) {
@@ -165,12 +197,18 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage
return chatPrompt
}
const messages = [
const messages: BaseMessagePromptTemplateLike[] = [
SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage),
new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)
new MessagesPlaceholder(memory.memoryKey ?? 'chat_history')
]
if (humanImageMessages.length) messages.push(...(humanImageMessages as any[]))
// OpenAI works better when separate images into standalone human messages
if (model instanceof ChatOpenAI && humanImageMessages.length) {
messages.push(HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`))
messages.push(new HumanMessage({ content: [...humanImageMessages] }))
} else if (humanImageMessages.length) {
messages.push(HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages]))
}
const chatPrompt = ChatPromptTemplate.fromMessages(messages)
@@ -179,32 +217,23 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage
const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
const chatHistory = options.chatHistory
let model = nodeData.inputs?.model as ChatOpenAI
let model = nodeData.inputs?.model as BaseChatModel
const memory = nodeData.inputs?.memory as FlowiseMemory
const memoryKey = memory.memoryKey ?? 'chat_history'
let humanImageMessages: HumanMessage[] = []
if (model instanceof ChatOpenAI) {
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
let messageContent: MessageContentImageUrl[] = []
if (llmSupportsVision(model)) {
messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
const visionChatModel = model as IVisionChatModal
if (messageContent?.length) {
// Change model to gpt-4-vision
model.modelName = 'gpt-4-vision-preview'
// Change default max token to higher when using gpt-4-vision
model.maxTokens = 1024
for (const msg of messageContent) {
humanImageMessages.push(new HumanMessage({ content: [msg] }))
}
visionChatModel.setVisionModel()
} else {
// revert to previous values if image upload is empty
model.modelName = model.configuredModel
model.maxTokens = model.configuredMaxToken
visionChatModel.revertToOriginalModel()
}
}
const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages)
const chatPrompt = prepareChatPrompt(nodeData, messageContent)
let promptVariables = {}
const promptValuesRaw = (chatPrompt as any).promptValues
if (promptValuesRaw) {
@@ -228,7 +257,7 @@ const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: s
},
...promptVariables
},
prepareChatPrompt(nodeData, humanImageMessages),
prepareChatPrompt(nodeData, messageContent),
model,
new StringOutputParser()
])
packages/components/nodes/chains/LLMChain/LLMChain.ts
+30 -18
View File
@@ -1,16 +1,15 @@
import { BaseLanguageModel, BaseLanguageModelCallOptions } from '@langchain/core/language_models/base'
import { BaseLLMOutputParser, BaseOutputParser } from '@langchain/core/output_parsers'
import { HumanMessage } from '@langchain/core/messages'
import { ChatPromptTemplate, FewShotPromptTemplate, PromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts'
import { ChatPromptTemplate, FewShotPromptTemplate, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
import { OutputFixingParser } from 'langchain/output_parsers'
import { LLMChain } from 'langchain/chains'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { IVisionChatModal, ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers'
import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
import { addImagesToMessages } from '../../../src/multiModalUtils'
import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
class LLMChain_Chains implements INode {
label: string
@@ -164,7 +163,6 @@ const runPrediction = async (
const socketIO = isStreaming ? options.socketIO : undefined
const socketIOClientId = isStreaming ? options.socketIOClientId : ''
const moderations = nodeData.inputs?.inputModeration as Moderation[]
let model = nodeData.inputs?.model as ChatOpenAI
if (moderations && moderations.length > 0) {
try {
@@ -183,24 +181,39 @@ const runPrediction = async (
* TO: { "value": "hello i am ben\n\n\thow are you?" }
*/
const promptValues = handleEscapeCharacters(promptValuesRaw, true)
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
if (chain.llm instanceof ChatOpenAI) {
const chatOpenAI = chain.llm as ChatOpenAI
if (llmSupportsVision(chain.llm)) {
const visionChatModel = chain.llm as IVisionChatModal
const messageContent = addImagesToMessages(nodeData, options, visionChatModel.multiModalOption)
if (messageContent?.length) {
// Change model to gpt-4-vision && max token to higher when using gpt-4-vision
chatOpenAI.modelName = 'gpt-4-vision-preview'
chatOpenAI.maxTokens = 1024
visionChatModel.setVisionModel()
// Add image to the message
if (chain.prompt instanceof PromptTemplate) {
const existingPromptTemplate = chain.prompt.template as string
let newChatPromptTemplate = ChatPromptTemplate.fromMessages([
HumanMessagePromptTemplate.fromTemplate(existingPromptTemplate)
const msg = HumanMessagePromptTemplate.fromTemplate([
...messageContent,
{
text: existingPromptTemplate
}
])
newChatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
chain.prompt = newChatPromptTemplate
msg.inputVariables = chain.prompt.inputVariables
chain.prompt = ChatPromptTemplate.fromMessages([msg])
} else if (chain.prompt instanceof ChatPromptTemplate) {
chain.prompt.promptMessages.push(new HumanMessage({ content: messageContent }))
if (chain.prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) {
const lastMessage = chain.prompt.promptMessages.pop() as HumanMessagePromptTemplate
const template = (lastMessage.prompt as PromptTemplate).template as string
const msg = HumanMessagePromptTemplate.fromTemplate([
...messageContent,
{
text: template
}
])
msg.inputVariables = lastMessage.inputVariables
chain.prompt.promptMessages.push(msg)
} else {
chain.prompt.promptMessages.push(new HumanMessage({ content: messageContent }))
}
} else if (chain.prompt instanceof FewShotPromptTemplate) {
let existingFewShotPromptTemplate = chain.prompt.examplePrompt.template as string
let newFewShotPromptTemplate = ChatPromptTemplate.fromMessages([
@@ -212,8 +225,7 @@ const runPrediction = async (
}
} else {
// revert to previous values if image upload is empty
chatOpenAI.modelName = model.configuredModel
chatOpenAI.maxTokens = model.configuredMaxToken
visionChatModel.revertToOriginalModel()
}
}
packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts
+24 -5
View File
@@ -1,8 +1,9 @@
import { AnthropicInput, ChatAnthropic } from '@langchain/anthropic'
import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic'
import { BaseCache } from '@langchain/core/caches'
import { BaseLLMParams } from '@langchain/core/language_models/llms'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { ChatAnthropic } from './FlowiseChatAntrhopic'
class ChatAnthropic_ChatModels implements INode {
label: string
@@ -19,12 +20,12 @@ class ChatAnthropic_ChatModels implements INode {
constructor() {
this.label = 'ChatAnthropic'
this.name = 'chatAnthropic'
this.version = 3.0
this.version = 4.0
this.type = 'ChatAnthropic'
this.icon = 'Anthropic.svg'
this.category = 'Chat Models'
this.description = 'Wrapper around ChatAnthropic large language models that use the Chat endpoint'
this.baseClasses = [this.type, ...getBaseClasses(ChatAnthropic)]
this.baseClasses = [this.type, ...getBaseClasses(LangchainChatAnthropic)]
this.credential = {
label: 'Connect Credential',
name: 'credential',
@@ -147,6 +148,15 @@ class ChatAnthropic_ChatModels implements INode {
step: 0.1,
optional: true,
additionalParams: true
},
{
label: 'Allow Image Uploads',
name: 'allowImageUploads',
type: 'boolean',
description:
'Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent',
default: false,
optional: true
}
]
}
@@ -163,6 +173,8 @@ class ChatAnthropic_ChatModels implements INode {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const anthropicApiKey = getCredentialParam('anthropicApiKey', credentialData, nodeData)
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
const obj: Partial<AnthropicInput> & BaseLLMParams & { anthropicApiKey?: string } = {
temperature: parseFloat(temperature),
modelName,
@@ -175,7 +187,14 @@ class ChatAnthropic_ChatModels implements INode {
if (topK) obj.topK = parseFloat(topK)
if (cache) obj.cache = cache
const model = new ChatAnthropic(obj)
const multiModalOption: IMultiModalOption = {
image: {
allowImageUploads: allowImageUploads ?? false
}
}
const model = new ChatAnthropic(nodeData.id, obj)
model.setMultiModalOption(multiModalOption)
return model
}
}
packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts
+33
View File
@@ -0,0 +1,33 @@
import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic'
import { IVisionChatModal, IMultiModalOption } from '../../../src'
import { BaseLLMParams } from '@langchain/core/language_models/llms'
export class ChatAnthropic extends LangchainChatAnthropic implements IVisionChatModal {
configuredModel: string
configuredMaxToken: number
multiModalOption: IMultiModalOption
id: string
constructor(id: string, fields: Partial<AnthropicInput> & BaseLLMParams & { anthropicApiKey?: string }) {
super(fields)
this.id = id
this.configuredModel = fields?.modelName || 'claude-3-opus-20240229'
this.configuredMaxToken = fields?.maxTokens ?? 256
}
revertToOriginalModel(): void {
super.modelName = this.configuredModel
super.maxTokens = this.configuredMaxToken
}
setMultiModalOption(multiModalOption: IMultiModalOption): void {
this.multiModalOption = multiModalOption
}
setVisionModel(): void {
if (!this.modelName.startsWith('claude-3')) {
super.modelName = 'claude-3-opus-20240229'
super.maxTokens = 1024
}
}
}
packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts
+2 -3
View File
@@ -228,7 +228,7 @@ class ChatOpenAI_ChatModels implements INode {
const obj: Partial<OpenAIChatInput> &
Partial<AzureOpenAIInput> &
BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption } = {
BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput } = {
temperature: parseFloat(temperature),
modelName,
openAIApiKey,
@@ -265,10 +265,9 @@ class ChatOpenAI_ChatModels implements INode {
imageResolution
}
}
obj.multiModalOption = multiModalOption
const model = new ChatOpenAI(nodeData.id, obj)
model.setMultiModalOption(multiModalOption)
return model
}
}
packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts
+19 -19
View File
@@ -1,39 +1,39 @@
import type { ClientOptions } from 'openai'
import {
ChatOpenAI as LangchainChatOpenAI,
OpenAIChatInput,
LegacyOpenAIInput,
AzureOpenAIInput,
ChatOpenAICallOptions
} from '@langchain/openai'
import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, LegacyOpenAIInput, AzureOpenAIInput } from '@langchain/openai'
import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
import { BaseMessageLike } from '@langchain/core/messages'
import { Callbacks } from '@langchain/core/callbacks/manager'
import { LLMResult } from '@langchain/core/outputs'
import { IMultiModalOption } from '../../../src'
import { IMultiModalOption, IVisionChatModal } from '../../../src'
export class ChatOpenAI extends LangchainChatOpenAI {
export class ChatOpenAI extends LangchainChatOpenAI implements IVisionChatModal {
configuredModel: string
configuredMaxToken?: number
multiModalOption?: IMultiModalOption
configuredMaxToken: number
multiModalOption: IMultiModalOption
id: string
constructor(
id: string,
fields?: Partial<OpenAIChatInput> &
Partial<AzureOpenAIInput> &
BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption },
BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput },
/** @deprecated */
configuration?: ClientOptions & LegacyOpenAIInput
) {
super(fields, configuration)
this.id = id
this.multiModalOption = fields?.multiModalOption
this.configuredModel = fields?.modelName ?? 'gpt-3.5-turbo'
this.configuredMaxToken = fields?.maxTokens
this.configuredMaxToken = fields?.maxTokens ?? 256
}
async generate(messages: BaseMessageLike[][], options?: string[] | ChatOpenAICallOptions, callbacks?: Callbacks): Promise<LLMResult> {
return super.generate(messages, options, callbacks)
revertToOriginalModel(): void {
super.modelName = this.configuredModel
super.maxTokens = this.configuredMaxToken
}
setMultiModalOption(multiModalOption: IMultiModalOption): void {
this.multiModalOption = multiModalOption
}
setVisionModel(): void {
super.modelName = 'gpt-4-vision-preview'
super.maxTokens = 1024
}
}
packages/components/package.json
+1 -1
View File
@@ -29,7 +29,7 @@
"@google-ai/generativelanguage": "^0.2.1",
"@google/generative-ai": "^0.1.3",
"@huggingface/inference": "^2.6.1",
"@langchain/anthropic": "^0.0.10",
"@langchain/anthropic": "^0.1.4",
"@langchain/cohere": "^0.0.5",
"@langchain/community": "^0.0.30",
"@langchain/google-genai": "^0.0.10",
packages/components/src/Interface.ts
+11
View File
@@ -270,3 +270,14 @@ export abstract class FlowiseSummaryMemory extends ConversationSummaryMemory imp
abstract addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId?: string): Promise<void>
abstract clearChatMessages(overrideSessionId?: string): Promise<void>
}
export interface IVisionChatModal {
id: string
configuredModel: string
configuredMaxToken: number
multiModalOption: IMultiModalOption
setVisionModel(): void
revertToOriginalModel(): void
setMultiModalOption(multiModalOption: IMultiModalOption): void
}
packages/components/src/multiModalUtils.ts
+4 -3
View File
@@ -1,5 +1,4 @@
import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface'
import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI'
import { IVisionChatModal, ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface'
import path from 'path'
import { getStoragePath } from './utils'
import fs from 'fs'
@@ -12,7 +11,7 @@ export const addImagesToMessages = (
const imageContent: MessageContentImageUrl[] = []
let model = nodeData.inputs?.model
if (model instanceof ChatOpenAI && multiModalOption) {
if (llmSupportsVision(model) && multiModalOption) {
// Image Uploaded
if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) {
const imageUploads = getImageUploads(options.uploads)
@@ -46,3 +45,5 @@ export const getAudioUploads = (uploads: IFileUpload[]) => {
export const getImageUploads = (uploads: IFileUpload[]) => {
return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('image/'))
}
export const llmSupportsVision = (value: any): value is IVisionChatModal => !!value?.multiModalOption
packages/server/marketplaces/chatflows/Claude LLM.json
+12 -2
View File
@@ -160,7 +160,7 @@
"data": {
"id": "chatAnthropic_0",
"label": "ChatAnthropic",
"version": 3,
"version": 4,
"name": "chatAnthropic",
"type": "ChatAnthropic",
"baseClasses": ["ChatAnthropic", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -288,6 +288,15 @@
"optional": true,
"additionalParams": true,
"id": "chatAnthropic_0-input-topK-number"
},
{
"label": "Allow Image Uploads",
"name": "allowImageUploads",
"type": "boolean",
"description": "Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
"default": false,
"optional": true,
"id": "chatAnthropic_0-input-allowImageUploads-boolean"
}
],
"inputAnchors": [
@@ -305,7 +314,8 @@
"temperature": 0.9,
"maxTokensToSample": "",
"topP": "",
"topK": ""
"topK": "",
"allowImageUploads": true
},
"outputAnchors": [
{
packages/server/src/index.ts
+1 -1
View File
@@ -1533,7 +1533,7 @@ export class App {
if (!chatflow) return `Chatflow ${chatflowid} not found`
const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent']
const uploadProcessingNodes = ['chatOpenAI']
const uploadProcessingNodes = ['chatOpenAI', 'chatAnthropic']
const flowObj = JSON.parse(chatflow.flowData)
const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = []