mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-29 13:01:14 +03:00
GPT Vision: Added multi model capabilities to ChatOpenAI and ConversationChain.
This commit is contained in:
vinodkiran
@@ -1,4 +1,4 @@
|
||||
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { ConversationChain } from 'langchain/chains'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from 'langchain/prompts'
|
||||
@@ -8,6 +8,8 @@ import { flatten } from 'lodash'
|
||||
import { Document } from 'langchain/document'
|
||||
import { RunnableSequence } from 'langchain/schema/runnable'
|
||||
import { StringOutputParser } from 'langchain/schema/output_parser'
|
||||
import { addImagesToMessages, processSpeechToText } from '../../../src/MultiModalUtils'
|
||||
import { HumanMessage } from 'langchain/schema'
|
||||
|
||||
let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.`
|
||||
const inputKey = 'input'
|
||||
@@ -67,13 +69,15 @@ class ConversationChain_Chains implements INode {
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const chain = prepareChain(nodeData, this.sessionId, options.chatHistory)
|
||||
const chain = prepareChain(nodeData, options, this.sessionId)
|
||||
return chain
|
||||
}
|
||||
|
||||
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
|
||||
const memory = nodeData.inputs?.memory
|
||||
const chain = prepareChain(nodeData, this.sessionId, options.chatHistory)
|
||||
input = await processSpeechToText(nodeData, input, options)
|
||||
|
||||
const chain = prepareChain(nodeData, options, this.sessionId)
|
||||
|
||||
const loggerHandler = new ConsoleCallbackHandler(options.logger)
|
||||
const callbacks = await additionalCallbacks(nodeData, options)
|
||||
@@ -105,7 +109,7 @@ class ConversationChain_Chains implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
const prepareChatPrompt = (nodeData: INodeData) => {
|
||||
const prepareChatPrompt = (nodeData: INodeData, options: ICommonObject) => {
|
||||
const memory = nodeData.inputs?.memory as FlowiseMemory
|
||||
const prompt = nodeData.inputs?.systemMessagePrompt as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
@@ -128,16 +132,31 @@ const prepareChatPrompt = (nodeData: INodeData) => {
|
||||
|
||||
if (finalText) systemMessage = `${systemMessage}\nThe AI has the following context:\n${finalText}`
|
||||
|
||||
const chatPrompt = ChatPromptTemplate.fromMessages([
|
||||
// TODO: add audio uploads
|
||||
// if (options.uploads.length > 0) {
|
||||
// const audioUploads = getAudioUploads(options.uploads)
|
||||
// for (const upload of audioUploads) {
|
||||
// await this.processAudioWithWhisper(upload, chatMessages)
|
||||
// }
|
||||
// }
|
||||
const imageContent = addImagesToMessages(nodeData, options)
|
||||
|
||||
//TODO, this should not be any[], what interface should it be?
|
||||
let promptMessages: any[] = [
|
||||
SystemMessagePromptTemplate.fromTemplate(prompt ? `${prompt}\n${systemMessage}` : systemMessage),
|
||||
new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
|
||||
HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)
|
||||
])
|
||||
]
|
||||
if (imageContent.length > 0) {
|
||||
promptMessages.push(new HumanMessage({ content: imageContent }))
|
||||
}
|
||||
const chatPrompt = ChatPromptTemplate.fromMessages(promptMessages)
|
||||
|
||||
return chatPrompt
|
||||
}
|
||||
|
||||
const prepareChain = (nodeData: INodeData, sessionId?: string, chatHistory: IMessage[] = []) => {
|
||||
const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
|
||||
const chatHistory = options.chatHistory
|
||||
const model = nodeData.inputs?.model as BaseChatModel
|
||||
const memory = nodeData.inputs?.memory as FlowiseMemory
|
||||
const memoryKey = memory.memoryKey ?? 'chat_history'
|
||||
@@ -150,7 +169,7 @@ const prepareChain = (nodeData: INodeData, sessionId?: string, chatHistory: IMes
|
||||
return history
|
||||
}
|
||||
},
|
||||
prepareChatPrompt(nodeData),
|
||||
prepareChatPrompt(nodeData, options),
|
||||
model,
|
||||
new StringOutputParser()
|
||||
])
|
||||
|
||||
@@ -19,7 +19,7 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
constructor() {
|
||||
this.label = 'ChatOpenAI'
|
||||
this.name = 'chatOpenAI'
|
||||
this.version = 2.0
|
||||
this.version = 3.0
|
||||
this.type = 'ChatOpenAI'
|
||||
this.icon = 'openai.svg'
|
||||
this.category = 'Chat Models'
|
||||
@@ -152,6 +152,73 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Audio Uploads',
|
||||
name: 'allowAudioUploads',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Speech to Text',
|
||||
name: 'allowSpeechToText',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
// TODO: only show when speechToText is true
|
||||
{
|
||||
label: 'Speech to Text Method',
|
||||
description: 'How to turn audio into text',
|
||||
name: 'speechToTextMode',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Transcriptions',
|
||||
name: 'transcriptions',
|
||||
description:
|
||||
'Transcribe audio into whatever language the audio is in. Default method when Speech to Text is turned on.'
|
||||
},
|
||||
{
|
||||
label: 'Translations',
|
||||
name: 'translations',
|
||||
description: 'Translate and transcribe the audio into english.'
|
||||
}
|
||||
],
|
||||
optional: false,
|
||||
default: 'transcriptions',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Image Resolution',
|
||||
description: 'This parameter controls the resolution in which the model views the image.',
|
||||
name: 'imageResolution',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'low'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'high'
|
||||
},
|
||||
{
|
||||
label: 'Auto',
|
||||
name: 'auto'
|
||||
}
|
||||
],
|
||||
default: 'low',
|
||||
optional: false,
|
||||
additionalParams: true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -168,6 +235,12 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
const basePath = nodeData.inputs?.basepath as string
|
||||
const baseOptions = nodeData.inputs?.baseOptions
|
||||
|
||||
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
|
||||
const allowAudioUploads = nodeData.inputs?.allowAudioUploads as boolean
|
||||
const allowSpeechToText = nodeData.inputs?.allowSpeechToText as boolean
|
||||
const speechToTextMode = nodeData.inputs?.speechToTextMode as string
|
||||
const imageResolution = nodeData.inputs?.imageResolution as string
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
|
||||
|
||||
@@ -200,6 +273,20 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
basePath,
|
||||
baseOptions: parsedBaseOptions
|
||||
})
|
||||
|
||||
const multiModal = {
|
||||
allowImageUploads: allowImageUploads ?? false,
|
||||
allowAudioUploads: allowAudioUploads ?? false,
|
||||
allowSpeechToText: allowSpeechToText ?? false,
|
||||
imageResolution,
|
||||
speechToTextMode
|
||||
}
|
||||
Object.defineProperty(model, 'multiModal', {
|
||||
enumerable: true,
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: multiModal
|
||||
})
|
||||
return model
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user