mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 13:00:56 +03:00
Merge branch 'main' into FEATURE/Vision
# Conflicts: # packages/components/nodes/chains/ConversationChain/ConversationChain.ts # packages/server/src/index.ts # packages/server/src/utils/index.ts
This commit is contained in:
Henry
@@ -1,14 +1,15 @@
|
||||
import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { ConversationChain } from 'langchain/chains'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
|
||||
import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from 'langchain/prompts'
|
||||
import { BaseChatModel } from 'langchain/chat_models/base'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import { flatten } from 'lodash'
|
||||
import { Document } from 'langchain/document'
|
||||
import { RunnableSequence } from 'langchain/schema/runnable'
|
||||
import { StringOutputParser } from 'langchain/schema/output_parser'
|
||||
import { injectChainNodeData } from '../../../src/MultiModalUtils'
|
||||
import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
|
||||
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
|
||||
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
|
||||
import { injectChainNodeData } from '../../../src/multiModalUtils'
|
||||
|
||||
let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.`
|
||||
const inputKey = 'input'
|
||||
@@ -28,7 +29,7 @@ class ConversationChain_Chains implements INode {
|
||||
constructor(fields?: { sessionId?: string }) {
|
||||
this.label = 'Conversation Chain'
|
||||
this.name = 'conversationChain'
|
||||
this.version = 1.0
|
||||
this.version = 3.0
|
||||
this.type = 'ConversationChain'
|
||||
this.icon = 'conv.svg'
|
||||
this.category = 'Chains'
|
||||
@@ -45,6 +46,14 @@ class ConversationChain_Chains implements INode {
|
||||
name: 'memory',
|
||||
type: 'BaseMemory'
|
||||
},
|
||||
{
|
||||
label: 'Chat Prompt Template',
|
||||
name: 'chatPromptTemplate',
|
||||
type: 'ChatPromptTemplate',
|
||||
description: 'Override existing prompt with Chat Prompt Template. Human Message must includes {input} variable',
|
||||
optional: true
|
||||
},
|
||||
/* Deprecated
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
@@ -53,15 +62,25 @@ class ConversationChain_Chains implements INode {
|
||||
'Include whole document into the context window, if you get maximum context length error, please use model with higher context window like Claude 100k, or gpt4 32k',
|
||||
optional: true,
|
||||
list: true
|
||||
},*/
|
||||
{
|
||||
label: 'Input Moderation',
|
||||
description: 'Detect text that could generate harmful output and prevent it from being sent to the language model',
|
||||
name: 'inputModeration',
|
||||
type: 'Moderation',
|
||||
optional: true,
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'System Message',
|
||||
name: 'systemMessagePrompt',
|
||||
type: 'string',
|
||||
rows: 4,
|
||||
description: 'If Chat Prompt Template is provided, this will be ignored',
|
||||
additionalParams: true,
|
||||
optional: true,
|
||||
placeholder: 'You are a helpful assistant that write codes'
|
||||
default: systemMessage,
|
||||
placeholder: systemMessage
|
||||
}
|
||||
]
|
||||
this.sessionId = fields?.sessionId
|
||||
@@ -72,22 +91,40 @@ class ConversationChain_Chains implements INode {
|
||||
return chain
|
||||
}
|
||||
|
||||
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
|
||||
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
|
||||
const memory = nodeData.inputs?.memory
|
||||
injectChainNodeData(nodeData, options)
|
||||
|
||||
const chain = prepareChain(nodeData, options, this.sessionId)
|
||||
const moderations = nodeData.inputs?.inputModeration as Moderation[]
|
||||
|
||||
if (moderations && moderations.length > 0) {
|
||||
try {
|
||||
// Use the output of the moderation chain as input for the LLM chain
|
||||
input = await checkInputs(moderations, input)
|
||||
} catch (e) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
|
||||
return formatResponse(e.message)
|
||||
}
|
||||
}
|
||||
|
||||
const loggerHandler = new ConsoleCallbackHandler(options.logger)
|
||||
const callbacks = await additionalCallbacks(nodeData, options)
|
||||
const additionalCallback = await additionalCallbacks(nodeData, options)
|
||||
|
||||
let res = ''
|
||||
let callbacks = [loggerHandler, ...additionalCallback]
|
||||
|
||||
if (process.env.DEBUG === 'true') {
|
||||
callbacks.push(new LCConsoleCallbackHandler())
|
||||
}
|
||||
|
||||
if (options.socketIO && options.socketIOClientId) {
|
||||
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
|
||||
res = await chain.invoke({ input }, { callbacks: [loggerHandler, handler, ...callbacks] })
|
||||
callbacks.push(handler)
|
||||
res = await chain.invoke({ input }, { callbacks })
|
||||
} else {
|
||||
res = await chain.invoke({ input }, { callbacks: [loggerHandler, ...callbacks] })
|
||||
res = await chain.invoke({ input }, { callbacks })
|
||||
}
|
||||
|
||||
await memory.addChatMessages(
|
||||
@@ -108,36 +145,33 @@ class ConversationChain_Chains implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
const prepareChatPrompt = (nodeData: INodeData, options: ICommonObject) => {
|
||||
const prepareChatPrompt = (nodeData: INodeData) => {
|
||||
const memory = nodeData.inputs?.memory as FlowiseMemory
|
||||
const prompt = nodeData.inputs?.systemMessagePrompt as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) {
|
||||
const sysPrompt = chatPromptTemplate.promptMessages[0]
|
||||
const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1]
|
||||
const chatPrompt = ChatPromptTemplate.fromMessages([
|
||||
sysPrompt,
|
||||
new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
|
||||
humanPrompt
|
||||
])
|
||||
|
||||
if ((chatPromptTemplate as any).promptValues) {
|
||||
// @ts-ignore
|
||||
chatPrompt.promptValues = (chatPromptTemplate as any).promptValues
|
||||
}
|
||||
|
||||
return chatPrompt
|
||||
}
|
||||
|
||||
let finalText = ''
|
||||
for (let i = 0; i < finalDocs.length; i += 1) {
|
||||
finalText += finalDocs[i].pageContent
|
||||
}
|
||||
|
||||
const replaceChar: string[] = ['{', '}']
|
||||
for (const char of replaceChar) finalText = finalText.replaceAll(char, '')
|
||||
|
||||
if (finalText) systemMessage = `${systemMessage}\nThe AI has the following context:\n${finalText}`
|
||||
|
||||
//TODO, this should not be any[], what interface should it be?
|
||||
let promptMessages: any[] = [
|
||||
SystemMessagePromptTemplate.fromTemplate(prompt ? `${prompt}\n${systemMessage}` : systemMessage),
|
||||
const chatPrompt = ChatPromptTemplate.fromMessages([
|
||||
SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage),
|
||||
new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
|
||||
HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)
|
||||
]
|
||||
const chatPrompt = ChatPromptTemplate.fromMessages(promptMessages)
|
||||
])
|
||||
|
||||
return chatPrompt
|
||||
}
|
||||
@@ -148,15 +182,31 @@ const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: s
|
||||
const memory = nodeData.inputs?.memory as FlowiseMemory
|
||||
const memoryKey = memory.memoryKey ?? 'chat_history'
|
||||
|
||||
const chatPrompt = prepareChatPrompt(nodeData)
|
||||
let promptVariables = {}
|
||||
const promptValuesRaw = (chatPrompt as any).promptValues
|
||||
if (promptValuesRaw) {
|
||||
const promptValues = handleEscapeCharacters(promptValuesRaw, true)
|
||||
for (const val in promptValues) {
|
||||
promptVariables = {
|
||||
...promptVariables,
|
||||
[val]: () => {
|
||||
return promptValues[val]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const conversationChain = RunnableSequence.from([
|
||||
{
|
||||
[inputKey]: (input: { input: string }) => input.input,
|
||||
[memoryKey]: async () => {
|
||||
const history = await memory.getChatMessages(sessionId, true, chatHistory)
|
||||
return history
|
||||
}
|
||||
},
|
||||
...promptVariables
|
||||
},
|
||||
prepareChatPrompt(nodeData, options),
|
||||
prepareChatPrompt(nodeData),
|
||||
model,
|
||||
new StringOutputParser()
|
||||
])
|
||||
|
||||
+9
-2
@@ -13,6 +13,7 @@ import { applyPatch } from 'fast-json-patch'
|
||||
import { convertBaseMessagetoIMessage, getBaseClasses } from '../../../src/utils'
|
||||
import { ConsoleCallbackHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams, MemoryMethods } from '../../../src/Interface'
|
||||
import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
|
||||
|
||||
type RetrievalChainInput = {
|
||||
chat_history: string
|
||||
@@ -176,11 +177,17 @@ class ConversationalRetrievalQAChain_Chains implements INode {
|
||||
const history = ((await memory.getChatMessages(this.sessionId, false, options.chatHistory)) as IMessage[]) ?? []
|
||||
|
||||
const loggerHandler = new ConsoleCallbackHandler(options.logger)
|
||||
const callbacks = await additionalCallbacks(nodeData, options)
|
||||
const additionalCallback = await additionalCallbacks(nodeData, options)
|
||||
|
||||
let callbacks = [loggerHandler, ...additionalCallback]
|
||||
|
||||
if (process.env.DEBUG === 'true') {
|
||||
callbacks.push(new LCConsoleCallbackHandler())
|
||||
}
|
||||
|
||||
const stream = answerChain.streamLog(
|
||||
{ question: input, chat_history: history },
|
||||
{ callbacks: [loggerHandler, ...callbacks] },
|
||||
{ callbacks },
|
||||
{
|
||||
includeNames: [sourceRunnableName]
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import { formatResponse, injectOutputParser } from '../../outputparsers/OutputPa
|
||||
import { BaseLLMOutputParser } from 'langchain/schema/output_parser'
|
||||
import { OutputFixingParser } from 'langchain/output_parsers'
|
||||
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
|
||||
import { injectChainNodeData } from '../../../src/MultiModalUtils'
|
||||
import { injectChainNodeData } from '../../../src/multiModalUtils'
|
||||
|
||||
class LLMChain_Chains implements INode {
|
||||
label: string
|
||||
@@ -83,7 +83,7 @@ class LLMChain_Chains implements INode {
|
||||
const model = nodeData.inputs?.model as BaseLanguageModel
|
||||
const prompt = nodeData.inputs?.prompt
|
||||
const output = nodeData.outputs?.output as string
|
||||
const promptValues = prompt.promptValues as ICommonObject
|
||||
let promptValues: ICommonObject | undefined = nodeData.inputs?.prompt.promptValues as ICommonObject
|
||||
const llmOutputParser = nodeData.inputs?.outputParser as BaseOutputParser
|
||||
this.outputParser = llmOutputParser
|
||||
if (llmOutputParser) {
|
||||
@@ -108,17 +108,25 @@ class LLMChain_Chains implements INode {
|
||||
verbose: process.env.DEBUG === 'true'
|
||||
})
|
||||
const inputVariables = chain.prompt.inputVariables as string[] // ["product"]
|
||||
injectChainNodeData(nodeData, options)
|
||||
promptValues = injectOutputParser(this.outputParser, chain, promptValues)
|
||||
const res = await runPrediction(inputVariables, chain, input, promptValues, options, nodeData)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\x1b[92m\x1b[1m\n*****OUTPUT PREDICTION*****\n\x1b[0m\x1b[0m')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(res)
|
||||
|
||||
let finalRes = res
|
||||
if (this.outputParser && typeof res === 'object' && Object.prototype.hasOwnProperty.call(res, 'json')) {
|
||||
finalRes = (res as ICommonObject).json
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply string transformation to convert special chars:
|
||||
* FROM: hello i am ben\n\n\thow are you?
|
||||
* TO: hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?
|
||||
*/
|
||||
return handleEscapeCharacters(res, false)
|
||||
return handleEscapeCharacters(finalRes, false)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,340 +0,0 @@
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils'
|
||||
import { OpenAIMultiModalChainInput, VLLMChain } from './VLLMChain'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
|
||||
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
|
||||
|
||||
class OpenAIMultiModalChain_Chains implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
type: string
|
||||
icon: string
|
||||
badge: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
description: string
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
credential: INodeParams
|
||||
|
||||
constructor() {
|
||||
this.label = 'Open AI MultiModal Chain'
|
||||
this.name = 'openAIMultiModalChain'
|
||||
this.version = 1.0
|
||||
this.type = 'OpenAIMultiModalChain'
|
||||
this.icon = 'chain.svg'
|
||||
this.category = 'Chains'
|
||||
this.badge = 'BETA'
|
||||
this.description = 'Chain to query against Image and Audio Input.'
|
||||
this.baseClasses = [this.type, ...getBaseClasses(VLLMChain)]
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['openAIApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Prompt',
|
||||
name: 'prompt',
|
||||
type: 'BasePromptTemplate',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Input Moderation',
|
||||
description: 'Detect text that could generate harmful output and prevent it from being sent to the language model',
|
||||
name: 'inputModeration',
|
||||
type: 'Moderation',
|
||||
optional: true,
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Model Name',
|
||||
name: 'modelName',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'gpt-4-vision-preview',
|
||||
name: 'gpt-4-vision-preview'
|
||||
}
|
||||
],
|
||||
default: 'gpt-4-vision-preview'
|
||||
},
|
||||
{
|
||||
label: 'Speech to Text',
|
||||
name: 'speechToText',
|
||||
type: 'boolean',
|
||||
optional: true
|
||||
},
|
||||
// TODO: only show when speechToText is true
|
||||
{
|
||||
label: 'Speech to Text Method',
|
||||
description: 'How to turn audio into text',
|
||||
name: 'speechToTextMode',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Transcriptions',
|
||||
name: 'transcriptions',
|
||||
description:
|
||||
'Transcribe audio into whatever language the audio is in. Default method when Speech to Text is turned on.'
|
||||
},
|
||||
{
|
||||
label: 'Translations',
|
||||
name: 'translations',
|
||||
description: 'Translate and transcribe the audio into english.'
|
||||
}
|
||||
],
|
||||
optional: false,
|
||||
default: 'transcriptions',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Image Resolution',
|
||||
description: 'This parameter controls the resolution in which the model views the image.',
|
||||
name: 'imageResolution',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'low'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'high'
|
||||
},
|
||||
{
|
||||
label: 'Auto',
|
||||
name: 'auto'
|
||||
}
|
||||
],
|
||||
default: 'low',
|
||||
optional: false,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Temperature',
|
||||
name: 'temperature',
|
||||
type: 'number',
|
||||
step: 0.1,
|
||||
default: 0.9,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top Probability',
|
||||
name: 'topP',
|
||||
type: 'number',
|
||||
step: 0.1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Max Tokens',
|
||||
name: 'maxTokens',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Accepted Upload Types',
|
||||
name: 'allowedUploadTypes',
|
||||
type: 'string',
|
||||
default: 'image/gif;image/jpeg;image/png;image/webp;audio/mpeg;audio/x-wav;audio/mp4',
|
||||
hidden: true
|
||||
},
|
||||
{
|
||||
label: 'Maximum Upload Size (MB)',
|
||||
name: 'maxUploadSize',
|
||||
type: 'number',
|
||||
default: '5',
|
||||
hidden: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Open AI MultiModal Chain',
|
||||
name: 'openAIMultiModalChain',
|
||||
baseClasses: [this.type, ...getBaseClasses(VLLMChain)]
|
||||
},
|
||||
{
|
||||
label: 'Output Prediction',
|
||||
name: 'outputPrediction',
|
||||
baseClasses: ['string', 'json']
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, input: string, options: ICommonObject): Promise<any> {
|
||||
const prompt = nodeData.inputs?.prompt
|
||||
const output = nodeData.outputs?.output as string
|
||||
const imageResolution = nodeData.inputs?.imageResolution
|
||||
const promptValues = prompt.promptValues as ICommonObject
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
|
||||
const temperature = nodeData.inputs?.temperature as string
|
||||
const modelName = nodeData.inputs?.modelName as string
|
||||
const maxTokens = nodeData.inputs?.maxTokens as string
|
||||
const topP = nodeData.inputs?.topP as string
|
||||
const speechToText = nodeData.inputs?.speechToText as boolean
|
||||
|
||||
const fields: OpenAIMultiModalChainInput = {
|
||||
openAIApiKey: openAIApiKey,
|
||||
imageResolution: imageResolution,
|
||||
verbose: process.env.DEBUG === 'true',
|
||||
uploads: options.uploads,
|
||||
modelName: modelName
|
||||
}
|
||||
if (temperature) fields.temperature = parseFloat(temperature)
|
||||
if (maxTokens) fields.maxTokens = parseInt(maxTokens, 10)
|
||||
if (topP) fields.topP = parseFloat(topP)
|
||||
if (speechToText) {
|
||||
const speechToTextMode = nodeData.inputs?.speechToTextMode ?? 'transcriptions'
|
||||
if (speechToTextMode) fields.speechToTextMode = speechToTextMode
|
||||
}
|
||||
|
||||
if (output === this.name) {
|
||||
const chain = new VLLMChain({
|
||||
...fields,
|
||||
prompt: prompt
|
||||
})
|
||||
return chain
|
||||
} else if (output === 'outputPrediction') {
|
||||
const chain = new VLLMChain({
|
||||
...fields
|
||||
})
|
||||
const inputVariables: string[] = prompt.inputVariables as string[] // ["product"]
|
||||
const res = await runPrediction(inputVariables, chain, input, promptValues, options, nodeData)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\x1b[92m\x1b[1m\n*****OUTPUT PREDICTION*****\n\x1b[0m\x1b[0m')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(res)
|
||||
/**
|
||||
* Apply string transformation to convert special chars:
|
||||
* FROM: hello i am ben\n\n\thow are you?
|
||||
* TO: hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?
|
||||
*/
|
||||
return handleEscapeCharacters(res, false)
|
||||
}
|
||||
}
|
||||
|
||||
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
|
||||
const prompt = nodeData.inputs?.prompt
|
||||
const inputVariables: string[] = prompt.inputVariables as string[] // ["product"]
|
||||
const chain = nodeData.instance as VLLMChain
|
||||
let promptValues: ICommonObject | undefined = nodeData.inputs?.prompt.promptValues as ICommonObject
|
||||
const res = await runPrediction(inputVariables, chain, input, promptValues, options, nodeData)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\x1b[93m\x1b[1m\n*****FINAL RESULT*****\n\x1b[0m\x1b[0m')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(res)
|
||||
return res
|
||||
}
|
||||
}
|
||||
|
||||
const runPrediction = async (
|
||||
inputVariables: string[],
|
||||
chain: VLLMChain,
|
||||
input: string,
|
||||
promptValuesRaw: ICommonObject | undefined,
|
||||
options: ICommonObject,
|
||||
nodeData: INodeData
|
||||
) => {
|
||||
const loggerHandler = new ConsoleCallbackHandler(options.logger)
|
||||
const callbacks = await additionalCallbacks(nodeData, options)
|
||||
|
||||
const isStreaming = options.socketIO && options.socketIOClientId
|
||||
const socketIO = isStreaming ? options.socketIO : undefined
|
||||
const socketIOClientId = isStreaming ? options.socketIOClientId : ''
|
||||
const moderations = nodeData.inputs?.inputModeration as Moderation[]
|
||||
const speechToText = nodeData.inputs?.speechToText as boolean
|
||||
|
||||
if (options?.uploads) {
|
||||
if (options.uploads.length === 1 && input.length === 0) {
|
||||
if (speechToText) {
|
||||
//special case, text input is empty, but we have an upload (recorded audio)
|
||||
const convertedText = await chain.processAudioWithWisper(options.uploads[0], undefined)
|
||||
//so we use the upload as input
|
||||
input = convertedText
|
||||
}
|
||||
// do not send the audio file to the model
|
||||
} else {
|
||||
chain.uploads = options.uploads
|
||||
}
|
||||
}
|
||||
|
||||
if (moderations && moderations.length > 0) {
|
||||
try {
|
||||
// Use the output of the moderation chain as input for the LLM chain
|
||||
input = await checkInputs(moderations, input)
|
||||
} catch (e) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
streamResponse(isStreaming, e.message, socketIO, socketIOClientId)
|
||||
return formatResponse(e.message)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply string transformation to reverse converted special chars:
|
||||
* FROM: { "value": "hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?" }
|
||||
* TO: { "value": "hello i am ben\n\n\thow are you?" }
|
||||
*/
|
||||
const promptValues = handleEscapeCharacters(promptValuesRaw, true)
|
||||
if (promptValues && inputVariables.length > 0) {
|
||||
let seen: string[] = []
|
||||
|
||||
for (const variable of inputVariables) {
|
||||
seen.push(variable)
|
||||
if (promptValues[variable]) {
|
||||
chain.inputKey = variable
|
||||
seen.pop()
|
||||
}
|
||||
}
|
||||
|
||||
if (seen.length === 0) {
|
||||
// All inputVariables have fixed values specified
|
||||
const options = { ...promptValues }
|
||||
if (isStreaming) {
|
||||
const handler = new CustomChainHandler(socketIO, socketIOClientId)
|
||||
const res = await chain.call(options, [loggerHandler, handler, ...callbacks])
|
||||
return formatResponse(res?.text)
|
||||
} else {
|
||||
const res = await chain.call(options, [loggerHandler, ...callbacks])
|
||||
return formatResponse(res?.text)
|
||||
}
|
||||
} else if (seen.length === 1) {
|
||||
// If one inputVariable is not specify, use input (user's question) as value
|
||||
const lastValue = seen.pop()
|
||||
if (!lastValue) throw new Error('Please provide Prompt Values')
|
||||
chain.inputKey = lastValue as string
|
||||
const options = {
|
||||
...promptValues,
|
||||
[lastValue]: input
|
||||
}
|
||||
if (isStreaming) {
|
||||
const handler = new CustomChainHandler(socketIO, socketIOClientId)
|
||||
const res = await chain.call(options, [loggerHandler, handler, ...callbacks])
|
||||
return formatResponse(res?.text)
|
||||
} else {
|
||||
const res = await chain.call(options, [loggerHandler, ...callbacks])
|
||||
return formatResponse(res?.text)
|
||||
}
|
||||
} else {
|
||||
throw new Error(`Please provide Prompt Values for: ${seen.join(', ')}`)
|
||||
}
|
||||
} else {
|
||||
if (isStreaming) {
|
||||
const handler = new CustomChainHandler(socketIO, socketIOClientId)
|
||||
const res = await chain.run(input, [loggerHandler, handler, ...callbacks])
|
||||
return formatResponse(res)
|
||||
} else {
|
||||
const res = await chain.run(input, [loggerHandler, ...callbacks])
|
||||
return formatResponse(res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: OpenAIMultiModalChain_Chains }
|
||||
@@ -1,216 +0,0 @@
|
||||
import { OpenAI as OpenAIClient, ClientOptions, OpenAI } from 'openai'
|
||||
import { BaseChain, ChainInputs } from 'langchain/chains'
|
||||
import { ChainValues } from 'langchain/schema'
|
||||
import { BasePromptTemplate, ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate } from 'langchain/prompts'
|
||||
import path from 'path'
|
||||
import { getUserHome } from '../../../src/utils'
|
||||
import fs from 'fs'
|
||||
import { ChatCompletionContentPart, ChatCompletionMessageParam } from 'openai/src/resources/chat/completions'
|
||||
import ChatCompletionCreateParamsNonStreaming = OpenAI.ChatCompletionCreateParamsNonStreaming
|
||||
import { IFileUpload } from '../../../src'
|
||||
|
||||
/**
|
||||
* Interface for the input parameters of the OpenAIVisionChain class.
|
||||
*/
|
||||
export interface OpenAIMultiModalChainInput extends ChainInputs {
|
||||
openAIApiKey?: string
|
||||
openAIOrganization?: string
|
||||
throwError?: boolean
|
||||
prompt?: BasePromptTemplate
|
||||
configuration?: ClientOptions
|
||||
uploads?: IFileUpload[]
|
||||
imageResolution?: 'auto' | 'low' | 'high'
|
||||
temperature?: number
|
||||
modelName?: string
|
||||
maxTokens?: number
|
||||
topP?: number
|
||||
speechToTextMode?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Class representing a chain for generating text from an image using the OpenAI
|
||||
* Vision API. It extends the BaseChain class and implements the
|
||||
* OpenAIVisionChainInput interface.
|
||||
*/
|
||||
export class VLLMChain extends BaseChain implements OpenAIMultiModalChainInput {
|
||||
static lc_name() {
|
||||
return 'VLLMChain'
|
||||
}
|
||||
prompt: BasePromptTemplate | undefined
|
||||
|
||||
inputKey = 'input'
|
||||
outputKey = 'text'
|
||||
uploads?: IFileUpload[]
|
||||
imageResolution: 'auto' | 'low' | 'high'
|
||||
openAIApiKey?: string
|
||||
openAIOrganization?: string
|
||||
clientConfig: ClientOptions
|
||||
client: OpenAIClient
|
||||
throwError: boolean
|
||||
temperature?: number
|
||||
modelName?: string
|
||||
maxTokens?: number
|
||||
topP?: number
|
||||
|
||||
speechToTextMode?: any
|
||||
|
||||
constructor(fields: OpenAIMultiModalChainInput) {
|
||||
super(fields)
|
||||
this.throwError = fields?.throwError ?? false
|
||||
this.imageResolution = fields?.imageResolution ?? 'low'
|
||||
this.openAIApiKey = fields?.openAIApiKey
|
||||
this.prompt = fields?.prompt
|
||||
this.temperature = fields?.temperature
|
||||
this.modelName = fields?.modelName
|
||||
this.maxTokens = fields?.maxTokens
|
||||
this.topP = fields?.topP
|
||||
this.uploads = fields?.uploads ?? []
|
||||
this.speechToTextMode = fields?.speechToTextMode ?? {}
|
||||
if (!this.openAIApiKey) {
|
||||
throw new Error('OpenAI API key not found')
|
||||
}
|
||||
|
||||
this.openAIOrganization = fields?.openAIOrganization
|
||||
|
||||
this.clientConfig = {
|
||||
...fields?.configuration,
|
||||
apiKey: this.openAIApiKey,
|
||||
organization: this.openAIOrganization
|
||||
}
|
||||
|
||||
this.client = new OpenAIClient(this.clientConfig)
|
||||
}
|
||||
|
||||
async _call(values: ChainValues): Promise<ChainValues> {
|
||||
const userInput = values[this.inputKey]
|
||||
|
||||
const vRequest: ChatCompletionCreateParamsNonStreaming = {
|
||||
model: 'gpt-4-vision-preview',
|
||||
temperature: this.temperature,
|
||||
top_p: this.topP,
|
||||
messages: []
|
||||
}
|
||||
if (this.maxTokens) vRequest.max_tokens = this.maxTokens
|
||||
else vRequest.max_tokens = 1024
|
||||
|
||||
const chatMessages: ChatCompletionContentPart[] = []
|
||||
const userRole: ChatCompletionMessageParam = { role: 'user', content: [] }
|
||||
chatMessages.push({
|
||||
type: 'text',
|
||||
text: userInput
|
||||
})
|
||||
if (this.speechToTextMode && this.uploads && this.uploads.length > 0) {
|
||||
const audioUploads = this.getAudioUploads(this.uploads)
|
||||
for (const upload of audioUploads) {
|
||||
await this.processAudioWithWisper(upload, chatMessages)
|
||||
}
|
||||
}
|
||||
if (this.uploads && this.uploads.length > 0) {
|
||||
const imageUploads = this.getImageUploads(this.uploads)
|
||||
for (const upload of imageUploads) {
|
||||
let bf = upload.data
|
||||
if (upload.type == 'stored-file') {
|
||||
const filePath = path.join(getUserHome(), '.flowise', 'gptvision', upload.data, upload.name)
|
||||
|
||||
// as the image is stored in the server, read the file and convert it to base64
|
||||
const contents = fs.readFileSync(filePath)
|
||||
bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64')
|
||||
}
|
||||
chatMessages.push({
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: bf,
|
||||
detail: this.imageResolution
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
userRole.content = chatMessages
|
||||
vRequest.messages.push(userRole)
|
||||
if (this.prompt && this.prompt instanceof ChatPromptTemplate) {
|
||||
let chatPrompt = this.prompt as ChatPromptTemplate
|
||||
chatPrompt.promptMessages.forEach((message: any) => {
|
||||
if (message instanceof SystemMessagePromptTemplate) {
|
||||
vRequest.messages.push({
|
||||
role: 'system',
|
||||
content: (message.prompt as any).template
|
||||
})
|
||||
} else if (message instanceof HumanMessagePromptTemplate) {
|
||||
vRequest.messages.push({
|
||||
role: 'user',
|
||||
content: (message.prompt as any).template
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
let response
|
||||
try {
|
||||
response = await this.client.chat.completions.create(vRequest)
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
throw error
|
||||
} else {
|
||||
throw new Error(error as string)
|
||||
}
|
||||
}
|
||||
const output = response.choices[0]
|
||||
return {
|
||||
[this.outputKey]: output.message.content
|
||||
}
|
||||
}
|
||||
|
||||
public async processAudioWithWisper(upload: IFileUpload, chatMessages: ChatCompletionContentPart[] | undefined): Promise<string> {
|
||||
const filePath = path.join(getUserHome(), '.flowise', 'gptvision', upload.data, upload.name)
|
||||
|
||||
// as the image is stored in the server, read the file and convert it to base64
|
||||
const audio_file = fs.createReadStream(filePath)
|
||||
if (this.speechToTextMode === 'transcriptions') {
|
||||
const transcription = await this.client.audio.transcriptions.create({
|
||||
file: audio_file,
|
||||
model: 'whisper-1'
|
||||
})
|
||||
if (chatMessages) {
|
||||
chatMessages.push({
|
||||
type: 'text',
|
||||
text: transcription.text
|
||||
})
|
||||
}
|
||||
return transcription.text
|
||||
} else if (this.speechToTextMode === 'translations') {
|
||||
const translation = await this.client.audio.translations.create({
|
||||
file: audio_file,
|
||||
model: 'whisper-1'
|
||||
})
|
||||
if (chatMessages) {
|
||||
chatMessages.push({
|
||||
type: 'text',
|
||||
text: translation.text
|
||||
})
|
||||
}
|
||||
return translation.text
|
||||
}
|
||||
//should never get here
|
||||
return ''
|
||||
}
|
||||
|
||||
getAudioUploads = (urls: any[]) => {
|
||||
return urls.filter((url: any) => url.mime.startsWith('audio/'))
|
||||
}
|
||||
|
||||
getImageUploads = (urls: any[]) => {
|
||||
return urls.filter((url: any) => url.mime.startsWith('image/'))
|
||||
}
|
||||
|
||||
_chainType() {
|
||||
return 'vision_chain'
|
||||
}
|
||||
|
||||
get inputKeys() {
|
||||
return this.prompt?.inputVariables ?? [this.inputKey]
|
||||
}
|
||||
|
||||
get outputKeys(): string[] {
|
||||
return [this.outputKey]
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-dna" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||||
<path d="M14.828 14.828a4 4 0 1 0 -5.656 -5.656a4 4 0 0 0 5.656 5.656z"></path>
|
||||
<path d="M9.172 20.485a4 4 0 1 0 -5.657 -5.657"></path>
|
||||
<path d="M14.828 3.515a4 4 0 0 0 5.657 5.657"></path>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 489 B |
Reference in New Issue
Block a user