farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

reverting all image upload logic to individual chains/agents

Browse Source
This commit is contained in:
vinodkiran
2024-02-19 15:27:19 -08:00
parent 8bad360796
commit b31e8715f4
6 changed files with 177 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/src/multiModalUtils.ts
+26 -72
View File
@@ -1,84 +1,38 @@
import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface'
import { ChatOpenAI as LangchainChatOpenAI } from 'langchain/chat_models/openai'
import path from 'path'
import { getStoragePath } from './utils'
import fs from 'fs'
import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI'
import { LLMChain } from 'langchain/chains'
import { RunnableBinding, RunnableSequence } from 'langchain/schema/runnable'
import { AgentExecutor as LcAgentExecutor, ChatAgent, RunnableAgent } from 'langchain/agents'
import { AgentExecutor } from './agents'
export interface MultiModalOptions {
nodeOptions: ICommonObject
}
export const injectLLMChainNodeData = (nodeData: INodeData, options: ICommonObject) => {
let llmChain = nodeData.instance as LLMChain
;(llmChain.llm as ChatOpenAI).lc_kwargs.chainData = { nodeOptions: getUploadsFromOptions(options) }
}
export const injectAgentExecutorNodeData = (agentExecutor: AgentExecutor, nodeData: INodeData, options: ICommonObject) => {
if (agentExecutor.agent instanceof RunnableAgent && agentExecutor.agent.runnable instanceof RunnableSequence) {
let rs = agentExecutor.agent.runnable as RunnableSequence
injectRunnableNodeData(rs, nodeData, options)
}
}
export const injectLcAgentExecutorNodeData = (agentExecutor: LcAgentExecutor, nodeData: INodeData, options: ICommonObject) => {
if (agentExecutor.agent instanceof ChatAgent) {
let llmChain = agentExecutor.agent.llmChain as LLMChain
;(llmChain.llm as ChatOpenAI).lc_kwargs.chainData = { nodeOptions: getUploadsFromOptions(options) }
}
}
export const injectRunnableNodeData = (runnableSequence: RunnableSequence, nodeData: INodeData, options: ICommonObject) => {
runnableSequence.steps.forEach((step) => {
if (step instanceof ChatOpenAI) {
;(step as ChatOpenAI).lc_kwargs.chainData = { nodeOptions: getUploadsFromOptions(options) }
}
if (step instanceof RunnableBinding) {
if ((step as RunnableBinding<any, any>).bound instanceof ChatOpenAI) {
;((step as RunnableBinding<any, any>).bound as ChatOpenAI).lc_kwargs.chainData = {
nodeOptions: getUploadsFromOptions(options)
}
}
}
})
}
const getUploadsFromOptions = (options: ICommonObject): ICommonObject => {
if (options?.uploads) {
return {
uploads: options.uploads,
chatflowid: options.chatflowid,
chatId: options.chatId
}
}
return {}
}
export const addImagesToMessages = (options: ICommonObject, multiModalOption?: IMultiModalOption): MessageContentImageUrl[] => {
export const addImagesToMessages = (
nodeData: INodeData,
options: ICommonObject,
multiModalOption?: IMultiModalOption
): MessageContentImageUrl[] => {
const imageContent: MessageContentImageUrl[] = []
let model = nodeData.inputs?.model
// Image Uploaded
if (multiModalOption?.image && multiModalOption?.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) {
const imageUploads = getImageUploads(options.uploads)
for (const upload of imageUploads) {
if (upload.type == 'stored-file') {
const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name)
if (model instanceof LangchainChatOpenAI && multiModalOption) {
// Image Uploaded
if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) {
const imageUploads = getImageUploads(options.uploads)
for (const upload of imageUploads) {
let bf = upload.data
if (upload.type == 'stored-file') {
const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name)
// as the image is stored in the server, read the file and convert it to base64
const contents = fs.readFileSync(filePath)
let bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64')
// as the image is stored in the server, read the file and convert it to base64
const contents = fs.readFileSync(filePath)
bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64')
imageContent.push({
type: 'image_url',
image_url: {
url: bf,
detail: multiModalOption?.image.imageResolution ?? 'low'
}
})
imageContent.push({
type: 'image_url',
image_url: {
url: bf,
detail: multiModalOption.image.imageResolution ?? 'low'
}
})
}
}
}
}