diff --git a/packages/components/nodes/chains/VisionChain/VLLMChain.ts b/packages/components/nodes/chains/VisionChain/VLLMChain.ts index 59a2483a..f9b92e53 100644 --- a/packages/components/nodes/chains/VisionChain/VLLMChain.ts +++ b/packages/components/nodes/chains/VisionChain/VLLMChain.ts @@ -3,6 +3,9 @@ import { BaseChain, ChainInputs } from 'langchain/chains' import { ChainValues } from 'langchain/schema' import { BasePromptTemplate, ChatPromptTemplate, SystemMessagePromptTemplate } from 'langchain/prompts' import { ChatOpenAI } from 'langchain/chat_models/openai' +import path from 'path' +import { getUserHome } from '../../../src/utils' +import fs from 'fs' /** * Interface for the input parameters of the OpenAIVisionChain class. @@ -89,10 +92,18 @@ export class VLLMChain extends BaseChain implements OpenAIVisionChainInput { }) if (this.imageUrls && this.imageUrls.length > 0) { this.imageUrls.forEach((imageUrl: any) => { + let bf = imageUrl?.data + if (imageUrl.type == 'stored-file') { + const filePath = path.join(getUserHome(), '.flowise', 'gptvision', imageUrl.data) + + // as the image is stored in the server, read the file and convert it to base64 + const contents = fs.readFileSync(filePath) + bf = 'data:' + imageUrl.mime + ';base64,' + contents.toString('base64') + } userRole.content.push({ type: 'image_url', image_url: { - url: imageUrl?.data, + url: bf, detail: this.imageResolution } }) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 70978c6a..83b018d5 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1349,17 +1349,20 @@ export class App { if (incomingInput.uploads) { // @ts-ignore - ;(incomingInput.uploads as any[]).forEach((url: any) => { - if (url.type === 'file') { - const filename = url.name - const bf = url.data + ;(incomingInput.uploads as any[]).forEach((upload: any) => { + if (upload.type === 'file') { + const filename = upload.name const filePath = path.join(getUserHome(), '.flowise', 'gptvision', filename) if (!fs.existsSync(path.join(getUserHome(), '.flowise', 'gptvision'))) { fs.mkdirSync(path.dirname(filePath), { recursive: true }) } + const splitDataURI = upload.data.split(',') + //const fname = splitDataURI.pop()?.split(':')[1] ?? '' + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf) - fs.unlinkSync(filePath) - url.data = bf.toString('base64') + // don't need to store the file contents in chatmessage, just the filename + upload.data = filename //bf.toString('base64') + upload.type = 'stored-file' } }) } diff --git a/packages/ui/src/views/chatmessage/ChatMessage.js b/packages/ui/src/views/chatmessage/ChatMessage.js index 0243f252..92c73699 100644 --- a/packages/ui/src/views/chatmessage/ChatMessage.js +++ b/packages/ui/src/views/chatmessage/ChatMessage.js @@ -128,7 +128,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { data: result, preview: URL.createObjectURL(file), type: 'file', - name: name + name: name, + mime: file.type }) } reader.readAsDataURL(file) @@ -138,9 +139,11 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { const newFiles = await Promise.all(files) setPreviews((prevPreviews) => [...prevPreviews, ...newFiles]) + // if (newFiles.length > 0) { + // document.getElementById('messagelist').style.height = '80%' + // } } if (e.dataTransfer.items) { - const newUploads = [] for (const item of e.dataTransfer.items) { if (item.kind === 'string' && item.type.match('^text/uri-list')) { item.getAsString((s) => { @@ -194,7 +197,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { data: result, preview: URL.createObjectURL(file), type: 'file', - name: name + name: name, + mime: file.type }) } reader.readAsDataURL(file) @@ -312,7 +316,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { urls.push({ data: item.data, type: item.type, - name: item.name + name: item.name, + mime: item.mime }) }) clearPreviews() @@ -510,7 +515,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { )}
-
+
{messages && messages.map((message, index) => { return ( @@ -710,31 +715,33 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
- {previews && previews.length > 0 && ( - - {previews.map((item, index) => ( - - - - -
setSourceDialogOpen(false)} /> )