GPT Vision: Storing filenames only in chat message

packages/components/nodes/chains/VisionChain/VLLMChain.ts

@@ -3,6 +3,9 @@ import { BaseChain, ChainInputs } from 'langchain/chains'
 import { ChainValues } from 'langchain/schema'
 import { BasePromptTemplate, ChatPromptTemplate, SystemMessagePromptTemplate } from 'langchain/prompts'
 import { ChatOpenAI } from 'langchain/chat_models/openai'
+import path from 'path'
+import { getUserHome } from '../../../src/utils'
+import fs from 'fs'
 
 /**
  * Interface for the input parameters of the OpenAIVisionChain class.
@@ -89,10 +92,18 @@ export class VLLMChain extends BaseChain implements OpenAIVisionChainInput {
         })
         if (this.imageUrls && this.imageUrls.length > 0) {
             this.imageUrls.forEach((imageUrl: any) => {
+                let bf = imageUrl?.data
+                if (imageUrl.type == 'stored-file') {
+                    const filePath = path.join(getUserHome(), '.flowise', 'gptvision', imageUrl.data)
+
+                    // as the image is stored in the server, read the file and convert it to base64
+                    const contents = fs.readFileSync(filePath)
+                    bf = 'data:' + imageUrl.mime + ';base64,' + contents.toString('base64')
+                }
                 userRole.content.push({
                     type: 'image_url',
                     image_url: {
-                        url: imageUrl?.data,
+                        url: bf,
                         detail: this.imageResolution
                     }
                 })

packages/server/src/index.ts

@@ -1349,17 +1349,20 @@ export class App {
 
             if (incomingInput.uploads) {
                 // @ts-ignore
-                ;(incomingInput.uploads as any[]).forEach((url: any) => {
+                ;(incomingInput.uploads as any[]).forEach((upload: any) => {
-                    if (url.type === 'file') {
+                    if (upload.type === 'file') {
-                        const filename = url.name
+                        const filename = upload.name
-                        const bf = url.data
                         const filePath = path.join(getUserHome(), '.flowise', 'gptvision', filename)
                         if (!fs.existsSync(path.join(getUserHome(), '.flowise', 'gptvision'))) {
                             fs.mkdirSync(path.dirname(filePath), { recursive: true })
                         }
+                        const splitDataURI = upload.data.split(',')
+                        //const fname = splitDataURI.pop()?.split(':')[1] ?? ''
+                        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
                         if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf)
-                        fs.unlinkSync(filePath)
+                        // don't need to store the file contents in chatmessage, just the filename
-                        url.data = bf.toString('base64')
+                        upload.data = filename //bf.toString('base64')
+                        upload.type = 'stored-file'
                     }
                 })
             }

packages/ui/src/views/chatmessage/ChatMessage.js

@@ -128,7 +128,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
                                 data: result,
                                 preview: URL.createObjectURL(file),
                                 type: 'file',
-                                name: name
+                                name: name,
+                                mime: file.type
                             })
                         }
                         reader.readAsDataURL(file)
@@ -138,9 +139,11 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
 
             const newFiles = await Promise.all(files)
             setPreviews((prevPreviews) => [...prevPreviews, ...newFiles])
+            // if (newFiles.length > 0) {
+            //     document.getElementById('messagelist').style.height = '80%'
+            // }
         }
         if (e.dataTransfer.items) {
-            const newUploads = []
             for (const item of e.dataTransfer.items) {
                 if (item.kind === 'string' && item.type.match('^text/uri-list')) {
                     item.getAsString((s) => {
@@ -194,7 +197,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
                             data: result,
                             preview: URL.createObjectURL(file),
                             type: 'file',
-                            name: name
+                            name: name,
+                            mime: file.type
                         })
                     }
                     reader.readAsDataURL(file)
@@ -312,7 +316,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
             urls.push({
                 data: item.data,
                 type: item.type,
-                name: item.name
+                name: item.name,
+                mime: item.mime
             })
         })
         clearPreviews()
@@ -510,7 +515,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
                 </Box>
             )}
             <div className={`${isDialog ? 'cloud-dialog' : 'cloud'}`}>
-                <div ref={ps} className={'messagelist'}>
+                <div ref={ps} id='messagelist' className={'messagelist'}>
                     {messages &&
                         messages.map((message, index) => {
                             return (
@@ -710,31 +715,33 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
                     </form>
                 </div>
             </div>
-            {previews && previews.length > 0 && (
+            <div className='preview-container'>
-                <Grid className='preview-container' container spacing={2} sx={{ p: 1, mt: '5px', ml: '1px' }}>
+                {previews && previews.length > 0 && (
-                    {previews.map((item, index) => (
+                    <Grid container spacing={2} sx={{ p: 1, mt: '5px', ml: '1px' }}>
-                        <Grid item xs={12} sm={6} md={3} key={index}>
+                        {previews.map((item, index) => (
-                            <Card variant='outlined' sx={{ maxWidth: 64 }}>
+                            <Grid item xs={12} sm={6} md={3} key={index}>
-                                <CardMedia
+                                <Card variant='outlined' sx={{ maxWidth: 64 }}>
-                                    component='img'
+                                    <CardMedia
-                                    image={item.preview}
+                                        component='img'
-                                    sx={{ height: 64 }}
+                                        image={item.preview}
-                                    alt={`preview ${index}`}
+                                        sx={{ height: 64 }}
-                                    style={previewStyle}
+                                        alt={`preview ${index}`}
-                                />
+                                        style={previewStyle}
-                                <CardActions className='center' sx={{ padding: 0, margin: 0 }}>
-                                    <Button
-                                        startIcon={<DeleteIcon />}
-                                        onClick={() => handleDeletePreview(item)}
-                                        size='small'
-                                        variant='text'
                                     />
-                                </CardActions>
+                                    <CardActions className='center' sx={{ padding: 0, margin: 0 }}>
-                            </Card>
+                                        <Button
-                        </Grid>
+                                            startIcon={<DeleteIcon />}
-                    ))}
+                                            onClick={() => handleDeletePreview(item)}
-                </Grid>
+                                            size='small'
-            )}
+                                            variant='text'
+                                        />
+                                    </CardActions>
+                                </Card>
+                            </Grid>
+                        ))}
+                    </Grid>
+                )}
+            </div>
             <SourceDocDialog show={sourceDialogOpen} dialogProps={sourceDialogProps} onCancel={() => setSourceDialogOpen(false)} />
         </div>
     )