farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-27 13:00:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

GPT Vision: Storing filenames only in chat message

Browse Source
This commit is contained in:
vinodkiran
2023-12-07 14:26:17 +05:30
parent dc265eb472
commit b492153f8a
3 changed files with 57 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/chains/VisionChain/VLLMChain.ts
+12 -1
View File
@@ -3,6 +3,9 @@ import { BaseChain, ChainInputs } from 'langchain/chains'
import { ChainValues } from 'langchain/schema'
import { BasePromptTemplate, ChatPromptTemplate, SystemMessagePromptTemplate } from 'langchain/prompts'
import { ChatOpenAI } from 'langchain/chat_models/openai'
import path from 'path'
import { getUserHome } from '../../../src/utils'
import fs from 'fs'
/**
* Interface for the input parameters of the OpenAIVisionChain class.
@@ -89,10 +92,18 @@ export class VLLMChain extends BaseChain implements OpenAIVisionChainInput {
})
if (this.imageUrls && this.imageUrls.length > 0) {
this.imageUrls.forEach((imageUrl: any) => {
let bf = imageUrl?.data
if (imageUrl.type == 'stored-file') {
const filePath = path.join(getUserHome(), '.flowise', 'gptvision', imageUrl.data)
// as the image is stored in the server, read the file and convert it to base64
const contents = fs.readFileSync(filePath)
bf = 'data:' + imageUrl.mime + ';base64,' + contents.toString('base64')
}
userRole.content.push({
type: 'image_url',
image_url: {
url: imageUrl?.data,
url: bf,
detail: this.imageResolution
}
})
packages/server/src/index.ts
+9 -6
View File
@@ -1349,17 +1349,20 @@ export class App {
if (incomingInput.uploads) {
// @ts-ignore
;(incomingInput.uploads as any[]).forEach((url: any) => {
if (url.type === 'file') {
const filename = url.name
const bf = url.data
;(incomingInput.uploads as any[]).forEach((upload: any) => {
if (upload.type === 'file') {
const filename = upload.name
const filePath = path.join(getUserHome(), '.flowise', 'gptvision', filename)
if (!fs.existsSync(path.join(getUserHome(), '.flowise', 'gptvision'))) {
fs.mkdirSync(path.dirname(filePath), { recursive: true })
}
const splitDataURI = upload.data.split(',')
//const fname = splitDataURI.pop()?.split(':')[1] ?? ''
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf)
fs.unlinkSync(filePath)
url.data = bf.toString('base64')
// don't need to store the file contents in chatmessage, just the filename
upload.data = filename //bf.toString('base64')
upload.type = 'stored-file'
}
})
}
packages/ui/src/views/chatmessage/ChatMessage.js
+36 -29
View File
@@ -128,7 +128,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
data: result,
preview: URL.createObjectURL(file),
type: 'file',
name: name
name: name,
mime: file.type
})
}
reader.readAsDataURL(file)
@@ -138,9 +139,11 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
const newFiles = await Promise.all(files)
setPreviews((prevPreviews) => [...prevPreviews, ...newFiles])
// if (newFiles.length > 0) {
// document.getElementById('messagelist').style.height = '80%'
// }
}
if (e.dataTransfer.items) {
const newUploads = []
for (const item of e.dataTransfer.items) {
if (item.kind === 'string' && item.type.match('^text/uri-list')) {
item.getAsString((s) => {
@@ -194,7 +197,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
data: result,
preview: URL.createObjectURL(file),
type: 'file',
name: name
name: name,
mime: file.type
})
}
reader.readAsDataURL(file)
@@ -312,7 +316,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
urls.push({
data: item.data,
type: item.type,
name: item.name
name: item.name,
mime: item.mime
})
})
clearPreviews()
@@ -510,7 +515,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
</Box>
)}
<div className={`${isDialog ? 'cloud-dialog' : 'cloud'}`}>
<div ref={ps} className={'messagelist'}>
<div ref={ps} id='messagelist' className={'messagelist'}>
{messages &&
messages.map((message, index) => {
return (
@@ -710,31 +715,33 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
</form>
</div>
</div>
{previews && previews.length > 0 && (
<Grid className='preview-container' container spacing={2} sx={{ p: 1, mt: '5px', ml: '1px' }}>
{previews.map((item, index) => (
<Grid item xs={12} sm={6} md={3} key={index}>
<Card variant='outlined' sx={{ maxWidth: 64 }}>
<CardMedia
component='img'
image={item.preview}
sx={{ height: 64 }}
alt={`preview ${index}`}
style={previewStyle}
/>
<CardActions className='center' sx={{ padding: 0, margin: 0 }}>
<Button
startIcon={<DeleteIcon />}
onClick={() => handleDeletePreview(item)}
size='small'
variant='text'
<div className='preview-container'>
{previews && previews.length > 0 && (
<Grid container spacing={2} sx={{ p: 1, mt: '5px', ml: '1px' }}>
{previews.map((item, index) => (
<Grid item xs={12} sm={6} md={3} key={index}>
<Card variant='outlined' sx={{ maxWidth: 64 }}>
<CardMedia
component='img'
image={item.preview}
sx={{ height: 64 }}
alt={`preview ${index}`}
style={previewStyle}
/>
</CardActions>
</Card>
</Grid>
))}
</Grid>
)}
<CardActions className='center' sx={{ padding: 0, margin: 0 }}>
<Button
startIcon={<DeleteIcon />}
onClick={() => handleDeletePreview(item)}
size='small'
variant='text'
/>
</CardActions>
</Card>
</Grid>
))}
</Grid>
)}
</div>
<SourceDocDialog show={sourceDialogOpen} dialogProps={sourceDialogProps} onCancel={() => setSourceDialogOpen(false)} />
</div>
)