farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 13:00:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/Append uploaded images to message history (#3105)

Browse Source 
append uploaded images to message history
This commit is contained in:
Henry Heng
2024-09-02 11:27:45 +01:00
committed by GitHub
parent b28885240d
commit 4c1378c670
13 changed files with 60 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/src/utils.ts
+48 -3
View File
@@ -5,9 +5,10 @@ import * as path from 'path'
import { JSDOM } from 'jsdom'
import { z } from 'zod'
import { DataSource } from 'typeorm'
import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable } from './Interface'
import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable, MessageContentImageUrl } from './Interface'
import { AES, enc } from 'crypto-js'
import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages'
import { getFileFromStorage } from './storageUtils'
export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank
@@ -601,14 +602,58 @@ export const getUserHome = (): string => {
* @param {IChatMessage[]} chatmessages
* @returns {BaseMessage[]}
*/
export const mapChatMessageToBaseMessage = (chatmessages: any[] = []): BaseMessage[] => {
export const mapChatMessageToBaseMessage = async (chatmessages: any[] = []): Promise<BaseMessage[]> => {
const chatHistory = []
for (const message of chatmessages) {
if (message.role === 'apiMessage' || message.type === 'apiMessage') {
chatHistory.push(new AIMessage(message.content || ''))
} else if (message.role === 'userMessage' || message.role === 'userMessage') {
chatHistory.push(new HumanMessage(message.content || ''))
// check for image uploads
if (message.fileUploads) {
// example: [{"type":"stored-file","name":"0_DiXc4ZklSTo3M8J4.jpg","mime":"image/jpeg"}]
try {
const uploads = JSON.parse(message.fileUploads)
const imageContents: MessageContentImageUrl[] = []
for (const upload of uploads) {
if (upload.type === 'stored-file') {
const fileData = await getFileFromStorage(upload.name, message.chatflowid, message.chatId)
// as the image is stored in the server, read the file and convert it to base64
const bf = 'data:' + upload.mime + ';base64,' + fileData.toString('base64')
imageContents.push({
type: 'image_url',
image_url: {
url: bf
}
})
} else if (upload.type === 'url') {
imageContents.push({
type: 'image_url',
image_url: {
url: upload.data
}
})
}
}
chatHistory.push(
new HumanMessage({
content: [
{
type: 'text',
text: message.content
},
...imageContents
]
})
)
} catch (e) {
// failed to parse fileUploads, continue with text only
chatHistory.push(new HumanMessage(message.content || ''))
}
} else {
chatHistory.push(new HumanMessage(message.content || ''))
}
}
}
return chatHistory