farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-29 05:01:10 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/Code Interpreter (#3183)

Browse Source 
* Base changes for ServerSide Events (instead of socket.io)

* lint fixes

* adding of interface and separate methods for streaming events

* lint

* first draft, handles both internal and external prediction end points.

* lint fixes

* additional internal end point for streaming and associated changes

* return streamresponse as true to build agent flow

* 1) JSON formatting for internal events
2) other fixes

* 1) convert internal event to metadata to maintain consistency with external response

* fix action and metadata streaming

* fix for error when agent flow is aborted

* prevent subflows from streaming and other code cleanup

* prevent streaming from enclosed tools

* add fix for preventing chaintool streaming

* update lock file

* add open when hidden to sse

* Streaming errors

* Streaming errors

* add fix for showing error message

* add code interpreter

* add artifacts to view message dialog

* Update pnpm-lock.yaml

---------

Co-authored-by: Vinod Paidimarry <vinodkiran@outlook.in>
This commit is contained in:
Henry Heng
2024-09-17 08:44:56 +01:00
committed by GitHub
parent 26444ac3ae
commit b02f279e9d
21 changed files with 729 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts
+17 -16
View File
@@ -208,6 +208,7 @@ class OpenAIAssistant_Agents implements INode {
const usedTools: IUsedTool[] = []
const fileAnnotations = []
const artifacts = []
const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({
id: selectedAssistantId
@@ -439,21 +440,23 @@ class OpenAIAssistant_Agents implements INode {
const fileId = chunk.image_file.file_id
const fileObj = await openai.files.retrieve(fileId)
const buffer = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId)
const base64String = Buffer.from(buffer).toString('base64')
// TODO: Use a file path and retrieve image on the fly. Storing as base64 to localStorage and database will easily hit limits
const imgHTML = `<img src="data:image/png;base64,${base64String}" width="100%" height="max-content" alt="${fileObj.filename}" /><br/>`
text += imgHTML
const filePath = await downloadImg(
openai,
fileId,
`${fileObj.filename}.png`,
options.chatflowid,
options.chatId
)
artifacts.push({ type: 'png', data: filePath })
if (!isStreamingStarted) {
isStreamingStarted = true
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, imgHTML)
sseStreamer.streamStartEvent(chatId, ' ')
}
}
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, imgHTML)
sseStreamer.streamArtifactsEvent(chatId, artifacts)
}
}
}
@@ -565,6 +568,7 @@ class OpenAIAssistant_Agents implements INode {
return {
text,
usedTools,
artifacts,
fileAnnotations,
assistant: { assistantId: openAIAssistantId, threadId, runId: runThreadId, messages: messageData }
}
@@ -769,12 +773,8 @@ class OpenAIAssistant_Agents implements INode {
const fileId = content.image_file.file_id
const fileObj = await openai.files.retrieve(fileId)
const buffer = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId)
const base64String = Buffer.from(buffer).toString('base64')
// TODO: Use a file path and retrieve image on the fly. Storing as base64 to localStorage and database will easily hit limits
const imgHTML = `<img src="data:image/png;base64,${base64String}" width="100%" height="max-content" alt="${fileObj.filename}" /><br/>`
returnVal += imgHTML
const filePath = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId)
artifacts.push({ type: 'png', data: filePath })
}
}
@@ -787,6 +787,7 @@ class OpenAIAssistant_Agents implements INode {
return {
text: returnVal,
usedTools,
artifacts,
fileAnnotations,
assistant: { assistantId: openAIAssistantId, threadId, runId: runThreadId, messages: messageData }
}
@@ -807,9 +808,9 @@ const downloadImg = async (openai: OpenAI, fileId: string, fileName: string, ...
const image_data_buffer = Buffer.from(image_data)
const mime = 'image/png'
await addSingleFileToStorage(mime, image_data_buffer, fileName, ...paths)
const res = await addSingleFileToStorage(mime, image_data_buffer, fileName, ...paths)
return image_data_buffer
return res
}
const downloadFile = async (openAIApiKey: string, fileObj: any, fileName: string, ...paths: string[]) => {
packages/components/nodes/agents/ToolAgent/ToolAgent.ts
+14 -1
View File
@@ -134,6 +134,7 @@ class ToolAgent_Agents implements INode {
let res: ChainValues = {}
let sourceDocuments: ICommonObject[] = []
let usedTools: IUsedTool[] = []
let artifacts = []
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
@@ -150,6 +151,12 @@ class ToolAgent_Agents implements INode {
}
usedTools = res.usedTools
}
if (res.artifacts) {
if (sseStreamer) {
sseStreamer.streamArtifactsEvent(chatId, flatten(res.artifacts))
}
artifacts = res.artifacts
}
// If the tool is set to returnDirect, stream the output to the client
if (res.usedTools && res.usedTools.length) {
let inputTools = nodeData.inputs?.tools
@@ -169,6 +176,9 @@ class ToolAgent_Agents implements INode {
if (res.usedTools) {
usedTools = res.usedTools
}
if (res.artifacts) {
artifacts = res.artifacts
}
}
let output = res?.output
@@ -203,7 +213,7 @@ class ToolAgent_Agents implements INode {
let finalRes = output
if (sourceDocuments.length || usedTools.length) {
if (sourceDocuments.length || usedTools.length || artifacts.length) {
const finalRes: ICommonObject = { text: output }
if (sourceDocuments.length) {
finalRes.sourceDocuments = flatten(sourceDocuments)
@@ -211,6 +221,9 @@ class ToolAgent_Agents implements INode {
if (usedTools.length) {
finalRes.usedTools = usedTools
}
if (artifacts.length) {
finalRes.artifacts = artifacts
}
return finalRes
}