farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 17:01:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/sse (#3125)

Browse Source 
* Base changes for ServerSide Events (instead of socket.io)

* lint fixes

* adding of interface and separate methods for streaming events

* lint

* first draft, handles both internal and external prediction end points.

* lint fixes

* additional internal end point for streaming and associated changes

* return streamresponse as true to build agent flow

* 1) JSON formatting for internal events
2) other fixes

* 1) convert internal event to metadata to maintain consistency with external response

* fix action and metadata streaming

* fix for error when agent flow is aborted

* prevent subflows from streaming and other code cleanup

* prevent streaming from enclosed tools

* add fix for preventing chaintool streaming

* update lock file

* add open when hidden to sse

* Streaming errors

* Streaming errors

* add fix for showing error message

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Vinod Kiran
2024-09-17 12:31:25 +05:30
committed by GitHub
parent 7a4c7efcab
commit 26444ac3ae
47 changed files with 1021 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/agents/AirtableAgent/AirtableAgent.ts
+10 -4
View File
@@ -2,7 +2,7 @@ import axios from 'axios'
import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { AgentExecutor } from 'langchain/agents'
import { LLMChain } from 'langchain/chains'
import { ICommonObject, INode, INodeData, INodeParams, PromptTemplate } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer, PromptTemplate } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { LoadPyodide, finalSystemPrompt, systemPrompt } from './core'
@@ -104,11 +104,17 @@ class Airtable_Agents implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
return formatResponse(e.message)
}
}
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const accessToken = getCredentialParam('accessToken', credentialData, nodeData)
@@ -123,7 +129,6 @@ class Airtable_Agents implements INode {
let base64String = Buffer.from(JSON.stringify(airtableData)).toString('base64')
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
const callbacks = await additionalCallbacks(nodeData, options)
const pyodide = await LoadPyodide()
@@ -194,7 +199,8 @@ json.dumps(my_dict)`
answer: finalResult
}
if (options.socketIO && options.socketIOClientId) {
if (options.shouldStreamResponse) {
const handler = new CustomChainHandler(shouldStreamResponse ? sseStreamer : undefined, chatId)
const result = await chain.call(inputs, [loggerHandler, handler, ...callbacks])
return result?.text
} else {
packages/components/nodes/agents/AutoGPT/AutoGPT.ts
+3 -1
View File
@@ -113,7 +113,9 @@ class AutoGPT_Agents implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
return formatResponse(e.message)
}
}
packages/components/nodes/agents/BabyAGI/BabyAGI.ts
+3 -1
View File
@@ -73,7 +73,9 @@ class BabyAGI_Agents implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
return formatResponse(e.message)
}
}
packages/components/nodes/agents/CSVAgent/CSVAgent.ts
+10 -4
View File
@@ -2,7 +2,7 @@ import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { AgentExecutor } from 'langchain/agents'
import { LLMChain } from 'langchain/chains'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { ICommonObject, INode, INodeData, INodeParams, PromptTemplate } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer, PromptTemplate } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { LoadPyodide, finalSystemPrompt, systemPrompt } from './core'
import { checkInputs, Moderation } from '../../moderation/Moderation'
@@ -90,13 +90,18 @@ class CSV_Agents implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
return formatResponse(e.message)
}
}
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
const callbacks = await additionalCallbacks(nodeData, options)
let files: string[] = []
@@ -203,7 +208,8 @@ json.dumps(my_dict)`
answer: finalResult
}
if (options.socketIO && options.socketIOClientId) {
if (options.shouldStreamResponse) {
const handler = new CustomChainHandler(shouldStreamResponse ? sseStreamer : undefined, chatId)
const result = await chain.call(inputs, [loggerHandler, handler, ...callbacks])
return result?.text
} else {
packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts
+27 -7
View File
@@ -9,7 +9,16 @@ import { RunnableSequence } from '@langchain/core/runnables'
import { ChatConversationalAgent } from 'langchain/agents'
import { getBaseClasses } from '../../../src/utils'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { IVisionChatModal, FlowiseMemory, ICommonObject, INode, INodeData, INodeParams, IUsedTool } from '../../../src/Interface'
import {
IVisionChatModal,
FlowiseMemory,
ICommonObject,
INode,
INodeData,
INodeParams,
IUsedTool,
IServerSideEventStreamer
} from '../../../src/Interface'
import { AgentExecutor } from '../../../src/agents'
import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
import { checkInputs, Moderation } from '../../moderation/Moderation'
@@ -106,12 +115,18 @@ class ConversationalAgent_Agents implements INode {
const memory = nodeData.inputs?.memory as FlowiseMemory
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the BabyAGI agent
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
return formatResponse(e.message)
}
@@ -125,15 +140,17 @@ class ConversationalAgent_Agents implements INode {
let sourceDocuments: ICommonObject[] = []
let usedTools: IUsedTool[] = []
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
if (options.shouldStreamResponse) {
const handler = new CustomChainHandler(shouldStreamResponse ? sseStreamer : undefined, chatId)
res = await executor.invoke({ input }, { callbacks: [loggerHandler, handler, ...callbacks] })
if (res.sourceDocuments) {
options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', flatten(res.sourceDocuments))
if (options.sseStreamer) {
sseStreamer.streamSourceDocumentsEvent(options.chatId, flatten(res.sourceDocuments))
}
sourceDocuments = res.sourceDocuments
}
if (res.usedTools) {
options.socketIO.to(options.socketIOClientId).emit('usedTools', res.usedTools)
sseStreamer.streamUsedToolsEvent(options.chatId, res.usedTools)
usedTools = res.usedTools
}
// If the tool is set to returnDirect, stream the output to the client
@@ -142,11 +159,14 @@ class ConversationalAgent_Agents implements INode {
inputTools = flatten(inputTools)
for (const tool of res.usedTools) {
const inputTool = inputTools.find((inputTool: Tool) => inputTool.name === tool.tool)
if (inputTool && inputTool.returnDirect) {
options.socketIO.to(options.socketIOClientId).emit('token', tool.toolOutput)
if (inputTool && inputTool.returnDirect && options.sseStreamer) {
sseStreamer.streamTokenEvent(options.chatId, tool.toolOutput)
}
}
}
if (sseStreamer) {
sseStreamer.streamEndEvent(options.chatId)
}
} else {
res = await executor.invoke({ input }, { callbacks: [loggerHandler, ...callbacks] })
if (res.sourceDocuments) {
packages/components/nodes/agents/ConversationalRetrievalToolAgent/ConversationalRetrievalToolAgent.ts
+20 -8
View File
@@ -7,7 +7,16 @@ import { ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, Pr
import { formatToOpenAIToolMessages } from 'langchain/agents/format_scratchpad/openai_tools'
import { getBaseClasses } from '../../../src/utils'
import { type ToolsAgentStep } from 'langchain/agents/openai/output_parser'
import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams, IUsedTool, IVisionChatModal } from '../../../src/Interface'
import {
FlowiseMemory,
ICommonObject,
INode,
INodeData,
INodeParams,
IServerSideEventStreamer,
IUsedTool,
IVisionChatModal
} from '../../../src/Interface'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { AgentExecutor, ToolCallingAgentOutputParser } from '../../../src/agents'
import { Moderation, checkInputs, streamResponse } from '../../moderation/Moderation'
@@ -104,7 +113,9 @@ class ConversationalRetrievalToolAgent_Agents implements INode {
const memory = nodeData.inputs?.memory as FlowiseMemory
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const isStreamable = options.socketIO && options.socketIOClientId
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
@@ -112,8 +123,9 @@ class ConversationalRetrievalToolAgent_Agents implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
if (isStreamable)
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
streamResponse(sseStreamer, chatId, e.message)
}
return formatResponse(e.message)
}
}
@@ -127,15 +139,15 @@ class ConversationalRetrievalToolAgent_Agents implements INode {
let sourceDocuments: ICommonObject[] = []
let usedTools: IUsedTool[] = []
if (isStreamable) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
res = await executor.invoke({ input }, { callbacks: [loggerHandler, handler, ...callbacks] })
if (res.sourceDocuments) {
options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', flatten(res.sourceDocuments))
sseStreamer.streamSourceDocumentsEvent(chatId, flatten(res.sourceDocuments))
sourceDocuments = res.sourceDocuments
}
if (res.usedTools) {
options.socketIO.to(options.socketIOClientId).emit('usedTools', res.usedTools)
sseStreamer.streamUsedToolsEvent(chatId, res.usedTools)
usedTools = res.usedTools
}
} else {
packages/components/nodes/agents/LlamaIndexAgents/OpenAIToolAgent/OpenAIToolAgent_LlamaIndex.ts
+23 -7
View File
@@ -1,7 +1,16 @@
import { flatten } from 'lodash'
import { ChatMessage, OpenAI, OpenAIAgent } from 'llamaindex'
import { getBaseClasses } from '../../../../src/utils'
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams, IUsedTool } from '../../../../src/Interface'
import {
FlowiseMemory,
ICommonObject,
IMessage,
INode,
INodeData,
INodeParams,
IServerSideEventStreamer,
IUsedTool
} from '../../../../src/Interface'
class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
label: string
@@ -67,7 +76,9 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
let tools = nodeData.inputs?.tools
tools = flatten(tools)
const isStreamingEnabled = options.socketIO && options.socketIOClientId
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
const chatHistory = [] as ChatMessage[]
@@ -104,7 +115,7 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
let isStreamingStarted = false
const usedTools: IUsedTool[] = []
if (isStreamingEnabled) {
if (shouldStreamResponse) {
const stream = await agent.chat({
message: input,
chatHistory,
@@ -116,7 +127,9 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
text += chunk.response.delta
if (!isStreamingStarted) {
isStreamingStarted = true
options.socketIO.to(options.socketIOClientId).emit('start', chunk.response.delta)
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, chunk.response.delta)
}
if (chunk.sources.length) {
for (const sourceTool of chunk.sources) {
usedTools.push({
@@ -125,11 +138,14 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
toolOutput: sourceTool.output as any
})
}
options.socketIO.to(options.socketIOClientId).emit('usedTools', usedTools)
if (sseStreamer) {
sseStreamer.streamUsedToolsEvent(chatId, usedTools)
}
}
}
options.socketIO.to(options.socketIOClientId).emit('token', chunk.response.delta)
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, chunk.response.delta)
}
}
} else {
const response = await agent.chat({ message: input, chatHistory, verbose: process.env.DEBUG === 'true' ? true : false })
packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts
+54 -22
View File
@@ -1,4 +1,13 @@
import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams, IUsedTool } from '../../../src/Interface'
import {
ICommonObject,
IDatabaseEntity,
INode,
INodeData,
INodeOptionsValue,
INodeParams,
IServerSideEventStreamer,
IUsedTool
} from '../../../src/Interface'
import OpenAI from 'openai'
import { DataSource } from 'typeorm'
import { getCredentialData, getCredentialParam } from '../../../src/utils'
@@ -176,16 +185,19 @@ class OpenAIAssistant_Agents implements INode {
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const _toolChoice = nodeData.inputs?.toolChoice as string
const parallelToolCalls = nodeData.inputs?.parallelToolCalls as boolean
const isStreaming = options.socketIO && options.socketIOClientId
const socketIO = isStreaming ? options.socketIO : undefined
const socketIOClientId = isStreaming ? options.socketIOClientId : ''
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
streamResponse(isStreaming, e.message, socketIO, socketIOClientId)
if (shouldStreamResponse) {
streamResponse(sseStreamer, chatId, e.message)
}
return formatResponse(e.message)
}
}
@@ -307,7 +319,7 @@ class OpenAIAssistant_Agents implements INode {
}
}
if (isStreaming) {
if (shouldStreamResponse) {
const streamThread = await openai.beta.threads.runs.create(threadId, {
assistant_id: retrievedAssistant.id,
stream: true,
@@ -389,26 +401,37 @@ class OpenAIAssistant_Agents implements INode {
if (message_content.value) {
if (!isStreamingStarted) {
isStreamingStarted = true
socketIO.to(socketIOClientId).emit('start', message_content.value)
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, message_content.value)
}
}
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, message_content.value)
}
socketIO.to(socketIOClientId).emit('token', message_content.value)
}
if (fileAnnotations.length) {
if (!isStreamingStarted) {
isStreamingStarted = true
socketIO.to(socketIOClientId).emit('start', '')
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, ' ')
}
}
if (sseStreamer) {
sseStreamer.streamFileAnnotationsEvent(chatId, fileAnnotations)
}
socketIO.to(socketIOClientId).emit('fileAnnotations', fileAnnotations)
}
} else {
text += chunk.text?.value
if (!isStreamingStarted) {
isStreamingStarted = true
socketIO.to(socketIOClientId).emit('start', chunk.text?.value)
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, chunk.text?.value || '')
}
}
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, chunk.text?.value || '')
}
socketIO.to(socketIOClientId).emit('token', chunk.text?.value)
}
}
@@ -425,10 +448,13 @@ class OpenAIAssistant_Agents implements INode {
if (!isStreamingStarted) {
isStreamingStarted = true
socketIO.to(socketIOClientId).emit('start', imgHTML)
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, imgHTML)
}
}
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, imgHTML)
}
socketIO.to(socketIOClientId).emit('token', imgHTML)
}
}
@@ -495,15 +521,19 @@ class OpenAIAssistant_Agents implements INode {
text += chunk.text.value
if (!isStreamingStarted) {
isStreamingStarted = true
socketIO.to(socketIOClientId).emit('start', chunk.text.value)
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, chunk.text.value)
}
}
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, chunk.text.value)
}
socketIO.to(socketIOClientId).emit('token', chunk.text.value)
}
}
}
socketIO.to(socketIOClientId).emit('usedTools', usedTools)
if (sseStreamer) {
sseStreamer.streamUsedToolsEvent(chatId, usedTools)
}
} catch (error) {
console.error('Error submitting tool outputs:', error)
await openai.beta.threads.runs.cancel(threadId, runThreadId)
@@ -574,7 +604,9 @@ class OpenAIAssistant_Agents implements INode {
// Start tool analytics
const toolIds = await analyticHandlers.onToolStart(tool.name, actions[i].toolInput, parentIds)
if (socketIO && socketIOClientId) socketIO.to(socketIOClientId).emit('tool', tool.name)
if (shouldStreamResponse && sseStreamer) {
sseStreamer.streamToolEvent(chatId, tool.name)
}
try {
const toolOutput = await tool.call(actions[i].toolInput, undefined, undefined, {
packages/components/nodes/agents/ReActAgentChat/ReActAgentChat.ts
+3 -1
View File
@@ -88,7 +88,9 @@ class ReActAgentChat_Agents implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
return formatResponse(e.message)
}
}
packages/components/nodes/agents/ReActAgentLLM/ReActAgentLLM.ts
+3 -1
View File
@@ -77,7 +77,9 @@ class ReActAgentLLM_Agents implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
return formatResponse(e.message)
}
}
packages/components/nodes/agents/ToolAgent/ToolAgent.ts
+26 -10
View File
@@ -8,7 +8,16 @@ import { ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, Pr
import { formatToOpenAIToolMessages } from 'langchain/agents/format_scratchpad/openai_tools'
import { type ToolsAgentStep } from 'langchain/agents/openai/output_parser'
import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams, IUsedTool, IVisionChatModal } from '../../../src/Interface'
import {
FlowiseMemory,
ICommonObject,
INode,
INodeData,
INodeParams,
IServerSideEventStreamer,
IUsedTool,
IVisionChatModal
} from '../../../src/Interface'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { AgentExecutor, ToolCallingAgentOutputParser } from '../../../src/agents'
import { Moderation, checkInputs, streamResponse } from '../../moderation/Moderation'
@@ -100,7 +109,9 @@ class ToolAgent_Agents implements INode {
const memory = nodeData.inputs?.memory as FlowiseMemory
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const isStreamable = options.socketIO && options.socketIOClientId
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
@@ -108,8 +119,9 @@ class ToolAgent_Agents implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
if (isStreamable)
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
streamResponse(sseStreamer, chatId, e.message)
}
return formatResponse(e.message)
}
}
@@ -123,15 +135,19 @@ class ToolAgent_Agents implements INode {
let sourceDocuments: ICommonObject[] = []
let usedTools: IUsedTool[] = []
if (isStreamable) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
res = await executor.invoke({ input }, { callbacks: [loggerHandler, handler, ...callbacks] })
if (res.sourceDocuments) {
options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', flatten(res.sourceDocuments))
if (sseStreamer) {
sseStreamer.streamSourceDocumentsEvent(chatId, flatten(res.sourceDocuments))
}
sourceDocuments = res.sourceDocuments
}
if (res.usedTools) {
options.socketIO.to(options.socketIOClientId).emit('usedTools', res.usedTools)
if (sseStreamer) {
sseStreamer.streamUsedToolsEvent(chatId, flatten(res.usedTools))
}
usedTools = res.usedTools
}
// If the tool is set to returnDirect, stream the output to the client
@@ -140,8 +156,8 @@ class ToolAgent_Agents implements INode {
inputTools = flatten(inputTools)
for (const tool of res.usedTools) {
const inputTool = inputTools.find((inputTool: Tool) => inputTool.name === tool.tool)
if (inputTool && inputTool.returnDirect) {
options.socketIO.to(options.socketIOClientId).emit('token', tool.toolOutput)
if (inputTool && inputTool.returnDirect && shouldStreamResponse) {
sseStreamer.streamTokenEvent(chatId, tool.toolOutput)
}
}
}
packages/components/nodes/agents/XMLAgent/XMLAgent.ts
+28 -7
View File
@@ -7,7 +7,16 @@ import { Tool } from '@langchain/core/tools'
import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
import { formatLogToMessage } from 'langchain/agents/format_scratchpad/log_to_message'
import { getBaseClasses } from '../../../src/utils'
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams, IUsedTool } from '../../../src/Interface'
import {
FlowiseMemory,
ICommonObject,
IMessage,
INode,
INodeData,
INodeParams,
IServerSideEventStreamer,
IUsedTool
} from '../../../src/Interface'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { AgentExecutor, XMLAgentOutputParser } from '../../../src/agents'
import { Moderation, checkInputs } from '../../moderation/Moderation'
@@ -112,13 +121,19 @@ class XMLAgent_Agents implements INode {
const memory = nodeData.inputs?.memory as FlowiseMemory
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the OpenAI Function Agent
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
return formatResponse(e.message)
}
}
@@ -131,15 +146,19 @@ class XMLAgent_Agents implements INode {
let sourceDocuments: ICommonObject[] = []
let usedTools: IUsedTool[] = []
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
res = await executor.invoke({ input }, { callbacks: [loggerHandler, handler, ...callbacks] })
if (res.sourceDocuments) {
options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', flatten(res.sourceDocuments))
if (sseStreamer) {
sseStreamer.streamSourceDocumentsEvent(chatId, flatten(res.sourceDocuments))
}
sourceDocuments = res.sourceDocuments
}
if (res.usedTools) {
options.socketIO.to(options.socketIOClientId).emit('usedTools', res.usedTools)
if (sseStreamer) {
sseStreamer.streamUsedToolsEvent(chatId, flatten(res.usedTools))
}
usedTools = res.usedTools
}
// If the tool is set to returnDirect, stream the output to the client
@@ -149,7 +168,9 @@ class XMLAgent_Agents implements INode {
for (const tool of res.usedTools) {
const inputTool = inputTools.find((inputTool: Tool) => inputTool.name === tool.tool)
if (inputTool && inputTool.returnDirect) {
options.socketIO.to(options.socketIOClientId).emit('token', tool.toolOutput)
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, tool.toolOutput)
}
}
}
}
packages/components/nodes/chains/ApiChain/GETApiChain.ts
+6 -3
View File
@@ -2,7 +2,7 @@ import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { PromptTemplate } from '@langchain/core/prompts'
import { APIChain } from 'langchain/chains'
import { getBaseClasses } from '../../../src/utils'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
export const API_URL_RAW_PROMPT_TEMPLATE = `You are given the below API Documentation:
@@ -100,9 +100,12 @@ class GETApiChain_Chains implements INode {
const chain = await getAPIChain(apiDocs, model, headers, urlPrompt, ansPrompt)
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId, 2)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
const res = await chain.run(input, [loggerHandler, handler, ...callbacks])
return res
} else {
packages/components/nodes/chains/ApiChain/OpenAPIChain.ts
+10 -4
View File
@@ -1,5 +1,5 @@
import { APIChain, createOpenAPIChain } from 'langchain/chains'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
@@ -74,18 +74,24 @@ class OpenApiChain_Chains implements INode {
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the OpenAPI chain
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
if (options.shouldStreamResponse) {
streamResponse(sseStreamer, chatId, e.message)
}
return formatResponse(e.message)
}
}
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
const res = await chain.run(input, [loggerHandler, handler, ...callbacks])
return res
} else {
packages/components/nodes/chains/ApiChain/POSTApiChain.ts
+7 -3
View File
@@ -2,7 +2,7 @@ import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { PromptTemplate } from '@langchain/core/prompts'
import { API_RESPONSE_RAW_PROMPT_TEMPLATE, API_URL_RAW_PROMPT_TEMPLATE, APIChain } from './postCore'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class POSTApiChain_Chains implements INode {
@@ -90,8 +90,12 @@ class POSTApiChain_Chains implements INode {
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId, 2)
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
const res = await chain.run(input, [loggerHandler, handler, ...callbacks])
return res
} else {
packages/components/nodes/chains/ConversationChain/ConversationChain.ts
+11 -4
View File
@@ -23,7 +23,8 @@ import {
INode,
INodeData,
INodeParams,
MessageContentImageUrl
MessageContentImageUrl,
IServerSideEventStreamer
} from '../../../src/Interface'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
@@ -114,13 +115,19 @@ class ConversationChain_Chains implements INode {
const chain = await prepareChain(nodeData, options, this.sessionId)
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the LLM chain
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
if (options.shouldStreamResponse) {
streamResponse(options.sseStreamer, options.chatId, e.message)
}
return formatResponse(e.message)
}
}
@@ -135,8 +142,8 @@ class ConversationChain_Chains implements INode {
callbacks.push(new LCConsoleCallbackHandler())
}
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
callbacks.push(handler)
res = await chain.invoke({ input }, { callbacks })
} else {
packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts
+27 -8
View File
@@ -22,7 +22,8 @@ import {
INodeData,
INodeParams,
IDatabaseEntity,
MemoryMethods
MemoryMethods,
IServerSideEventStreamer
} from '../../../src/Interface'
import { QA_TEMPLATE, REPHRASE_TEMPLATE, RESPONSE_TEMPLATE } from './prompts'
@@ -181,6 +182,10 @@ class ConversationalRetrievalQAChain_Chains implements INode {
const databaseEntities = options.databaseEntities as IDatabaseEntity
const chatflowid = options.chatflowid as string
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
let customResponsePrompt = responsePrompt
// If the deprecated systemMessagePrompt is still exists
if (systemMessagePrompt) {
@@ -205,7 +210,9 @@ class ConversationalRetrievalQAChain_Chains implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
if (options.shouldStreamResponse) {
streamResponse(options.sseStreamer, options.chatId, e.message)
}
return formatResponse(e.message)
}
}
@@ -234,18 +241,22 @@ class ConversationalRetrievalQAChain_Chains implements INode {
let sourceDocuments: ICommonObject[] = []
let text = ''
let isStreamingStarted = false
const isStreamingEnabled = options.socketIO && options.socketIOClientId
for await (const chunk of stream) {
streamedResponse = applyPatch(streamedResponse, chunk.ops).newDocument
if (streamedResponse.final_output) {
text = streamedResponse.final_output?.output
if (isStreamingEnabled) options.socketIO.to(options.socketIOClientId).emit('end')
if (Array.isArray(streamedResponse?.logs?.[sourceRunnableName]?.final_output?.output)) {
sourceDocuments = streamedResponse?.logs?.[sourceRunnableName]?.final_output?.output
if (isStreamingEnabled && returnSourceDocuments)
options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', sourceDocuments)
if (shouldStreamResponse && returnSourceDocuments) {
if (sseStreamer) {
sseStreamer.streamSourceDocumentsEvent(chatId, sourceDocuments)
}
}
}
if (shouldStreamResponse && sseStreamer) {
sseStreamer.streamEndEvent(chatId)
}
}
@@ -258,9 +269,17 @@ class ConversationalRetrievalQAChain_Chains implements INode {
if (!isStreamingStarted) {
isStreamingStarted = true
if (isStreamingEnabled) options.socketIO.to(options.socketIOClientId).emit('start', token)
if (shouldStreamResponse) {
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, token)
}
}
}
if (shouldStreamResponse) {
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, token)
}
}
if (isStreamingEnabled) options.socketIO.to(options.socketIOClientId).emit('token', token)
}
}
packages/components/nodes/chains/LLMChain/LLMChain.ts
+24 -11
View File
@@ -4,7 +4,15 @@ import { HumanMessage } from '@langchain/core/messages'
import { ChatPromptTemplate, FewShotPromptTemplate, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
import { OutputFixingParser } from 'langchain/output_parsers'
import { LLMChain } from 'langchain/chains'
import { IVisionChatModal, ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import {
IVisionChatModal,
ICommonObject,
INode,
INodeData,
INodeOutputsValue,
INodeParams,
IServerSideEventStreamer
} from '../../../src/Interface'
import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
@@ -162,18 +170,22 @@ const runPrediction = async (
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
const isStreaming = !disableStreaming && options.socketIO && options.socketIOClientId
const socketIO = isStreaming ? options.socketIO : undefined
const socketIOClientId = isStreaming ? options.socketIOClientId : ''
const moderations = nodeData.inputs?.inputModeration as Moderation[]
// this is true if the prediction is external and the client has requested streaming='true'
const shouldStreamResponse = !disableStreaming && options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the LLM chain
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
streamResponse(isStreaming, e.message, socketIO, socketIOClientId)
if (shouldStreamResponse) {
streamResponse(sseStreamer, chatId, e.message)
}
return formatResponse(e.message)
}
}
@@ -245,8 +257,8 @@ const runPrediction = async (
if (seen.length === 0) {
// All inputVariables have fixed values specified
const options = { ...promptValues }
if (isStreaming) {
const handler = new CustomChainHandler(socketIO, socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
const res = await chain.call(options, [loggerHandler, handler, ...callbacks])
return formatResponse(res?.text)
} else {
@@ -261,8 +273,8 @@ const runPrediction = async (
...promptValues,
[lastValue]: input
}
if (isStreaming) {
const handler = new CustomChainHandler(socketIO, socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
const res = await chain.call(options, [loggerHandler, handler, ...callbacks])
return formatResponse(res?.text)
} else {
@@ -273,8 +285,9 @@ const runPrediction = async (
throw new Error(`Please provide Prompt Values for: ${seen.join(', ')}`)
}
} else {
if (isStreaming) {
const handler = new CustomChainHandler(socketIO, socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
const res = await chain.run(input, [loggerHandler, handler, ...callbacks])
return formatResponse(res)
} else {
packages/components/nodes/chains/MultiPromptChain/MultiPromptChain.ts
+12 -4
View File
@@ -1,6 +1,6 @@
import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { MultiPromptChain } from 'langchain/chains'
import { ICommonObject, INode, INodeData, INodeParams, PromptRetriever } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer, PromptRetriever } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
@@ -75,13 +75,21 @@ class MultiPromptChain_Chains implements INode {
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
const chain = nodeData.instance as MultiPromptChain
const moderations = nodeData.inputs?.inputModeration as Moderation[]
// this is true if the prediction is external and the client has requested streaming='true'
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the Multi Prompt Chain
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
if (options.shouldStreamResponse) {
streamResponse(options.sseStreamer, options.chatId, e.message)
}
return formatResponse(e.message)
}
}
@@ -90,8 +98,8 @@ class MultiPromptChain_Chains implements INode {
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId, 2)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId, 2)
const res = await chain.call(obj, [loggerHandler, handler, ...callbacks])
return res?.text
} else {
packages/components/nodes/chains/MultiRetrievalQAChain/MultiRetrievalQAChain.ts
+11 -4
View File
@@ -1,6 +1,6 @@
import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { MultiRetrievalQAChain } from 'langchain/chains'
import { ICommonObject, INode, INodeData, INodeParams, VectorStoreRetriever } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer, VectorStoreRetriever } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
@@ -83,13 +83,20 @@ class MultiRetrievalQAChain_Chains implements INode {
const chain = nodeData.instance as MultiRetrievalQAChain
const returnSourceDocuments = nodeData.inputs?.returnSourceDocuments as boolean
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the Multi Retrieval QA Chain
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
if (options.shouldStreamResponse) {
streamResponse(options.sseStreamer, options.chatId, e.message)
}
return formatResponse(e.message)
}
}
@@ -97,8 +104,8 @@ class MultiRetrievalQAChain_Chains implements INode {
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId, 2, returnSourceDocuments)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId, 2, returnSourceDocuments)
const res = await chain.call(obj, [loggerHandler, handler, ...callbacks])
if (res.text && res.sourceDocuments) return res
return res?.text
packages/components/nodes/chains/RetrievalQAChain/RetrievalQAChain.ts
+11 -4
View File
@@ -2,7 +2,7 @@ import { BaseRetriever } from '@langchain/core/retrievers'
import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { RetrievalQAChain } from 'langchain/chains'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
@@ -60,13 +60,20 @@ class RetrievalQAChain_Chains implements INode {
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
const chain = nodeData.instance as RetrievalQAChain
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the Retrieval QA Chain
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
streamResponse(sseStreamer, chatId, e.message)
}
return formatResponse(e.message)
}
}
@@ -76,8 +83,8 @@ class RetrievalQAChain_Chains implements INode {
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
const res = await chain.call(obj, [loggerHandler, handler, ...callbacks])
return res?.text
} else {
packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts
+13 -4
View File
@@ -4,7 +4,7 @@ import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { PromptTemplate, PromptTemplateInput } from '@langchain/core/prompts'
import { SqlDatabaseChain, SqlDatabaseChainInput, DEFAULT_SQL_DATABASE_PROMPT } from 'langchain/chains/sql_db'
import { SqlDatabase } from 'langchain/sql_db'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { getBaseClasses, getInputVariables } from '../../../src/utils'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
@@ -166,13 +166,21 @@ class SqlDatabaseChain_Chains implements INode {
const topK = nodeData.inputs?.topK as number
const customPrompt = nodeData.inputs?.customPrompt as string
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the Sql Database Chain
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
streamResponse(sseStreamer, chatId, e.message)
}
// streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
return formatResponse(e.message)
}
}
@@ -190,8 +198,9 @@ class SqlDatabaseChain_Chains implements INode {
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId, 2)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId, 2)
const res = await chain.run(input, [loggerHandler, handler, ...callbacks])
return res
} else {
packages/components/nodes/chains/VectaraChain/VectaraChain.ts
+3 -1
View File
@@ -269,7 +269,9 @@ class VectaraChain_Chains implements INode {
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
return formatResponse(e.message)
}
}
packages/components/nodes/chains/VectorDBQAChain/VectorDBQAChain.ts
+10 -4
View File
@@ -2,7 +2,7 @@ import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { VectorStore } from '@langchain/core/vectorstores'
import { VectorDBQAChain } from 'langchain/chains'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { checkInputs, Moderation } from '../../moderation/Moderation'
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
@@ -64,13 +64,19 @@ class VectorDBQAChain_Chains implements INode {
const chain = nodeData.instance as VectorDBQAChain
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (moderations && moderations.length > 0) {
try {
// Use the output of the moderation chain as input for the VectorDB QA Chain
input = await checkInputs(moderations, input)
} catch (e) {
await new Promise((resolve) => setTimeout(resolve, 500))
//streamResponse(options.socketIO && options.socketIOClientId, e.message, options.socketIO, options.socketIOClientId)
// if (options.shouldStreamResponse) {
// streamResponse(options.sseStreamer, options.chatId, e.message)
// }
return formatResponse(e.message)
}
}
@@ -81,8 +87,8 @@ class VectorDBQAChain_Chains implements INode {
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
if (options.socketIO && options.socketIOClientId) {
const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
if (shouldStreamResponse) {
const handler = new CustomChainHandler(sseStreamer, chatId)
const res = await chain.call(obj, [loggerHandler, handler, ...callbacks])
return res?.text
} else {
packages/components/nodes/engine/ChatEngine/ContextChatEngine.ts
+24 -6
View File
@@ -1,4 +1,13 @@
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import {
FlowiseMemory,
ICommonObject,
IMessage,
INode,
INodeData,
INodeOutputsValue,
INodeParams,
IServerSideEventStreamer
} from '../../../src/Interface'
import { Metadata, BaseRetriever, LLM, ContextChatEngine, ChatMessage, NodeWithScore } from 'llamaindex'
import { reformatSourceDocuments } from '../EngineUtils'
@@ -103,24 +112,33 @@ class ContextChatEngine_LlamaIndex implements INode {
let isStreamingStarted = false
let sourceDocuments: ICommonObject[] = []
let sourceNodes: NodeWithScore<Metadata>[] = []
const isStreamingEnabled = options.socketIO && options.socketIOClientId
if (isStreamingEnabled) {
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (shouldStreamResponse) {
const stream = await chatEngine.chat({ message: input, chatHistory, stream: true })
for await (const chunk of stream) {
text += chunk.response
if (chunk.sourceNodes) sourceNodes = chunk.sourceNodes
if (!isStreamingStarted) {
isStreamingStarted = true
options.socketIO.to(options.socketIOClientId).emit('start', chunk.response)
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, chunk.response)
}
}
options.socketIO.to(options.socketIOClientId).emit('token', chunk.response)
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, chunk.response)
}
}
if (returnSourceDocuments) {
sourceDocuments = reformatSourceDocuments(sourceNodes)
options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', sourceDocuments)
if (sseStreamer) {
sseStreamer.streamSourceDocumentsEvent(chatId, sourceDocuments)
}
}
} else {
const response = await chatEngine.chat({ message: input, chatHistory })
packages/components/nodes/engine/ChatEngine/SimpleChatEngine.ts
+21 -6
View File
@@ -1,4 +1,13 @@
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import {
FlowiseMemory,
ICommonObject,
IMessage,
INode,
INodeData,
INodeOutputsValue,
INodeParams,
IServerSideEventStreamer
} from '../../../src/Interface'
import { LLM, ChatMessage, SimpleChatEngine } from 'llamaindex'
class SimpleChatEngine_LlamaIndex implements INode {
@@ -86,18 +95,24 @@ class SimpleChatEngine_LlamaIndex implements INode {
let text = ''
let isStreamingStarted = false
const isStreamingEnabled = options.socketIO && options.socketIOClientId
if (isStreamingEnabled) {
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (shouldStreamResponse) {
const stream = await chatEngine.chat({ message: input, chatHistory, stream: true })
for await (const chunk of stream) {
text += chunk.response
if (!isStreamingStarted) {
isStreamingStarted = true
options.socketIO.to(options.socketIOClientId).emit('start', chunk.response)
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, chunk.response)
}
}
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, chunk.response)
}
options.socketIO.to(options.socketIOClientId).emit('token', chunk.response)
}
} else {
const response = await chatEngine.chat({ message: input, chatHistory })
packages/components/nodes/engine/QueryEngine/QueryEngine.ts
+15 -7
View File
@@ -1,4 +1,4 @@
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
import {
RetrieverQueryEngine,
ResponseSynthesizer,
@@ -71,24 +71,32 @@ class QueryEngine_LlamaIndex implements INode {
let sourceDocuments: ICommonObject[] = []
let sourceNodes: NodeWithScore<Metadata>[] = []
let isStreamingStarted = false
const isStreamingEnabled = options.socketIO && options.socketIOClientId
if (isStreamingEnabled) {
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (shouldStreamResponse) {
const stream = await queryEngine.query({ query: input, stream: true })
for await (const chunk of stream) {
text += chunk.response
if (chunk.sourceNodes) sourceNodes = chunk.sourceNodes
if (!isStreamingStarted) {
isStreamingStarted = true
options.socketIO.to(options.socketIOClientId).emit('start', chunk.response)
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, chunk.response)
}
}
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, chunk.response)
}
options.socketIO.to(options.socketIOClientId).emit('token', chunk.response)
}
if (returnSourceDocuments) {
sourceDocuments = reformatSourceDocuments(sourceNodes)
options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', sourceDocuments)
if (sseStreamer) {
sseStreamer.streamSourceDocumentsEvent(chatId, sourceDocuments)
}
}
} else {
const response = await queryEngine.query({ query: input })
packages/components/nodes/engine/SubQuestionQueryEngine/SubQuestionQueryEngine.ts
+15 -7
View File
@@ -1,5 +1,5 @@
import { flatten } from 'lodash'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
import {
TreeSummarize,
SimpleResponseBuilder,
@@ -88,24 +88,32 @@ class SubQuestionQueryEngine_LlamaIndex implements INode {
let sourceDocuments: ICommonObject[] = []
let sourceNodes: NodeWithScore<Metadata>[] = []
let isStreamingStarted = false
const isStreamingEnabled = options.socketIO && options.socketIOClientId
if (isStreamingEnabled) {
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
if (shouldStreamResponse) {
const stream = await queryEngine.query({ query: input, stream: true })
for await (const chunk of stream) {
text += chunk.response
if (chunk.sourceNodes) sourceNodes = chunk.sourceNodes
if (!isStreamingStarted) {
isStreamingStarted = true
options.socketIO.to(options.socketIOClientId).emit('start', chunk.response)
if (sseStreamer) {
sseStreamer.streamStartEvent(chatId, chunk.response)
}
}
if (sseStreamer) {
sseStreamer.streamTokenEvent(chatId, chunk.response)
}
options.socketIO.to(options.socketIOClientId).emit('token', chunk.response)
}
if (returnSourceDocuments) {
sourceDocuments = reformatSourceDocuments(sourceNodes)
options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', sourceDocuments)
if (sseStreamer) {
sseStreamer.streamSourceDocumentsEvent(chatId, sourceDocuments)
}
}
} else {
const response = await queryEngine.query({ query: input })
packages/components/nodes/moderation/Moderation.ts
+10 -12
View File
@@ -1,4 +1,4 @@
import { Server } from 'socket.io'
import { IServerSideEventStreamer } from '../../src'
export abstract class Moderation {
abstract checkForViolations(input: string): Promise<string>
@@ -13,15 +13,13 @@ export const checkInputs = async (inputModerations: Moderation[], input: string)
// is this the correct location for this function?
// should we have a utils files that all node components can use?
export const streamResponse = (isStreaming: any, response: string, socketIO: Server, socketIOClientId: string) => {
if (isStreaming) {
const result = response.split(/(\s+)/)
result.forEach((token: string, index: number) => {
if (index === 0) {
socketIO.to(socketIOClientId).emit('start', token)
}
socketIO.to(socketIOClientId).emit('token', token)
})
socketIO.to(socketIOClientId).emit('end')
}
export const streamResponse = (sseStreamer: IServerSideEventStreamer, chatId: string, response: string) => {
const result = response.split(/(\s+)/)
result.forEach((token: string, index: number) => {
if (index === 0) {
sseStreamer.streamStartEvent(chatId, token)
}
sseStreamer.streamTokenEvent(chatId, token)
})
sseStreamer.streamEndEvent(chatId)
}
packages/components/nodes/tools/ChainTool/core.ts
+32 -2
View File
@@ -1,6 +1,7 @@
import { DynamicTool, DynamicToolInput } from '@langchain/core/tools'
import { BaseChain } from 'langchain/chains'
import { handleEscapeCharacters } from '../../../src/utils'
import { CustomChainHandler } from '../../../src'
export interface ChainToolInput extends Omit<DynamicToolInput, 'func'> {
chain: BaseChain
@@ -13,13 +14,42 @@ export class ChainTool extends DynamicTool {
super({
...rest,
func: async (input, runManager) => {
// To enable LLM Chain which has promptValues
// prevent sending SSE events of the sub-chain
const sseStreamer = runManager?.handlers.find((handler) => handler instanceof CustomChainHandler)?.sseStreamer
if (runManager) {
const callbacks = runManager.handlers
for (let i = 0; i < callbacks.length; i += 1) {
if (callbacks[i] instanceof CustomChainHandler) {
;(callbacks[i] as any).sseStreamer = undefined
}
}
}
if ((chain as any).prompt && (chain as any).prompt.promptValues) {
const promptValues = handleEscapeCharacters((chain as any).prompt.promptValues, true)
const values = await chain.call(promptValues, runManager?.getChild())
if (runManager && sseStreamer) {
const callbacks = runManager.handlers
for (let i = 0; i < callbacks.length; i += 1) {
if (callbacks[i] instanceof CustomChainHandler) {
;(callbacks[i] as any).sseStreamer = sseStreamer
}
}
}
return values?.text
}
return chain.run(input, runManager?.getChild())
const values = chain.run(input, runManager?.getChild())
if (runManager && sseStreamer) {
const callbacks = runManager.handlers
for (let i = 0; i < callbacks.length; i += 1) {
if (callbacks[i] instanceof CustomChainHandler) {
;(callbacks[i] as any).sseStreamer = sseStreamer
}
}
}
return values
}
})
this.chain = chain
packages/components/nodes/tools/ChatflowTool/ChatflowTool.ts
+10
View File
@@ -7,6 +7,7 @@ import { StructuredTool } from '@langchain/core/tools'
import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
import { availableDependencies, defaultAllowBuiltInDep, getCredentialData, getCredentialParam } from '../../../src/utils'
import { v4 as uuidv4 } from 'uuid'
import { CustomChainHandler } from '../../../src'
class ChatflowTool_Tools implements INode {
label: string
@@ -248,6 +249,15 @@ class ChatflowTool extends StructuredTool {
} catch (e) {
throw new Error(`Received tool input did not match expected schema: ${JSON.stringify(arg)}`)
}
// iterate over the callbacks and the sse streamer
if (config.callbacks instanceof CallbackManager) {
const callbacks = config.callbacks.handlers
for (let i = 0; i < callbacks.length; i += 1) {
if (callbacks[i] instanceof CustomChainHandler) {
;(callbacks[i] as any).sseStreamer = undefined
}
}
}
const callbackManager_ = await CallbackManager.configure(
config.callbacks,
this.callbacks,