Feature/sse (#3125)

* Base changes for ServerSide Events (instead of socket.io)

* lint fixes

* adding of interface and separate methods for streaming events

* lint

* first draft, handles both internal and external prediction end points.

* lint fixes

* additional internal end point for streaming and associated changes

* return streamresponse as true to build agent flow

* 1) JSON formatting for internal events
2) other fixes

* 1) convert internal event to metadata to maintain consistency with external response

* fix action and metadata streaming

* fix for error when agent flow is aborted

* prevent subflows from streaming and other code cleanup

* prevent streaming from enclosed tools

* add fix for preventing chaintool streaming

* update lock file

* add open when hidden to sse

* Streaming errors

* Streaming errors

* add fix for showing error message

---------

Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/src/controllers/internal-predictions/index.ts

@@ -1,16 +1,45 @@
 import { Request, Response, NextFunction } from 'express'
 import { utilBuildChatflow } from '../../utils/buildChatflow'
+import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
+import { getErrorMessage } from '../../errors/utils'
 
 // Send input message and get prediction result (Internal)
 const createInternalPrediction = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        const apiResponse = await utilBuildChatflow(req, req.io, true)
-        return res.json(apiResponse)
+        if (req.body.streaming || req.body.streaming === 'true') {
+            createAndStreamInternalPrediction(req, res, next)
+            return
+        } else {
+            const apiResponse = await utilBuildChatflow(req, true)
+            return res.json(apiResponse)
+        }
     } catch (error) {
         next(error)
     }
 }
 
+// Send input message and stream prediction result using SSE (Internal)
+const createAndStreamInternalPrediction = async (req: Request, res: Response, next: NextFunction) => {
+    const chatId = req.body.chatId
+    const sseStreamer = getRunningExpressApp().sseStreamer
+    try {
+        sseStreamer.addClient(chatId, res)
+        res.setHeader('Content-Type', 'text/event-stream')
+        res.setHeader('Cache-Control', 'no-cache')
+        res.setHeader('Connection', 'keep-alive')
+        res.flushHeaders()
+
+        const apiResponse = await utilBuildChatflow(req, true)
+        sseStreamer.streamMetadataEvent(apiResponse.chatId, apiResponse)
+    } catch (error) {
+        if (chatId) {
+            sseStreamer.streamErrorEvent(chatId, getErrorMessage(error))
+        }
+        next(error)
+    } finally {
+        sseStreamer.removeClient(chatId)
+    }
+}
 export default {
     createInternalPrediction
 }

packages/server/src/controllers/predictions/index.ts

@@ -5,6 +5,9 @@ import logger from '../../utils/logger'
 import predictionsServices from '../../services/predictions'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
+import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
+import { v4 as uuidv4 } from 'uuid'
+import { getErrorMessage } from '../../errors/utils'
 
 // Send input message and get prediction result (External)
 const createPrediction = async (req: Request, res: Response, next: NextFunction) => {
@@ -46,9 +49,36 @@ const createPrediction = async (req: Request, res: Response, next: NextFunction)
             }
         }
         if (isDomainAllowed) {
-            //@ts-ignore
-            const apiResponse = await predictionsServices.buildChatflow(req, req?.io)
-            return res.json(apiResponse)
+            const streamable = await chatflowsService.checkIfChatflowIsValidForStreaming(req.params.id)
+            const isStreamingRequested = req.body.streaming === 'true' || req.body.streaming === true
+            if (streamable?.isStreaming && isStreamingRequested) {
+                const sseStreamer = getRunningExpressApp().sseStreamer
+                let chatId = req.body.chatId
+                if (!req.body.chatId) {
+                    chatId = req.body.chatId ?? req.body.overrideConfig?.sessionId ?? uuidv4()
+                    req.body.chatId = chatId
+                }
+                try {
+                    sseStreamer.addExternalClient(chatId, res)
+                    res.setHeader('Content-Type', 'text/event-stream')
+                    res.setHeader('Cache-Control', 'no-cache')
+                    res.setHeader('Connection', 'keep-alive')
+                    res.flushHeaders()
+
+                    const apiResponse = await predictionsServices.buildChatflow(req)
+                    sseStreamer.streamMetadataEvent(apiResponse.chatId, apiResponse)
+                } catch (error) {
+                    if (chatId) {
+                        sseStreamer.streamErrorEvent(chatId, getErrorMessage(error))
+                    }
+                    next(error)
+                } finally {
+                    sseStreamer.removeClient(chatId)
+                }
+            } else {
+                const apiResponse = await predictionsServices.buildChatflow(req)
+                return res.json(apiResponse)
+            }
         } else {
             throw new InternalFlowiseError(StatusCodes.UNAUTHORIZED, `This site is not allowed to access this chatbot`)
         }