Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

packages/server/src/utils/SSEStreamer.ts

@@ -1,4 +1,3 @@
-import express from 'express'
 import { Response } from 'express'
 import { IServerSideEventStreamer } from 'flowise-components'
 
@@ -13,11 +12,6 @@ type Client = {
 
 export class SSEStreamer implements IServerSideEventStreamer {
     clients: { [id: string]: Client } = {}
-    app: express.Application
-
-    constructor(app: express.Application) {
-        this.app = app
-    }
 
     addExternalClient(chatId: string, res: Response) {
         this.clients[chatId] = { clientType: 'EXTERNAL', response: res, started: false }
@@ -40,18 +34,6 @@ export class SSEStreamer implements IServerSideEventStreamer {
         }
     }
 
-    // Send SSE message to a specific client
-    streamEvent(chatId: string, data: string) {
-        const client = this.clients[chatId]
-        if (client) {
-            const clientResponse = {
-                event: 'start',
-                data: data
-            }
-            client.response.write('message:\ndata:' + JSON.stringify(clientResponse) + '\n\n')
-        }
-    }
-
     streamCustomEvent(chatId: string, eventType: string, data: any) {
         const client = this.clients[chatId]
         if (client) {