Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

packages/server/src/CachePool.ts

@@ -1,19 +1,51 @@
-import { IActiveCache } from './Interface'
+import { IActiveCache, MODE } from './Interface'
+import Redis from 'ioredis'
 
 /**
  * This pool is to keep track of in-memory cache used for LLM and Embeddings
  */
 export class CachePool {
+    private redisClient: Redis | null = null
     activeLLMCache: IActiveCache = {}
     activeEmbeddingCache: IActiveCache = {}
 
+    constructor() {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (process.env.REDIS_URL) {
+                this.redisClient = new Redis(process.env.REDIS_URL)
+            } else {
+                this.redisClient = new Redis({
+                    host: process.env.REDIS_HOST || 'localhost',
+                    port: parseInt(process.env.REDIS_PORT || '6379'),
+                    username: process.env.REDIS_USERNAME || undefined,
+                    password: process.env.REDIS_PASSWORD || undefined,
+                    tls:
+                        process.env.REDIS_TLS === 'true'
+                            ? {
+                                  cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
+                                  key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
+                                  ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+                              }
+                            : undefined
+                })
+            }
+        }
+    }
+
     /**
      * Add to the llm cache pool
      * @param {string} chatflowid
      * @param {Map<any, any>} value
      */
-    addLLMCache(chatflowid: string, value: Map<any, any>) {
-        this.activeLLMCache[chatflowid] = value
+    async addLLMCache(chatflowid: string, value: Map<any, any>) {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (this.redisClient) {
+                const serializedValue = JSON.stringify(Array.from(value.entries()))
+                await this.redisClient.set(`llmCache:${chatflowid}`, serializedValue)
+            }
+        } else {
+            this.activeLLMCache[chatflowid] = value
+        }
     }
 
     /**
@@ -21,24 +53,60 @@ export class CachePool {
      * @param {string} chatflowid
      * @param {Map<any, any>} value
      */
-    addEmbeddingCache(chatflowid: string, value: Map<any, any>) {
-        this.activeEmbeddingCache[chatflowid] = value
+    async addEmbeddingCache(chatflowid: string, value: Map<any, any>) {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (this.redisClient) {
+                const serializedValue = JSON.stringify(Array.from(value.entries()))
+                await this.redisClient.set(`embeddingCache:${chatflowid}`, serializedValue)
+            }
+        } else {
+            this.activeEmbeddingCache[chatflowid] = value
+        }
     }
 
     /**
      * Get item from llm cache pool
      * @param {string} chatflowid
      */
-    getLLMCache(chatflowid: string): Map<any, any> | undefined {
-        return this.activeLLMCache[chatflowid]
+    async getLLMCache(chatflowid: string): Promise<Map<any, any> | undefined> {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (this.redisClient) {
+                const serializedValue = await this.redisClient.get(`llmCache:${chatflowid}`)
+                if (serializedValue) {
+                    return new Map(JSON.parse(serializedValue))
+                }
+            }
+        } else {
+            return this.activeLLMCache[chatflowid]
+        }
+        return undefined
     }
 
     /**
      * Get item from embedding cache pool
      * @param {string} chatflowid
      */
-    getEmbeddingCache(chatflowid: string): Map<any, any> | undefined {
-        return this.activeEmbeddingCache[chatflowid]
+    async getEmbeddingCache(chatflowid: string): Promise<Map<any, any> | undefined> {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (this.redisClient) {
+                const serializedValue = await this.redisClient.get(`embeddingCache:${chatflowid}`)
+                if (serializedValue) {
+                    return new Map(JSON.parse(serializedValue))
+                }
+            }
+        } else {
+            return this.activeEmbeddingCache[chatflowid]
+        }
+        return undefined
+    }
+
+    /**
+     * Close Redis connection if applicable
+     */
+    async close() {
+        if (this.redisClient) {
+            await this.redisClient.quit()
+        }
     }
 }