Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

packages/server/src/AbortControllerPool.ts

@@ -0,0 +1,45 @@
+/**
+ * This pool is to keep track of abort controllers mapped to chatflowid_chatid
+ */
+export class AbortControllerPool {
+    abortControllers: Record<string, AbortController> = {}
+
+    /**
+     * Add to the pool
+     * @param {string} id
+     * @param {AbortController} abortController
+     */
+    add(id: string, abortController: AbortController) {
+        this.abortControllers[id] = abortController
+    }
+
+    /**
+     * Remove from the pool
+     * @param {string} id
+     */
+    remove(id: string) {
+        if (Object.prototype.hasOwnProperty.call(this.abortControllers, id)) {
+            delete this.abortControllers[id]
+        }
+    }
+
+    /**
+     * Get the abort controller
+     * @param {string} id
+     */
+    get(id: string) {
+        return this.abortControllers[id]
+    }
+
+    /**
+     * Abort
+     * @param {string} id
+     */
+    abort(id: string) {
+        const abortController = this.abortControllers[id]
+        if (abortController) {
+            abortController.abort()
+            this.remove(id)
+        }
+    }
+}