Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

docker/.env.example

@@ -32,8 +32,6 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
 # FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
 # FLOWISE_FILE_SIZE_LIMIT=50mb
 
-# DISABLE_CHATFLOW_REUSE=true
-
 # DEBUG=true
 # LOG_LEVEL=info (error | warn | info | verbose | debug)
 # TOOL_FUNCTION_BUILTIN_DEP=crypto,fs
@@ -79,4 +77,21 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
 # see https://www.npmjs.com/package/global-agent for more details
 # GLOBAL_AGENT_HTTP_PROXY=CorporateHttpProxyUrl
 # GLOBAL_AGENT_HTTPS_PROXY=CorporateHttpsProxyUrl
-# GLOBAL_AGENT_NO_PROXY=ExceptionHostsToBypassProxyIfNeeded
+# GLOBAL_AGENT_NO_PROXY=ExceptionHostsToBypassProxyIfNeeded
+
+######################
+# QUEUE CONFIGURATION
+#######################
+# MODE=queue #(queue | main)
+# QUEUE_NAME=flowise-queue
+# QUEUE_REDIS_EVENT_STREAM_MAX_LEN=100000
+# WORKER_CONCURRENCY=100000
+# REDIS_URL=
+# REDIS_HOST=localhost
+# REDIS_PORT=6379
+# REDIS_USERNAME=
+# REDIS_PASSWORD=
+# REDIS_TLS=
+# REDIS_CERT=
+# REDIS_KEY=
+# REDIS_CA=