Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

packages/server/.env.example

@@ -28,8 +28,6 @@ PORT=3000
 # FLOWISE_PASSWORD=1234
 # FLOWISE_FILE_SIZE_LIMIT=50mb
 
-# DISABLE_CHATFLOW_REUSE=true
-
 # DEBUG=true
 # LOG_PATH=/your_log_path/.flowise/logs
 # LOG_LEVEL=info (error | warn | info | verbose | debug)
@@ -77,3 +75,20 @@ PORT=3000
 # GLOBAL_AGENT_HTTP_PROXY=CorporateHttpProxyUrl
 # GLOBAL_AGENT_HTTPS_PROXY=CorporateHttpsProxyUrl
 # GLOBAL_AGENT_NO_PROXY=ExceptionHostsToBypassProxyIfNeeded
+
+######################
+# QUEUE CONFIGURATION
+#######################
+# MODE=queue #(queue | main)
+# QUEUE_NAME=flowise-queue
+# QUEUE_REDIS_EVENT_STREAM_MAX_LEN=100000
+# WORKER_CONCURRENCY=100000
+# REDIS_URL=
+# REDIS_HOST=localhost
+# REDIS_PORT=6379
+# REDIS_USERNAME=
+# REDIS_PASSWORD=
+# REDIS_TLS=
+# REDIS_CERT=
+# REDIS_KEY=
+# REDIS_CA=