Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

docker/.env.example

@@ -32,8 +32,6 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
 # FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
 # FLOWISE_FILE_SIZE_LIMIT=50mb
 
-# DISABLE_CHATFLOW_REUSE=true
-
 # DEBUG=true
 # LOG_LEVEL=info (error | warn | info | verbose | debug)
 # TOOL_FUNCTION_BUILTIN_DEP=crypto,fs
@@ -79,4 +77,21 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
 # see https://www.npmjs.com/package/global-agent for more details
 # GLOBAL_AGENT_HTTP_PROXY=CorporateHttpProxyUrl
 # GLOBAL_AGENT_HTTPS_PROXY=CorporateHttpsProxyUrl
-# GLOBAL_AGENT_NO_PROXY=ExceptionHostsToBypassProxyIfNeeded
+# GLOBAL_AGENT_NO_PROXY=ExceptionHostsToBypassProxyIfNeeded
+
+######################
+# QUEUE CONFIGURATION
+#######################
+# MODE=queue #(queue | main)
+# QUEUE_NAME=flowise-queue
+# QUEUE_REDIS_EVENT_STREAM_MAX_LEN=100000
+# WORKER_CONCURRENCY=100000
+# REDIS_URL=
+# REDIS_HOST=localhost
+# REDIS_PORT=6379
+# REDIS_USERNAME=
+# REDIS_PASSWORD=
+# REDIS_TLS=
+# REDIS_CERT=
+# REDIS_KEY=
+# REDIS_CA=

docker/docker-compose.yml

@@ -34,6 +34,19 @@ services:
             - GLOBAL_AGENT_HTTPS_PROXY=${GLOBAL_AGENT_HTTPS_PROXY}
             - GLOBAL_AGENT_NO_PROXY=${GLOBAL_AGENT_NO_PROXY}
             - DISABLED_NODES=${DISABLED_NODES}
+            - MODE=${MODE}
+            - WORKER_CONCURRENCY=${WORKER_CONCURRENCY}
+            - QUEUE_NAME=${QUEUE_NAME}
+            - QUEUE_REDIS_EVENT_STREAM_MAX_LEN=${QUEUE_REDIS_EVENT_STREAM_MAX_LEN}
+            - REDIS_URL=${REDIS_URL}
+            - REDIS_HOST=${REDIS_HOST}
+            - REDIS_PORT=${REDIS_PORT}
+            - REDIS_PASSWORD=${REDIS_PASSWORD}
+            - REDIS_USERNAME=${REDIS_USERNAME}
+            - REDIS_TLS=${REDIS_TLS}
+            - REDIS_CERT=${REDIS_CERT}
+            - REDIS_KEY=${REDIS_KEY}
+            - REDIS_CA=${REDIS_CA}
         ports:
             - '${PORT}:${PORT}'
         volumes:

docker/worker/README.md

@@ -0,0 +1,24 @@
+# Flowise Worker
+
+By utilizing worker instances when operating in queue mode, Flowise can be scaled horizontally by adding more workers to handle increased workloads or scaled down by removing workers when demand decreases.
+
+Here’s an overview of the process:
+
+1. The primary Flowise instance sends an execution ID to a message broker, Redis, which maintains a queue of pending executions, allowing the next available worker to process them.
+2. A worker from the pool retrieves a message from Redis.
+   The worker starts execute the actual job.
+3. Once the execution is completed, the worker alerts the main instance that the execution is finished.
+
+# How to use
+
+## Setting up Main Server:
+
+1. Follow [setup guide](https://github.com/FlowiseAI/Flowise/blob/main/docker/README.md)
+2. In the `.env.example`, setup all the necessary env variables for `QUEUE CONFIGURATION`
+
+## Setting up Worker:
+
+1. Copy paste the same `.env` file used to setup main server. Change the `PORT` to other available port numbers. Ex: 5566
+2. `docker compose up -d`
+3. Open [http://localhost:5566](http://localhost:5566)
+4. You can bring the worker container down by `docker compose stop`

docker/worker/docker-compose.yml

@@ -0,0 +1,54 @@
+version: '3.1'
+
+services:
+    flowise:
+        image: flowiseai/flowise
+        restart: always
+        environment:
+            - PORT=${PORT}
+            - CORS_ORIGINS=${CORS_ORIGINS}
+            - IFRAME_ORIGINS=${IFRAME_ORIGINS}
+            - FLOWISE_USERNAME=${FLOWISE_USERNAME}
+            - FLOWISE_PASSWORD=${FLOWISE_PASSWORD}
+            - FLOWISE_FILE_SIZE_LIMIT=${FLOWISE_FILE_SIZE_LIMIT}
+            - DEBUG=${DEBUG}
+            - DATABASE_PATH=${DATABASE_PATH}
+            - DATABASE_TYPE=${DATABASE_TYPE}
+            - DATABASE_PORT=${DATABASE_PORT}
+            - DATABASE_HOST=${DATABASE_HOST}
+            - DATABASE_NAME=${DATABASE_NAME}
+            - DATABASE_USER=${DATABASE_USER}
+            - DATABASE_PASSWORD=${DATABASE_PASSWORD}
+            - DATABASE_SSL=${DATABASE_SSL}
+            - DATABASE_SSL_KEY_BASE64=${DATABASE_SSL_KEY_BASE64}
+            - APIKEY_STORAGE_TYPE=${APIKEY_STORAGE_TYPE}
+            - APIKEY_PATH=${APIKEY_PATH}
+            - SECRETKEY_PATH=${SECRETKEY_PATH}
+            - FLOWISE_SECRETKEY_OVERWRITE=${FLOWISE_SECRETKEY_OVERWRITE}
+            - LOG_LEVEL=${LOG_LEVEL}
+            - LOG_PATH=${LOG_PATH}
+            - BLOB_STORAGE_PATH=${BLOB_STORAGE_PATH}
+            - DISABLE_FLOWISE_TELEMETRY=${DISABLE_FLOWISE_TELEMETRY}
+            - MODEL_LIST_CONFIG_JSON=${MODEL_LIST_CONFIG_JSON}
+            - GLOBAL_AGENT_HTTP_PROXY=${GLOBAL_AGENT_HTTP_PROXY}
+            - GLOBAL_AGENT_HTTPS_PROXY=${GLOBAL_AGENT_HTTPS_PROXY}
+            - GLOBAL_AGENT_NO_PROXY=${GLOBAL_AGENT_NO_PROXY}
+            - DISABLED_NODES=${DISABLED_NODES}
+            - MODE=${MODE}
+            - WORKER_CONCURRENCY=${WORKER_CONCURRENCY}
+            - QUEUE_NAME=${QUEUE_NAME}
+            - QUEUE_REDIS_EVENT_STREAM_MAX_LEN=${QUEUE_REDIS_EVENT_STREAM_MAX_LEN}
+            - REDIS_URL=${REDIS_URL}
+            - REDIS_HOST=${REDIS_HOST}
+            - REDIS_PORT=${REDIS_PORT}
+            - REDIS_PASSWORD=${REDIS_PASSWORD}
+            - REDIS_USERNAME=${REDIS_USERNAME}
+            - REDIS_TLS=${REDIS_TLS}
+            - REDIS_CERT=${REDIS_CERT}
+            - REDIS_KEY=${REDIS_KEY}
+            - REDIS_CA=${REDIS_CA}
+        ports:
+            - '${PORT}:${PORT}'
+        volumes:
+            - ~/.flowise:/root/.flowise
+        entrypoint: /bin/sh -c "sleep 3; flowise worker"