Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing * Update pnpm-lock.yaml * update queue manager * remove singleton patterns, add redis to cache pool * add bull board ui * update rate limit handler * update redis configuration * Merge add rate limit redis prefix * update rate limit queue events * update preview loader to queue * refractor namings to constants * update env variable for queue * update worker shutdown gracefully
2026-06-27 21:00:42 +03:00 · 2025-01-23 14:08:02 +00:00
parent 14adb936f2
commit a2a475ba7a
59 changed files with 38958 additions and 36985 deletions
@@ -0,0 +1,127 @@
+import { BaseQueue } from './BaseQueue'
+import { PredictionQueue } from './PredictionQueue'
+import { UpsertQueue } from './UpsertQueue'
+import { IComponentNodes } from '../Interface'
+import { Telemetry } from '../utils/telemetry'
+import { CachePool } from '../CachePool'
+import { DataSource } from 'typeorm'
+import { AbortControllerPool } from '../AbortControllerPool'
+import { QueueEventsProducer, RedisOptions } from 'bullmq'
+import { createBullBoard } from 'bull-board'
+import { BullMQAdapter } from 'bull-board/bullMQAdapter'
+import { Express } from 'express'
+
+const QUEUE_NAME = process.env.QUEUE_NAME || 'flowise-queue'
+
+type QUEUE_TYPE = 'prediction' | 'upsert'
+
+export class QueueManager {
+    private static instance: QueueManager
+    private queues: Map<string, BaseQueue> = new Map()
+    private connection: RedisOptions
+    private bullBoardRouter?: Express
+    private predictionQueueEventsProducer?: QueueEventsProducer
+
+    private constructor() {
+        let tlsOpts = undefined
+        if (process.env.REDIS_URL && process.env.REDIS_URL.startsWith('rediss://')) {
+            tlsOpts = {
+                rejectUnauthorized: false
+            }
+        } else if (process.env.REDIS_TLS === 'true') {
+            tlsOpts = {
+                cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
+                key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
+                ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+            }
+        }
+        this.connection = {
+            url: process.env.REDIS_URL || undefined,
+            host: process.env.REDIS_HOST || 'localhost',
+            port: parseInt(process.env.REDIS_PORT || '6379'),
+            username: process.env.REDIS_USERNAME || undefined,
+            password: process.env.REDIS_PASSWORD || undefined,
+            tls: tlsOpts
+        }
+    }
+
+    public static getInstance(): QueueManager {
+        if (!QueueManager.instance) {
+            QueueManager.instance = new QueueManager()
+        }
+        return QueueManager.instance
+    }
+
+    public registerQueue(name: string, queue: BaseQueue) {
+        this.queues.set(name, queue)
+    }
+
+    public getConnection() {
+        return this.connection
+    }
+
+    public getQueue(name: QUEUE_TYPE): BaseQueue {
+        const queue = this.queues.get(name)
+        if (!queue) throw new Error(`Queue ${name} not found`)
+        return queue
+    }
+
+    public getPredictionQueueEventsProducer(): QueueEventsProducer {
+        if (!this.predictionQueueEventsProducer) throw new Error('Prediction queue events producer not found')
+        return this.predictionQueueEventsProducer
+    }
+
+    public getBullBoardRouter(): Express {
+        if (!this.bullBoardRouter) throw new Error('BullBoard router not found')
+        return this.bullBoardRouter
+    }
+
+    public async getAllJobCounts(): Promise<{ [queueName: string]: { [status: string]: number } }> {
+        const counts: { [queueName: string]: { [status: string]: number } } = {}
+
+        for (const [name, queue] of this.queues) {
+            counts[name] = await queue.getJobCounts()
+        }
+
+        return counts
+    }
+
+    public setupAllQueues({
+        componentNodes,
+        telemetry,
+        cachePool,
+        appDataSource,
+        abortControllerPool
+    }: {
+        componentNodes: IComponentNodes
+        telemetry: Telemetry
+        cachePool: CachePool
+        appDataSource: DataSource
+        abortControllerPool: AbortControllerPool
+    }) {
+        const predictionQueueName = `${QUEUE_NAME}-prediction`
+        const predictionQueue = new PredictionQueue(predictionQueueName, this.connection, {
+            componentNodes,
+            telemetry,
+            cachePool,
+            appDataSource,
+            abortControllerPool
+        })
+        this.registerQueue('prediction', predictionQueue)
+        this.predictionQueueEventsProducer = new QueueEventsProducer(predictionQueue.getQueueName(), {
+            connection: this.connection
+        })
+
+        const upsertionQueueName = `${QUEUE_NAME}-upsertion`
+        const upsertionQueue = new UpsertQueue(upsertionQueueName, this.connection, {
+            componentNodes,
+            telemetry,
+            cachePool,
+            appDataSource
+        })
+        this.registerQueue('upsert', upsertionQueue)
+
+        const bullboard = createBullBoard([new BullMQAdapter(predictionQueue.getQueue()), new BullMQAdapter(upsertionQueue.getQueue())])
+        this.bullBoardRouter = bullboard.router
+    }
+}