import { BaseQueue } from './BaseQueue' import { PredictionQueue } from './PredictionQueue' import { UpsertQueue } from './UpsertQueue' import { IComponentNodes } from '../Interface' import { Telemetry } from '../utils/telemetry' import { CachePool } from '../CachePool' import { DataSource } from 'typeorm' import { AbortControllerPool } from '../AbortControllerPool' import { QueueEventsProducer, RedisOptions } from 'bullmq' import { createBullBoard } from '@bull-board/api' import { BullMQAdapter } from '@bull-board/api/bullMQAdapter' import { Express } from 'express' import { UsageCacheManager } from '../UsageCacheManager' import { ExpressAdapter } from '@bull-board/express' import logger from '../utils/logger' const QUEUE_NAME = process.env.QUEUE_NAME || 'flowise-queue' type QUEUE_TYPE = 'prediction' | 'upsert' export class QueueManager { private static instance: QueueManager private queues: Map = new Map() private connection: RedisOptions private bullBoardRouter?: Express private predictionQueueEventsProducer?: QueueEventsProducer private constructor() { if (process.env.REDIS_URL) { let tlsOpts = undefined if (process.env.REDIS_URL.startsWith('rediss://')) { tlsOpts = { rejectUnauthorized: false } } else if (process.env.REDIS_TLS === 'true') { tlsOpts = { cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined, key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined, ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined } } this.connection = { url: process.env.REDIS_URL, tls: tlsOpts, enableReadyCheck: true, keepAlive: process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10)) ? parseInt(process.env.REDIS_KEEP_ALIVE, 10) : undefined } logger.info( `[QueueManager] Connecting to Redis using URL: ${process.env.REDIS_URL.replace(/\/\/[^:]+:[^@]+@/, '//[CREDENTIALS]@')}` ) } else { let tlsOpts = undefined if (process.env.REDIS_TLS === 'true') { tlsOpts = { cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined, key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined, ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined } } this.connection = { host: process.env.REDIS_HOST || 'localhost', port: parseInt(process.env.REDIS_PORT || '6379'), username: process.env.REDIS_USERNAME || undefined, password: process.env.REDIS_PASSWORD || undefined, tls: tlsOpts, enableReadyCheck: true, keepAlive: process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10)) ? parseInt(process.env.REDIS_KEEP_ALIVE, 10) : undefined } logger.info( `[QueueManager] Connecting to Redis using host:port: ${process.env.REDIS_HOST || 'localhost'}:${ process.env.REDIS_PORT || '6379' }` ) } } public static getInstance(): QueueManager { if (!QueueManager.instance) { QueueManager.instance = new QueueManager() } return QueueManager.instance } public registerQueue(name: string, queue: BaseQueue) { this.queues.set(name, queue) } public getConnection() { return this.connection } public getQueue(name: QUEUE_TYPE): BaseQueue { const queue = this.queues.get(name) if (!queue) throw new Error(`Queue ${name} not found`) return queue } public getPredictionQueueEventsProducer(): QueueEventsProducer { if (!this.predictionQueueEventsProducer) throw new Error('Prediction queue events producer not found') return this.predictionQueueEventsProducer } public getBullBoardRouter(): Express { if (!this.bullBoardRouter) throw new Error('BullBoard router not found') return this.bullBoardRouter } public async getAllJobCounts(): Promise<{ [queueName: string]: { [status: string]: number } }> { const counts: { [queueName: string]: { [status: string]: number } } = {} for (const [name, queue] of this.queues) { counts[name] = await queue.getJobCounts() } return counts } public setupAllQueues({ componentNodes, telemetry, cachePool, appDataSource, abortControllerPool, usageCacheManager, serverAdapter }: { componentNodes: IComponentNodes telemetry: Telemetry cachePool: CachePool appDataSource: DataSource abortControllerPool: AbortControllerPool usageCacheManager: UsageCacheManager serverAdapter?: ExpressAdapter }) { const predictionQueueName = `${QUEUE_NAME}-prediction` const predictionQueue = new PredictionQueue(predictionQueueName, this.connection, { componentNodes, telemetry, cachePool, appDataSource, abortControllerPool, usageCacheManager }) this.registerQueue('prediction', predictionQueue) // Add connection event logging for prediction queue if (predictionQueue.getQueue().opts.connection) { const connInfo = predictionQueue.getQueue().opts.connection || {} const connInfoString = JSON.stringify(connInfo) .replace(/"username":"[^"]*"/g, '"username":"[REDACTED]"') .replace(/"password":"[^"]*"/g, '"password":"[REDACTED]"') logger.info(`[QueueManager] Prediction queue connected to Redis: ${connInfoString}`) } this.predictionQueueEventsProducer = new QueueEventsProducer(predictionQueue.getQueueName(), { connection: this.connection }) const upsertionQueueName = `${QUEUE_NAME}-upsertion` const upsertionQueue = new UpsertQueue(upsertionQueueName, this.connection, { componentNodes, telemetry, cachePool, appDataSource, usageCacheManager }) this.registerQueue('upsert', upsertionQueue) // Add connection event logging for upsert queue if (upsertionQueue.getQueue().opts.connection) { const connInfo = upsertionQueue.getQueue().opts.connection || {} const connInfoString = JSON.stringify(connInfo) .replace(/"username":"[^"]*"/g, '"username":"[REDACTED]"') .replace(/"password":"[^"]*"/g, '"password":"[REDACTED]"') logger.info(`[QueueManager] Upsert queue connected to Redis: ${connInfoString}`) } if (serverAdapter) { createBullBoard({ queues: [new BullMQAdapter(predictionQueue.getQueue()), new BullMQAdapter(upsertionQueue.getQueue())], serverAdapter: serverAdapter }) this.bullBoardRouter = serverAdapter.getRouter() } } }