Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

packages/server/src/AbortControllerPool.ts

@@ -0,0 +1,45 @@
+/**
+ * This pool is to keep track of abort controllers mapped to chatflowid_chatid
+ */
+export class AbortControllerPool {
+    abortControllers: Record<string, AbortController> = {}
+
+    /**
+     * Add to the pool
+     * @param {string} id
+     * @param {AbortController} abortController
+     */
+    add(id: string, abortController: AbortController) {
+        this.abortControllers[id] = abortController
+    }
+
+    /**
+     * Remove from the pool
+     * @param {string} id
+     */
+    remove(id: string) {
+        if (Object.prototype.hasOwnProperty.call(this.abortControllers, id)) {
+            delete this.abortControllers[id]
+        }
+    }
+
+    /**
+     * Get the abort controller
+     * @param {string} id
+     */
+    get(id: string) {
+        return this.abortControllers[id]
+    }
+
+    /**
+     * Abort
+     * @param {string} id
+     */
+    abort(id: string) {
+        const abortController = this.abortControllers[id]
+        if (abortController) {
+            abortController.abort()
+            this.remove(id)
+        }
+    }
+}

packages/server/src/CachePool.ts

@@ -1,19 +1,51 @@
-import { IActiveCache } from './Interface'
+import { IActiveCache, MODE } from './Interface'
+import Redis from 'ioredis'
 
 /**
  * This pool is to keep track of in-memory cache used for LLM and Embeddings
  */
 export class CachePool {
+    private redisClient: Redis | null = null
     activeLLMCache: IActiveCache = {}
     activeEmbeddingCache: IActiveCache = {}
 
+    constructor() {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (process.env.REDIS_URL) {
+                this.redisClient = new Redis(process.env.REDIS_URL)
+            } else {
+                this.redisClient = new Redis({
+                    host: process.env.REDIS_HOST || 'localhost',
+                    port: parseInt(process.env.REDIS_PORT || '6379'),
+                    username: process.env.REDIS_USERNAME || undefined,
+                    password: process.env.REDIS_PASSWORD || undefined,
+                    tls:
+                        process.env.REDIS_TLS === 'true'
+                            ? {
+                                  cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
+                                  key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
+                                  ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+                              }
+                            : undefined
+                })
+            }
+        }
+    }
+
     /**
      * Add to the llm cache pool
      * @param {string} chatflowid
      * @param {Map<any, any>} value
      */
-    addLLMCache(chatflowid: string, value: Map<any, any>) {
-        this.activeLLMCache[chatflowid] = value
+    async addLLMCache(chatflowid: string, value: Map<any, any>) {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (this.redisClient) {
+                const serializedValue = JSON.stringify(Array.from(value.entries()))
+                await this.redisClient.set(`llmCache:${chatflowid}`, serializedValue)
+            }
+        } else {
+            this.activeLLMCache[chatflowid] = value
+        }
     }
 
     /**
@@ -21,24 +53,60 @@ export class CachePool {
      * @param {string} chatflowid
      * @param {Map<any, any>} value
      */
-    addEmbeddingCache(chatflowid: string, value: Map<any, any>) {
-        this.activeEmbeddingCache[chatflowid] = value
+    async addEmbeddingCache(chatflowid: string, value: Map<any, any>) {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (this.redisClient) {
+                const serializedValue = JSON.stringify(Array.from(value.entries()))
+                await this.redisClient.set(`embeddingCache:${chatflowid}`, serializedValue)
+            }
+        } else {
+            this.activeEmbeddingCache[chatflowid] = value
+        }
     }
 
     /**
      * Get item from llm cache pool
      * @param {string} chatflowid
      */
-    getLLMCache(chatflowid: string): Map<any, any> | undefined {
-        return this.activeLLMCache[chatflowid]
+    async getLLMCache(chatflowid: string): Promise<Map<any, any> | undefined> {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (this.redisClient) {
+                const serializedValue = await this.redisClient.get(`llmCache:${chatflowid}`)
+                if (serializedValue) {
+                    return new Map(JSON.parse(serializedValue))
+                }
+            }
+        } else {
+            return this.activeLLMCache[chatflowid]
+        }
+        return undefined
     }
 
     /**
      * Get item from embedding cache pool
      * @param {string} chatflowid
      */
-    getEmbeddingCache(chatflowid: string): Map<any, any> | undefined {
-        return this.activeEmbeddingCache[chatflowid]
+    async getEmbeddingCache(chatflowid: string): Promise<Map<any, any> | undefined> {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (this.redisClient) {
+                const serializedValue = await this.redisClient.get(`embeddingCache:${chatflowid}`)
+                if (serializedValue) {
+                    return new Map(JSON.parse(serializedValue))
+                }
+            }
+        } else {
+            return this.activeEmbeddingCache[chatflowid]
+        }
+        return undefined
+    }
+
+    /**
+     * Close Redis connection if applicable
+     */
+    async close() {
+        if (this.redisClient) {
+            await this.redisClient.quit()
+        }
     }
 }
 

packages/server/src/ChatflowPool.ts

@@ -1,59 +0,0 @@
-import { ICommonObject } from 'flowise-components'
-import { IActiveChatflows, INodeData, IReactFlowNode } from './Interface'
-import logger from './utils/logger'
-
-/**
- * This pool is to keep track of active chatflow pools
- * so we can prevent building langchain flow all over again
- */
-export class ChatflowPool {
-    activeChatflows: IActiveChatflows = {}
-
-    /**
-     * Add to the pool
-     * @param {string} chatflowid
-     * @param {INodeData} endingNodeData
-     * @param {IReactFlowNode[]} startingNodes
-     * @param {ICommonObject} overrideConfig
-     */
-    add(
-        chatflowid: string,
-        endingNodeData: INodeData | undefined,
-        startingNodes: IReactFlowNode[],
-        overrideConfig?: ICommonObject,
-        chatId?: string
-    ) {
-        this.activeChatflows[chatflowid] = {
-            startingNodes,
-            endingNodeData,
-            inSync: true
-        }
-        if (overrideConfig) this.activeChatflows[chatflowid].overrideConfig = overrideConfig
-        if (chatId) this.activeChatflows[chatflowid].chatId = chatId
-
-        logger.info(`[server]: Chatflow ${chatflowid} added into ChatflowPool`)
-    }
-
-    /**
-     * Update to the pool
-     * @param {string} chatflowid
-     * @param {boolean} inSync
-     */
-    updateInSync(chatflowid: string, inSync: boolean) {
-        if (Object.prototype.hasOwnProperty.call(this.activeChatflows, chatflowid)) {
-            this.activeChatflows[chatflowid].inSync = inSync
-            logger.info(`[server]: Chatflow ${chatflowid} updated inSync=${inSync} in ChatflowPool`)
-        }
-    }
-
-    /**
-     * Remove from the pool
-     * @param {string} chatflowid
-     */
-    async remove(chatflowid: string) {
-        if (Object.prototype.hasOwnProperty.call(this.activeChatflows, chatflowid)) {
-            delete this.activeChatflows[chatflowid]
-            logger.info(`[server]: Chatflow ${chatflowid} removed from ChatflowPool`)
-        }
-    }
-}

packages/server/src/Interface.DocumentStore.ts

@@ -1,5 +1,9 @@
 import { ICommonObject } from 'flowise-components'
 import { DocumentStore } from './database/entities/DocumentStore'
+import { DataSource } from 'typeorm'
+import { IComponentNodes } from './Interface'
+import { Telemetry } from './utils/telemetry'
+import { CachePool } from './CachePool'
 
 export enum DocumentStoreStatus {
     EMPTY_SYNC = 'EMPTY',
@@ -112,6 +116,38 @@ export interface IDocumentStoreWhereUsed {
     name: string
 }
 
+export interface IUpsertQueueAppServer {
+    appDataSource: DataSource
+    componentNodes: IComponentNodes
+    telemetry: Telemetry
+    cachePool?: CachePool
+}
+
+export interface IExecuteDocStoreUpsert extends IUpsertQueueAppServer {
+    storeId: string
+    totalItems: IDocumentStoreUpsertData[]
+    files: Express.Multer.File[]
+    isRefreshAPI: boolean
+}
+
+export interface IExecutePreviewLoader extends Omit<IUpsertQueueAppServer, 'telemetry'> {
+    data: IDocumentStoreLoaderForPreview
+    isPreviewOnly: boolean
+    telemetry?: Telemetry
+}
+
+export interface IExecuteProcessLoader extends IUpsertQueueAppServer {
+    data: IDocumentStoreLoaderForPreview
+    docLoaderId: string
+    isProcessWithoutUpsert: boolean
+}
+
+export interface IExecuteVectorStoreInsert extends IUpsertQueueAppServer {
+    data: ICommonObject
+    isStrictSave: boolean
+    isVectorStoreInsert: boolean
+}
+
 const getFileName = (fileBase64: string) => {
     let fileNames = []
     if (fileBase64.startsWith('FILE-STORAGE::')) {

packages/server/src/Interface.ts

@@ -1,4 +1,15 @@
-import { IAction, ICommonObject, IFileUpload, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components'
+import {
+    IAction,
+    ICommonObject,
+    IFileUpload,
+    INode,
+    INodeData as INodeDataFromComponent,
+    INodeParams,
+    IServerSideEventStreamer
+} from 'flowise-components'
+import { DataSource } from 'typeorm'
+import { CachePool } from './CachePool'
+import { Telemetry } from './utils/telemetry'
 
 export type MessageType = 'apiMessage' | 'userMessage'
 
@@ -6,6 +17,11 @@ export type ChatflowType = 'CHATFLOW' | 'MULTIAGENT' | 'ASSISTANT'
 
 export type AssistantType = 'CUSTOM' | 'OPENAI' | 'AZURE'
 
+export enum MODE {
+    QUEUE = 'queue',
+    MAIN = 'main'
+}
+
 export enum ChatType {
     INTERNAL = 'INTERNAL',
     EXTERNAL = 'EXTERNAL'
@@ -28,6 +44,7 @@ export interface IChatFlow {
     isPublic?: boolean
     apikeyid?: string
     analytic?: string
+    speechToText?: string
     chatbotConfig?: string
     followUpPrompts?: string
     apiConfig?: string
@@ -226,6 +243,7 @@ export interface IncomingInput {
     leadEmail?: string
     history?: IMessage[]
     action?: IAction
+    streaming?: boolean
 }
 
 export interface IActiveChatflows {
@@ -290,6 +308,34 @@ export interface ICustomTemplate {
     usecases?: string
 }
 
+export interface IFlowConfig {
+    chatflowid: string
+    chatId: string
+    sessionId: string
+    chatHistory: IMessage[]
+    apiMessageId: string
+    overrideConfig?: ICommonObject
+}
+
+export interface IPredictionQueueAppServer {
+    appDataSource: DataSource
+    componentNodes: IComponentNodes
+    sseStreamer: IServerSideEventStreamer
+    telemetry: Telemetry
+    cachePool: CachePool
+}
+
+export interface IExecuteFlowParams extends IPredictionQueueAppServer {
+    incomingInput: IncomingInput
+    chatflow: IChatFlow
+    chatId: string
+    baseURL: string
+    isInternal: boolean
+    signal?: AbortController
+    files?: Express.Multer.File[]
+    isUpsert?: boolean
+}
+
 export interface INodeOverrides {
     [key: string]: {
         label: string

packages/server/src/commands/base.ts

@@ -0,0 +1,201 @@
+import { Command, Flags } from '@oclif/core'
+import path from 'path'
+import dotenv from 'dotenv'
+import logger from '../utils/logger'
+
+dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true })
+
+enum EXIT_CODE {
+    SUCCESS = 0,
+    FAILED = 1
+}
+
+export abstract class BaseCommand extends Command {
+    static flags = {
+        FLOWISE_USERNAME: Flags.string(),
+        FLOWISE_PASSWORD: Flags.string(),
+        FLOWISE_FILE_SIZE_LIMIT: Flags.string(),
+        PORT: Flags.string(),
+        CORS_ORIGINS: Flags.string(),
+        IFRAME_ORIGINS: Flags.string(),
+        DEBUG: Flags.string(),
+        BLOB_STORAGE_PATH: Flags.string(),
+        APIKEY_STORAGE_TYPE: Flags.string(),
+        APIKEY_PATH: Flags.string(),
+        LOG_PATH: Flags.string(),
+        LOG_LEVEL: Flags.string(),
+        TOOL_FUNCTION_BUILTIN_DEP: Flags.string(),
+        TOOL_FUNCTION_EXTERNAL_DEP: Flags.string(),
+        NUMBER_OF_PROXIES: Flags.string(),
+        DATABASE_TYPE: Flags.string(),
+        DATABASE_PATH: Flags.string(),
+        DATABASE_PORT: Flags.string(),
+        DATABASE_HOST: Flags.string(),
+        DATABASE_NAME: Flags.string(),
+        DATABASE_USER: Flags.string(),
+        DATABASE_PASSWORD: Flags.string(),
+        DATABASE_SSL: Flags.string(),
+        DATABASE_SSL_KEY_BASE64: Flags.string(),
+        LANGCHAIN_TRACING_V2: Flags.string(),
+        LANGCHAIN_ENDPOINT: Flags.string(),
+        LANGCHAIN_API_KEY: Flags.string(),
+        LANGCHAIN_PROJECT: Flags.string(),
+        DISABLE_FLOWISE_TELEMETRY: Flags.string(),
+        MODEL_LIST_CONFIG_JSON: Flags.string(),
+        STORAGE_TYPE: Flags.string(),
+        S3_STORAGE_BUCKET_NAME: Flags.string(),
+        S3_STORAGE_ACCESS_KEY_ID: Flags.string(),
+        S3_STORAGE_SECRET_ACCESS_KEY: Flags.string(),
+        S3_STORAGE_REGION: Flags.string(),
+        S3_ENDPOINT_URL: Flags.string(),
+        S3_FORCE_PATH_STYLE: Flags.string(),
+        SHOW_COMMUNITY_NODES: Flags.string(),
+        SECRETKEY_STORAGE_TYPE: Flags.string(),
+        SECRETKEY_PATH: Flags.string(),
+        FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
+        SECRETKEY_AWS_ACCESS_KEY: Flags.string(),
+        SECRETKEY_AWS_SECRET_KEY: Flags.string(),
+        SECRETKEY_AWS_REGION: Flags.string(),
+        DISABLED_NODES: Flags.string(),
+        MODE: Flags.string(),
+        WORKER_CONCURRENCY: Flags.string(),
+        QUEUE_NAME: Flags.string(),
+        QUEUE_REDIS_EVENT_STREAM_MAX_LEN: Flags.string(),
+        REDIS_URL: Flags.string(),
+        REDIS_HOST: Flags.string(),
+        REDIS_PORT: Flags.string(),
+        REDIS_USERNAME: Flags.string(),
+        REDIS_PASSWORD: Flags.string(),
+        REDIS_TLS: Flags.string(),
+        REDIS_CERT: Flags.string(),
+        REDIS_KEY: Flags.string(),
+        REDIS_CA: Flags.string()
+    }
+
+    protected async stopProcess() {
+        // Overridden method by child class
+    }
+
+    protected onTerminate() {
+        return async () => {
+            try {
+                // Shut down the app after timeout if it ever stuck removing pools
+                setTimeout(async () => {
+                    logger.info('Flowise was forced to shut down after 30 secs')
+                    await this.failExit()
+                }, 30000)
+
+                await this.stopProcess()
+            } catch (error) {
+                logger.error('There was an error shutting down Flowise...', error)
+            }
+        }
+    }
+
+    protected async gracefullyExit() {
+        process.exit(EXIT_CODE.SUCCESS)
+    }
+
+    protected async failExit() {
+        process.exit(EXIT_CODE.FAILED)
+    }
+
+    async init(): Promise<void> {
+        await super.init()
+
+        process.on('SIGTERM', this.onTerminate())
+        process.on('SIGINT', this.onTerminate())
+
+        // Prevent throw new Error from crashing the app
+        // TODO: Get rid of this and send proper error message to ui
+        process.on('uncaughtException', (err) => {
+            logger.error('uncaughtException: ', err)
+        })
+
+        process.on('unhandledRejection', (err) => {
+            logger.error('unhandledRejection: ', err)
+        })
+
+        const { flags } = await this.parse(BaseCommand)
+        if (flags.PORT) process.env.PORT = flags.PORT
+        if (flags.CORS_ORIGINS) process.env.CORS_ORIGINS = flags.CORS_ORIGINS
+        if (flags.IFRAME_ORIGINS) process.env.IFRAME_ORIGINS = flags.IFRAME_ORIGINS
+        if (flags.DEBUG) process.env.DEBUG = flags.DEBUG
+        if (flags.NUMBER_OF_PROXIES) process.env.NUMBER_OF_PROXIES = flags.NUMBER_OF_PROXIES
+        if (flags.SHOW_COMMUNITY_NODES) process.env.SHOW_COMMUNITY_NODES = flags.SHOW_COMMUNITY_NODES
+        if (flags.DISABLED_NODES) process.env.DISABLED_NODES = flags.DISABLED_NODES
+
+        // Authorization
+        if (flags.FLOWISE_USERNAME) process.env.FLOWISE_USERNAME = flags.FLOWISE_USERNAME
+        if (flags.FLOWISE_PASSWORD) process.env.FLOWISE_PASSWORD = flags.FLOWISE_PASSWORD
+        if (flags.APIKEY_STORAGE_TYPE) process.env.APIKEY_STORAGE_TYPE = flags.APIKEY_STORAGE_TYPE
+        if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH
+
+        // API Configuration
+        if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT
+
+        // Credentials
+        if (flags.SECRETKEY_STORAGE_TYPE) process.env.SECRETKEY_STORAGE_TYPE = flags.SECRETKEY_STORAGE_TYPE
+        if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
+        if (flags.FLOWISE_SECRETKEY_OVERWRITE) process.env.FLOWISE_SECRETKEY_OVERWRITE = flags.FLOWISE_SECRETKEY_OVERWRITE
+        if (flags.SECRETKEY_AWS_ACCESS_KEY) process.env.SECRETKEY_AWS_ACCESS_KEY = flags.SECRETKEY_AWS_ACCESS_KEY
+        if (flags.SECRETKEY_AWS_SECRET_KEY) process.env.SECRETKEY_AWS_SECRET_KEY = flags.SECRETKEY_AWS_SECRET_KEY
+        if (flags.SECRETKEY_AWS_REGION) process.env.SECRETKEY_AWS_REGION = flags.SECRETKEY_AWS_REGION
+
+        // Logs
+        if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
+        if (flags.LOG_LEVEL) process.env.LOG_LEVEL = flags.LOG_LEVEL
+
+        // Tool functions
+        if (flags.TOOL_FUNCTION_BUILTIN_DEP) process.env.TOOL_FUNCTION_BUILTIN_DEP = flags.TOOL_FUNCTION_BUILTIN_DEP
+        if (flags.TOOL_FUNCTION_EXTERNAL_DEP) process.env.TOOL_FUNCTION_EXTERNAL_DEP = flags.TOOL_FUNCTION_EXTERNAL_DEP
+
+        // Database config
+        if (flags.DATABASE_TYPE) process.env.DATABASE_TYPE = flags.DATABASE_TYPE
+        if (flags.DATABASE_PATH) process.env.DATABASE_PATH = flags.DATABASE_PATH
+        if (flags.DATABASE_PORT) process.env.DATABASE_PORT = flags.DATABASE_PORT
+        if (flags.DATABASE_HOST) process.env.DATABASE_HOST = flags.DATABASE_HOST
+        if (flags.DATABASE_NAME) process.env.DATABASE_NAME = flags.DATABASE_NAME
+        if (flags.DATABASE_USER) process.env.DATABASE_USER = flags.DATABASE_USER
+        if (flags.DATABASE_PASSWORD) process.env.DATABASE_PASSWORD = flags.DATABASE_PASSWORD
+        if (flags.DATABASE_SSL) process.env.DATABASE_SSL = flags.DATABASE_SSL
+        if (flags.DATABASE_SSL_KEY_BASE64) process.env.DATABASE_SSL_KEY_BASE64 = flags.DATABASE_SSL_KEY_BASE64
+
+        // Langsmith tracing
+        if (flags.LANGCHAIN_TRACING_V2) process.env.LANGCHAIN_TRACING_V2 = flags.LANGCHAIN_TRACING_V2
+        if (flags.LANGCHAIN_ENDPOINT) process.env.LANGCHAIN_ENDPOINT = flags.LANGCHAIN_ENDPOINT
+        if (flags.LANGCHAIN_API_KEY) process.env.LANGCHAIN_API_KEY = flags.LANGCHAIN_API_KEY
+        if (flags.LANGCHAIN_PROJECT) process.env.LANGCHAIN_PROJECT = flags.LANGCHAIN_PROJECT
+
+        // Telemetry
+        if (flags.DISABLE_FLOWISE_TELEMETRY) process.env.DISABLE_FLOWISE_TELEMETRY = flags.DISABLE_FLOWISE_TELEMETRY
+
+        // Model list config
+        if (flags.MODEL_LIST_CONFIG_JSON) process.env.MODEL_LIST_CONFIG_JSON = flags.MODEL_LIST_CONFIG_JSON
+
+        // Storage
+        if (flags.STORAGE_TYPE) process.env.STORAGE_TYPE = flags.STORAGE_TYPE
+        if (flags.BLOB_STORAGE_PATH) process.env.BLOB_STORAGE_PATH = flags.BLOB_STORAGE_PATH
+        if (flags.S3_STORAGE_BUCKET_NAME) process.env.S3_STORAGE_BUCKET_NAME = flags.S3_STORAGE_BUCKET_NAME
+        if (flags.S3_STORAGE_ACCESS_KEY_ID) process.env.S3_STORAGE_ACCESS_KEY_ID = flags.S3_STORAGE_ACCESS_KEY_ID
+        if (flags.S3_STORAGE_SECRET_ACCESS_KEY) process.env.S3_STORAGE_SECRET_ACCESS_KEY = flags.S3_STORAGE_SECRET_ACCESS_KEY
+        if (flags.S3_STORAGE_REGION) process.env.S3_STORAGE_REGION = flags.S3_STORAGE_REGION
+        if (flags.S3_ENDPOINT_URL) process.env.S3_ENDPOINT_URL = flags.S3_ENDPOINT_URL
+        if (flags.S3_FORCE_PATH_STYLE) process.env.S3_FORCE_PATH_STYLE = flags.S3_FORCE_PATH_STYLE
+
+        // Queue
+        if (flags.MODE) process.env.MODE = flags.MODE
+        if (flags.REDIS_URL) process.env.REDIS_URL = flags.REDIS_URL
+        if (flags.REDIS_HOST) process.env.REDIS_HOST = flags.REDIS_HOST
+        if (flags.REDIS_PORT) process.env.REDIS_PORT = flags.REDIS_PORT
+        if (flags.REDIS_USERNAME) process.env.REDIS_USERNAME = flags.REDIS_USERNAME
+        if (flags.REDIS_PASSWORD) process.env.REDIS_PASSWORD = flags.REDIS_PASSWORD
+        if (flags.REDIS_TLS) process.env.REDIS_TLS = flags.REDIS_TLS
+        if (flags.REDIS_CERT) process.env.REDIS_CERT = flags.REDIS_CERT
+        if (flags.REDIS_KEY) process.env.REDIS_KEY = flags.REDIS_KEY
+        if (flags.REDIS_CA) process.env.REDIS_CA = flags.REDIS_CA
+        if (flags.WORKER_CONCURRENCY) process.env.WORKER_CONCURRENCY = flags.WORKER_CONCURRENCY
+        if (flags.QUEUE_NAME) process.env.QUEUE_NAME = flags.QUEUE_NAME
+        if (flags.QUEUE_REDIS_EVENT_STREAM_MAX_LEN) process.env.QUEUE_REDIS_EVENT_STREAM_MAX_LEN = flags.QUEUE_REDIS_EVENT_STREAM
+    }
+}

packages/server/src/commands/start.ts

@@ -1,181 +1,33 @@
-import { Command, Flags } from '@oclif/core'
-import path from 'path'
 import * as Server from '../index'
 import * as DataSource from '../DataSource'
-import dotenv from 'dotenv'
 import logger from '../utils/logger'
+import { BaseCommand } from './base'
 
-dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true })
+export default class Start extends BaseCommand {
+    async run(): Promise<void> {
+        logger.info('Starting Flowise...')
+        await DataSource.init()
+        await Server.start()
+    }
 
-enum EXIT_CODE {
-    SUCCESS = 0,
-    FAILED = 1
-}
-let processExitCode = EXIT_CODE.SUCCESS
-
-export default class Start extends Command {
-    static args = []
-    static flags = {
-        FLOWISE_USERNAME: Flags.string(),
-        FLOWISE_PASSWORD: Flags.string(),
-        FLOWISE_FILE_SIZE_LIMIT: Flags.string(),
-        PORT: Flags.string(),
-        CORS_ORIGINS: Flags.string(),
-        IFRAME_ORIGINS: Flags.string(),
-        DEBUG: Flags.string(),
-        BLOB_STORAGE_PATH: Flags.string(),
-        APIKEY_STORAGE_TYPE: Flags.string(),
-        APIKEY_PATH: Flags.string(),
-        LOG_PATH: Flags.string(),
-        LOG_LEVEL: Flags.string(),
-        TOOL_FUNCTION_BUILTIN_DEP: Flags.string(),
-        TOOL_FUNCTION_EXTERNAL_DEP: Flags.string(),
-        NUMBER_OF_PROXIES: Flags.string(),
-        DISABLE_CHATFLOW_REUSE: Flags.string(),
-        DATABASE_TYPE: Flags.string(),
-        DATABASE_PATH: Flags.string(),
-        DATABASE_PORT: Flags.string(),
-        DATABASE_HOST: Flags.string(),
-        DATABASE_NAME: Flags.string(),
-        DATABASE_USER: Flags.string(),
-        DATABASE_PASSWORD: Flags.string(),
-        DATABASE_SSL: Flags.string(),
-        DATABASE_SSL_KEY_BASE64: Flags.string(),
-        LANGCHAIN_TRACING_V2: Flags.string(),
-        LANGCHAIN_ENDPOINT: Flags.string(),
-        LANGCHAIN_API_KEY: Flags.string(),
-        LANGCHAIN_PROJECT: Flags.string(),
-        DISABLE_FLOWISE_TELEMETRY: Flags.string(),
-        MODEL_LIST_CONFIG_JSON: Flags.string(),
-        STORAGE_TYPE: Flags.string(),
-        S3_STORAGE_BUCKET_NAME: Flags.string(),
-        S3_STORAGE_ACCESS_KEY_ID: Flags.string(),
-        S3_STORAGE_SECRET_ACCESS_KEY: Flags.string(),
-        S3_STORAGE_REGION: Flags.string(),
-        S3_ENDPOINT_URL: Flags.string(),
-        S3_FORCE_PATH_STYLE: Flags.string(),
-        SHOW_COMMUNITY_NODES: Flags.string(),
-        SECRETKEY_STORAGE_TYPE: Flags.string(),
-        SECRETKEY_PATH: Flags.string(),
-        FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
-        SECRETKEY_AWS_ACCESS_KEY: Flags.string(),
-        SECRETKEY_AWS_SECRET_KEY: Flags.string(),
-        SECRETKEY_AWS_REGION: Flags.string(),
-        DISABLED_NODES: Flags.string()
+    async catch(error: Error) {
+        if (error.stack) logger.error(error.stack)
+        await new Promise((resolve) => {
+            setTimeout(resolve, 1000)
+        })
+        await this.failExit()
     }
 
     async stopProcess() {
-        logger.info('Shutting down Flowise...')
         try {
-            // Shut down the app after timeout if it ever stuck removing pools
-            setTimeout(() => {
-                logger.info('Flowise was forced to shut down after 30 secs')
-                process.exit(processExitCode)
-            }, 30000)
-
-            // Removing pools
+            logger.info(`Shutting down Flowise...`)
             const serverApp = Server.getInstance()
             if (serverApp) await serverApp.stopApp()
         } catch (error) {
             logger.error('There was an error shutting down Flowise...', error)
+            await this.failExit()
         }
-        process.exit(processExitCode)
-    }
 
-    async run(): Promise<void> {
-        process.on('SIGTERM', this.stopProcess)
-        process.on('SIGINT', this.stopProcess)
-
-        // Prevent throw new Error from crashing the app
-        // TODO: Get rid of this and send proper error message to ui
-        process.on('uncaughtException', (err) => {
-            logger.error('uncaughtException: ', err)
-        })
-
-        process.on('unhandledRejection', (err) => {
-            logger.error('unhandledRejection: ', err)
-        })
-
-        const { flags } = await this.parse(Start)
-
-        if (flags.PORT) process.env.PORT = flags.PORT
-        if (flags.CORS_ORIGINS) process.env.CORS_ORIGINS = flags.CORS_ORIGINS
-        if (flags.IFRAME_ORIGINS) process.env.IFRAME_ORIGINS = flags.IFRAME_ORIGINS
-        if (flags.DEBUG) process.env.DEBUG = flags.DEBUG
-        if (flags.NUMBER_OF_PROXIES) process.env.NUMBER_OF_PROXIES = flags.NUMBER_OF_PROXIES
-        if (flags.DISABLE_CHATFLOW_REUSE) process.env.DISABLE_CHATFLOW_REUSE = flags.DISABLE_CHATFLOW_REUSE
-        if (flags.SHOW_COMMUNITY_NODES) process.env.SHOW_COMMUNITY_NODES = flags.SHOW_COMMUNITY_NODES
-        if (flags.DISABLED_NODES) process.env.DISABLED_NODES = flags.DISABLED_NODES
-
-        // Authorization
-        if (flags.FLOWISE_USERNAME) process.env.FLOWISE_USERNAME = flags.FLOWISE_USERNAME
-        if (flags.FLOWISE_PASSWORD) process.env.FLOWISE_PASSWORD = flags.FLOWISE_PASSWORD
-        if (flags.APIKEY_STORAGE_TYPE) process.env.APIKEY_STORAGE_TYPE = flags.APIKEY_STORAGE_TYPE
-        if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH
-
-        // API Configuration
-        if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT
-
-        // Credentials
-        if (flags.SECRETKEY_STORAGE_TYPE) process.env.SECRETKEY_STORAGE_TYPE = flags.SECRETKEY_STORAGE_TYPE
-        if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
-        if (flags.FLOWISE_SECRETKEY_OVERWRITE) process.env.FLOWISE_SECRETKEY_OVERWRITE = flags.FLOWISE_SECRETKEY_OVERWRITE
-        if (flags.SECRETKEY_AWS_ACCESS_KEY) process.env.SECRETKEY_AWS_ACCESS_KEY = flags.SECRETKEY_AWS_ACCESS_KEY
-        if (flags.SECRETKEY_AWS_SECRET_KEY) process.env.SECRETKEY_AWS_SECRET_KEY = flags.SECRETKEY_AWS_SECRET_KEY
-        if (flags.SECRETKEY_AWS_REGION) process.env.SECRETKEY_AWS_REGION = flags.SECRETKEY_AWS_REGION
-
-        // Logs
-        if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
-        if (flags.LOG_LEVEL) process.env.LOG_LEVEL = flags.LOG_LEVEL
-
-        // Tool functions
-        if (flags.TOOL_FUNCTION_BUILTIN_DEP) process.env.TOOL_FUNCTION_BUILTIN_DEP = flags.TOOL_FUNCTION_BUILTIN_DEP
-        if (flags.TOOL_FUNCTION_EXTERNAL_DEP) process.env.TOOL_FUNCTION_EXTERNAL_DEP = flags.TOOL_FUNCTION_EXTERNAL_DEP
-
-        // Database config
-        if (flags.DATABASE_TYPE) process.env.DATABASE_TYPE = flags.DATABASE_TYPE
-        if (flags.DATABASE_PATH) process.env.DATABASE_PATH = flags.DATABASE_PATH
-        if (flags.DATABASE_PORT) process.env.DATABASE_PORT = flags.DATABASE_PORT
-        if (flags.DATABASE_HOST) process.env.DATABASE_HOST = flags.DATABASE_HOST
-        if (flags.DATABASE_NAME) process.env.DATABASE_NAME = flags.DATABASE_NAME
-        if (flags.DATABASE_USER) process.env.DATABASE_USER = flags.DATABASE_USER
-        if (flags.DATABASE_PASSWORD) process.env.DATABASE_PASSWORD = flags.DATABASE_PASSWORD
-        if (flags.DATABASE_SSL) process.env.DATABASE_SSL = flags.DATABASE_SSL
-        if (flags.DATABASE_SSL_KEY_BASE64) process.env.DATABASE_SSL_KEY_BASE64 = flags.DATABASE_SSL_KEY_BASE64
-
-        // Langsmith tracing
-        if (flags.LANGCHAIN_TRACING_V2) process.env.LANGCHAIN_TRACING_V2 = flags.LANGCHAIN_TRACING_V2
-        if (flags.LANGCHAIN_ENDPOINT) process.env.LANGCHAIN_ENDPOINT = flags.LANGCHAIN_ENDPOINT
-        if (flags.LANGCHAIN_API_KEY) process.env.LANGCHAIN_API_KEY = flags.LANGCHAIN_API_KEY
-        if (flags.LANGCHAIN_PROJECT) process.env.LANGCHAIN_PROJECT = flags.LANGCHAIN_PROJECT
-
-        // Telemetry
-        if (flags.DISABLE_FLOWISE_TELEMETRY) process.env.DISABLE_FLOWISE_TELEMETRY = flags.DISABLE_FLOWISE_TELEMETRY
-
-        // Model list config
-        if (flags.MODEL_LIST_CONFIG_JSON) process.env.MODEL_LIST_CONFIG_JSON = flags.MODEL_LIST_CONFIG_JSON
-
-        // Storage
-        if (flags.STORAGE_TYPE) process.env.STORAGE_TYPE = flags.STORAGE_TYPE
-        if (flags.BLOB_STORAGE_PATH) process.env.BLOB_STORAGE_PATH = flags.BLOB_STORAGE_PATH
-        if (flags.S3_STORAGE_BUCKET_NAME) process.env.S3_STORAGE_BUCKET_NAME = flags.S3_STORAGE_BUCKET_NAME
-        if (flags.S3_STORAGE_ACCESS_KEY_ID) process.env.S3_STORAGE_ACCESS_KEY_ID = flags.S3_STORAGE_ACCESS_KEY_ID
-        if (flags.S3_STORAGE_SECRET_ACCESS_KEY) process.env.S3_STORAGE_SECRET_ACCESS_KEY = flags.S3_STORAGE_SECRET_ACCESS_KEY
-        if (flags.S3_STORAGE_REGION) process.env.S3_STORAGE_REGION = flags.S3_STORAGE_REGION
-        if (flags.S3_ENDPOINT_URL) process.env.S3_ENDPOINT_URL = flags.S3_ENDPOINT_URL
-        if (flags.S3_FORCE_PATH_STYLE) process.env.S3_FORCE_PATH_STYLE = flags.S3_FORCE_PATH_STYLE
-
-        await (async () => {
-            try {
-                logger.info('Starting Flowise...')
-                await DataSource.init()
-                await Server.start()
-            } catch (error) {
-                logger.error('There was an error starting Flowise...', error)
-                processExitCode = EXIT_CODE.FAILED
-                // @ts-ignore
-                process.emit('SIGINT')
-            }
-        })()
+        await this.gracefullyExit()
     }
 }

packages/server/src/commands/worker.ts

@@ -0,0 +1,103 @@
+import logger from '../utils/logger'
+import { QueueManager } from '../queue/QueueManager'
+import { BaseCommand } from './base'
+import { getDataSource } from '../DataSource'
+import { Telemetry } from '../utils/telemetry'
+import { NodesPool } from '../NodesPool'
+import { CachePool } from '../CachePool'
+import { QueueEvents, QueueEventsListener } from 'bullmq'
+import { AbortControllerPool } from '../AbortControllerPool'
+
+interface CustomListener extends QueueEventsListener {
+    abort: (args: { id: string }, id: string) => void
+}
+
+export default class Worker extends BaseCommand {
+    predictionWorkerId: string
+    upsertionWorkerId: string
+
+    async run(): Promise<void> {
+        logger.info('Starting Flowise Worker...')
+
+        const { appDataSource, telemetry, componentNodes, cachePool, abortControllerPool } = await this.prepareData()
+
+        const queueManager = QueueManager.getInstance()
+        queueManager.setupAllQueues({
+            componentNodes,
+            telemetry,
+            cachePool,
+            appDataSource,
+            abortControllerPool
+        })
+
+        /** Prediction */
+        const predictionQueue = queueManager.getQueue('prediction')
+        const predictionWorker = predictionQueue.createWorker()
+        this.predictionWorkerId = predictionWorker.id
+        logger.info(`Prediction Worker ${this.predictionWorkerId} created`)
+
+        const predictionQueueName = predictionQueue.getQueueName()
+        const queueEvents = new QueueEvents(predictionQueueName, { connection: queueManager.getConnection() })
+
+        queueEvents.on<CustomListener>('abort', async ({ id }: { id: string }) => {
+            abortControllerPool.abort(id)
+        })
+
+        /** Upsertion */
+        const upsertionQueue = queueManager.getQueue('upsert')
+        const upsertionWorker = upsertionQueue.createWorker()
+        this.upsertionWorkerId = upsertionWorker.id
+        logger.info(`Upsertion Worker ${this.upsertionWorkerId} created`)
+
+        // Keep the process running
+        process.stdin.resume()
+    }
+
+    async prepareData() {
+        // Init database
+        const appDataSource = getDataSource()
+        await appDataSource.initialize()
+        await appDataSource.runMigrations({ transaction: 'each' })
+
+        // Initialize abortcontroller pool
+        const abortControllerPool = new AbortControllerPool()
+
+        // Init telemetry
+        const telemetry = new Telemetry()
+
+        // Initialize nodes pool
+        const nodesPool = new NodesPool()
+        await nodesPool.initialize()
+
+        // Initialize cache pool
+        const cachePool = new CachePool()
+
+        return { appDataSource, telemetry, componentNodes: nodesPool.componentNodes, cachePool, abortControllerPool }
+    }
+
+    async catch(error: Error) {
+        if (error.stack) logger.error(error.stack)
+        await new Promise((resolve) => {
+            setTimeout(resolve, 1000)
+        })
+        await this.failExit()
+    }
+
+    async stopProcess() {
+        try {
+            const queueManager = QueueManager.getInstance()
+            const predictionWorker = queueManager.getQueue('prediction').getWorker()
+            logger.info(`Shutting down Flowise Prediction Worker ${this.predictionWorkerId}...`)
+            await predictionWorker.close()
+
+            const upsertWorker = queueManager.getQueue('upsert').getWorker()
+            logger.info(`Shutting down Flowise Upsertion Worker ${this.upsertionWorkerId}...`)
+            await upsertWorker.close()
+        } catch (error) {
+            logger.error('There was an error shutting down Flowise Worker...', error)
+            await this.failExit()
+        }
+
+        await this.gracefullyExit()
+    }
+}

packages/server/src/controllers/chatflows/index.ts

@@ -2,7 +2,7 @@ import { NextFunction, Request, Response } from 'express'
 import { StatusCodes } from 'http-status-codes'
 import apiKeyService from '../../services/apikey'
 import { ChatFlow } from '../../database/entities/ChatFlow'
-import { updateRateLimiter } from '../../utils/rateLimit'
+import { RateLimiterManager } from '../../utils/rateLimit'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { ChatflowType } from '../../Interface'
 import chatflowsService from '../../services/chatflows'
@@ -130,7 +130,8 @@ const updateChatflow = async (req: Request, res: Response, next: NextFunction) =
         Object.assign(updateChatFlow, body)
 
         updateChatFlow.id = chatflow.id
-        updateRateLimiter(updateChatFlow)
+        const rateLimiterManager = RateLimiterManager.getInstance()
+        await rateLimiterManager.updateRateLimiter(updateChatFlow)
 
         const apiResponse = await chatflowsService.updateChatflow(chatflow, updateChatFlow)
         return res.json(apiResponse)

packages/server/src/controllers/documentstore/index.ts

@@ -4,15 +4,8 @@ import documentStoreService from '../../services/documentstore'
 import { DocumentStore } from '../../database/entities/DocumentStore'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { DocumentStoreDTO } from '../../Interface'
-import { getRateLimiter } from '../../utils/rateLimit'
-
-const getRateLimiterMiddleware = async (req: Request, res: Response, next: NextFunction) => {
-    try {
-        return getRateLimiter(req, res, next)
-    } catch (error) {
-        next(error)
-    }
-}
+import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
+import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../../Interface.Metrics'
 
 const createDocumentStore = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -90,8 +83,14 @@ const getDocumentStoreFileChunks = async (req: Request, res: Response, next: Nex
                 `Error: documentStoreController.getDocumentStoreFileChunks - fileId not provided!`
             )
         }
+        const appDataSource = getRunningExpressApp().AppDataSource
         const page = req.params.pageNo ? parseInt(req.params.pageNo) : 1
-        const apiResponse = await documentStoreService.getDocumentStoreFileChunks(req.params.storeId, req.params.fileId, page)
+        const apiResponse = await documentStoreService.getDocumentStoreFileChunks(
+            appDataSource,
+            req.params.storeId,
+            req.params.fileId,
+            page
+        )
         return res.json(apiResponse)
     } catch (error) {
         next(error)
@@ -171,6 +170,7 @@ const editDocumentStoreFileChunk = async (req: Request, res: Response, next: Nex
 
 const saveProcessingLoader = async (req: Request, res: Response, next: NextFunction) => {
     try {
+        const appServer = getRunningExpressApp()
         if (typeof req.body === 'undefined') {
             throw new InternalFlowiseError(
                 StatusCodes.PRECONDITION_FAILED,
@@ -178,7 +178,7 @@ const saveProcessingLoader = async (req: Request, res: Response, next: NextFunct
             )
         }
         const body = req.body
-        const apiResponse = await documentStoreService.saveProcessingLoader(body)
+        const apiResponse = await documentStoreService.saveProcessingLoader(appServer.AppDataSource, body)
         return res.json(apiResponse)
     } catch (error) {
         next(error)
@@ -201,7 +201,7 @@ const processLoader = async (req: Request, res: Response, next: NextFunction) =>
         }
         const docLoaderId = req.params.loaderId
         const body = req.body
-        const apiResponse = await documentStoreService.processLoader(body, docLoaderId)
+        const apiResponse = await documentStoreService.processLoaderMiddleware(body, docLoaderId)
         return res.json(apiResponse)
     } catch (error) {
         next(error)
@@ -264,7 +264,7 @@ const previewFileChunks = async (req: Request, res: Response, next: NextFunction
         }
         const body = req.body
         body.preview = true
-        const apiResponse = await documentStoreService.previewChunks(body)
+        const apiResponse = await documentStoreService.previewChunksMiddleware(body)
         return res.json(apiResponse)
     } catch (error) {
         next(error)
@@ -286,9 +286,15 @@ const insertIntoVectorStore = async (req: Request, res: Response, next: NextFunc
             throw new Error('Error: documentStoreController.insertIntoVectorStore - body not provided!')
         }
         const body = req.body
-        const apiResponse = await documentStoreService.insertIntoVectorStore(body)
+        const apiResponse = await documentStoreService.insertIntoVectorStoreMiddleware(body)
+        getRunningExpressApp().metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
+            status: FLOWISE_COUNTER_STATUS.SUCCESS
+        })
         return res.json(DocumentStoreDTO.fromEntity(apiResponse))
     } catch (error) {
+        getRunningExpressApp().metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
+            status: FLOWISE_COUNTER_STATUS.FAILURE
+        })
         next(error)
     }
 }
@@ -327,7 +333,9 @@ const saveVectorStoreConfig = async (req: Request, res: Response, next: NextFunc
             throw new Error('Error: documentStoreController.saveVectorStoreConfig - body not provided!')
         }
         const body = req.body
-        const apiResponse = await documentStoreService.saveVectorStoreConfig(body)
+        const appDataSource = getRunningExpressApp().AppDataSource
+        const componentNodes = getRunningExpressApp().nodesPool.componentNodes
+        const apiResponse = await documentStoreService.saveVectorStoreConfig(appDataSource, componentNodes, body)
         return res.json(apiResponse)
     } catch (error) {
         next(error)
@@ -388,8 +396,14 @@ const upsertDocStoreMiddleware = async (req: Request, res: Response, next: NextF
         const body = req.body
         const files = (req.files as Express.Multer.File[]) || []
         const apiResponse = await documentStoreService.upsertDocStoreMiddleware(req.params.id, body, files)
+        getRunningExpressApp().metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
+            status: FLOWISE_COUNTER_STATUS.SUCCESS
+        })
         return res.json(apiResponse)
     } catch (error) {
+        getRunningExpressApp().metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
+            status: FLOWISE_COUNTER_STATUS.FAILURE
+        })
         next(error)
     }
 }
@@ -404,8 +418,14 @@ const refreshDocStoreMiddleware = async (req: Request, res: Response, next: Next
         }
         const body = req.body
         const apiResponse = await documentStoreService.refreshDocStoreMiddleware(req.params.id, body)
+        getRunningExpressApp().metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
+            status: FLOWISE_COUNTER_STATUS.SUCCESS
+        })
         return res.json(apiResponse)
     } catch (error) {
+        getRunningExpressApp().metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
+            status: FLOWISE_COUNTER_STATUS.FAILURE
+        })
         next(error)
     }
 }
@@ -470,7 +490,6 @@ export default {
     queryVectorStore,
     deleteVectorStoreFromStore,
     updateVectorStoreConfigOnly,
-    getRateLimiterMiddleware,
     upsertDocStoreMiddleware,
     refreshDocStoreMiddleware,
     saveProcessingLoader,

packages/server/src/controllers/internal-predictions/index.ts

@@ -2,6 +2,7 @@ import { Request, Response, NextFunction } from 'express'
 import { utilBuildChatflow } from '../../utils/buildChatflow'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { getErrorMessage } from '../../errors/utils'
+import { MODE } from '../../Interface'
 
 // Send input message and get prediction result (Internal)
 const createInternalPrediction = async (req: Request, res: Response, next: NextFunction) => {
@@ -11,7 +12,7 @@ const createInternalPrediction = async (req: Request, res: Response, next: NextF
             return
         } else {
             const apiResponse = await utilBuildChatflow(req, true)
-            return res.json(apiResponse)
+            if (apiResponse) return res.json(apiResponse)
         }
     } catch (error) {
         next(error)
@@ -22,6 +23,7 @@ const createInternalPrediction = async (req: Request, res: Response, next: NextF
 const createAndStreamInternalPrediction = async (req: Request, res: Response, next: NextFunction) => {
     const chatId = req.body.chatId
     const sseStreamer = getRunningExpressApp().sseStreamer
+
     try {
         sseStreamer.addClient(chatId, res)
         res.setHeader('Content-Type', 'text/event-stream')
@@ -30,6 +32,10 @@ const createAndStreamInternalPrediction = async (req: Request, res: Response, ne
         res.setHeader('X-Accel-Buffering', 'no') //nginx config: https://serverfault.com/a/801629
         res.flushHeaders()
 
+        if (process.env.MODE === MODE.QUEUE) {
+            getRunningExpressApp().redisSubscriber.subscribe(chatId)
+        }
+
         const apiResponse = await utilBuildChatflow(req, true)
         sseStreamer.streamMetadataEvent(apiResponse.chatId, apiResponse)
     } catch (error) {

packages/server/src/controllers/predictions/index.ts

@@ -1,5 +1,5 @@
 import { Request, Response, NextFunction } from 'express'
-import { getRateLimiter } from '../../utils/rateLimit'
+import { RateLimiterManager } from '../../utils/rateLimit'
 import chatflowsService from '../../services/chatflows'
 import logger from '../../utils/logger'
 import predictionsServices from '../../services/predictions'
@@ -8,6 +8,7 @@ import { StatusCodes } from 'http-status-codes'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { v4 as uuidv4 } from 'uuid'
 import { getErrorMessage } from '../../errors/utils'
+import { MODE } from '../../Interface'
 
 // Send input message and get prediction result (External)
 const createPrediction = async (req: Request, res: Response, next: NextFunction) => {
@@ -55,6 +56,7 @@ const createPrediction = async (req: Request, res: Response, next: NextFunction)
             const isStreamingRequested = req.body.streaming === 'true' || req.body.streaming === true
             if (streamable?.isStreaming && isStreamingRequested) {
                 const sseStreamer = getRunningExpressApp().sseStreamer
+
                 let chatId = req.body.chatId
                 if (!req.body.chatId) {
                     chatId = req.body.chatId ?? req.body.overrideConfig?.sessionId ?? uuidv4()
@@ -68,6 +70,10 @@ const createPrediction = async (req: Request, res: Response, next: NextFunction)
                     res.setHeader('X-Accel-Buffering', 'no') //nginx config: https://serverfault.com/a/801629
                     res.flushHeaders()
 
+                    if (process.env.MODE === MODE.QUEUE) {
+                        getRunningExpressApp().redisSubscriber.subscribe(chatId)
+                    }
+
                     const apiResponse = await predictionsServices.buildChatflow(req)
                     sseStreamer.streamMetadataEvent(apiResponse.chatId, apiResponse)
                 } catch (error) {
@@ -96,7 +102,7 @@ const createPrediction = async (req: Request, res: Response, next: NextFunction)
 
 const getRateLimiterMiddleware = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        return getRateLimiter(req, res, next)
+        return RateLimiterManager.getInstance().getRateLimiter()(req, res, next)
     } catch (error) {
         next(error)
     }

packages/server/src/controllers/vectors/index.ts

@@ -1,10 +1,10 @@
 import { Request, Response, NextFunction } from 'express'
 import vectorsService from '../../services/vectors'
-import { getRateLimiter } from '../../utils/rateLimit'
+import { RateLimiterManager } from '../../utils/rateLimit'
 
 const getRateLimiterMiddleware = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        return getRateLimiter(req, res, next)
+        return RateLimiterManager.getInstance().getRateLimiter()(req, res, next)
     } catch (error) {
         next(error)
     }

packages/server/src/index.ts

@@ -4,17 +4,16 @@ import path from 'path'
 import cors from 'cors'
 import http from 'http'
 import basicAuth from 'express-basic-auth'
-import { Server } from 'socket.io'
 import { DataSource } from 'typeorm'
-import { IChatFlow } from './Interface'
+import { MODE } from './Interface'
 import { getNodeModulesPackagePath, getEncryptionKey } from './utils'
 import logger, { expressRequestLogger } from './utils/logger'
 import { getDataSource } from './DataSource'
 import { NodesPool } from './NodesPool'
 import { ChatFlow } from './database/entities/ChatFlow'
-import { ChatflowPool } from './ChatflowPool'
 import { CachePool } from './CachePool'
-import { initializeRateLimiter } from './utils/rateLimit'
+import { AbortControllerPool } from './AbortControllerPool'
+import { RateLimiterManager } from './utils/rateLimit'
 import { getAPIKeys } from './utils/apiKey'
 import { sanitizeMiddleware, getCorsOptions, getAllowedIframeOrigins } from './utils/XSS'
 import { Telemetry } from './utils/telemetry'
@@ -25,14 +24,13 @@ import { validateAPIKey } from './utils/validateKey'
 import { IMetricsProvider } from './Interface.Metrics'
 import { Prometheus } from './metrics/Prometheus'
 import { OpenTelemetry } from './metrics/OpenTelemetry'
+import { QueueManager } from './queue/QueueManager'
+import { RedisEventSubscriber } from './queue/RedisEventSubscriber'
 import { WHITELIST_URLS } from './utils/constants'
 import 'global-agent/bootstrap'
 
 declare global {
     namespace Express {
-        interface Request {
-            io?: Server
-        }
         namespace Multer {
             interface File {
                 bucket: string
@@ -53,12 +51,15 @@ declare global {
 export class App {
     app: express.Application
     nodesPool: NodesPool
-    chatflowPool: ChatflowPool
+    abortControllerPool: AbortControllerPool
     cachePool: CachePool
     telemetry: Telemetry
+    rateLimiterManager: RateLimiterManager
     AppDataSource: DataSource = getDataSource()
     sseStreamer: SSEStreamer
     metricsProvider: IMetricsProvider
+    queueManager: QueueManager
+    redisSubscriber: RedisEventSubscriber
 
     constructor() {
         this.app = express()
@@ -77,8 +78,8 @@ export class App {
             this.nodesPool = new NodesPool()
             await this.nodesPool.initialize()
 
-            // Initialize chatflow pool
-            this.chatflowPool = new ChatflowPool()
+            // Initialize abort controllers pool
+            this.abortControllerPool = new AbortControllerPool()
 
             // Initialize API keys
             await getAPIKeys()
@@ -87,21 +88,39 @@ export class App {
             await getEncryptionKey()
 
             // Initialize Rate Limit
-            const AllChatFlow: IChatFlow[] = await getAllChatFlow()
-            await initializeRateLimiter(AllChatFlow)
+            this.rateLimiterManager = RateLimiterManager.getInstance()
+            await this.rateLimiterManager.initializeRateLimiters(await getDataSource().getRepository(ChatFlow).find())
 
             // Initialize cache pool
             this.cachePool = new CachePool()
 
             // Initialize telemetry
             this.telemetry = new Telemetry()
+
+            // Initialize SSE Streamer
+            this.sseStreamer = new SSEStreamer()
+
+            // Init Queues
+            if (process.env.MODE === MODE.QUEUE) {
+                this.queueManager = QueueManager.getInstance()
+                this.queueManager.setupAllQueues({
+                    componentNodes: this.nodesPool.componentNodes,
+                    telemetry: this.telemetry,
+                    cachePool: this.cachePool,
+                    appDataSource: this.AppDataSource,
+                    abortControllerPool: this.abortControllerPool
+                })
+                this.redisSubscriber = new RedisEventSubscriber(this.sseStreamer)
+                await this.redisSubscriber.connect()
+            }
+
             logger.info('📦 [server]: Data Source has been initialized!')
         } catch (error) {
             logger.error('❌ [server]: Error during Data Source initialization:', error)
         }
     }
 
-    async config(socketIO?: Server) {
+    async config() {
         // Limit is needed to allow sending/receiving base64 encoded string
         const flowise_file_size_limit = process.env.FLOWISE_FILE_SIZE_LIMIT || '50mb'
         this.app.use(express.json({ limit: flowise_file_size_limit }))
@@ -133,12 +152,6 @@ export class App {
         // Add the sanitizeMiddleware to guard against XSS
         this.app.use(sanitizeMiddleware)
 
-        // Make io accessible to our router on req.io
-        this.app.use((req, res, next) => {
-            req.io = socketIO
-            next()
-        })
-
         const whitelistURLs = WHITELIST_URLS
         const URL_CASE_INSENSITIVE_REGEX: RegExp = /\/api\/v1\//i
         const URL_CASE_SENSITIVE_REGEX: RegExp = /\/api\/v1\//
@@ -227,7 +240,6 @@ export class App {
         }
 
         this.app.use('/api/v1', flowiseApiV1Router)
-        this.sseStreamer = new SSEStreamer(this.app)
 
         // ----------------------------------------
         // Configure number of proxies in Host Environment
@@ -239,6 +251,10 @@ export class App {
             })
         })
 
+        if (process.env.MODE === MODE.QUEUE) {
+            this.app.use('/admin/queues', this.queueManager.getBullBoardRouter())
+        }
+
         // ----------------------------------------
         // Serve UI static
         // ----------------------------------------
@@ -262,6 +278,9 @@ export class App {
         try {
             const removePromises: any[] = []
             removePromises.push(this.telemetry.flush())
+            if (this.queueManager) {
+                removePromises.push(this.redisSubscriber.disconnect())
+            }
             await Promise.all(removePromises)
         } catch (e) {
             logger.error(`❌[server]: Flowise Server shut down error: ${e}`)
@@ -271,10 +290,6 @@ export class App {
 
 let serverApp: App | undefined
 
-export async function getAllChatFlow(): Promise<IChatFlow[]> {
-    return await getDataSource().getRepository(ChatFlow).find()
-}
-
 export async function start(): Promise<void> {
     serverApp = new App()
 
@@ -282,12 +297,8 @@ export async function start(): Promise<void> {
     const port = parseInt(process.env.PORT || '', 10) || 3000
     const server = http.createServer(serverApp.app)
 
-    const io = new Server(server, {
-        cors: getCorsOptions()
-    })
-
     await serverApp.initDatabase()
-    await serverApp.config(io)
+    await serverApp.config()
 
     server.listen(port, host, () => {
         logger.info(`⚡️ [server]: Flowise Server is listening at ${host ? 'http://' + host : ''}:${port}`)

packages/server/src/queue/BaseQueue.ts

@@ -0,0 +1,81 @@
+import { Queue, Worker, Job, QueueEvents, RedisOptions } from 'bullmq'
+import { v4 as uuidv4 } from 'uuid'
+import logger from '../utils/logger'
+
+const QUEUE_REDIS_EVENT_STREAM_MAX_LEN = process.env.QUEUE_REDIS_EVENT_STREAM_MAX_LEN
+    ? parseInt(process.env.QUEUE_REDIS_EVENT_STREAM_MAX_LEN)
+    : 10000
+const WORKER_CONCURRENCY = process.env.WORKER_CONCURRENCY ? parseInt(process.env.WORKER_CONCURRENCY) : 100000
+
+export abstract class BaseQueue {
+    protected queue: Queue
+    protected queueEvents: QueueEvents
+    protected connection: RedisOptions
+    private worker: Worker
+
+    constructor(queueName: string, connection: RedisOptions) {
+        this.connection = connection
+        this.queue = new Queue(queueName, {
+            connection: this.connection,
+            streams: { events: { maxLen: QUEUE_REDIS_EVENT_STREAM_MAX_LEN } }
+        })
+        this.queueEvents = new QueueEvents(queueName, { connection: this.connection })
+    }
+
+    abstract processJob(data: any): Promise<any>
+
+    abstract getQueueName(): string
+
+    abstract getQueue(): Queue
+
+    public getWorker(): Worker {
+        return this.worker
+    }
+
+    public async addJob(jobData: any): Promise<Job> {
+        const jobId = jobData.id || uuidv4()
+        return await this.queue.add(jobId, jobData, { removeOnFail: true })
+    }
+
+    public createWorker(concurrency: number = WORKER_CONCURRENCY): Worker {
+        this.worker = new Worker(
+            this.queue.name,
+            async (job: Job) => {
+                const start = new Date().getTime()
+                logger.info(`Processing job ${job.id} in ${this.queue.name} at ${new Date().toISOString()}`)
+                const result = await this.processJob(job.data)
+                const end = new Date().getTime()
+                logger.info(`Completed job ${job.id} in ${this.queue.name} at ${new Date().toISOString()} (${end - start}ms)`)
+                return result
+            },
+            {
+                connection: this.connection,
+                concurrency
+            }
+        )
+        return this.worker
+    }
+
+    public async getJobs(): Promise<Job[]> {
+        return await this.queue.getJobs()
+    }
+
+    public async getJobCounts(): Promise<{ [index: string]: number }> {
+        return await this.queue.getJobCounts()
+    }
+
+    public async getJobByName(jobName: string): Promise<Job> {
+        const jobs = await this.queue.getJobs()
+        const job = jobs.find((job) => job.name === jobName)
+        if (!job) throw new Error(`Job name ${jobName} not found`)
+        return job
+    }
+
+    public getQueueEvents(): QueueEvents {
+        return this.queueEvents
+    }
+
+    public async clearQueue(): Promise<void> {
+        await this.queue.obliterate({ force: true })
+    }
+}

packages/server/src/queue/PredictionQueue.ts

@@ -0,0 +1,64 @@
+import { DataSource } from 'typeorm'
+import { executeFlow } from '../utils/buildChatflow'
+import { IComponentNodes, IExecuteFlowParams } from '../Interface'
+import { Telemetry } from '../utils/telemetry'
+import { CachePool } from '../CachePool'
+import { RedisEventPublisher } from './RedisEventPublisher'
+import { AbortControllerPool } from '../AbortControllerPool'
+import { BaseQueue } from './BaseQueue'
+import { RedisOptions } from 'bullmq'
+
+interface PredictionQueueOptions {
+    appDataSource: DataSource
+    telemetry: Telemetry
+    cachePool: CachePool
+    componentNodes: IComponentNodes
+    abortControllerPool: AbortControllerPool
+}
+
+export class PredictionQueue extends BaseQueue {
+    private componentNodes: IComponentNodes
+    private telemetry: Telemetry
+    private cachePool: CachePool
+    private appDataSource: DataSource
+    private abortControllerPool: AbortControllerPool
+    private redisPublisher: RedisEventPublisher
+    private queueName: string
+
+    constructor(name: string, connection: RedisOptions, options: PredictionQueueOptions) {
+        super(name, connection)
+        this.queueName = name
+        this.componentNodes = options.componentNodes || {}
+        this.telemetry = options.telemetry
+        this.cachePool = options.cachePool
+        this.appDataSource = options.appDataSource
+        this.abortControllerPool = options.abortControllerPool
+        this.redisPublisher = new RedisEventPublisher()
+        this.redisPublisher.connect()
+    }
+
+    public getQueueName() {
+        return this.queueName
+    }
+
+    public getQueue() {
+        return this.queue
+    }
+
+    async processJob(data: IExecuteFlowParams) {
+        if (this.appDataSource) data.appDataSource = this.appDataSource
+        if (this.telemetry) data.telemetry = this.telemetry
+        if (this.cachePool) data.cachePool = this.cachePool
+        if (this.componentNodes) data.componentNodes = this.componentNodes
+        if (this.redisPublisher) data.sseStreamer = this.redisPublisher
+
+        if (this.abortControllerPool) {
+            const abortControllerId = `${data.chatflow.id}_${data.chatId}`
+            const signal = new AbortController()
+            this.abortControllerPool.add(abortControllerId, signal)
+            data.signal = signal
+        }
+
+        return await executeFlow(data)
+    }
+}

packages/server/src/queue/QueueManager.ts

@@ -0,0 +1,127 @@
+import { BaseQueue } from './BaseQueue'
+import { PredictionQueue } from './PredictionQueue'
+import { UpsertQueue } from './UpsertQueue'
+import { IComponentNodes } from '../Interface'
+import { Telemetry } from '../utils/telemetry'
+import { CachePool } from '../CachePool'
+import { DataSource } from 'typeorm'
+import { AbortControllerPool } from '../AbortControllerPool'
+import { QueueEventsProducer, RedisOptions } from 'bullmq'
+import { createBullBoard } from 'bull-board'
+import { BullMQAdapter } from 'bull-board/bullMQAdapter'
+import { Express } from 'express'
+
+const QUEUE_NAME = process.env.QUEUE_NAME || 'flowise-queue'
+
+type QUEUE_TYPE = 'prediction' | 'upsert'
+
+export class QueueManager {
+    private static instance: QueueManager
+    private queues: Map<string, BaseQueue> = new Map()
+    private connection: RedisOptions
+    private bullBoardRouter?: Express
+    private predictionQueueEventsProducer?: QueueEventsProducer
+
+    private constructor() {
+        let tlsOpts = undefined
+        if (process.env.REDIS_URL && process.env.REDIS_URL.startsWith('rediss://')) {
+            tlsOpts = {
+                rejectUnauthorized: false
+            }
+        } else if (process.env.REDIS_TLS === 'true') {
+            tlsOpts = {
+                cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
+                key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
+                ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+            }
+        }
+        this.connection = {
+            url: process.env.REDIS_URL || undefined,
+            host: process.env.REDIS_HOST || 'localhost',
+            port: parseInt(process.env.REDIS_PORT || '6379'),
+            username: process.env.REDIS_USERNAME || undefined,
+            password: process.env.REDIS_PASSWORD || undefined,
+            tls: tlsOpts
+        }
+    }
+
+    public static getInstance(): QueueManager {
+        if (!QueueManager.instance) {
+            QueueManager.instance = new QueueManager()
+        }
+        return QueueManager.instance
+    }
+
+    public registerQueue(name: string, queue: BaseQueue) {
+        this.queues.set(name, queue)
+    }
+
+    public getConnection() {
+        return this.connection
+    }
+
+    public getQueue(name: QUEUE_TYPE): BaseQueue {
+        const queue = this.queues.get(name)
+        if (!queue) throw new Error(`Queue ${name} not found`)
+        return queue
+    }
+
+    public getPredictionQueueEventsProducer(): QueueEventsProducer {
+        if (!this.predictionQueueEventsProducer) throw new Error('Prediction queue events producer not found')
+        return this.predictionQueueEventsProducer
+    }
+
+    public getBullBoardRouter(): Express {
+        if (!this.bullBoardRouter) throw new Error('BullBoard router not found')
+        return this.bullBoardRouter
+    }
+
+    public async getAllJobCounts(): Promise<{ [queueName: string]: { [status: string]: number } }> {
+        const counts: { [queueName: string]: { [status: string]: number } } = {}
+
+        for (const [name, queue] of this.queues) {
+            counts[name] = await queue.getJobCounts()
+        }
+
+        return counts
+    }
+
+    public setupAllQueues({
+        componentNodes,
+        telemetry,
+        cachePool,
+        appDataSource,
+        abortControllerPool
+    }: {
+        componentNodes: IComponentNodes
+        telemetry: Telemetry
+        cachePool: CachePool
+        appDataSource: DataSource
+        abortControllerPool: AbortControllerPool
+    }) {
+        const predictionQueueName = `${QUEUE_NAME}-prediction`
+        const predictionQueue = new PredictionQueue(predictionQueueName, this.connection, {
+            componentNodes,
+            telemetry,
+            cachePool,
+            appDataSource,
+            abortControllerPool
+        })
+        this.registerQueue('prediction', predictionQueue)
+        this.predictionQueueEventsProducer = new QueueEventsProducer(predictionQueue.getQueueName(), {
+            connection: this.connection
+        })
+
+        const upsertionQueueName = `${QUEUE_NAME}-upsertion`
+        const upsertionQueue = new UpsertQueue(upsertionQueueName, this.connection, {
+            componentNodes,
+            telemetry,
+            cachePool,
+            appDataSource
+        })
+        this.registerQueue('upsert', upsertionQueue)
+
+        const bullboard = createBullBoard([new BullMQAdapter(predictionQueue.getQueue()), new BullMQAdapter(upsertionQueue.getQueue())])
+        this.bullBoardRouter = bullboard.router
+    }
+}

packages/server/src/queue/RedisEventPublisher.ts

@@ -0,0 +1,262 @@
+import { IServerSideEventStreamer } from 'flowise-components'
+import { createClient } from 'redis'
+
+export class RedisEventPublisher implements IServerSideEventStreamer {
+    private redisPublisher: ReturnType<typeof createClient>
+
+    constructor() {
+        if (process.env.REDIS_URL) {
+            this.redisPublisher = createClient({
+                url: process.env.REDIS_URL
+            })
+        } else {
+            this.redisPublisher = createClient({
+                username: process.env.REDIS_USERNAME || undefined,
+                password: process.env.REDIS_PASSWORD || undefined,
+                socket: {
+                    host: process.env.REDIS_HOST || 'localhost',
+                    port: parseInt(process.env.REDIS_PORT || '6379'),
+                    tls: process.env.REDIS_TLS === 'true',
+                    cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
+                    key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
+                    ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+                }
+            })
+        }
+    }
+
+    async connect() {
+        await this.redisPublisher.connect()
+    }
+
+    streamCustomEvent(chatId: string, eventType: string, data: any) {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType,
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming custom event:', error)
+        }
+    }
+
+    streamStartEvent(chatId: string, data: string) {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'start',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming start event:', error)
+        }
+    }
+
+    streamTokenEvent(chatId: string, data: string) {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'token',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming token event:', error)
+        }
+    }
+
+    streamSourceDocumentsEvent(chatId: string, data: any) {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'sourceDocuments',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming sourceDocuments event:', error)
+        }
+    }
+
+    streamArtifactsEvent(chatId: string, data: any) {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'artifacts',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming artifacts event:', error)
+        }
+    }
+
+    streamUsedToolsEvent(chatId: string, data: any) {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'usedTools',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming usedTools event:', error)
+        }
+    }
+
+    streamFileAnnotationsEvent(chatId: string, data: any) {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'fileAnnotations',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming fileAnnotations event:', error)
+        }
+    }
+
+    streamToolEvent(chatId: string, data: any): void {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'tool',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming tool event:', error)
+        }
+    }
+
+    streamAgentReasoningEvent(chatId: string, data: any): void {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'agentReasoning',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming agentReasoning event:', error)
+        }
+    }
+
+    streamNextAgentEvent(chatId: string, data: any): void {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'nextAgent',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming nextAgent event:', error)
+        }
+    }
+
+    streamActionEvent(chatId: string, data: any): void {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'action',
+                    data
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming action event:', error)
+        }
+    }
+
+    streamAbortEvent(chatId: string): void {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'abort',
+                    data: '[DONE]'
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming abort event:', error)
+        }
+    }
+
+    streamEndEvent(_: string) {
+        // placeholder for future use
+    }
+
+    streamErrorEvent(chatId: string, msg: string) {
+        try {
+            this.redisPublisher.publish(
+                chatId,
+                JSON.stringify({
+                    chatId,
+                    eventType: 'error',
+                    data: msg
+                })
+            )
+        } catch (error) {
+            console.error('Error streaming error event:', error)
+        }
+    }
+
+    streamMetadataEvent(chatId: string, apiResponse: any) {
+        try {
+            const metadataJson: any = {}
+            if (apiResponse.chatId) {
+                metadataJson['chatId'] = apiResponse.chatId
+            }
+            if (apiResponse.chatMessageId) {
+                metadataJson['chatMessageId'] = apiResponse.chatMessageId
+            }
+            if (apiResponse.question) {
+                metadataJson['question'] = apiResponse.question
+            }
+            if (apiResponse.sessionId) {
+                metadataJson['sessionId'] = apiResponse.sessionId
+            }
+            if (apiResponse.memoryType) {
+                metadataJson['memoryType'] = apiResponse.memoryType
+            }
+            if (Object.keys(metadataJson).length > 0) {
+                this.streamCustomEvent(chatId, 'metadata', metadataJson)
+            }
+        } catch (error) {
+            console.error('Error streaming metadata event:', error)
+        }
+    }
+
+    async disconnect() {
+        if (this.redisPublisher) {
+            await this.redisPublisher.quit()
+        }
+    }
+}

packages/server/src/queue/RedisEventSubscriber.ts

@@ -0,0 +1,108 @@
+import { createClient } from 'redis'
+import { SSEStreamer } from '../utils/SSEStreamer'
+
+export class RedisEventSubscriber {
+    private redisSubscriber: ReturnType<typeof createClient>
+    private sseStreamer: SSEStreamer
+    private subscribedChannels: Set<string> = new Set()
+
+    constructor(sseStreamer: SSEStreamer) {
+        if (process.env.REDIS_URL) {
+            this.redisSubscriber = createClient({
+                url: process.env.REDIS_URL
+            })
+        } else {
+            this.redisSubscriber = createClient({
+                username: process.env.REDIS_USERNAME || undefined,
+                password: process.env.REDIS_PASSWORD || undefined,
+                socket: {
+                    host: process.env.REDIS_HOST || 'localhost',
+                    port: parseInt(process.env.REDIS_PORT || '6379'),
+                    tls: process.env.REDIS_TLS === 'true',
+                    cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
+                    key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
+                    ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+                }
+            })
+        }
+        this.sseStreamer = sseStreamer
+    }
+
+    async connect() {
+        await this.redisSubscriber.connect()
+    }
+
+    subscribe(channel: string) {
+        // Subscribe to the Redis channel for job events
+        if (!this.redisSubscriber) {
+            throw new Error('Redis subscriber not connected.')
+        }
+
+        // Check if already subscribed
+        if (this.subscribedChannels.has(channel)) {
+            return // Prevent duplicate subscription
+        }
+
+        this.redisSubscriber.subscribe(channel, (message) => {
+            this.handleEvent(message)
+        })
+
+        // Mark the channel as subscribed
+        this.subscribedChannels.add(channel)
+    }
+
+    private handleEvent(message: string) {
+        // Parse the message from Redis
+        const event = JSON.parse(message)
+        const { eventType, chatId, data } = event
+
+        // Stream the event to the client
+        switch (eventType) {
+            case 'start':
+                this.sseStreamer.streamStartEvent(chatId, data)
+                break
+            case 'token':
+                this.sseStreamer.streamTokenEvent(chatId, data)
+                break
+            case 'sourceDocuments':
+                this.sseStreamer.streamSourceDocumentsEvent(chatId, data)
+                break
+            case 'artifacts':
+                this.sseStreamer.streamArtifactsEvent(chatId, data)
+                break
+            case 'usedTools':
+                this.sseStreamer.streamUsedToolsEvent(chatId, data)
+                break
+            case 'fileAnnotations':
+                this.sseStreamer.streamFileAnnotationsEvent(chatId, data)
+                break
+            case 'tool':
+                this.sseStreamer.streamToolEvent(chatId, data)
+                break
+            case 'agentReasoning':
+                this.sseStreamer.streamAgentReasoningEvent(chatId, data)
+                break
+            case 'nextAgent':
+                this.sseStreamer.streamNextAgentEvent(chatId, data)
+                break
+            case 'action':
+                this.sseStreamer.streamActionEvent(chatId, data)
+                break
+            case 'abort':
+                this.sseStreamer.streamAbortEvent(chatId)
+                break
+            case 'error':
+                this.sseStreamer.streamErrorEvent(chatId, data)
+                break
+            case 'metadata':
+                this.sseStreamer.streamMetadataEvent(chatId, data)
+                break
+        }
+    }
+
+    async disconnect() {
+        if (this.redisSubscriber) {
+            await this.redisSubscriber.quit()
+        }
+    }
+}

packages/server/src/queue/UpsertQueue.ts

@@ -0,0 +1,85 @@
+import { DataSource } from 'typeorm'
+import {
+    IComponentNodes,
+    IExecuteDocStoreUpsert,
+    IExecuteFlowParams,
+    IExecutePreviewLoader,
+    IExecuteProcessLoader,
+    IExecuteVectorStoreInsert
+} from '../Interface'
+import { Telemetry } from '../utils/telemetry'
+import { CachePool } from '../CachePool'
+import { BaseQueue } from './BaseQueue'
+import { executeUpsert } from '../utils/upsertVector'
+import { executeDocStoreUpsert, insertIntoVectorStore, previewChunks, processLoader } from '../services/documentstore'
+import { RedisOptions } from 'bullmq'
+import logger from '../utils/logger'
+
+interface UpsertQueueOptions {
+    appDataSource: DataSource
+    telemetry: Telemetry
+    cachePool: CachePool
+    componentNodes: IComponentNodes
+}
+
+export class UpsertQueue extends BaseQueue {
+    private componentNodes: IComponentNodes
+    private telemetry: Telemetry
+    private cachePool: CachePool
+    private appDataSource: DataSource
+    private queueName: string
+
+    constructor(name: string, connection: RedisOptions, options: UpsertQueueOptions) {
+        super(name, connection)
+        this.queueName = name
+        this.componentNodes = options.componentNodes || {}
+        this.telemetry = options.telemetry
+        this.cachePool = options.cachePool
+        this.appDataSource = options.appDataSource
+    }
+
+    public getQueueName() {
+        return this.queueName
+    }
+
+    public getQueue() {
+        return this.queue
+    }
+
+    async processJob(
+        data: IExecuteFlowParams | IExecuteDocStoreUpsert | IExecuteProcessLoader | IExecuteVectorStoreInsert | IExecutePreviewLoader
+    ) {
+        if (this.appDataSource) data.appDataSource = this.appDataSource
+        if (this.telemetry) data.telemetry = this.telemetry
+        if (this.cachePool) data.cachePool = this.cachePool
+        if (this.componentNodes) data.componentNodes = this.componentNodes
+
+        // document-store/loader/preview
+        if (Object.prototype.hasOwnProperty.call(data, 'isPreviewOnly')) {
+            logger.info('Previewing loader...')
+            return await previewChunks(data as IExecutePreviewLoader)
+        }
+
+        // document-store/loader/process/:loaderId
+        if (Object.prototype.hasOwnProperty.call(data, 'isProcessWithoutUpsert')) {
+            logger.info('Processing loader...')
+            return await processLoader(data as IExecuteProcessLoader)
+        }
+
+        // document-store/vectorstore/insert/:loaderId
+        if (Object.prototype.hasOwnProperty.call(data, 'isVectorStoreInsert')) {
+            logger.info('Inserting vector store...')
+            return await insertIntoVectorStore(data as IExecuteVectorStoreInsert)
+        }
+
+        // document-store/upsert/:storeId
+        if (Object.prototype.hasOwnProperty.call(data, 'storeId')) {
+            logger.info('Upserting to vector store via document loader...')
+            return await executeDocStoreUpsert(data as IExecuteDocStoreUpsert)
+        }
+
+        // upsert-vector/:chatflowid
+        logger.info('Upserting to vector store via chatflow...')
+        return await executeUpsert(data as IExecuteFlowParams)
+    }
+}

packages/server/src/services/chat-messages/index.ts

@@ -1,6 +1,6 @@
 import { DeleteResult, FindOptionsWhere } from 'typeorm'
 import { StatusCodes } from 'http-status-codes'
-import { ChatMessageRatingType, ChatType, IChatMessage } from '../../Interface'
+import { ChatMessageRatingType, ChatType, IChatMessage, MODE } from '../../Interface'
 import { utilGetChatMessage } from '../../utils/getChatMessage'
 import { utilAddChatMessage } from '../../utils/addChatMesage'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
@@ -160,16 +160,15 @@ const removeChatMessagesByMessageIds = async (
 const abortChatMessage = async (chatId: string, chatflowid: string) => {
     try {
         const appServer = getRunningExpressApp()
+        const id = `${chatflowid}_${chatId}`
 
-        const endingNodeData = appServer.chatflowPool.activeChatflows[`${chatflowid}_${chatId}`]?.endingNodeData as any
-
-        if (endingNodeData && endingNodeData.signal) {
-            try {
-                endingNodeData.signal.abort()
-                await appServer.chatflowPool.remove(`${chatflowid}_${chatId}`)
-            } catch (e) {
-                logger.error(`[server]: Error aborting chat message for ${chatflowid}, chatId ${chatId}: ${e}`)
-            }
+        if (process.env.MODE === MODE.QUEUE) {
+            await appServer.queueManager.getPredictionQueueEventsProducer().publishEvent({
+                eventName: 'abort',
+                id
+            })
+        } else {
+            appServer.abortControllerPool.abort(id)
         }
     } catch (error) {
         throw new InternalFlowiseError(

packages/server/src/services/chatflows/index.ts

@@ -267,12 +267,6 @@ const updateChatflow = async (chatflow: ChatFlow, updateChatFlow: ChatFlow): Pro
         await _checkAndUpdateDocumentStoreUsage(newDbChatflow)
         const dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).save(newDbChatflow)
 
-        // chatFlowPool is initialized only when a flow is opened
-        // if the user attempts to rename/update category without opening any flow, chatFlowPool will be undefined
-        if (appServer.chatflowPool) {
-            // Update chatflowpool inSync to false, to build flow from scratch again because data has been changed
-            appServer.chatflowPool.updateInSync(chatflow.id, false)
-        }
         return dbResponse
     } catch (error) {
         throw new InternalFlowiseError(

packages/server/src/services/documentstore/index.ts

@@ -18,6 +18,7 @@ import {
     addLoaderSource,
     ChatType,
     DocumentStoreStatus,
+    IComponentNodes,
     IDocumentStoreFileChunkPagedResponse,
     IDocumentStoreLoader,
     IDocumentStoreLoaderFile,
@@ -25,8 +26,13 @@ import {
     IDocumentStoreRefreshData,
     IDocumentStoreUpsertData,
     IDocumentStoreWhereUsed,
+    IExecuteDocStoreUpsert,
+    IExecuteProcessLoader,
+    IExecuteVectorStoreInsert,
     INodeData,
-    IOverrideConfig
+    MODE,
+    IOverrideConfig,
+    IExecutePreviewLoader
 } from '../../Interface'
 import { DocumentStoreFileChunk } from '../../database/entities/DocumentStoreFileChunk'
 import { v4 as uuidv4 } from 'uuid'
@@ -38,12 +44,12 @@ import { StatusCodes } from 'http-status-codes'
 import { getErrorMessage } from '../../errors/utils'
 import { ChatFlow } from '../../database/entities/ChatFlow'
 import { Document } from '@langchain/core/documents'
-import { App } from '../../index'
 import { UpsertHistory } from '../../database/entities/UpsertHistory'
 import { cloneDeep, omit } from 'lodash'
-import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../../Interface.Metrics'
 import { DOCUMENTSTORE_TOOL_DESCRIPTION_PROMPT_GENERATOR } from '../../utils/prompt'
-import { INPUT_PARAMS_TYPE } from '../../utils/constants'
+import { DataSource } from 'typeorm'
+import { Telemetry } from '../../utils/telemetry'
+import { INPUT_PARAMS_TYPE, OMIT_QUEUE_JOB_DATA } from '../../utils/constants'
 
 const DOCUMENT_STORE_BASE_FOLDER = 'docustore'
 
@@ -185,10 +191,9 @@ const getUsedChatflowNames = async (entity: DocumentStore) => {
 }
 
 // Get chunks for a specific loader or store
-const getDocumentStoreFileChunks = async (storeId: string, docId: string, pageNo: number = 1) => {
+const getDocumentStoreFileChunks = async (appDataSource: DataSource, storeId: string, docId: string, pageNo: number = 1) => {
     try {
-        const appServer = getRunningExpressApp()
-        const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
+        const entity = await appDataSource.getRepository(DocumentStore).findOneBy({
             id: storeId
         })
         if (!entity) {
@@ -230,10 +235,10 @@ const getDocumentStoreFileChunks = async (storeId: string, docId: string, pageNo
         if (docId === 'all') {
             whereCondition = { storeId: storeId }
         }
-        const count = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).count({
+        const count = await appDataSource.getRepository(DocumentStoreFileChunk).count({
             where: whereCondition
         })
-        const chunksWithCount = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).find({
+        const chunksWithCount = await appDataSource.getRepository(DocumentStoreFileChunk).find({
             skip,
             take,
             where: whereCondition,
@@ -326,7 +331,7 @@ const deleteDocumentStoreFileChunk = async (storeId: string, docId: string, chun
         found.totalChars -= tbdChunk.pageContent.length
         entity.loaders = JSON.stringify(loaders)
         await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
-        return getDocumentStoreFileChunks(storeId, docId)
+        return getDocumentStoreFileChunks(appServer.AppDataSource, storeId, docId)
     } catch (error) {
         throw new InternalFlowiseError(
             StatusCodes.INTERNAL_SERVER_ERROR,
@@ -338,6 +343,8 @@ const deleteDocumentStoreFileChunk = async (storeId: string, docId: string, chun
 const deleteVectorStoreFromStore = async (storeId: string) => {
     try {
         const appServer = getRunningExpressApp()
+        const componentNodes = appServer.nodesPool.componentNodes
+
         const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
             id: storeId
         })
@@ -370,7 +377,7 @@ const deleteVectorStoreFromStore = async (storeId: string) => {
         // Get Record Manager Instance
         const recordManagerConfig = JSON.parse(entity.recordManagerConfig)
         const recordManagerObj = await _createRecordManagerObject(
-            appServer,
+            componentNodes,
             { recordManagerName: recordManagerConfig.name, recordManagerConfig: recordManagerConfig.config },
             options
         )
@@ -378,7 +385,7 @@ const deleteVectorStoreFromStore = async (storeId: string) => {
         // Get Embeddings Instance
         const embeddingConfig = JSON.parse(entity.embeddingConfig)
         const embeddingObj = await _createEmbeddingsObject(
-            appServer,
+            componentNodes,
             { embeddingName: embeddingConfig.name, embeddingConfig: embeddingConfig.config },
             options
         )
@@ -386,7 +393,7 @@ const deleteVectorStoreFromStore = async (storeId: string) => {
         // Get Vector Store Node Data
         const vectorStoreConfig = JSON.parse(entity.vectorStoreConfig)
         const vStoreNodeData = _createVectorStoreNodeData(
-            appServer,
+            componentNodes,
             { vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config },
             embeddingObj,
             recordManagerObj
@@ -394,7 +401,7 @@ const deleteVectorStoreFromStore = async (storeId: string) => {
 
         // Get Vector Store Instance
         const vectorStoreObj = await _createVectorStoreObject(
-            appServer,
+            componentNodes,
             { vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config },
             vStoreNodeData
         )
@@ -440,7 +447,7 @@ const editDocumentStoreFileChunk = async (storeId: string, docId: string, chunkI
         await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).save(editChunk)
         entity.loaders = JSON.stringify(loaders)
         await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
-        return getDocumentStoreFileChunks(storeId, docId)
+        return getDocumentStoreFileChunks(appServer.AppDataSource, storeId, docId)
     } catch (error) {
         throw new InternalFlowiseError(
             StatusCodes.INTERNAL_SERVER_ERROR,
@@ -449,7 +456,6 @@ const editDocumentStoreFileChunk = async (storeId: string, docId: string, chunkI
     }
 }
 
-// Update documentStore
 const updateDocumentStore = async (documentStore: DocumentStore, updatedDocumentStore: DocumentStore) => {
     try {
         const appServer = getRunningExpressApp()
@@ -484,12 +490,11 @@ const _saveFileToStorage = async (fileBase64: string, entity: DocumentStore) =>
     }
 }
 
-const _splitIntoChunks = async (data: IDocumentStoreLoaderForPreview) => {
+const _splitIntoChunks = async (appDataSource: DataSource, componentNodes: IComponentNodes, data: IDocumentStoreLoaderForPreview) => {
     try {
-        const appServer = getRunningExpressApp()
         let splitterInstance = null
         if (data.splitterId && data.splitterConfig && Object.keys(data.splitterConfig).length > 0) {
-            const nodeInstanceFilePath = appServer.nodesPool.componentNodes[data.splitterId].filePath as string
+            const nodeInstanceFilePath = componentNodes[data.splitterId].filePath as string
             const nodeModule = await import(nodeInstanceFilePath)
             const newNodeInstance = new nodeModule.nodeClass()
             let nodeData = {
@@ -499,7 +504,7 @@ const _splitIntoChunks = async (data: IDocumentStoreLoaderForPreview) => {
             splitterInstance = await newNodeInstance.init(nodeData)
         }
         if (!data.loaderId) return []
-        const nodeInstanceFilePath = appServer.nodesPool.componentNodes[data.loaderId].filePath as string
+        const nodeInstanceFilePath = componentNodes[data.loaderId].filePath as string
         const nodeModule = await import(nodeInstanceFilePath)
         // doc loader configs
         const nodeData = {
@@ -509,7 +514,7 @@ const _splitIntoChunks = async (data: IDocumentStoreLoaderForPreview) => {
         }
         const options: ICommonObject = {
             chatflowid: uuidv4(),
-            appDataSource: appServer.AppDataSource,
+            appDataSource,
             databaseEntities,
             logger
         }
@@ -524,7 +529,7 @@ const _splitIntoChunks = async (data: IDocumentStoreLoaderForPreview) => {
     }
 }
 
-const _normalizeFilePaths = async (data: IDocumentStoreLoaderForPreview, entity: DocumentStore | null) => {
+const _normalizeFilePaths = async (appDataSource: DataSource, data: IDocumentStoreLoaderForPreview, entity: DocumentStore | null) => {
     const keys = Object.getOwnPropertyNames(data.loaderConfig)
     let rehydrated = false
     for (let i = 0; i < keys.length; i++) {
@@ -538,8 +543,7 @@ const _normalizeFilePaths = async (data: IDocumentStoreLoaderForPreview, entity:
         let documentStoreEntity: DocumentStore | null = entity
         if (input.startsWith('FILE-STORAGE::')) {
             if (!documentStoreEntity) {
-                const appServer = getRunningExpressApp()
-                documentStoreEntity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
+                documentStoreEntity = await appDataSource.getRepository(DocumentStore).findOneBy({
                     id: data.storeId
                 })
                 if (!documentStoreEntity) {
@@ -573,7 +577,43 @@ const _normalizeFilePaths = async (data: IDocumentStoreLoaderForPreview, entity:
     data.rehydrated = rehydrated
 }
 
-const previewChunks = async (data: IDocumentStoreLoaderForPreview) => {
+const previewChunksMiddleware = async (data: IDocumentStoreLoaderForPreview) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const appDataSource = appServer.AppDataSource
+        const componentNodes = appServer.nodesPool.componentNodes
+
+        const executeData: IExecutePreviewLoader = {
+            appDataSource,
+            componentNodes,
+            data,
+            isPreviewOnly: true
+        }
+
+        if (process.env.MODE === MODE.QUEUE) {
+            const upsertQueue = appServer.queueManager.getQueue('upsert')
+            const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
+            logger.debug(`[server]: Job added to queue: ${job.id}`)
+
+            const queueEvents = upsertQueue.getQueueEvents()
+            const result = await job.waitUntilFinished(queueEvents)
+
+            if (!result) {
+                throw new Error('Job execution failed')
+            }
+            return result
+        }
+
+        return await previewChunks(executeData)
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.previewChunksMiddleware - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+export const previewChunks = async ({ appDataSource, componentNodes, data }: IExecutePreviewLoader) => {
     try {
         if (data.preview) {
             if (
@@ -585,9 +625,9 @@ const previewChunks = async (data: IDocumentStoreLoaderForPreview) => {
             }
         }
         if (!data.rehydrated) {
-            await _normalizeFilePaths(data, null)
+            await _normalizeFilePaths(appDataSource, data, null)
         }
-        let docs = await _splitIntoChunks(data)
+        let docs = await _splitIntoChunks(appDataSource, componentNodes, data)
         const totalChunks = docs.length
         // if -1, return all chunks
         if (data.previewChunkCount === -1) data.previewChunkCount = totalChunks
@@ -605,10 +645,9 @@ const previewChunks = async (data: IDocumentStoreLoaderForPreview) => {
     }
 }
 
-const saveProcessingLoader = async (data: IDocumentStoreLoaderForPreview): Promise<IDocumentStoreLoader> => {
+const saveProcessingLoader = async (appDataSource: DataSource, data: IDocumentStoreLoaderForPreview): Promise<IDocumentStoreLoader> => {
     try {
-        const appServer = getRunningExpressApp()
-        const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
+        const entity = await appDataSource.getRepository(DocumentStore).findOneBy({
             id: data.storeId
         })
         if (!entity) {
@@ -670,7 +709,7 @@ const saveProcessingLoader = async (data: IDocumentStoreLoaderForPreview): Promi
             existingLoaders.push(loader)
             entity.loaders = JSON.stringify(existingLoaders)
         }
-        await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
+        await appDataSource.getRepository(DocumentStore).save(entity)
         const newLoaders = JSON.parse(entity.loaders)
         const newLoader = newLoaders.find((ldr: IDocumentStoreLoader) => ldr.id === newDocLoaderId)
         if (!newLoader) {
@@ -686,21 +725,51 @@ const saveProcessingLoader = async (data: IDocumentStoreLoaderForPreview): Promi
     }
 }
 
-const processLoader = async (data: IDocumentStoreLoaderForPreview, docLoaderId: string) => {
+export const processLoader = async ({ appDataSource, componentNodes, data, docLoaderId }: IExecuteProcessLoader) => {
+    const entity = await appDataSource.getRepository(DocumentStore).findOneBy({
+        id: data.storeId
+    })
+    if (!entity) {
+        throw new InternalFlowiseError(
+            StatusCodes.NOT_FOUND,
+            `Error: documentStoreServices.processLoader - Document store ${data.storeId} not found`
+        )
+    }
+    await _saveChunksToStorage(appDataSource, componentNodes, data, entity, docLoaderId)
+    return getDocumentStoreFileChunks(appDataSource, data.storeId as string, docLoaderId)
+}
+
+const processLoaderMiddleware = async (data: IDocumentStoreLoaderForPreview, docLoaderId: string) => {
     try {
         const appServer = getRunningExpressApp()
-        const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
-            id: data.storeId
-        })
-        if (!entity) {
-            throw new InternalFlowiseError(
-                StatusCodes.NOT_FOUND,
-                `Error: documentStoreServices.processLoader - Document store ${data.storeId} not found`
-            )
+        const appDataSource = appServer.AppDataSource
+        const componentNodes = appServer.nodesPool.componentNodes
+        const telemetry = appServer.telemetry
+
+        const executeData: IExecuteProcessLoader = {
+            appDataSource,
+            componentNodes,
+            data,
+            docLoaderId,
+            isProcessWithoutUpsert: true,
+            telemetry
         }
-        // this method will run async, will have to be moved to a worker thread
-        await _saveChunksToStorage(data, entity, docLoaderId)
-        return getDocumentStoreFileChunks(data.storeId as string, docLoaderId)
+
+        if (process.env.MODE === MODE.QUEUE) {
+            const upsertQueue = appServer.queueManager.getQueue('upsert')
+            const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
+            logger.debug(`[server]: Job added to queue: ${job.id}`)
+
+            const queueEvents = upsertQueue.getQueueEvents()
+            const result = await job.waitUntilFinished(queueEvents)
+
+            if (!result) {
+                throw new Error('Job execution failed')
+            }
+            return result
+        }
+
+        return await processLoader(executeData)
     } catch (error) {
         throw new InternalFlowiseError(
             StatusCodes.INTERNAL_SERVER_ERROR,
@@ -709,16 +778,26 @@ const processLoader = async (data: IDocumentStoreLoaderForPreview, docLoaderId:
     }
 }
 
-const _saveChunksToStorage = async (data: IDocumentStoreLoaderForPreview, entity: DocumentStore, newLoaderId: string) => {
+const _saveChunksToStorage = async (
+    appDataSource: DataSource,
+    componentNodes: IComponentNodes,
+    data: IDocumentStoreLoaderForPreview,
+    entity: DocumentStore,
+    newLoaderId: string
+) => {
     const re = new RegExp('^data.*;base64', 'i')
 
     try {
-        const appServer = getRunningExpressApp()
         //step 1: restore the full paths, if any
-        await _normalizeFilePaths(data, entity)
+        await _normalizeFilePaths(appDataSource, data, entity)
 
         //step 2: split the file into chunks
-        const response = await previewChunks(data)
+        const response = await previewChunks({
+            appDataSource,
+            componentNodes,
+            data,
+            isPreviewOnly: false
+        })
 
         //step 3: remove all files associated with the loader
         const existingLoaders = JSON.parse(entity.loaders)
@@ -786,7 +865,7 @@ const _saveChunksToStorage = async (data: IDocumentStoreLoaderForPreview, entity
         }
 
         //step 7: remove all previous chunks
-        await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).delete({ docId: newLoaderId })
+        await appDataSource.getRepository(DocumentStoreFileChunk).delete({ docId: newLoaderId })
         if (response.chunks) {
             //step 8: now save the new chunks
             const totalChars = response.chunks.reduce((acc, chunk) => {
@@ -804,8 +883,8 @@ const _saveChunksToStorage = async (data: IDocumentStoreLoaderForPreview, entity
                     pageContent: chunk.pageContent,
                     metadata: JSON.stringify(chunk.metadata)
                 }
-                const dChunk = appServer.AppDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
-                await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
+                const dChunk = appDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
+                await appDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
             })
             // update the loader with the new metrics
             loader.totalChunks = response.totalChunks
@@ -818,7 +897,7 @@ const _saveChunksToStorage = async (data: IDocumentStoreLoaderForPreview, entity
         entity.loaders = JSON.stringify(existingLoaders)
 
         //step 9: update the entity in the database
-        await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
+        await appDataSource.getRepository(DocumentStore).save(entity)
 
         return
     } catch (error) {
@@ -917,10 +996,9 @@ const updateVectorStoreConfigOnly = async (data: ICommonObject) => {
         )
     }
 }
-const saveVectorStoreConfig = async (data: ICommonObject, isStrictSave = true) => {
+const saveVectorStoreConfig = async (appDataSource: DataSource, data: ICommonObject, isStrictSave = true) => {
     try {
-        const appServer = getRunningExpressApp()
-        const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
+        const entity = await appDataSource.getRepository(DocumentStore).findOneBy({
             id: data.storeId
         })
         if (!entity) {
@@ -971,7 +1049,7 @@ const saveVectorStoreConfig = async (data: ICommonObject, isStrictSave = true) =
             // this also means that the store is not yet sync'ed to vector store
             entity.status = DocumentStoreStatus.SYNC
         }
-        await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
+        await appDataSource.getRepository(DocumentStore).save(entity)
         return entity
     } catch (error) {
         throw new InternalFlowiseError(
@@ -981,15 +1059,19 @@ const saveVectorStoreConfig = async (data: ICommonObject, isStrictSave = true) =
     }
 }
 
-const insertIntoVectorStore = async (data: ICommonObject, isStrictSave = true) => {
+export const insertIntoVectorStore = async ({
+    appDataSource,
+    componentNodes,
+    telemetry,
+    data,
+    isStrictSave
+}: IExecuteVectorStoreInsert) => {
     try {
-        const appServer = getRunningExpressApp()
-        const entity = await saveVectorStoreConfig(data, isStrictSave)
+        const entity = await saveVectorStoreConfig(appDataSource, data, isStrictSave)
         entity.status = DocumentStoreStatus.UPSERTING
-        await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
+        await appDataSource.getRepository(DocumentStore).save(entity)
 
-        // TODO: to be moved into a worker thread...
-        const indexResult = await _insertIntoVectorStoreWorkerThread(data, isStrictSave)
+        const indexResult = await _insertIntoVectorStoreWorkerThread(appDataSource, componentNodes, telemetry, data, isStrictSave)
         return indexResult
     } catch (error) {
         throw new InternalFlowiseError(
@@ -999,16 +1081,60 @@ const insertIntoVectorStore = async (data: ICommonObject, isStrictSave = true) =
     }
 }
 
-const _insertIntoVectorStoreWorkerThread = async (data: ICommonObject, isStrictSave = true) => {
+const insertIntoVectorStoreMiddleware = async (data: ICommonObject, isStrictSave = true) => {
     try {
         const appServer = getRunningExpressApp()
-        const entity = await saveVectorStoreConfig(data, isStrictSave)
+        const appDataSource = appServer.AppDataSource
+        const componentNodes = appServer.nodesPool.componentNodes
+        const telemetry = appServer.telemetry
+
+        const executeData: IExecuteVectorStoreInsert = {
+            appDataSource,
+            componentNodes,
+            telemetry,
+            data,
+            isStrictSave,
+            isVectorStoreInsert: true
+        }
+
+        if (process.env.MODE === MODE.QUEUE) {
+            const upsertQueue = appServer.queueManager.getQueue('upsert')
+            const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
+            logger.debug(`[server]: Job added to queue: ${job.id}`)
+
+            const queueEvents = upsertQueue.getQueueEvents()
+            const result = await job.waitUntilFinished(queueEvents)
+
+            if (!result) {
+                throw new Error('Job execution failed')
+            }
+            return result
+        } else {
+            return await insertIntoVectorStore(executeData)
+        }
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.insertIntoVectorStoreMiddleware - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const _insertIntoVectorStoreWorkerThread = async (
+    appDataSource: DataSource,
+    componentNodes: IComponentNodes,
+    telemetry: Telemetry,
+    data: ICommonObject,
+    isStrictSave = true
+) => {
+    try {
+        const entity = await saveVectorStoreConfig(appDataSource, data, isStrictSave)
         let upsertHistory: Record<string, any> = {}
         const chatflowid = data.storeId // fake chatflowid because this is not tied to any chatflow
 
         const options: ICommonObject = {
             chatflowid,
-            appDataSource: appServer.AppDataSource,
+            appDataSource,
             databaseEntities,
             logger
         }
@@ -1017,14 +1143,14 @@ const _insertIntoVectorStoreWorkerThread = async (data: ICommonObject, isStrictS
 
         // Get Record Manager Instance
         if (data.recordManagerName && data.recordManagerConfig) {
-            recordManagerObj = await _createRecordManagerObject(appServer, data, options, upsertHistory)
+            recordManagerObj = await _createRecordManagerObject(componentNodes, data, options, upsertHistory)
         }
 
         // Get Embeddings Instance
-        const embeddingObj = await _createEmbeddingsObject(appServer, data, options, upsertHistory)
+        const embeddingObj = await _createEmbeddingsObject(componentNodes, data, options, upsertHistory)
 
         // Get Vector Store Node Data
-        const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, recordManagerObj)
+        const vStoreNodeData = _createVectorStoreNodeData(componentNodes, data, embeddingObj, recordManagerObj)
 
         // Prepare docs for upserting
         const filterOptions: ICommonObject = {
@@ -1033,7 +1159,7 @@ const _insertIntoVectorStoreWorkerThread = async (data: ICommonObject, isStrictS
         if (data.docId) {
             filterOptions['docId'] = data.docId
         }
-        const chunks = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).find({
+        const chunks = await appDataSource.getRepository(DocumentStoreFileChunk).find({
             where: filterOptions
         })
         const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
@@ -1045,7 +1171,7 @@ const _insertIntoVectorStoreWorkerThread = async (data: ICommonObject, isStrictS
         vStoreNodeData.inputs.document = docs
 
         // Get Vector Store Instance
-        const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData, upsertHistory)
+        const vectorStoreObj = await _createVectorStoreObject(componentNodes, data, vStoreNodeData, upsertHistory)
         const indexResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
 
         // Save to DB
@@ -1056,20 +1182,19 @@ const _insertIntoVectorStoreWorkerThread = async (data: ICommonObject, isStrictS
             result.chatflowid = chatflowid
             const newUpsertHistory = new UpsertHistory()
             Object.assign(newUpsertHistory, result)
-            const upsertHistoryItem = appServer.AppDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
-            await appServer.AppDataSource.getRepository(UpsertHistory).save(upsertHistoryItem)
+            const upsertHistoryItem = appDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
+            await appDataSource.getRepository(UpsertHistory).save(upsertHistoryItem)
         }
 
-        await appServer.telemetry.sendTelemetry('vector_upserted', {
+        await telemetry.sendTelemetry('vector_upserted', {
             version: await getAppVersion(),
             chatlowId: chatflowid,
             type: ChatType.INTERNAL,
             flowGraph: omit(indexResult['result'], ['totalKeys', 'addedDocs'])
         })
-        appServer.metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, { status: FLOWISE_COUNTER_STATUS.SUCCESS })
 
         entity.status = DocumentStoreStatus.UPSERTED
-        await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
+        await appDataSource.getRepository(DocumentStore).save(entity)
 
         return indexResult ?? { result: 'Successfully Upserted' }
     } catch (error) {
@@ -1123,6 +1248,8 @@ const getRecordManagerProviders = async () => {
 const queryVectorStore = async (data: ICommonObject) => {
     try {
         const appServer = getRunningExpressApp()
+        const componentNodes = appServer.nodesPool.componentNodes
+
         const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
             id: data.storeId
         })
@@ -1147,7 +1274,7 @@ const queryVectorStore = async (data: ICommonObject) => {
         const embeddingConfig = JSON.parse(entity.embeddingConfig)
         data.embeddingName = embeddingConfig.name
         data.embeddingConfig = embeddingConfig.config
-        let embeddingObj = await _createEmbeddingsObject(appServer, data, options)
+        let embeddingObj = await _createEmbeddingsObject(componentNodes, data, options)
 
         const vsConfig = JSON.parse(entity.vectorStoreConfig)
         data.vectorStoreName = vsConfig.name
@@ -1156,10 +1283,10 @@ const queryVectorStore = async (data: ICommonObject) => {
             data.vectorStoreConfig = { ...vsConfig.config, ...data.inputs }
         }
 
-        const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, undefined)
+        const vStoreNodeData = _createVectorStoreNodeData(componentNodes, data, embeddingObj, undefined)
 
         // Get Vector Store Instance
-        const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData)
+        const vectorStoreObj = await _createVectorStoreObject(componentNodes, data, vStoreNodeData)
         const retriever = await vectorStoreObj.init(vStoreNodeData, '', options)
         if (!retriever) {
             throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create retriever`)
@@ -1208,13 +1335,13 @@ const queryVectorStore = async (data: ICommonObject) => {
 }
 
 const _createEmbeddingsObject = async (
-    appServer: App,
+    componentNodes: IComponentNodes,
     data: ICommonObject,
     options: ICommonObject,
     upsertHistory?: Record<string, any>
 ): Promise<any> => {
     // prepare embedding node data
-    const embeddingComponent = appServer.nodesPool.componentNodes[data.embeddingName]
+    const embeddingComponent = componentNodes[data.embeddingName]
     const embeddingNodeData: any = {
         inputs: { ...data.embeddingConfig },
         outputs: { output: 'document' },
@@ -1243,13 +1370,13 @@ const _createEmbeddingsObject = async (
 }
 
 const _createRecordManagerObject = async (
-    appServer: App,
+    componentNodes: IComponentNodes,
     data: ICommonObject,
     options: ICommonObject,
     upsertHistory?: Record<string, any>
 ) => {
     // prepare record manager node data
-    const recordManagerComponent = appServer.nodesPool.componentNodes[data.recordManagerName]
+    const recordManagerComponent = componentNodes[data.recordManagerName]
     const rmNodeData: any = {
         inputs: { ...data.recordManagerConfig },
         id: `${recordManagerComponent.name}_0`,
@@ -1276,8 +1403,8 @@ const _createRecordManagerObject = async (
     return recordManagerObj
 }
 
-const _createVectorStoreNodeData = (appServer: App, data: ICommonObject, embeddingObj: any, recordManagerObj?: any) => {
-    const vectorStoreComponent = appServer.nodesPool.componentNodes[data.vectorStoreName]
+const _createVectorStoreNodeData = (componentNodes: IComponentNodes, data: ICommonObject, embeddingObj: any, recordManagerObj?: any) => {
+    const vectorStoreComponent = componentNodes[data.vectorStoreName]
     const vStoreNodeData: any = {
         id: `${vectorStoreComponent.name}_0`,
         inputs: { ...data.vectorStoreConfig },
@@ -1306,25 +1433,27 @@ const _createVectorStoreNodeData = (appServer: App, data: ICommonObject, embeddi
 }
 
 const _createVectorStoreObject = async (
-    appServer: App,
+    componentNodes: IComponentNodes,
     data: ICommonObject,
     vStoreNodeData: INodeData,
     upsertHistory?: Record<string, any>
 ) => {
-    const vStoreNodeInstanceFilePath = appServer.nodesPool.componentNodes[data.vectorStoreName].filePath as string
+    const vStoreNodeInstanceFilePath = componentNodes[data.vectorStoreName].filePath as string
     const vStoreNodeModule = await import(vStoreNodeInstanceFilePath)
     const vStoreNodeInstance = new vStoreNodeModule.nodeClass()
     if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(vStoreNodeData, upsertHistory)
     return vStoreNodeInstance
 }
 
-const upsertDocStoreMiddleware = async (
+const upsertDocStore = async (
+    appDataSource: DataSource,
+    componentNodes: IComponentNodes,
+    telemetry: Telemetry,
     storeId: string,
     data: IDocumentStoreUpsertData,
     files: Express.Multer.File[] = [],
     isRefreshExisting = false
 ) => {
-    const appServer = getRunningExpressApp()
     const docId = data.docId
     let metadata = {}
     if (data.metadata) {
@@ -1342,7 +1471,7 @@ const upsertDocStoreMiddleware = async (
     const newRecordManager = typeof data.recordManager === 'string' ? JSON.parse(data.recordManager) : data.recordManager
 
     const getComponentLabelFromName = (nodeName: string) => {
-        const component = Object.values(appServer.nodesPool.componentNodes).find((node) => node.name === nodeName)
+        const component = Object.values(componentNodes).find((node) => node.name === nodeName)
         return component?.label || ''
     }
 
@@ -1365,7 +1494,7 @@ const upsertDocStoreMiddleware = async (
 
     // Step 1: Get existing loader
     if (docId) {
-        const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ id: storeId })
+        const entity = await appDataSource.getRepository(DocumentStore).findOneBy({ id: storeId })
         if (!entity) {
             throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
         }
@@ -1527,8 +1656,15 @@ const upsertDocStoreMiddleware = async (
     }
 
     try {
-        const newLoader = await saveProcessingLoader(processData)
-        const result = await processLoader(processData, newLoader.id || '')
+        const newLoader = await saveProcessingLoader(appDataSource, processData)
+        const result = await processLoader({
+            appDataSource,
+            componentNodes,
+            data: processData,
+            docLoaderId: newLoader.id || '',
+            isProcessWithoutUpsert: false,
+            telemetry
+        })
         const newDocId = result.docId
 
         const insertData = {
@@ -1542,10 +1678,74 @@ const upsertDocStoreMiddleware = async (
             recordManagerConfig
         }
 
-        const res = await insertIntoVectorStore(insertData, false)
+        const res = await insertIntoVectorStore({
+            appDataSource,
+            componentNodes,
+            telemetry,
+            data: insertData,
+            isStrictSave: false,
+            isVectorStoreInsert: true
+        })
         res.docId = newDocId
 
         return res
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.upsertDocStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+export const executeDocStoreUpsert = async ({
+    appDataSource,
+    componentNodes,
+    telemetry,
+    storeId,
+    totalItems,
+    files,
+    isRefreshAPI
+}: IExecuteDocStoreUpsert) => {
+    const results = []
+    for (const item of totalItems) {
+        const res = await upsertDocStore(appDataSource, componentNodes, telemetry, storeId, item, files, isRefreshAPI)
+        results.push(res)
+    }
+    return isRefreshAPI ? results : results[0]
+}
+
+const upsertDocStoreMiddleware = async (storeId: string, data: IDocumentStoreUpsertData, files: Express.Multer.File[] = []) => {
+    const appServer = getRunningExpressApp()
+    const componentNodes = appServer.nodesPool.componentNodes
+    const appDataSource = appServer.AppDataSource
+    const telemetry = appServer.telemetry
+
+    try {
+        const executeData: IExecuteDocStoreUpsert = {
+            appDataSource,
+            componentNodes,
+            telemetry,
+            storeId,
+            totalItems: [data],
+            files,
+            isRefreshAPI: false
+        }
+
+        if (process.env.MODE === MODE.QUEUE) {
+            const upsertQueue = appServer.queueManager.getQueue('upsert')
+            const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
+            logger.debug(`[server]: Job added to queue: ${job.id}`)
+
+            const queueEvents = upsertQueue.getQueueEvents()
+            const result = await job.waitUntilFinished(queueEvents)
+
+            if (!result) {
+                throw new Error('Job execution failed')
+            }
+            return result
+        } else {
+            return await executeDocStoreUpsert(executeData)
+        }
     } catch (error) {
         throw new InternalFlowiseError(
             StatusCodes.INTERNAL_SERVER_ERROR,
@@ -1556,9 +1756,11 @@ const upsertDocStoreMiddleware = async (
 
 const refreshDocStoreMiddleware = async (storeId: string, data?: IDocumentStoreRefreshData) => {
     const appServer = getRunningExpressApp()
+    const componentNodes = appServer.nodesPool.componentNodes
+    const appDataSource = appServer.AppDataSource
+    const telemetry = appServer.telemetry
 
     try {
-        const results = []
         let totalItems: IDocumentStoreUpsertData[] = []
 
         if (!data || !data.items || data.items.length === 0) {
@@ -1577,12 +1779,31 @@ const refreshDocStoreMiddleware = async (storeId: string, data?: IDocumentStoreR
             totalItems = data.items
         }
 
-        for (const item of totalItems) {
-            const res = await upsertDocStoreMiddleware(storeId, item, [], true)
-            results.push(res)
+        const executeData: IExecuteDocStoreUpsert = {
+            appDataSource,
+            componentNodes,
+            telemetry,
+            storeId,
+            totalItems,
+            files: [],
+            isRefreshAPI: true
         }
 
-        return results
+        if (process.env.MODE === MODE.QUEUE) {
+            const upsertQueue = appServer.queueManager.getQueue('upsert')
+            const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
+            logger.debug(`[server]: Job added to queue: ${job.id}`)
+
+            const queueEvents = upsertQueue.getQueueEvents()
+            const result = await job.waitUntilFinished(queueEvents)
+
+            if (!result) {
+                throw new Error('Job execution failed')
+            }
+            return result
+        } else {
+            return await executeDocStoreUpsert(executeData)
+        }
     } catch (error) {
         throw new InternalFlowiseError(
             StatusCodes.INTERNAL_SERVER_ERROR,
@@ -1799,13 +2020,13 @@ export default {
     getUsedChatflowNames,
     getDocumentStoreFileChunks,
     updateDocumentStore,
-    previewChunks,
+    previewChunksMiddleware,
     saveProcessingLoader,
-    processLoader,
+    processLoaderMiddleware,
     deleteDocumentStoreFileChunk,
     editDocumentStoreFileChunk,
     getDocumentLoaders,
-    insertIntoVectorStore,
+    insertIntoVectorStoreMiddleware,
     getEmbeddingProviders,
     getVectorStoreProviders,
     getRecordManagerProviders,

packages/server/src/services/openai-realtime/index.ts

@@ -95,7 +95,6 @@ const buildAndInitTool = async (chatflowid: string, _chatId?: string, _apiMessag
     const flowDataObj: ICommonObject = { chatflowid, chatId }
 
     const reactFlowNodeData: INodeData = await resolveVariables(
-        appServer.AppDataSource,
         nodeToExecute.data,
         reactFlowNodes,
         '',

packages/server/src/utils/SSEStreamer.ts

@@ -1,4 +1,3 @@
-import express from 'express'
 import { Response } from 'express'
 import { IServerSideEventStreamer } from 'flowise-components'
 
@@ -13,11 +12,6 @@ type Client = {
 
 export class SSEStreamer implements IServerSideEventStreamer {
     clients: { [id: string]: Client } = {}
-    app: express.Application
-
-    constructor(app: express.Application) {
-        this.app = app
-    }
 
     addExternalClient(chatId: string, res: Response) {
         this.clients[chatId] = { clientType: 'EXTERNAL', response: res, started: false }
@@ -40,18 +34,6 @@ export class SSEStreamer implements IServerSideEventStreamer {
         }
     }
 
-    // Send SSE message to a specific client
-    streamEvent(chatId: string, data: string) {
-        const client = this.clients[chatId]
-        if (client) {
-            const clientResponse = {
-                event: 'start',
-                data: data
-            }
-            client.response.write('message:\ndata:' + JSON.stringify(clientResponse) + '\n\n')
-        }
-    }
-
     streamCustomEvent(chatId: string, eventType: string, data: any) {
         const client = this.clients[chatId]
         if (client) {

packages/server/src/utils/addChatMesage.ts

@@ -1,3 +1,4 @@
+import { DataSource } from 'typeorm'
 import { ChatMessage } from '../database/entities/ChatMessage'
 import { IChatMessage } from '../Interface'
 import { getRunningExpressApp } from '../utils/getRunningExpressApp'
@@ -6,14 +7,14 @@ import { getRunningExpressApp } from '../utils/getRunningExpressApp'
  * Method that add chat messages.
  * @param {Partial<IChatMessage>} chatMessage
  */
-export const utilAddChatMessage = async (chatMessage: Partial<IChatMessage>): Promise<ChatMessage> => {
-    const appServer = getRunningExpressApp()
+export const utilAddChatMessage = async (chatMessage: Partial<IChatMessage>, appDataSource?: DataSource): Promise<ChatMessage> => {
+    const dataSource = appDataSource ?? getRunningExpressApp().AppDataSource
     const newChatMessage = new ChatMessage()
     Object.assign(newChatMessage, chatMessage)
     if (!newChatMessage.createdDate) {
         newChatMessage.createdDate = new Date()
     }
-    const chatmessage = await appServer.AppDataSource.getRepository(ChatMessage).create(newChatMessage)
-    const dbResponse = await appServer.AppDataSource.getRepository(ChatMessage).save(chatmessage)
+    const chatmessage = await dataSource.getRepository(ChatMessage).create(newChatMessage)
+    const dbResponse = await dataSource.getRepository(ChatMessage).save(chatmessage)
     return dbResponse
 }

packages/server/src/utils/buildAgentGraph.ts

@@ -19,144 +19,77 @@ import { StatusCodes } from 'http-status-codes'
 import { v4 as uuidv4 } from 'uuid'
 import { StructuredTool } from '@langchain/core/tools'
 import { BaseMessage, HumanMessage, AIMessage, AIMessageChunk, ToolMessage } from '@langchain/core/messages'
-import {
-    IChatFlow,
-    IComponentNodes,
-    IDepthQueue,
-    IReactFlowNode,
-    IReactFlowObject,
-    IReactFlowEdge,
-    IMessage,
-    IncomingInput
-} from '../Interface'
-import {
-    buildFlow,
-    getStartingNodes,
-    getEndingNodes,
-    constructGraphs,
-    databaseEntities,
-    getSessionChatHistory,
-    getMemorySessionId,
-    clearSessionMemory,
-    getAPIOverrideConfig
-} from '../utils'
-import { getRunningExpressApp } from './getRunningExpressApp'
+import { IChatFlow, IComponentNodes, IDepthQueue, IReactFlowNode, IReactFlowEdge, IMessage, IncomingInput, IFlowConfig } from '../Interface'
+import { databaseEntities, clearSessionMemory, getAPIOverrideConfig } from '../utils'
 import { replaceInputsWithConfig, resolveVariables } from '.'
 import { InternalFlowiseError } from '../errors/internalFlowiseError'
 import { getErrorMessage } from '../errors/utils'
 import logger from './logger'
 import { Variable } from '../database/entities/Variable'
+import { DataSource } from 'typeorm'
+import { CachePool } from '../CachePool'
 
 /**
  * Build Agent Graph
- * @param {IChatFlow} chatflow
- * @param {string} chatId
- * @param {string} sessionId
- * @param {ICommonObject} incomingInput
- * @param {boolean} isInternal
- * @param {string} baseURL
  */
-export const buildAgentGraph = async (
-    chatflow: IChatFlow,
-    chatId: string,
-    apiMessageId: string,
-    sessionId: string,
-    incomingInput: IncomingInput,
-    isInternal: boolean,
-    baseURL?: string,
-    sseStreamer?: IServerSideEventStreamer,
-    shouldStreamResponse?: boolean,
-    uploadedFilesContent?: string
-): Promise<any> => {
+export const buildAgentGraph = async ({
+    agentflow,
+    flowConfig,
+    incomingInput,
+    nodes,
+    edges,
+    initializedNodes,
+    endingNodeIds,
+    startingNodeIds,
+    depthQueue,
+    chatHistory,
+    uploadedFilesContent,
+    appDataSource,
+    componentNodes,
+    sseStreamer,
+    shouldStreamResponse,
+    cachePool,
+    baseURL,
+    signal
+}: {
+    agentflow: IChatFlow
+    flowConfig: IFlowConfig
+    incomingInput: IncomingInput
+    nodes: IReactFlowNode[]
+    edges: IReactFlowEdge[]
+    initializedNodes: IReactFlowNode[]
+    endingNodeIds: string[]
+    startingNodeIds: string[]
+    depthQueue: IDepthQueue
+    chatHistory: IMessage[]
+    uploadedFilesContent: string
+    appDataSource: DataSource
+    componentNodes: IComponentNodes
+    sseStreamer: IServerSideEventStreamer
+    shouldStreamResponse: boolean
+    cachePool: CachePool
+    baseURL: string
+    signal?: AbortController
+}): Promise<any> => {
     try {
-        const appServer = getRunningExpressApp()
-        const chatflowid = chatflow.id
-
-        /*** Get chatflows and prepare data  ***/
-        const flowData = chatflow.flowData
-        const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
-        const nodes = parsedFlowData.nodes
-        const edges = parsedFlowData.edges
-
-        /*** Get Ending Node with Directed Graph  ***/
-        const { graph, nodeDependencies } = constructGraphs(nodes, edges)
-        const directedGraph = graph
-
-        const endingNodes = getEndingNodes(nodeDependencies, directedGraph, nodes)
-
-        /*** Get Starting Nodes with Reversed Graph ***/
-        const constructedObj = constructGraphs(nodes, edges, { isReversed: true })
-        const nonDirectedGraph = constructedObj.graph
-        let startingNodeIds: string[] = []
-        let depthQueue: IDepthQueue = {}
-        const endingNodeIds = endingNodes.map((n) => n.id)
-        for (const endingNodeId of endingNodeIds) {
-            const resx = getStartingNodes(nonDirectedGraph, endingNodeId)
-            startingNodeIds.push(...resx.startingNodeIds)
-            depthQueue = Object.assign(depthQueue, resx.depthQueue)
-        }
-        startingNodeIds = [...new Set(startingNodeIds)]
-
-        /*** Get Memory Node for Chat History ***/
-        let chatHistory: IMessage[] = []
-        const agentMemoryList = ['agentMemory', 'sqliteAgentMemory', 'postgresAgentMemory', 'mySQLAgentMemory']
-        const memoryNode = nodes.find((node) => agentMemoryList.includes(node.data.name))
-        if (memoryNode) {
-            chatHistory = await getSessionChatHistory(
-                chatflowid,
-                getMemorySessionId(memoryNode, incomingInput, chatId, isInternal),
-                memoryNode,
-                appServer.nodesPool.componentNodes,
-                appServer.AppDataSource,
-                databaseEntities,
-                logger,
-                incomingInput.history
-            )
-        }
-
-        /*** Get API Config ***/
-        const availableVariables = await appServer.AppDataSource.getRepository(Variable).find()
-        const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
-
-        // Initialize nodes like ChatModels, Tools, etc.
-        const reactFlowNodes: IReactFlowNode[] = await buildFlow({
-            startingNodeIds,
-            reactFlowNodes: nodes,
-            reactFlowEdges: edges,
-            apiMessageId,
-            graph,
-            depthQueue,
-            componentNodes: appServer.nodesPool.componentNodes,
-            question: incomingInput.question,
-            uploadedFilesContent,
-            chatHistory,
-            chatId,
-            sessionId,
-            chatflowid,
-            appDataSource: appServer.AppDataSource,
-            overrideConfig: incomingInput?.overrideConfig,
-            apiOverrideStatus,
-            nodeOverrides,
-            availableVariables,
-            variableOverrides,
-            cachePool: appServer.cachePool,
-            isUpsert: false,
-            uploads: incomingInput.uploads,
-            baseURL
-        })
+        const chatflowid = flowConfig.chatflowid
+        const chatId = flowConfig.chatId
+        const sessionId = flowConfig.sessionId
+        const analytic = agentflow.analytic
+        const uploads = incomingInput.uploads
 
         const options = {
             chatId,
             sessionId,
             chatflowid,
             logger,
-            analytic: chatflow.analytic,
-            appDataSource: appServer.AppDataSource,
-            databaseEntities: databaseEntities,
-            cachePool: appServer.cachePool,
-            uploads: incomingInput.uploads,
+            analytic,
+            appDataSource,
+            databaseEntities,
+            cachePool,
+            uploads,
             baseURL,
-            signal: new AbortController()
+            signal: signal ?? new AbortController()
         }
 
         let streamResults
@@ -171,9 +104,9 @@ export const buildAgentGraph = async (
         let totalUsedTools: IUsedTool[] = []
         let totalArtifacts: ICommonObject[] = []
 
-        const workerNodes = reactFlowNodes.filter((node) => node.data.name === 'worker')
-        const supervisorNodes = reactFlowNodes.filter((node) => node.data.name === 'supervisor')
-        const seqAgentNodes = reactFlowNodes.filter((node) => node.data.category === 'Sequential Agents')
+        const workerNodes = initializedNodes.filter((node) => node.data.name === 'worker')
+        const supervisorNodes = initializedNodes.filter((node) => node.data.name === 'supervisor')
+        const seqAgentNodes = initializedNodes.filter((node) => node.data.category === 'Sequential Agents')
 
         const mapNameToLabel: Record<string, { label: string; nodeName: string }> = {}
 
@@ -189,11 +122,12 @@ export const buildAgentGraph = async (
         try {
             if (!seqAgentNodes.length) {
                 streamResults = await compileMultiAgentsGraph({
-                    chatflow,
+                    agentflow,
+                    appDataSource,
                     mapNameToLabel,
-                    reactFlowNodes,
+                    reactFlowNodes: initializedNodes,
                     workerNodeIds: endingNodeIds,
-                    componentNodes: appServer.nodesPool.componentNodes,
+                    componentNodes,
                     options,
                     startingNodeIds,
                     question: incomingInput.question,
@@ -208,10 +142,11 @@ export const buildAgentGraph = async (
                 isSequential = true
                 streamResults = await compileSeqAgentsGraph({
                     depthQueue,
-                    chatflow,
-                    reactFlowNodes,
+                    agentflow,
+                    appDataSource,
+                    reactFlowNodes: initializedNodes,
                     reactFlowEdges: edges,
-                    componentNodes: appServer.nodesPool.componentNodes,
+                    componentNodes,
                     options,
                     question: incomingInput.question,
                     prependHistoryMessages: incomingInput.history,
@@ -275,7 +210,7 @@ export const buildAgentGraph = async (
                                 )
 
                                 inputEdges.forEach((edge) => {
-                                    const parentNode = reactFlowNodes.find((nd) => nd.id === edge.source)
+                                    const parentNode = initializedNodes.find((nd) => nd.id === edge.source)
                                     if (parentNode) {
                                         if (parentNode.data.name.includes('seqCondition')) {
                                             const newMessages = messages.slice(0, -1)
@@ -366,7 +301,7 @@ export const buildAgentGraph = async (
                     // If last message is an AI Message with tool calls, that means the last node was interrupted
                     if (lastMessageRaw.tool_calls && lastMessageRaw.tool_calls.length > 0) {
                         // The last node that got interrupted
-                        const node = reactFlowNodes.find((node) => node.id === lastMessageRaw.additional_kwargs.nodeId)
+                        const node = initializedNodes.find((node) => node.id === lastMessageRaw.additional_kwargs.nodeId)
 
                         // Find the next tool node that is connected to the interrupted node, to get the approve/reject button text
                         const tooNodeId = edges.find(
@@ -374,7 +309,7 @@ export const buildAgentGraph = async (
                                 edge.target.includes('seqToolNode') &&
                                 edge.source === (lastMessageRaw.additional_kwargs && lastMessageRaw.additional_kwargs.nodeId)
                         )?.target
-                        const connectedToolNode = reactFlowNodes.find((node) => node.id === tooNodeId)
+                        const connectedToolNode = initializedNodes.find((node) => node.id === tooNodeId)
 
                         // Map raw tool calls to used tools, to be shown on interrupted message
                         const mappedToolCalls = lastMessageRaw.tool_calls.map((toolCall) => {
@@ -449,7 +384,7 @@ export const buildAgentGraph = async (
             }
         } catch (e) {
             // clear agent memory because checkpoints were saved during runtime
-            await clearSessionMemory(nodes, appServer.nodesPool.componentNodes, chatId, appServer.AppDataSource, sessionId)
+            await clearSessionMemory(nodes, componentNodes, chatId, appDataSource, sessionId)
             if (getErrorMessage(e).includes('Aborted')) {
                 if (shouldStreamResponse && sseStreamer) {
                     sseStreamer.streamAbortEvent(chatId)
@@ -466,7 +401,8 @@ export const buildAgentGraph = async (
 }
 
 type MultiAgentsGraphParams = {
-    chatflow: IChatFlow
+    agentflow: IChatFlow
+    appDataSource: DataSource
     mapNameToLabel: Record<string, { label: string; nodeName: string }>
     reactFlowNodes: IReactFlowNode[]
     workerNodeIds: string[]
@@ -484,13 +420,13 @@ type MultiAgentsGraphParams = {
 
 const compileMultiAgentsGraph = async (params: MultiAgentsGraphParams) => {
     const {
-        chatflow,
+        agentflow,
+        appDataSource,
         mapNameToLabel,
         reactFlowNodes,
         workerNodeIds,
         componentNodes,
         options,
-        startingNodeIds,
         prependHistoryMessages = [],
         chatHistory = [],
         overrideConfig = {},
@@ -501,7 +437,6 @@ const compileMultiAgentsGraph = async (params: MultiAgentsGraphParams) => {
 
     let question = params.question
 
-    const appServer = getRunningExpressApp()
     const channels: ITeamState = {
         messages: {
             value: (x: BaseMessage[], y: BaseMessage[]) => x.concat(y),
@@ -522,8 +457,8 @@ const compileMultiAgentsGraph = async (params: MultiAgentsGraphParams) => {
     const workerNodes = reactFlowNodes.filter((node) => workerNodeIds.includes(node.data.id))
 
     /*** Get API Config ***/
-    const availableVariables = await appServer.AppDataSource.getRepository(Variable).find()
-    const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
+    const availableVariables = await appDataSource.getRepository(Variable).find()
+    const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(agentflow)
 
     let supervisorWorkers: { [key: string]: IMultiAgentNode[] } = {}
 
@@ -537,7 +472,6 @@ const compileMultiAgentsGraph = async (params: MultiAgentsGraphParams) => {
         if (overrideConfig && apiOverrideStatus)
             flowNodeData = replaceInputsWithConfig(flowNodeData, overrideConfig, nodeOverrides, variableOverrides)
         flowNodeData = await resolveVariables(
-            appServer.AppDataSource,
             flowNodeData,
             reactFlowNodes,
             question,
@@ -579,7 +513,6 @@ const compileMultiAgentsGraph = async (params: MultiAgentsGraphParams) => {
         if (overrideConfig && apiOverrideStatus)
             flowNodeData = replaceInputsWithConfig(flowNodeData, overrideConfig, nodeOverrides, variableOverrides)
         flowNodeData = await resolveVariables(
-            appServer.AppDataSource,
             flowNodeData,
             reactFlowNodes,
             question,
@@ -626,15 +559,7 @@ const compileMultiAgentsGraph = async (params: MultiAgentsGraphParams) => {
 
             //@ts-ignore
             workflowGraph.addEdge(START, supervisorResult.name)
-
-            // Add agentflow to pool
             ;(workflowGraph as any).signal = options.signal
-            appServer.chatflowPool.add(
-                `${chatflow.id}_${options.chatId}`,
-                workflowGraph as any,
-                reactFlowNodes.filter((node) => startingNodeIds.includes(node.id)),
-                overrideConfig
-            )
 
             // Get memory
             let memory = supervisorResult?.checkpointMemory
@@ -685,7 +610,8 @@ const compileMultiAgentsGraph = async (params: MultiAgentsGraphParams) => {
 
 type SeqAgentsGraphParams = {
     depthQueue: IDepthQueue
-    chatflow: IChatFlow
+    agentflow: IChatFlow
+    appDataSource: DataSource
     reactFlowNodes: IReactFlowNode[]
     reactFlowEdges: IReactFlowEdge[]
     componentNodes: IComponentNodes
@@ -702,7 +628,8 @@ type SeqAgentsGraphParams = {
 const compileSeqAgentsGraph = async (params: SeqAgentsGraphParams) => {
     const {
         depthQueue,
-        chatflow,
+        agentflow,
+        appDataSource,
         reactFlowNodes,
         reactFlowEdges,
         componentNodes,
@@ -717,8 +644,6 @@ const compileSeqAgentsGraph = async (params: SeqAgentsGraphParams) => {
 
     let question = params.question
 
-    const appServer = getRunningExpressApp()
-
     let channels: ISeqAgentsState = {
         messages: {
             value: (x: BaseMessage[], y: BaseMessage[]) => x.concat(y),
@@ -761,8 +686,8 @@ const compileSeqAgentsGraph = async (params: SeqAgentsGraphParams) => {
     let interruptToolNodeNames = []
 
     /*** Get API Config ***/
-    const availableVariables = await appServer.AppDataSource.getRepository(Variable).find()
-    const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
+    const availableVariables = await appDataSource.getRepository(Variable).find()
+    const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(agentflow)
 
     const initiateNode = async (node: IReactFlowNode) => {
         const nodeInstanceFilePath = componentNodes[node.data.name].filePath as string
@@ -773,7 +698,6 @@ const compileSeqAgentsGraph = async (params: SeqAgentsGraphParams) => {
         if (overrideConfig && apiOverrideStatus)
             flowNodeData = replaceInputsWithConfig(flowNodeData, overrideConfig, nodeOverrides, variableOverrides)
         flowNodeData = await resolveVariables(
-            appServer.AppDataSource,
             flowNodeData,
             reactFlowNodes,
             question,
@@ -1059,14 +983,8 @@ const compileSeqAgentsGraph = async (params: SeqAgentsGraphParams) => {
             routeMessage
         )
     }
-    /*** Add agentflow to pool ***/
+
     ;(seqGraph as any).signal = options.signal
-    appServer.chatflowPool.add(
-        `${chatflow.id}_${options.chatId}`,
-        seqGraph as any,
-        reactFlowNodes.filter((node) => startAgentNodes.map((nd) => nd.id).includes(node.id)),
-        overrideConfig
-    )
 
     /*** Get memory ***/
     const startNode = reactFlowNodes.find((node: IReactFlowNode) => node.data.name === 'seqStart')

packages/server/src/utils/constants.ts

@@ -20,6 +20,8 @@ export const WHITELIST_URLS = [
     '/api/v1/metrics'
 ]
 
+export const OMIT_QUEUE_JOB_DATA = ['componentNodes', 'appDataSource', 'sseStreamer', 'telemetry', 'cachePool']
+
 export const INPUT_PARAMS_TYPE = [
     'asyncOptions',
     'options',

packages/server/src/utils/index.ts

@@ -560,7 +560,6 @@ export const buildFlow = async ({
             if (isUpsert) upsertHistory['flowData'] = saveUpsertFlowData(flowNodeData, upsertHistory)
 
             const reactFlowNodeData: INodeData = await resolveVariables(
-                appDataSource,
                 flowNodeData,
                 flowNodes,
                 question,
@@ -762,10 +761,9 @@ export const clearSessionMemory = async (
 }
 
 const getGlobalVariable = async (
-    appDataSource: DataSource,
     overrideConfig?: ICommonObject,
     availableVariables: IVariable[] = [],
-    variableOverrides?: ICommonObject[]
+    variableOverrides: ICommonObject[] = []
 ) => {
     // override variables defined in overrideConfig
     // nodeData.inputs.vars is an Object, check each property and override the variable
@@ -826,13 +824,12 @@ const getGlobalVariable = async (
  * @returns {string}
  */
 export const getVariableValue = async (
-    appDataSource: DataSource,
     paramValue: string | object,
     reactFlowNodes: IReactFlowNode[],
     question: string,
     chatHistory: IMessage[],
     isAcceptVariable = false,
-    flowData?: ICommonObject,
+    flowConfig?: ICommonObject,
     uploadedFilesContent?: string,
     availableVariables: IVariable[] = [],
     variableOverrides: ICommonObject[] = []
@@ -877,7 +874,7 @@ export const getVariableValue = async (
             }
 
             if (variableFullPath.startsWith('$vars.')) {
-                const vars = await getGlobalVariable(appDataSource, flowData, availableVariables, variableOverrides)
+                const vars = await getGlobalVariable(flowConfig, availableVariables, variableOverrides)
                 const variableValue = get(vars, variableFullPath.replace('$vars.', ''))
                 if (variableValue != null) {
                     variableDict[`{{${variableFullPath}}}`] = variableValue
@@ -885,8 +882,8 @@ export const getVariableValue = async (
                 }
             }
 
-            if (variableFullPath.startsWith('$flow.') && flowData) {
-                const variableValue = get(flowData, variableFullPath.replace('$flow.', ''))
+            if (variableFullPath.startsWith('$flow.') && flowConfig) {
+                const variableValue = get(flowConfig, variableFullPath.replace('$flow.', ''))
                 if (variableValue != null) {
                     variableDict[`{{${variableFullPath}}}`] = variableValue
                     returnVal = returnVal.split(`{{${variableFullPath}}}`).join(variableValue)
@@ -980,12 +977,11 @@ export const getVariableValue = async (
  * @returns {INodeData}
  */
 export const resolveVariables = async (
-    appDataSource: DataSource,
     reactFlowNodeData: INodeData,
     reactFlowNodes: IReactFlowNode[],
     question: string,
     chatHistory: IMessage[],
-    flowData?: ICommonObject,
+    flowConfig?: ICommonObject,
     uploadedFilesContent?: string,
     availableVariables: IVariable[] = [],
     variableOverrides: ICommonObject[] = []
@@ -1000,13 +996,12 @@ export const resolveVariables = async (
                 const resolvedInstances = []
                 for (const param of paramValue) {
                     const resolvedInstance = await getVariableValue(
-                        appDataSource,
                         param,
                         reactFlowNodes,
                         question,
                         chatHistory,
                         undefined,
-                        flowData,
+                        flowConfig,
                         uploadedFilesContent,
                         availableVariables,
                         variableOverrides
@@ -1017,13 +1012,12 @@ export const resolveVariables = async (
             } else {
                 const isAcceptVariable = reactFlowNodeData.inputParams.find((param) => param.name === key)?.acceptVariable ?? false
                 const resolvedInstance = await getVariableValue(
-                    appDataSource,
                     paramValue,
                     reactFlowNodes,
                     question,
                     chatHistory,
                     isAcceptVariable,
-                    flowData,
+                    flowConfig,
                     uploadedFilesContent,
                     availableVariables,
                     variableOverrides

packages/server/src/utils/rateLimit.ts

@@ -1,55 +1,174 @@
 import { NextFunction, Request, Response } from 'express'
 import { rateLimit, RateLimitRequestHandler } from 'express-rate-limit'
-import { IChatFlow } from '../Interface'
+import { IChatFlow, MODE } from '../Interface'
 import { Mutex } from 'async-mutex'
+import { RedisStore } from 'rate-limit-redis'
+import Redis from 'ioredis'
+import { QueueEvents, QueueEventsListener, QueueEventsProducer } from 'bullmq'
 
-let rateLimiters: Record<string, RateLimitRequestHandler> = {}
-const rateLimiterMutex = new Mutex()
+interface CustomListener extends QueueEventsListener {
+    updateRateLimiter: (args: { limitDuration: number; limitMax: number; limitMsg: string; id: string }) => void
+}
 
-async function addRateLimiter(id: string, duration: number, limit: number, message: string) {
-    const release = await rateLimiterMutex.acquire()
-    try {
-        rateLimiters[id] = rateLimit({
-            windowMs: duration * 1000,
-            max: limit,
-            handler: (_, res) => {
-                res.status(429).send(message)
+const QUEUE_NAME = 'ratelimit'
+const QUEUE_EVENT_NAME = 'updateRateLimiter'
+
+export class RateLimiterManager {
+    private rateLimiters: Record<string, RateLimitRequestHandler> = {}
+    private rateLimiterMutex: Mutex = new Mutex()
+    private redisClient: Redis
+    private static instance: RateLimiterManager
+    private queueEventsProducer: QueueEventsProducer
+    private queueEvents: QueueEvents
+
+    constructor() {
+        if (process.env.MODE === MODE.QUEUE) {
+            if (process.env.REDIS_URL) {
+                this.redisClient = new Redis(process.env.REDIS_URL)
+            } else {
+                this.redisClient = new Redis({
+                    host: process.env.REDIS_HOST || 'localhost',
+                    port: parseInt(process.env.REDIS_PORT || '6379'),
+                    username: process.env.REDIS_USERNAME || undefined,
+                    password: process.env.REDIS_PASSWORD || undefined,
+                    tls:
+                        process.env.REDIS_TLS === 'true'
+                            ? {
+                                  cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
+                                  key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
+                                  ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+                              }
+                            : undefined
+                })
             }
-        })
-    } finally {
-        release()
+            this.queueEventsProducer = new QueueEventsProducer(QUEUE_NAME, { connection: this.getConnection() })
+            this.queueEvents = new QueueEvents(QUEUE_NAME, { connection: this.getConnection() })
+        }
+    }
+
+    getConnection() {
+        let tlsOpts = undefined
+        if (process.env.REDIS_URL && process.env.REDIS_URL.startsWith('rediss://')) {
+            tlsOpts = {
+                rejectUnauthorized: false
+            }
+        } else if (process.env.REDIS_TLS === 'true') {
+            tlsOpts = {
+                cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
+                key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
+                ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+            }
+        }
+        return {
+            url: process.env.REDIS_URL || undefined,
+            host: process.env.REDIS_HOST || 'localhost',
+            port: parseInt(process.env.REDIS_PORT || '6379'),
+            username: process.env.REDIS_USERNAME || undefined,
+            password: process.env.REDIS_PASSWORD || undefined,
+            tls: tlsOpts
+        }
+    }
+
+    public static getInstance(): RateLimiterManager {
+        if (!RateLimiterManager.instance) {
+            RateLimiterManager.instance = new RateLimiterManager()
+        }
+        return RateLimiterManager.instance
+    }
+
+    public async addRateLimiter(id: string, duration: number, limit: number, message: string): Promise<void> {
+        const release = await this.rateLimiterMutex.acquire()
+        try {
+            if (process.env.MODE === MODE.QUEUE) {
+                this.rateLimiters[id] = rateLimit({
+                    windowMs: duration * 1000,
+                    max: limit,
+                    standardHeaders: true,
+                    legacyHeaders: false,
+                    message,
+                    store: new RedisStore({
+                        prefix: `rl:${id}`,
+                        // @ts-expect-error - Known issue: the `call` function is not present in @types/ioredis
+                        sendCommand: (...args: string[]) => this.redisClient.call(...args)
+                    })
+                })
+            } else {
+                this.rateLimiters[id] = rateLimit({
+                    windowMs: duration * 1000,
+                    max: limit,
+                    message
+                })
+            }
+        } finally {
+            release()
+        }
+    }
+
+    public removeRateLimiter(id: string): void {
+        if (this.rateLimiters[id]) {
+            delete this.rateLimiters[id]
+        }
+    }
+
+    public getRateLimiter(): (req: Request, res: Response, next: NextFunction) => void {
+        return (req: Request, res: Response, next: NextFunction) => {
+            const id = req.params.id
+            if (!this.rateLimiters[id]) return next()
+            const idRateLimiter = this.rateLimiters[id]
+            return idRateLimiter(req, res, next)
+        }
+    }
+
+    public async updateRateLimiter(chatFlow: IChatFlow, isInitialized?: boolean): Promise<void> {
+        if (!chatFlow.apiConfig) return
+        const apiConfig = JSON.parse(chatFlow.apiConfig)
+
+        const rateLimit: { limitDuration: number; limitMax: number; limitMsg: string; status?: boolean } = apiConfig.rateLimit
+        if (!rateLimit) return
+
+        const { limitDuration, limitMax, limitMsg, status } = rateLimit
+
+        if (!isInitialized && process.env.MODE === MODE.QUEUE && this.queueEventsProducer) {
+            await this.queueEventsProducer.publishEvent({
+                eventName: QUEUE_EVENT_NAME,
+                limitDuration,
+                limitMax,
+                limitMsg,
+                id: chatFlow.id
+            })
+        } else {
+            if (status === false) {
+                this.removeRateLimiter(chatFlow.id)
+            } else if (limitMax && limitDuration && limitMsg) {
+                await this.addRateLimiter(chatFlow.id, limitDuration, limitMax, limitMsg)
+            }
+        }
+    }
+
+    public async initializeRateLimiters(chatflows: IChatFlow[]): Promise<void> {
+        await Promise.all(
+            chatflows.map(async (chatFlow) => {
+                await this.updateRateLimiter(chatFlow, true)
+            })
+        )
+
+        if (process.env.MODE === MODE.QUEUE && this.queueEvents) {
+            this.queueEvents.on<CustomListener>(
+                QUEUE_EVENT_NAME,
+                async ({
+                    limitDuration,
+                    limitMax,
+                    limitMsg,
+                    id
+                }: {
+                    limitDuration: number
+                    limitMax: number
+                    limitMsg: string
+                    id: string
+                }) => {
+                    await this.addRateLimiter(id, limitDuration, limitMax, limitMsg)
+                }
+            )
+        }
     }
 }
-
-function removeRateLimit(id: string) {
-    if (rateLimiters[id]) {
-        delete rateLimiters[id]
-    }
-}
-
-export function getRateLimiter(req: Request, res: Response, next: NextFunction) {
-    const id = req.params.id
-    if (!rateLimiters[id]) return next()
-    const idRateLimiter = rateLimiters[id]
-    return idRateLimiter(req, res, next)
-}
-
-export async function updateRateLimiter(chatFlow: IChatFlow) {
-    if (!chatFlow.apiConfig) return
-    const apiConfig = JSON.parse(chatFlow.apiConfig)
-
-    const rateLimit: { limitDuration: number; limitMax: number; limitMsg: string; status?: boolean } = apiConfig.rateLimit
-    if (!rateLimit) return
-
-    const { limitDuration, limitMax, limitMsg, status } = rateLimit
-    if (status === false) removeRateLimit(chatFlow.id)
-    else if (limitMax && limitDuration && limitMsg) await addRateLimiter(chatFlow.id, limitDuration, limitMax, limitMsg)
-}
-
-export async function initializeRateLimiter(chatFlowPool: IChatFlow[]) {
-    await Promise.all(
-        chatFlowPool.map(async (chatFlow) => {
-            await updateRateLimiter(chatFlow)
-        })
-    )
-}

packages/server/src/utils/upsertVector.ts

@@ -23,7 +23,7 @@ import {
     getAPIOverrideConfig
 } from '../utils'
 import { validateChatflowAPIKey } from './validateKey'
-import { IncomingInput, INodeDirectedGraph, IReactFlowObject, ChatType } from '../Interface'
+import { IncomingInput, INodeDirectedGraph, IReactFlowObject, ChatType, IExecuteFlowParams, MODE } from '../Interface'
 import { ChatFlow } from '../database/entities/ChatFlow'
 import { getRunningExpressApp } from '../utils/getRunningExpressApp'
 import { UpsertHistory } from '../database/entities/UpsertHistory'
@@ -33,17 +33,182 @@ import { getErrorMessage } from '../errors/utils'
 import { v4 as uuidv4 } from 'uuid'
 import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../Interface.Metrics'
 import { Variable } from '../database/entities/Variable'
+import { OMIT_QUEUE_JOB_DATA } from './constants'
+
+export const executeUpsert = async ({
+    componentNodes,
+    incomingInput,
+    chatflow,
+    chatId,
+    appDataSource,
+    telemetry,
+    cachePool,
+    isInternal,
+    files
+}: IExecuteFlowParams) => {
+    const question = incomingInput.question
+    const overrideConfig = incomingInput.overrideConfig ?? {}
+    let stopNodeId = incomingInput?.stopNodeId ?? ''
+    const chatHistory: IMessage[] = []
+    const isUpsert = true
+    const chatflowid = chatflow.id
+    const apiMessageId = uuidv4()
+
+    if (files?.length) {
+        const overrideConfig: ICommonObject = { ...incomingInput }
+        for (const file of files) {
+            const fileNames: string[] = []
+            const fileBuffer = await getFileFromUpload(file.path ?? file.key)
+            // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
+            file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
+            const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid)
+
+            const fileInputFieldFromMimeType = mapMimeTypeToInputField(file.mimetype)
+
+            const fileExtension = path.extname(file.originalname)
+
+            const fileInputFieldFromExt = mapExtToInputField(fileExtension)
+
+            let fileInputField = 'txtFile'
+
+            if (fileInputFieldFromExt !== 'txtFile') {
+                fileInputField = fileInputFieldFromExt
+            } else if (fileInputFieldFromMimeType !== 'txtFile') {
+                fileInputField = fileInputFieldFromExt
+            }
+
+            if (overrideConfig[fileInputField]) {
+                const existingFileInputField = overrideConfig[fileInputField].replace('FILE-STORAGE::', '')
+                const existingFileInputFieldArray = JSON.parse(existingFileInputField)
+
+                const newFileInputField = storagePath.replace('FILE-STORAGE::', '')
+                const newFileInputFieldArray = JSON.parse(newFileInputField)
+
+                const updatedFieldArray = existingFileInputFieldArray.concat(newFileInputFieldArray)
+
+                overrideConfig[fileInputField] = `FILE-STORAGE::${JSON.stringify(updatedFieldArray)}`
+            } else {
+                overrideConfig[fileInputField] = storagePath
+            }
+
+            await removeSpecificFileFromUpload(file.path ?? file.key)
+        }
+        if (overrideConfig.vars && typeof overrideConfig.vars === 'string') {
+            overrideConfig.vars = JSON.parse(overrideConfig.vars)
+        }
+        incomingInput = {
+            ...incomingInput,
+            question: '',
+            overrideConfig,
+            stopNodeId,
+            chatId
+        }
+    }
+
+    /*** Get chatflows and prepare data  ***/
+    const flowData = chatflow.flowData
+    const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
+    const nodes = parsedFlowData.nodes
+    const edges = parsedFlowData.edges
+
+    /*** Get session ID ***/
+    const memoryNode = findMemoryNode(nodes, edges)
+    let sessionId = getMemorySessionId(memoryNode, incomingInput, chatId, isInternal)
+
+    /*** Find the 1 final vector store will be upserted  ***/
+    const vsNodes = nodes.filter((node) => node.data.category === 'Vector Stores')
+    const vsNodesWithFileUpload = vsNodes.filter((node) => node.data.inputs?.fileUpload)
+    if (vsNodesWithFileUpload.length > 1) {
+        throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, 'Multiple vector store nodes with fileUpload enabled')
+    } else if (vsNodesWithFileUpload.length === 1 && !stopNodeId) {
+        stopNodeId = vsNodesWithFileUpload[0].data.id
+    }
+
+    /*** Check if multiple vector store nodes exist, and if stopNodeId is specified ***/
+    if (vsNodes.length > 1 && !stopNodeId) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            'There are multiple vector nodes, please provide stopNodeId in body request'
+        )
+    } else if (vsNodes.length === 1 && !stopNodeId) {
+        stopNodeId = vsNodes[0].data.id
+    } else if (!vsNodes.length && !stopNodeId) {
+        throw new InternalFlowiseError(StatusCodes.NOT_FOUND, 'No vector node found')
+    }
+
+    /*** Get Starting Nodes with Reversed Graph ***/
+    const { graph } = constructGraphs(nodes, edges, { isReversed: true })
+    const nodeIds = getAllConnectedNodes(graph, stopNodeId)
+    const filteredGraph: INodeDirectedGraph = {}
+    for (const key of nodeIds) {
+        if (Object.prototype.hasOwnProperty.call(graph, key)) {
+            filteredGraph[key] = graph[key]
+        }
+    }
+    const { startingNodeIds, depthQueue } = getStartingNodes(filteredGraph, stopNodeId)
+
+    /*** Get API Config ***/
+    const availableVariables = await appDataSource.getRepository(Variable).find()
+    const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
+
+    const upsertedResult = await buildFlow({
+        startingNodeIds,
+        reactFlowNodes: nodes,
+        reactFlowEdges: edges,
+        apiMessageId,
+        graph: filteredGraph,
+        depthQueue,
+        componentNodes,
+        question,
+        chatHistory,
+        chatId,
+        sessionId,
+        chatflowid,
+        appDataSource,
+        overrideConfig,
+        apiOverrideStatus,
+        nodeOverrides,
+        availableVariables,
+        variableOverrides,
+        cachePool,
+        isUpsert,
+        stopNodeId
+    })
+
+    // Save to DB
+    if (upsertedResult['flowData'] && upsertedResult['result']) {
+        const result = cloneDeep(upsertedResult)
+        result['flowData'] = JSON.stringify(result['flowData'])
+        result['result'] = JSON.stringify(omit(result['result'], ['totalKeys', 'addedDocs']))
+        result.chatflowid = chatflowid
+        const newUpsertHistory = new UpsertHistory()
+        Object.assign(newUpsertHistory, result)
+        const upsertHistory = appDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
+        await appDataSource.getRepository(UpsertHistory).save(upsertHistory)
+    }
+
+    await telemetry.sendTelemetry('vector_upserted', {
+        version: await getAppVersion(),
+        chatlowId: chatflowid,
+        type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
+        flowGraph: getTelemetryFlowObj(nodes, edges),
+        stopNodeId
+    })
+
+    return upsertedResult['result'] ?? { result: 'Successfully Upserted' }
+}
+
 /**
  * Upsert documents
  * @param {Request} req
  * @param {boolean} isInternal
  */
 export const upsertVector = async (req: Request, isInternal: boolean = false) => {
+    const appServer = getRunningExpressApp()
     try {
-        const appServer = getRunningExpressApp()
         const chatflowid = req.params.id
-        let incomingInput: IncomingInput = req.body
 
+        // Check if chatflow exists
         const chatflow = await appServer.AppDataSource.getRepository(ChatFlow).findOneBy({
             id: chatflowid
         })
@@ -51,6 +216,12 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
             throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Chatflow ${chatflowid} not found`)
         }
 
+        const httpProtocol = req.get('x-forwarded-proto') || req.protocol
+        const baseURL = `${httpProtocol}://${req.get('host')}`
+        const incomingInput: IncomingInput = req.body
+        const chatId = incomingInput.chatId ?? incomingInput.overrideConfig?.sessionId ?? uuidv4()
+        const files = (req.files as Express.Multer.File[]) || []
+
         if (!isInternal) {
             const isKeyValidated = await validateChatflowAPIKey(req, chatflow)
             if (!isKeyValidated) {
@@ -58,168 +229,50 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
             }
         }
 
-        const files = (req.files as Express.Multer.File[]) || []
-
-        if (files.length) {
-            const overrideConfig: ICommonObject = { ...req.body }
-            for (const file of files) {
-                const fileNames: string[] = []
-                const fileBuffer = await getFileFromUpload(file.path ?? file.key)
-                // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
-                file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
-                const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid)
-
-                const fileInputFieldFromMimeType = mapMimeTypeToInputField(file.mimetype)
-
-                const fileExtension = path.extname(file.originalname)
-
-                const fileInputFieldFromExt = mapExtToInputField(fileExtension)
-
-                let fileInputField = 'txtFile'
-
-                if (fileInputFieldFromExt !== 'txtFile') {
-                    fileInputField = fileInputFieldFromExt
-                } else if (fileInputFieldFromMimeType !== 'txtFile') {
-                    fileInputField = fileInputFieldFromExt
-                }
-
-                if (overrideConfig[fileInputField]) {
-                    const existingFileInputField = overrideConfig[fileInputField].replace('FILE-STORAGE::', '')
-                    const existingFileInputFieldArray = JSON.parse(existingFileInputField)
-
-                    const newFileInputField = storagePath.replace('FILE-STORAGE::', '')
-                    const newFileInputFieldArray = JSON.parse(newFileInputField)
-
-                    const updatedFieldArray = existingFileInputFieldArray.concat(newFileInputFieldArray)
-
-                    overrideConfig[fileInputField] = `FILE-STORAGE::${JSON.stringify(updatedFieldArray)}`
-                } else {
-                    overrideConfig[fileInputField] = storagePath
-                }
-
-                await removeSpecificFileFromUpload(file.path ?? file.key)
-            }
-            if (overrideConfig.vars && typeof overrideConfig.vars === 'string') {
-                overrideConfig.vars = JSON.parse(overrideConfig.vars)
-            }
-            incomingInput = {
-                question: req.body.question ?? 'hello',
-                overrideConfig,
-                stopNodeId: req.body.stopNodeId
-            }
-            if (req.body.chatId) {
-                incomingInput.chatId = req.body.chatId
-            }
-        }
-
-        /*** Get chatflows and prepare data  ***/
-        const flowData = chatflow.flowData
-        const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
-        const nodes = parsedFlowData.nodes
-        const edges = parsedFlowData.edges
-
-        const apiMessageId = req.body.apiMessageId ?? uuidv4()
-
-        let stopNodeId = incomingInput?.stopNodeId ?? ''
-        let chatHistory: IMessage[] = []
-        let chatId = incomingInput.chatId ?? ''
-        let isUpsert = true
-
-        // Get session ID
-        const memoryNode = findMemoryNode(nodes, edges)
-        let sessionId = getMemorySessionId(memoryNode, incomingInput, chatId, isInternal)
-
-        const vsNodes = nodes.filter((node) => node.data.category === 'Vector Stores')
-
-        // Get StopNodeId for vector store which has fielUpload
-        const vsNodesWithFileUpload = vsNodes.filter((node) => node.data.inputs?.fileUpload)
-        if (vsNodesWithFileUpload.length > 1) {
-            throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, 'Multiple vector store nodes with fileUpload enabled')
-        } else if (vsNodesWithFileUpload.length === 1 && !stopNodeId) {
-            stopNodeId = vsNodesWithFileUpload[0].data.id
-        }
-
-        // Check if multiple vector store nodes exist, and if stopNodeId is specified
-        if (vsNodes.length > 1 && !stopNodeId) {
-            throw new InternalFlowiseError(
-                StatusCodes.INTERNAL_SERVER_ERROR,
-                'There are multiple vector nodes, please provide stopNodeId in body request'
-            )
-        } else if (vsNodes.length === 1 && !stopNodeId) {
-            stopNodeId = vsNodes[0].data.id
-        } else if (!vsNodes.length && !stopNodeId) {
-            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, 'No vector node found')
-        }
-
-        const { graph } = constructGraphs(nodes, edges, { isReversed: true })
-
-        const nodeIds = getAllConnectedNodes(graph, stopNodeId)
-
-        const filteredGraph: INodeDirectedGraph = {}
-        for (const key of nodeIds) {
-            if (Object.prototype.hasOwnProperty.call(graph, key)) {
-                filteredGraph[key] = graph[key]
-            }
-        }
-
-        const { startingNodeIds, depthQueue } = getStartingNodes(filteredGraph, stopNodeId)
-
-        /*** Get API Config ***/
-        const availableVariables = await appServer.AppDataSource.getRepository(Variable).find()
-        const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
-
-        const upsertedResult = await buildFlow({
-            startingNodeIds,
-            reactFlowNodes: nodes,
-            reactFlowEdges: edges,
-            apiMessageId,
-            graph: filteredGraph,
-            depthQueue,
+        const executeData: IExecuteFlowParams = {
             componentNodes: appServer.nodesPool.componentNodes,
-            question: incomingInput.question,
-            chatHistory,
+            incomingInput,
+            chatflow,
             chatId,
-            sessionId: sessionId ?? '',
-            chatflowid,
             appDataSource: appServer.AppDataSource,
-            overrideConfig: incomingInput?.overrideConfig,
-            apiOverrideStatus,
-            nodeOverrides,
-            availableVariables,
-            variableOverrides,
+            telemetry: appServer.telemetry,
             cachePool: appServer.cachePool,
-            isUpsert,
-            stopNodeId
-        })
-
-        const startingNodes = nodes.filter((nd) => startingNodeIds.includes(nd.data.id))
-
-        await appServer.chatflowPool.add(chatflowid, undefined, startingNodes, incomingInput?.overrideConfig, chatId)
-
-        // Save to DB
-        if (upsertedResult['flowData'] && upsertedResult['result']) {
-            const result = cloneDeep(upsertedResult)
-            result['flowData'] = JSON.stringify(result['flowData'])
-            result['result'] = JSON.stringify(omit(result['result'], ['totalKeys', 'addedDocs']))
-            result.chatflowid = chatflowid
-            const newUpsertHistory = new UpsertHistory()
-            Object.assign(newUpsertHistory, result)
-            const upsertHistory = appServer.AppDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
-            await appServer.AppDataSource.getRepository(UpsertHistory).save(upsertHistory)
+            sseStreamer: appServer.sseStreamer,
+            baseURL,
+            isInternal,
+            files,
+            isUpsert: true
         }
 
-        await appServer.telemetry.sendTelemetry('vector_upserted', {
-            version: await getAppVersion(),
-            chatlowId: chatflowid,
-            type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
-            flowGraph: getTelemetryFlowObj(nodes, edges),
-            stopNodeId
-        })
-        appServer.metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, { status: FLOWISE_COUNTER_STATUS.SUCCESS })
+        if (process.env.MODE === MODE.QUEUE) {
+            const upsertQueue = appServer.queueManager.getQueue('upsert')
 
-        return upsertedResult['result'] ?? { result: 'Successfully Upserted' }
+            const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
+            logger.debug(`[server]: Job added to queue: ${job.id}`)
+
+            const queueEvents = upsertQueue.getQueueEvents()
+            const result = await job.waitUntilFinished(queueEvents)
+
+            if (!result) {
+                throw new Error('Job execution failed')
+            }
+
+            appServer.metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
+                status: FLOWISE_COUNTER_STATUS.SUCCESS
+            })
+            return result
+        } else {
+            const result = await executeUpsert(executeData)
+
+            appServer.metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
+                status: FLOWISE_COUNTER_STATUS.SUCCESS
+            })
+            return result
+        }
     } catch (e) {
         logger.error('[server]: Error:', e)
+        appServer.metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, { status: FLOWISE_COUNTER_STATUS.FAILURE })
+
         if (e instanceof InternalFlowiseError && e.statusCode === StatusCodes.UNAUTHORIZED) {
             throw e
         } else {