Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

packages/components/nodes/vectorstores/Postgres/Postgres.ts

@@ -7,7 +7,7 @@ import { howToUseFileUpload } from '../VectorStoreUtils'
 import { VectorStore } from '@langchain/core/vectorstores'
 import { VectorStoreDriver } from './driver/Base'
 import { TypeORMDriver } from './driver/TypeORM'
-import { PGVectorDriver } from './driver/PGVector'
+// import { PGVectorDriver } from './driver/PGVector'
 import { getContentColumnName, getDatabase, getHost, getPort, getTableName } from './utils'
 
 const serverCredentialsExists = !!process.env.POSTGRES_VECTORSTORE_USER && !!process.env.POSTGRES_VECTORSTORE_PASSWORD
@@ -91,7 +91,7 @@ class Postgres_VectorStores implements INode {
                 additionalParams: true,
                 optional: true
             },
-            {
+            /*{
                 label: 'Driver',
                 name: 'driver',
                 type: 'options',
@@ -109,7 +109,7 @@ class Postgres_VectorStores implements INode {
                 ],
                 optional: true,
                 additionalParams: true
-            },
+            },*/
             {
                 label: 'Distance Strategy',
                 name: 'distanceStrategy',
@@ -300,14 +300,15 @@ class Postgres_VectorStores implements INode {
     }
 
     static getDriverFromConfig(nodeData: INodeData, options: ICommonObject): VectorStoreDriver {
-        switch (nodeData.inputs?.driver) {
+        /*switch (nodeData.inputs?.driver) {
             case 'typeorm':
                 return new TypeORMDriver(nodeData, options)
             case 'pgvector':
                 return new PGVectorDriver(nodeData, options)
             default:
                 return new TypeORMDriver(nodeData, options)
-        }
+        }*/
+        return new TypeORMDriver(nodeData, options)
     }
 }
 

packages/components/nodes/vectorstores/Postgres/driver/PGVector.ts

@@ -1,3 +1,7 @@
+/*
+* Temporary disabled due to increasing open connections without releasing them
+* Use TypeORM instead
+
 import { VectorStoreDriver } from './Base'
 import { FLOWISE_CHATID } from '../../../../src'
 import { DistanceStrategy, PGVectorStore, PGVectorStoreArgs } from '@langchain/community/vectorstores/pgvector'
@@ -120,3 +124,4 @@ export class PGVectorDriver extends VectorStoreDriver {
         return instance
     }
 }
+*/

packages/components/nodes/vectorstores/Postgres/driver/TypeORM.ts

@@ -51,7 +51,9 @@ export class TypeORMDriver extends VectorStoreDriver {
     }
 
     async instanciate(metadataFilters?: any) {
-        return this.adaptInstance(await TypeORMVectorStore.fromDataSource(this.getEmbeddings(), await this.getArgs()), metadataFilters)
+        // @ts-ignore
+        const instance = new TypeORMVectorStore(this.getEmbeddings(), await this.getArgs())
+        return this.adaptInstance(instance, metadataFilters)
     }
 
     async fromDocuments(documents: Document[]) {
@@ -77,7 +79,8 @@ export class TypeORMDriver extends VectorStoreDriver {
             [ERROR]: uncaughtException:  Illegal invocation TypeError: Illegal invocation at Socket.ref (node:net:1524:18) at Connection.ref (.../node_modules/pg/lib/connection.js:183:17) at Client.ref (.../node_modules/pg/lib/client.js:591:21) at BoundPool._pulseQueue (/node_modules/pg-pool/index.js:148:28) at .../node_modules/pg-pool/index.js:184:37 at process.processTicksAndRejections (node:internal/process/task_queues:77:11)
         */
         instance.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
-            return await TypeORMDriver.similaritySearchVectorWithScore(
+            await instance.appDataSource.initialize()
+            const res = await TypeORMDriver.similaritySearchVectorWithScore(
                 query,
                 k,
                 tableName,
@@ -85,6 +88,8 @@ export class TypeORMDriver extends VectorStoreDriver {
                 filter ?? metadataFilters,
                 this.computedOperatorString
             )
+            await instance.appDataSource.destroy()
+            return res
         }
 
         instance.delete = async (params: { ids: string[] }): Promise<void> => {
@@ -92,9 +97,12 @@ export class TypeORMDriver extends VectorStoreDriver {
 
             if (ids?.length) {
                 try {
+                    await instance.appDataSource.initialize()
                     instance.appDataSource.getRepository(instance.documentEntity).delete(ids)
                 } catch (e) {
                     console.error('Failed to delete')
+                } finally {
+                    await instance.appDataSource.destroy()
                 }
             }
         }
@@ -102,7 +110,10 @@ export class TypeORMDriver extends VectorStoreDriver {
         const baseAddVectorsFn = instance.addVectors.bind(instance)
 
         instance.addVectors = async (vectors, documents) => {
-            return baseAddVectorsFn(vectors, this.sanitizeDocuments(documents))
+            await instance.appDataSource.initialize()
+            const res = baseAddVectorsFn(vectors, this.sanitizeDocuments(documents))
+            await instance.appDataSource.destroy()
+            return res
         }
 
         return instance