farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 19:00:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/Add bullmq redis for message queue processing (#3568)

Browse Source 
* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully
This commit is contained in:
Henry Heng
2025-01-23 14:08:02 +00:00
committed by GitHub
parent 14adb936f2
commit a2a475ba7a
59 changed files with 38958 additions and 36985 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts
+42 -5
View File
@@ -138,7 +138,14 @@ class Elasticsearch_VectorStores implements INode {
})
// end of workaround
const elasticSearchClientArgs = prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
const { elasticClient, elasticSearchClientArgs } = prepareClientArgs(
endPoint,
cloudId,
credentialData,
nodeData,
similarityMeasure,
indexName
)
const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
try {
@@ -155,9 +162,11 @@ class Elasticsearch_VectorStores implements INode {
vectorStoreName: indexName
}
})
await elasticClient.close()
return res
} else {
await vectorStore.addDocuments(finalDocs)
await elasticClient.close()
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {
@@ -174,7 +183,14 @@ class Elasticsearch_VectorStores implements INode {
const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
const cloudId = getCredentialParam('cloudId', credentialData, nodeData)
const elasticSearchClientArgs = prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
const { elasticClient, elasticSearchClientArgs } = prepareClientArgs(
endPoint,
cloudId,
credentialData,
nodeData,
similarityMeasure,
indexName
)
const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
try {
@@ -186,8 +202,10 @@ class Elasticsearch_VectorStores implements INode {
await vectorStore.delete({ ids: keys })
await recordManager.deleteKeys(keys)
await elasticClient.close()
} else {
await vectorStore.delete({ ids })
await elasticClient.close()
}
} catch (e) {
throw new Error(e)
@@ -206,8 +224,22 @@ class Elasticsearch_VectorStores implements INode {
const k = topK ? parseFloat(topK) : 4
const output = nodeData.outputs?.output as string
const elasticSearchClientArgs = prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
const { elasticClient, elasticSearchClientArgs } = prepareClientArgs(
endPoint,
cloudId,
credentialData,
nodeData,
similarityMeasure,
indexName
)
const vectorStore = await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs)
const originalSimilaritySearchVectorWithScore = vectorStore.similaritySearchVectorWithScore
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
const results = await originalSimilaritySearchVectorWithScore.call(vectorStore, query, k, filter)
await elasticClient.close()
return results
}
if (output === 'retriever') {
return vectorStore.asRetriever(k)
@@ -289,12 +321,17 @@ const prepareClientArgs = (
similarity: 'l2_norm'
}
}
const elasticClient = new Client(elasticSearchClientOptions)
const elasticSearchClientArgs: ElasticClientArgs = {
client: new Client(elasticSearchClientOptions),
client: elasticClient,
indexName: indexName,
vectorSearchOptions: vectorSearchOptions
}
return elasticSearchClientArgs
return {
elasticClient,
elasticSearchClientArgs
}
}
module.exports = { nodeClass: Elasticsearch_VectorStores }
packages/components/nodes/vectorstores/Pinecone/Pinecone.ts
+5 -22
View File
@@ -1,5 +1,5 @@
import { flatten, isEqual } from 'lodash'
import { Pinecone, PineconeConfiguration } from '@pinecone-database/pinecone'
import { flatten } from 'lodash'
import { Pinecone } from '@pinecone-database/pinecone'
import { PineconeStoreParams, PineconeStore } from '@langchain/pinecone'
import { Embeddings } from '@langchain/core/embeddings'
import { Document } from '@langchain/core/documents'
@@ -9,23 +9,6 @@ import { FLOWISE_CHATID, getBaseClasses, getCredentialData, getCredentialParam }
import { addMMRInputParams, howToUseFileUpload, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
import { index } from '../../../src/indexing'
let pineconeClientSingleton: Pinecone
let pineconeClientOption: PineconeConfiguration
const getPineconeClient = (option: PineconeConfiguration) => {
if (!pineconeClientSingleton) {
// if client doesn't exists
pineconeClientSingleton = new Pinecone(option)
pineconeClientOption = option
return pineconeClientSingleton
} else if (pineconeClientSingleton && !isEqual(option, pineconeClientOption)) {
// if client exists but option changed
pineconeClientSingleton = new Pinecone(option)
return pineconeClientSingleton
}
return pineconeClientSingleton
}
class Pinecone_VectorStores implements INode {
label: string
name: string
@@ -155,7 +138,7 @@ class Pinecone_VectorStores implements INode {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
const client = getPineconeClient({ apiKey: pineconeApiKey })
const client = new Pinecone({ apiKey: pineconeApiKey })
const pineconeIndex = client.Index(_index)
@@ -211,7 +194,7 @@ class Pinecone_VectorStores implements INode {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
const client = getPineconeClient({ apiKey: pineconeApiKey })
const client = new Pinecone({ apiKey: pineconeApiKey })
const pineconeIndex = client.Index(_index)
@@ -253,7 +236,7 @@ class Pinecone_VectorStores implements INode {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
const client = getPineconeClient({ apiKey: pineconeApiKey })
const client = new Pinecone({ apiKey: pineconeApiKey })
const pineconeIndex = client.Index(index)
packages/components/nodes/vectorstores/Postgres/Postgres.ts
+6 -5
View File
@@ -7,7 +7,7 @@ import { howToUseFileUpload } from '../VectorStoreUtils'
import { VectorStore } from '@langchain/core/vectorstores'
import { VectorStoreDriver } from './driver/Base'
import { TypeORMDriver } from './driver/TypeORM'
import { PGVectorDriver } from './driver/PGVector'
// import { PGVectorDriver } from './driver/PGVector'
import { getContentColumnName, getDatabase, getHost, getPort, getTableName } from './utils'
const serverCredentialsExists = !!process.env.POSTGRES_VECTORSTORE_USER && !!process.env.POSTGRES_VECTORSTORE_PASSWORD
@@ -91,7 +91,7 @@ class Postgres_VectorStores implements INode {
additionalParams: true,
optional: true
},
{
/*{
label: 'Driver',
name: 'driver',
type: 'options',
@@ -109,7 +109,7 @@ class Postgres_VectorStores implements INode {
],
optional: true,
additionalParams: true
},
},*/
{
label: 'Distance Strategy',
name: 'distanceStrategy',
@@ -300,14 +300,15 @@ class Postgres_VectorStores implements INode {
}
static getDriverFromConfig(nodeData: INodeData, options: ICommonObject): VectorStoreDriver {
switch (nodeData.inputs?.driver) {
/*switch (nodeData.inputs?.driver) {
case 'typeorm':
return new TypeORMDriver(nodeData, options)
case 'pgvector':
return new PGVectorDriver(nodeData, options)
default:
return new TypeORMDriver(nodeData, options)
}
}*/
return new TypeORMDriver(nodeData, options)
}
}
packages/components/nodes/vectorstores/Postgres/driver/PGVector.ts
+5
View File
@@ -1,3 +1,7 @@
/*
* Temporary disabled due to increasing open connections without releasing them
* Use TypeORM instead
import { VectorStoreDriver } from './Base'
import { FLOWISE_CHATID } from '../../../../src'
import { DistanceStrategy, PGVectorStore, PGVectorStoreArgs } from '@langchain/community/vectorstores/pgvector'
@@ -120,3 +124,4 @@ export class PGVectorDriver extends VectorStoreDriver {
return instance
}
}
*/
packages/components/nodes/vectorstores/Postgres/driver/TypeORM.ts
+14 -3
View File
@@ -51,7 +51,9 @@ export class TypeORMDriver extends VectorStoreDriver {
}
async instanciate(metadataFilters?: any) {
return this.adaptInstance(await TypeORMVectorStore.fromDataSource(this.getEmbeddings(), await this.getArgs()), metadataFilters)
// @ts-ignore
const instance = new TypeORMVectorStore(this.getEmbeddings(), await this.getArgs())
return this.adaptInstance(instance, metadataFilters)
}
async fromDocuments(documents: Document[]) {
@@ -77,7 +79,8 @@ export class TypeORMDriver extends VectorStoreDriver {
[ERROR]: uncaughtException: Illegal invocation TypeError: Illegal invocation at Socket.ref (node:net:1524:18) at Connection.ref (.../node_modules/pg/lib/connection.js:183:17) at Client.ref (.../node_modules/pg/lib/client.js:591:21) at BoundPool._pulseQueue (/node_modules/pg-pool/index.js:148:28) at .../node_modules/pg-pool/index.js:184:37 at process.processTicksAndRejections (node:internal/process/task_queues:77:11)
*/
instance.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
return await TypeORMDriver.similaritySearchVectorWithScore(
await instance.appDataSource.initialize()
const res = await TypeORMDriver.similaritySearchVectorWithScore(
query,
k,
tableName,
@@ -85,6 +88,8 @@ export class TypeORMDriver extends VectorStoreDriver {
filter ?? metadataFilters,
this.computedOperatorString
)
await instance.appDataSource.destroy()
return res
}
instance.delete = async (params: { ids: string[] }): Promise<void> => {
@@ -92,9 +97,12 @@ export class TypeORMDriver extends VectorStoreDriver {
if (ids?.length) {
try {
await instance.appDataSource.initialize()
instance.appDataSource.getRepository(instance.documentEntity).delete(ids)
} catch (e) {
console.error('Failed to delete')
} finally {
await instance.appDataSource.destroy()
}
}
}
@@ -102,7 +110,10 @@ export class TypeORMDriver extends VectorStoreDriver {
const baseAddVectorsFn = instance.addVectors.bind(instance)
instance.addVectors = async (vectors, documents) => {
return baseAddVectorsFn(vectors, this.sanitizeDocuments(documents))
await instance.appDataSource.initialize()
const res = baseAddVectorsFn(vectors, this.sanitizeDocuments(documents))
await instance.appDataSource.destroy()
return res
}
return instance
packages/components/nodes/vectorstores/Redis/Redis.ts
+21 -28
View File
@@ -1,32 +1,11 @@
import { flatten, isEqual } from 'lodash'
import { createClient, SearchOptions, RedisClientOptions } from 'redis'
import { flatten } from 'lodash'
import { createClient, SearchOptions } from 'redis'
import { Embeddings } from '@langchain/core/embeddings'
import { RedisVectorStore, RedisVectorStoreConfig } from '@langchain/community/vectorstores/redis'
import { Document } from '@langchain/core/documents'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { escapeAllStrings, escapeSpecialChars, unEscapeSpecialChars } from './utils'
let redisClientSingleton: ReturnType<typeof createClient>
let redisClientOption: RedisClientOptions
const getRedisClient = async (option: RedisClientOptions) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = createClient(option)
await redisClientSingleton.connect()
redisClientOption = option
return redisClientSingleton
} else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = createClient(option)
await redisClientSingleton.connect()
redisClientOption = option
return redisClientSingleton
}
return redisClientSingleton
}
import { escapeSpecialChars, unEscapeSpecialChars } from './utils'
class Redis_VectorStores implements INode {
label: string
@@ -163,13 +142,13 @@ class Redis_VectorStores implements INode {
for (let i = 0; i < flattenDocs.length; i += 1) {
if (flattenDocs[i] && flattenDocs[i].pageContent) {
const document = new Document(flattenDocs[i])
escapeAllStrings(document.metadata)
finalDocs.push(document)
}
}
try {
const redisClient = await getRedisClient({ url: redisUrl })
const redisClient = createClient({ url: redisUrl })
await redisClient.connect()
const storeConfig: RedisVectorStoreConfig = {
redisClient: redisClient,
@@ -203,6 +182,8 @@ class Redis_VectorStores implements INode {
)
}
await redisClient.quit()
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
@@ -231,7 +212,7 @@ class Redis_VectorStores implements INode {
redisUrl = 'redis://' + username + ':' + password + '@' + host + ':' + portStr
}
const redisClient = await getRedisClient({ url: redisUrl })
const redisClient = createClient({ url: redisUrl })
const storeConfig: RedisVectorStoreConfig = {
redisClient: redisClient,
@@ -246,7 +227,19 @@ class Redis_VectorStores implements INode {
// Avoid Illegal invocation error
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
return await similaritySearchVectorWithScore(query, k, indexName, metadataKey, vectorKey, contentKey, redisClient, filter)
await redisClient.connect()
const results = await similaritySearchVectorWithScore(
query,
k,
indexName,
metadataKey,
vectorKey,
contentKey,
redisClient,
filter
)
await redisClient.quit()
return results
}
if (output === 'retriever') {