farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 23:01:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/Add bullmq redis for message queue processing (#3568)

Browse Source 
* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully
This commit is contained in:
Henry Heng
2025-01-23 14:08:02 +00:00
committed by GitHub
parent 14adb936f2
commit a2a475ba7a
59 changed files with 38958 additions and 36985 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts
+42 -5
View File
@@ -138,7 +138,14 @@ class Elasticsearch_VectorStores implements INode {
})
// end of workaround
const elasticSearchClientArgs = prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
const { elasticClient, elasticSearchClientArgs } = prepareClientArgs(
endPoint,
cloudId,
credentialData,
nodeData,
similarityMeasure,
indexName
)
const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
try {
@@ -155,9 +162,11 @@ class Elasticsearch_VectorStores implements INode {
vectorStoreName: indexName
}
})
await elasticClient.close()
return res
} else {
await vectorStore.addDocuments(finalDocs)
await elasticClient.close()
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {
@@ -174,7 +183,14 @@ class Elasticsearch_VectorStores implements INode {
const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
const cloudId = getCredentialParam('cloudId', credentialData, nodeData)
const elasticSearchClientArgs = prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
const { elasticClient, elasticSearchClientArgs } = prepareClientArgs(
endPoint,
cloudId,
credentialData,
nodeData,
similarityMeasure,
indexName
)
const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
try {
@@ -186,8 +202,10 @@ class Elasticsearch_VectorStores implements INode {
await vectorStore.delete({ ids: keys })
await recordManager.deleteKeys(keys)
await elasticClient.close()
} else {
await vectorStore.delete({ ids })
await elasticClient.close()
}
} catch (e) {
throw new Error(e)
@@ -206,8 +224,22 @@ class Elasticsearch_VectorStores implements INode {
const k = topK ? parseFloat(topK) : 4
const output = nodeData.outputs?.output as string
const elasticSearchClientArgs = prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
const { elasticClient, elasticSearchClientArgs } = prepareClientArgs(
endPoint,
cloudId,
credentialData,
nodeData,
similarityMeasure,
indexName
)
const vectorStore = await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs)
const originalSimilaritySearchVectorWithScore = vectorStore.similaritySearchVectorWithScore
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
const results = await originalSimilaritySearchVectorWithScore.call(vectorStore, query, k, filter)
await elasticClient.close()
return results
}
if (output === 'retriever') {
return vectorStore.asRetriever(k)
@@ -289,12 +321,17 @@ const prepareClientArgs = (
similarity: 'l2_norm'
}
}
const elasticClient = new Client(elasticSearchClientOptions)
const elasticSearchClientArgs: ElasticClientArgs = {
client: new Client(elasticSearchClientOptions),
client: elasticClient,
indexName: indexName,
vectorSearchOptions: vectorSearchOptions
}
return elasticSearchClientArgs
return {
elasticClient,
elasticSearchClientArgs
}
}
module.exports = { nodeClass: Elasticsearch_VectorStores }