farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 21:00:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/Add bullmq redis for message queue processing (#3568)

Browse Source 
* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully
This commit is contained in:
Henry Heng
2025-01-23 14:08:02 +00:00
committed by GitHub
parent 14adb936f2
commit a2a475ba7a
59 changed files with 38958 additions and 36985 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/vectorstores/Redis/Redis.ts
+21 -28
View File
@@ -1,32 +1,11 @@
import { flatten, isEqual } from 'lodash'
import { createClient, SearchOptions, RedisClientOptions } from 'redis'
import { flatten } from 'lodash'
import { createClient, SearchOptions } from 'redis'
import { Embeddings } from '@langchain/core/embeddings'
import { RedisVectorStore, RedisVectorStoreConfig } from '@langchain/community/vectorstores/redis'
import { Document } from '@langchain/core/documents'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { escapeAllStrings, escapeSpecialChars, unEscapeSpecialChars } from './utils'
let redisClientSingleton: ReturnType<typeof createClient>
let redisClientOption: RedisClientOptions
const getRedisClient = async (option: RedisClientOptions) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = createClient(option)
await redisClientSingleton.connect()
redisClientOption = option
return redisClientSingleton
} else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = createClient(option)
await redisClientSingleton.connect()
redisClientOption = option
return redisClientSingleton
}
return redisClientSingleton
}
import { escapeSpecialChars, unEscapeSpecialChars } from './utils'
class Redis_VectorStores implements INode {
label: string
@@ -163,13 +142,13 @@ class Redis_VectorStores implements INode {
for (let i = 0; i < flattenDocs.length; i += 1) {
if (flattenDocs[i] && flattenDocs[i].pageContent) {
const document = new Document(flattenDocs[i])
escapeAllStrings(document.metadata)
finalDocs.push(document)
}
}
try {
const redisClient = await getRedisClient({ url: redisUrl })
const redisClient = createClient({ url: redisUrl })
await redisClient.connect()
const storeConfig: RedisVectorStoreConfig = {
redisClient: redisClient,
@@ -203,6 +182,8 @@ class Redis_VectorStores implements INode {
)
}
await redisClient.quit()
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
@@ -231,7 +212,7 @@ class Redis_VectorStores implements INode {
redisUrl = 'redis://' + username + ':' + password + '@' + host + ':' + portStr
}
const redisClient = await getRedisClient({ url: redisUrl })
const redisClient = createClient({ url: redisUrl })
const storeConfig: RedisVectorStoreConfig = {
redisClient: redisClient,
@@ -246,7 +227,19 @@ class Redis_VectorStores implements INode {
// Avoid Illegal invocation error
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
return await similaritySearchVectorWithScore(query, k, indexName, metadataKey, vectorKey, contentKey, redisClient, filter)
await redisClient.connect()
const results = await similaritySearchVectorWithScore(
query,
k,
indexName,
metadataKey,
vectorKey,
contentKey,
redisClient,
filter
)
await redisClient.quit()
return results
}
if (output === 'retriever') {