Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

packages/components/nodes/cache/InMemoryCache/InMemoryCache.ts

@@ -27,17 +27,17 @@ class InMemoryCache implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const memoryMap = options.cachePool.getLLMCache(options.chatflowid) ?? new Map()
+        const memoryMap = (await options.cachePool.getLLMCache(options.chatflowid)) ?? new Map()
         const inMemCache = new InMemoryCacheExtended(memoryMap)
 
         inMemCache.lookup = async (prompt: string, llmKey: string): Promise<any | null> => {
-            const memory = options.cachePool.getLLMCache(options.chatflowid) ?? inMemCache.cache
+            const memory = (await options.cachePool.getLLMCache(options.chatflowid)) ?? inMemCache.cache
             return Promise.resolve(memory.get(getCacheKey(prompt, llmKey)) ?? null)
         }
 
         inMemCache.update = async (prompt: string, llmKey: string, value: any): Promise<void> => {
             inMemCache.cache.set(getCacheKey(prompt, llmKey), value)
-            options.cachePool.addLLMCache(options.chatflowid, inMemCache.cache)
+            await options.cachePool.addLLMCache(options.chatflowid, inMemCache.cache)
         }
         return inMemCache
     }

packages/components/nodes/cache/InMemoryCache/InMemoryEmbeddingCache.ts

@@ -43,11 +43,11 @@ class InMemoryEmbeddingCache implements INode {
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
         const namespace = nodeData.inputs?.namespace as string
         const underlyingEmbeddings = nodeData.inputs?.embeddings as Embeddings
-        const memoryMap = options.cachePool.getEmbeddingCache(options.chatflowid) ?? {}
+        const memoryMap = (await options.cachePool.getEmbeddingCache(options.chatflowid)) ?? {}
         const inMemCache = new InMemoryEmbeddingCacheExtended(memoryMap)
 
         inMemCache.mget = async (keys: string[]) => {
-            const memory = options.cachePool.getEmbeddingCache(options.chatflowid) ?? inMemCache.store
+            const memory = (await options.cachePool.getEmbeddingCache(options.chatflowid)) ?? inMemCache.store
             return keys.map((key) => memory[key])
         }
 
@@ -55,14 +55,14 @@ class InMemoryEmbeddingCache implements INode {
             for (const [key, value] of keyValuePairs) {
                 inMemCache.store[key] = value
             }
-            options.cachePool.addEmbeddingCache(options.chatflowid, inMemCache.store)
+            await options.cachePool.addEmbeddingCache(options.chatflowid, inMemCache.store)
         }
 
         inMemCache.mdelete = async (keys: string[]): Promise<void> => {
             for (const key of keys) {
                 delete inMemCache.store[key]
             }
-            options.cachePool.addEmbeddingCache(options.chatflowid, inMemCache.store)
+            await options.cachePool.addEmbeddingCache(options.chatflowid, inMemCache.store)
         }
 
         return CacheBackedEmbeddings.fromBytesStore(underlyingEmbeddings, inMemCache, {

packages/components/nodes/cache/RedisCache/RedisCache.ts

@@ -1,47 +1,10 @@
-import { Redis, RedisOptions } from 'ioredis'
-import { isEqual } from 'lodash'
+import { Redis } from 'ioredis'
 import hash from 'object-hash'
 import { RedisCache as LangchainRedisCache } from '@langchain/community/caches/ioredis'
 import { StoredGeneration, mapStoredMessageToChatMessage } from '@langchain/core/messages'
 import { Generation, ChatGeneration } from '@langchain/core/outputs'
 import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src'
 
-let redisClientSingleton: Redis
-let redisClientOption: RedisOptions
-let redisClientUrl: string
-
-const getRedisClientbyOption = (option: RedisOptions) => {
-    if (!redisClientSingleton) {
-        // if client doesn't exists
-        redisClientSingleton = new Redis(option)
-        redisClientOption = option
-        return redisClientSingleton
-    } else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
-        // if client exists but option changed
-        redisClientSingleton.quit()
-        redisClientSingleton = new Redis(option)
-        redisClientOption = option
-        return redisClientSingleton
-    }
-    return redisClientSingleton
-}
-
-const getRedisClientbyUrl = (url: string) => {
-    if (!redisClientSingleton) {
-        // if client doesn't exists
-        redisClientSingleton = new Redis(url)
-        redisClientUrl = url
-        return redisClientSingleton
-    } else if (redisClientSingleton && url !== redisClientUrl) {
-        // if client exists but option changed
-        redisClientSingleton.quit()
-        redisClientSingleton = new Redis(url)
-        redisClientUrl = url
-        return redisClientSingleton
-    }
-    return redisClientSingleton
-}
-
 class RedisCache implements INode {
     label: string
     name: string
@@ -85,33 +48,19 @@ class RedisCache implements INode {
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
         const ttl = nodeData.inputs?.ttl as string
 
-        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
-
-        let client: Redis
-        if (!redisUrl || redisUrl === '') {
-            const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
-            const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
-            const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
-            const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
-            const sslEnabled = getCredentialParam('redisCacheSslEnabled', credentialData, nodeData)
-
-            const tlsOptions = sslEnabled === true ? { tls: { rejectUnauthorized: false } } : {}
-
-            client = getRedisClientbyOption({
-                port: portStr ? parseInt(portStr) : 6379,
-                host,
-                username,
-                password,
-                ...tlsOptions
-            })
-        } else {
-            client = getRedisClientbyUrl(redisUrl)
-        }
-
+        let client = await getRedisClient(nodeData, options)
         const redisClient = new LangchainRedisCache(client)
 
         redisClient.lookup = async (prompt: string, llmKey: string) => {
+            try {
+                const pingResp = await client.ping()
+                if (pingResp !== 'PONG') {
+                    client = await getRedisClient(nodeData, options)
+                }
+            } catch (error) {
+                client = await getRedisClient(nodeData, options)
+            }
+
             let idx = 0
             let key = getCacheKey(prompt, llmKey, String(idx))
             let value = await client.get(key)
@@ -125,10 +74,21 @@ class RedisCache implements INode {
                 value = await client.get(key)
             }
 
+            client.quit()
+
             return generations.length > 0 ? generations : null
         }
 
         redisClient.update = async (prompt: string, llmKey: string, value: Generation[]) => {
+            try {
+                const pingResp = await client.ping()
+                if (pingResp !== 'PONG') {
+                    client = await getRedisClient(nodeData, options)
+                }
+            } catch (error) {
+                client = await getRedisClient(nodeData, options)
+            }
+
             for (let i = 0; i < value.length; i += 1) {
                 const key = getCacheKey(prompt, llmKey, String(i))
                 if (ttl) {
@@ -137,12 +97,43 @@ class RedisCache implements INode {
                     await client.set(key, JSON.stringify(serializeGeneration(value[i])))
                 }
             }
+
+            client.quit()
         }
 
+        client.quit()
+
         return redisClient
     }
 }
+const getRedisClient = async (nodeData: INodeData, options: ICommonObject) => {
+    let client: Redis
 
+    const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+    const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
+
+    if (!redisUrl || redisUrl === '') {
+        const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
+        const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
+        const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
+        const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
+        const sslEnabled = getCredentialParam('redisCacheSslEnabled', credentialData, nodeData)
+
+        const tlsOptions = sslEnabled === true ? { tls: { rejectUnauthorized: false } } : {}
+
+        client = new Redis({
+            port: portStr ? parseInt(portStr) : 6379,
+            host,
+            username,
+            password,
+            ...tlsOptions
+        })
+    } else {
+        client = new Redis(redisUrl)
+    }
+
+    return client
+}
 const getCacheKey = (...strings: string[]): string => hash(strings.join('_'))
 const deserializeStoredGeneration = (storedGeneration: StoredGeneration) => {
     if (storedGeneration.message !== undefined) {

packages/components/nodes/cache/RedisCache/RedisEmbeddingsCache.ts

@@ -1,45 +1,11 @@
-import { Redis, RedisOptions } from 'ioredis'
-import { isEqual } from 'lodash'
+import { Redis } from 'ioredis'
 import { RedisByteStore } from '@langchain/community/storage/ioredis'
-import { Embeddings } from '@langchain/core/embeddings'
-import { CacheBackedEmbeddings } from 'langchain/embeddings/cache_backed'
+import { Embeddings, EmbeddingsInterface } from '@langchain/core/embeddings'
+import { CacheBackedEmbeddingsFields } from 'langchain/embeddings/cache_backed'
 import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src'
-
-let redisClientSingleton: Redis
-let redisClientOption: RedisOptions
-let redisClientUrl: string
-
-const getRedisClientbyOption = (option: RedisOptions) => {
-    if (!redisClientSingleton) {
-        // if client doesn't exists
-        redisClientSingleton = new Redis(option)
-        redisClientOption = option
-        return redisClientSingleton
-    } else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
-        // if client exists but option changed
-        redisClientSingleton.quit()
-        redisClientSingleton = new Redis(option)
-        redisClientOption = option
-        return redisClientSingleton
-    }
-    return redisClientSingleton
-}
-
-const getRedisClientbyUrl = (url: string) => {
-    if (!redisClientSingleton) {
-        // if client doesn't exists
-        redisClientSingleton = new Redis(url)
-        redisClientUrl = url
-        return redisClientSingleton
-    } else if (redisClientSingleton && url !== redisClientUrl) {
-        // if client exists but option changed
-        redisClientSingleton.quit()
-        redisClientSingleton = new Redis(url)
-        redisClientUrl = url
-        return redisClientSingleton
-    }
-    return redisClientSingleton
-}
+import { BaseStore } from '@langchain/core/stores'
+import { insecureHash } from '@langchain/core/utils/hash'
+import { Document } from '@langchain/core/documents'
 
 class RedisEmbeddingsCache implements INode {
     label: string
@@ -112,7 +78,7 @@ class RedisEmbeddingsCache implements INode {
 
             const tlsOptions = sslEnabled === true ? { tls: { rejectUnauthorized: false } } : {}
 
-            client = getRedisClientbyOption({
+            client = new Redis({
                 port: portStr ? parseInt(portStr) : 6379,
                 host,
                 username,
@@ -120,7 +86,7 @@ class RedisEmbeddingsCache implements INode {
                 ...tlsOptions
             })
         } else {
-            client = getRedisClientbyUrl(redisUrl)
+            client = new Redis(redisUrl)
         }
 
         ttl ??= '3600'
@@ -130,10 +96,143 @@ class RedisEmbeddingsCache implements INode {
             ttl: ttlNumber
         })
 
-        return CacheBackedEmbeddings.fromBytesStore(underlyingEmbeddings, redisStore, {
-            namespace: namespace
+        const store = CacheBackedEmbeddings.fromBytesStore(underlyingEmbeddings, redisStore, {
+            namespace: namespace,
+            redisClient: client
+        })
+
+        return store
+    }
+}
+
+class CacheBackedEmbeddings extends Embeddings {
+    protected underlyingEmbeddings: EmbeddingsInterface
+
+    protected documentEmbeddingStore: BaseStore<string, number[]>
+
+    protected redisClient?: Redis
+
+    constructor(fields: CacheBackedEmbeddingsFields & { redisClient?: Redis }) {
+        super(fields)
+        this.underlyingEmbeddings = fields.underlyingEmbeddings
+        this.documentEmbeddingStore = fields.documentEmbeddingStore
+        this.redisClient = fields.redisClient
+    }
+
+    async embedQuery(document: string): Promise<number[]> {
+        const res = this.underlyingEmbeddings.embedQuery(document)
+        this.redisClient?.quit()
+        return res
+    }
+
+    async embedDocuments(documents: string[]): Promise<number[][]> {
+        const vectors = await this.documentEmbeddingStore.mget(documents)
+        const missingIndicies = []
+        const missingDocuments = []
+        for (let i = 0; i < vectors.length; i += 1) {
+            if (vectors[i] === undefined) {
+                missingIndicies.push(i)
+                missingDocuments.push(documents[i])
+            }
+        }
+        if (missingDocuments.length) {
+            const missingVectors = await this.underlyingEmbeddings.embedDocuments(missingDocuments)
+            const keyValuePairs: [string, number[]][] = missingDocuments.map((document, i) => [document, missingVectors[i]])
+            await this.documentEmbeddingStore.mset(keyValuePairs)
+            for (let i = 0; i < missingIndicies.length; i += 1) {
+                vectors[missingIndicies[i]] = missingVectors[i]
+            }
+        }
+        this.redisClient?.quit()
+        return vectors as number[][]
+    }
+
+    static fromBytesStore(
+        underlyingEmbeddings: EmbeddingsInterface,
+        documentEmbeddingStore: BaseStore<string, Uint8Array>,
+        options?: {
+            namespace?: string
+            redisClient?: Redis
+        }
+    ) {
+        const encoder = new TextEncoder()
+        const decoder = new TextDecoder()
+        const encoderBackedStore = new EncoderBackedStore<string, number[], Uint8Array>({
+            store: documentEmbeddingStore,
+            keyEncoder: (key) => (options?.namespace ?? '') + insecureHash(key),
+            valueSerializer: (value) => encoder.encode(JSON.stringify(value)),
+            valueDeserializer: (serializedValue) => JSON.parse(decoder.decode(serializedValue))
+        })
+        return new this({
+            underlyingEmbeddings,
+            documentEmbeddingStore: encoderBackedStore,
+            redisClient: options?.redisClient
         })
     }
 }
 
+class EncoderBackedStore<K, V, SerializedType = any> extends BaseStore<K, V> {
+    lc_namespace = ['langchain', 'storage']
+
+    store: BaseStore<string, SerializedType>
+
+    keyEncoder: (key: K) => string
+
+    valueSerializer: (value: V) => SerializedType
+
+    valueDeserializer: (value: SerializedType) => V
+
+    constructor(fields: {
+        store: BaseStore<string, SerializedType>
+        keyEncoder: (key: K) => string
+        valueSerializer: (value: V) => SerializedType
+        valueDeserializer: (value: SerializedType) => V
+    }) {
+        super(fields)
+        this.store = fields.store
+        this.keyEncoder = fields.keyEncoder
+        this.valueSerializer = fields.valueSerializer
+        this.valueDeserializer = fields.valueDeserializer
+    }
+
+    async mget(keys: K[]): Promise<(V | undefined)[]> {
+        const encodedKeys = keys.map(this.keyEncoder)
+        const values = await this.store.mget(encodedKeys)
+        return values.map((value) => {
+            if (value === undefined) {
+                return undefined
+            }
+            return this.valueDeserializer(value)
+        })
+    }
+
+    async mset(keyValuePairs: [K, V][]): Promise<void> {
+        const encodedPairs: [string, SerializedType][] = keyValuePairs.map(([key, value]) => [
+            this.keyEncoder(key),
+            this.valueSerializer(value)
+        ])
+        return this.store.mset(encodedPairs)
+    }
+
+    async mdelete(keys: K[]): Promise<void> {
+        const encodedKeys = keys.map(this.keyEncoder)
+        return this.store.mdelete(encodedKeys)
+    }
+
+    async *yieldKeys(prefix?: string | undefined): AsyncGenerator<string | K> {
+        yield* this.store.yieldKeys(prefix)
+    }
+}
+
+export function createDocumentStoreFromByteStore(store: BaseStore<string, Uint8Array>) {
+    const encoder = new TextEncoder()
+    const decoder = new TextDecoder()
+    return new EncoderBackedStore({
+        store,
+        keyEncoder: (key: string) => key,
+        valueSerializer: (doc: Document) => encoder.encode(JSON.stringify({ pageContent: doc.pageContent, metadata: doc.metadata })),
+        valueDeserializer: (bytes: Uint8Array) => new Document(JSON.parse(decoder.decode(bytes)))
+    })
+}
+
 module.exports = { nodeClass: RedisEmbeddingsCache }