From 931e14c0829ec673c1cb84c39af1117cf0f335d4 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Fri, 20 Oct 2023 20:03:38 +0530 Subject: [PATCH] New Feature : Redis Vector Store --- .../vectorstores/Redis/RedisSearchBase.ts | 115 ++++++++++++++++++ .../vectorstores/Redis/Redis_Existing.ts | 32 +++++ .../nodes/vectorstores/Redis/Redis_Upsert.ts | 46 +++++++ .../nodes/vectorstores/Redis/redis.svg | 1 + 4 files changed, 194 insertions(+) create mode 100644 packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts create mode 100644 packages/components/nodes/vectorstores/Redis/Redis_Existing.ts create mode 100644 packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts create mode 100644 packages/components/nodes/vectorstores/Redis/redis.svg diff --git a/packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts b/packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts new file mode 100644 index 00000000..c47c1e51 --- /dev/null +++ b/packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts @@ -0,0 +1,115 @@ +import { + getBaseClasses, + getCredentialData, + getCredentialParam, + ICommonObject, + INodeData, + INodeOutputsValue, + INodeParams +} from '../../../src' + +import { Embeddings } from 'langchain/embeddings/base' +import { VectorStore } from 'langchain/vectorstores/base' +import { Document } from 'langchain/document' +import { createClient } from 'redis' +import { RedisVectorStore } from 'langchain/vectorstores/redis' + +export abstract class RedisSearchBase { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + redisClient: ReturnType + + protected constructor() { + this.type = 'Redis' + this.icon = 'redis.svg' + this.category = 'Vector Stores' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['redisCacheUrlApi', 'redisCacheApi'] + } + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Index Name', + name: 'indexName', + placeholder: '', + type: 'string' + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Redis Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Redis Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(RedisVectorStore)] + } + ] + } + + abstract constructVectorStore( + embeddings: Embeddings, + indexName: string, + docs: Document>[] | undefined + ): Promise + + async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document>[] | undefined): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const indexName = nodeData.inputs?.indexName as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const topK = nodeData.inputs?.topK as string + const k = topK ? parseFloat(topK) : 4 + const output = nodeData.outputs?.output as string + + let redisUrl = getCredentialParam('redisUrl', credentialData, nodeData) + if (!redisUrl || redisUrl === '') { + const username = getCredentialParam('redisCacheUser', credentialData, nodeData) + const password = getCredentialParam('redisCachePwd', credentialData, nodeData) + const portStr = getCredentialParam('redisCachePort', credentialData, nodeData) + const host = getCredentialParam('redisCacheHost', credentialData, nodeData) + + redisUrl = 'redis://' + username + ':' + password + '@' + host + ':' + portStr + } + + this.redisClient = createClient({ url: redisUrl }) + await this.redisClient.connect() + + const vectorStore = await this.constructVectorStore(embeddings, indexName, docs) + + if (output === 'retriever') { + return vectorStore.asRetriever(k) + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} diff --git a/packages/components/nodes/vectorstores/Redis/Redis_Existing.ts b/packages/components/nodes/vectorstores/Redis/Redis_Existing.ts new file mode 100644 index 00000000..88166659 --- /dev/null +++ b/packages/components/nodes/vectorstores/Redis/Redis_Existing.ts @@ -0,0 +1,32 @@ +import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { VectorStore } from 'langchain/vectorstores/base' +import { RedisVectorStore, RedisVectorStoreConfig } from 'langchain/vectorstores/redis' +import { Document } from 'langchain/document' + +import { RedisSearchBase } from './RedisSearchBase' + +class RedisExisting_VectorStores extends RedisSearchBase implements INode { + constructor() { + super() + this.label = 'Redis Load Existing Index' + this.name = 'RedisIndex' + this.version = 1.0 + this.description = 'Load existing index from Redis (i.e: Document has been upserted)' + } + + async constructVectorStore(embeddings: Embeddings, indexName: string, _: Document>[]): Promise { + const storeConfig: RedisVectorStoreConfig = { + redisClient: this.redisClient, + indexName: indexName + } + + return new RedisVectorStore(embeddings, storeConfig) + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + return super.init(nodeData, _, options, undefined) + } +} + +module.exports = { nodeClass: RedisExisting_VectorStores } diff --git a/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts new file mode 100644 index 00000000..2af445eb --- /dev/null +++ b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts @@ -0,0 +1,46 @@ +import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' + +import { flatten } from 'lodash' +import { RedisSearchBase } from './RedisSearchBase' +import { VectorStore } from 'langchain/vectorstores/base' +import { RedisVectorStore, RedisVectorStoreConfig } from 'langchain/vectorstores/redis' + +class RedisUpsert_VectorStores extends RedisSearchBase implements INode { + constructor() { + super() + this.label = 'Redis Upsert Document' + this.name = 'RedisUpsert' + this.version = 1.0 + this.description = 'Upsert documents to Redis' + this.inputs.unshift({ + label: 'Document', + name: 'document', + type: 'Document', + list: true + }) + } + + async constructVectorStore(embeddings: Embeddings, indexName: string, docs: Document>[]): Promise { + const storeConfig: RedisVectorStoreConfig = { + redisClient: this.redisClient, + indexName: indexName + } + return await RedisVectorStore.fromDocuments(docs, embeddings, storeConfig) + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const docs = nodeData.inputs?.document as Document[] + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + finalDocs.push(new Document(flattenDocs[i])) + } + + return super.init(nodeData, _, options, flattenDocs) + } +} + +module.exports = { nodeClass: RedisUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/Redis/redis.svg b/packages/components/nodes/vectorstores/Redis/redis.svg new file mode 100644 index 00000000..90359069 --- /dev/null +++ b/packages/components/nodes/vectorstores/Redis/redis.svg @@ -0,0 +1 @@ + \ No newline at end of file