diff --git a/packages/components/credentials/MongoDBUrlApi.credential.ts b/packages/components/credentials/MongoDBUrlApi.credential.ts new file mode 100644 index 00000000..2f2cba38 --- /dev/null +++ b/packages/components/credentials/MongoDBUrlApi.credential.ts @@ -0,0 +1,25 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class MongoDBUrlApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'MongoDB ATLAS' + this.name = 'mongoDBUrlApi' + this.version = 1.0 + this.inputs = [ + { + label: 'ATLAS Connection URL', + name: 'mongoDBConnectUrl', + type: 'string', + placeholder: 'mongodb+srv://myDatabaseUser:D1fficultP%40ssw0rd@cluster0.example.mongodb.net/?retryWrites=true&w=majority' + } + ] + } +} + +module.exports = { credClass: MongoDBUrlApi } diff --git a/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts b/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts new file mode 100644 index 00000000..7de2ec34 --- /dev/null +++ b/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts @@ -0,0 +1,146 @@ +import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' +import { MongoDBChatMessageHistory } from 'langchain/stores/message/mongodb' +import { BufferMemory, BufferMemoryInput } from 'langchain/memory' +import { BaseMessage, mapStoredMessageToChatMessage } from 'langchain/schema' +import { MongoClient } from 'mongodb' + +class MongoDB_Memory implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'MongoDB Atlas Chat Memory' + this.name = 'MongoDBAtlasChatMemory' + this.version = 1.0 + this.type = 'MongoDBAtlasChatMemory' + this.icon = 'mongodb.png' + this.category = 'Memory' + this.description = 'Stores the conversation in MongoDB Atlas' + this.baseClasses = [this.type, ...getBaseClasses(BufferMemory)] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['mongoDBUrlApi'] + } + this.inputs = [ + { + label: 'Database', + name: 'databaseName', + placeholder: '', + type: 'string' + }, + { + label: 'Collection Name', + name: 'collectionName', + placeholder: '', + type: 'string' + }, + { + label: 'Session Id', + name: 'sessionId', + type: 'string', + description: 'If not specified, the first CHAT_MESSAGE_ID will be used as sessionId', + default: '', + additionalParams: true, + optional: true + }, + { + label: 'Memory Key', + name: 'memoryKey', + type: 'string', + default: 'chat_history', + additionalParams: true + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + return initializeMongoDB(nodeData, options) + } + + async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise { + const mongodbMemory = await initializeMongoDB(nodeData, options) + const sessionId = nodeData.inputs?.sessionId as string + const chatId = options?.chatId as string + options.logger.info(`Clearing MongoDB memory session ${sessionId ? sessionId : chatId}`) + await mongodbMemory.clear() + options.logger.info(`Successfully cleared MongoDB memory session ${sessionId ? sessionId : chatId}`) + } +} + +const initializeMongoDB = async (nodeData: INodeData, options: ICommonObject): Promise => { + const databaseName = nodeData.inputs?.databaseName as string + const collectionName = nodeData.inputs?.collectionName as string + const sessionId = nodeData.inputs?.sessionId as string + const memoryKey = nodeData.inputs?.memoryKey as string + const chatId = options?.chatId as string + + let isSessionIdUsingChatMessageId = false + if (!sessionId && chatId) isSessionIdUsingChatMessageId = true + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + let mongoDBConnectUrl = getCredentialParam('mongoDBConnectUrl', credentialData, nodeData) + + const client = new MongoClient(mongoDBConnectUrl) + await client.connect() + const collection = client.db(databaseName).collection(collectionName) + + const mongoDBChatMessageHistory = new MongoDBChatMessageHistory({ + collection, + sessionId: sessionId ? sessionId : chatId + }) + + mongoDBChatMessageHistory.getMessages = async (): Promise => { + const document = await collection.findOne({ + sessionId: (mongoDBChatMessageHistory as any).sessionId + }) + const messages = document?.messages || [] + return messages.map(mapStoredMessageToChatMessage) + } + + mongoDBChatMessageHistory.addMessage = async (message: BaseMessage): Promise => { + const messages = [message].map((msg) => msg.toDict()) + await collection.updateOne( + { sessionId: (mongoDBChatMessageHistory as any).sessionId }, + { + $push: { messages: { $each: messages } } + }, + { upsert: true } + ) + } + + mongoDBChatMessageHistory.clear = async (): Promise => { + await collection.deleteOne({ sessionId: (mongoDBChatMessageHistory as any).sessionId }) + } + + return new BufferMemoryExtended({ + memoryKey, + chatHistory: mongoDBChatMessageHistory, + returnMessages: true, + isSessionIdUsingChatMessageId + }) +} + +interface BufferMemoryExtendedInput { + isSessionIdUsingChatMessageId: boolean +} + +class BufferMemoryExtended extends BufferMemory { + isSessionIdUsingChatMessageId? = false + + constructor(fields: BufferMemoryInput & Partial) { + super(fields) + this.isSessionIdUsingChatMessageId = fields.isSessionIdUsingChatMessageId + } +} + +module.exports = { nodeClass: MongoDB_Memory } diff --git a/packages/components/nodes/memory/MongoDBMemory/mongodb.png b/packages/components/nodes/memory/MongoDBMemory/mongodb.png new file mode 100644 index 00000000..5586fe0a Binary files /dev/null and b/packages/components/nodes/memory/MongoDBMemory/mongodb.png differ diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts index d3965786..a8ccd49a 100644 --- a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts @@ -50,7 +50,7 @@ class ElasicsearchUpsert_VectorStores extends ElasticSearchBase implements INode delete d.metadata.loc }) // end of workaround - return super.init(nodeData, _, options, flattenDocs) + return super.init(nodeData, _, options, finalDocs) } } diff --git a/packages/components/nodes/vectorstores/MongoDB/MongoDBSearchBase.ts b/packages/components/nodes/vectorstores/MongoDB/MongoDBSearchBase.ts new file mode 100644 index 00000000..e9ef8e9a --- /dev/null +++ b/packages/components/nodes/vectorstores/MongoDB/MongoDBSearchBase.ts @@ -0,0 +1,145 @@ +import { + getBaseClasses, + getCredentialData, + getCredentialParam, + ICommonObject, + INodeData, + INodeOutputsValue, + INodeParams +} from '../../../src' + +import { Embeddings } from 'langchain/embeddings/base' +import { VectorStore } from 'langchain/vectorstores/base' +import { Document } from 'langchain/document' +import { MongoDBAtlasVectorSearch } from 'langchain/vectorstores/mongodb_atlas' +import { Collection, MongoClient } from 'mongodb' + +export abstract class MongoDBSearchBase { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + mongoClient: MongoClient + + protected constructor() { + this.type = 'MongoDB Atlas' + this.icon = 'mongodb.png' + this.category = 'Vector Stores' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['mongoDBUrlApi'] + } + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Database', + name: 'databaseName', + placeholder: '', + type: 'string' + }, + { + label: 'Collection Name', + name: 'collectionName', + placeholder: '', + type: 'string' + }, + { + label: 'Index Name', + name: 'indexName', + placeholder: '', + type: 'string' + }, + { + label: 'Content Field', + name: 'textKey', + description: 'Name of the field (column) that contains the actual content', + type: 'string', + default: 'text', + additionalParams: true, + optional: true + }, + { + label: 'Embedded Field', + name: 'embeddingKey', + description: 'Name of the field (column) that contains the Embedding', + type: 'string', + default: 'embedding', + additionalParams: true, + optional: true + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'MongoDB Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'MongoDB Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(MongoDBAtlasVectorSearch)] + } + ] + } + + abstract constructVectorStore( + embeddings: Embeddings, + collection: Collection, + indexName: string, + textKey: string, + embeddingKey: string, + docs: Document>[] | undefined + ): Promise + + async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document>[] | undefined): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const databaseName = nodeData.inputs?.databaseName as string + const collectionName = nodeData.inputs?.collectionName as string + const indexName = nodeData.inputs?.indexName as string + let textKey = nodeData.inputs?.textKey as string + let embeddingKey = nodeData.inputs?.embeddingKey as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const topK = nodeData.inputs?.topK as string + const k = topK ? parseFloat(topK) : 4 + const output = nodeData.outputs?.output as string + + let mongoDBConnectUrl = getCredentialParam('mongoDBConnectUrl', credentialData, nodeData) + + this.mongoClient = new MongoClient(mongoDBConnectUrl) + const collection = this.mongoClient.db(databaseName).collection(collectionName) + if (!textKey || textKey === '') textKey = 'text' + if (!embeddingKey || embeddingKey === '') embeddingKey = 'embedding' + const vectorStore = await this.constructVectorStore(embeddings, collection, indexName, textKey, embeddingKey, docs) + + if (output === 'retriever') { + return vectorStore.asRetriever(k) + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} diff --git a/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts new file mode 100644 index 00000000..7b06814a --- /dev/null +++ b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts @@ -0,0 +1,39 @@ +import { Collection } from 'mongodb' +import { MongoDBAtlasVectorSearch } from 'langchain/vectorstores/mongodb_atlas' +import { Embeddings } from 'langchain/embeddings/base' +import { VectorStore } from 'langchain/vectorstores/base' +import { Document } from 'langchain/document' +import { MongoDBSearchBase } from './MongoDBSearchBase' +import { ICommonObject, INode, INodeData } from '../../../src/Interface' + +class MongoDBExisting_VectorStores extends MongoDBSearchBase implements INode { + constructor() { + super() + this.label = 'MongoDB Atlas Load Existing Index' + this.name = 'MongoDBIndex' + this.version = 1.0 + this.description = 'Load existing data from MongoDB Atlas (i.e: Document has been upserted)' + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + return super.init(nodeData, _, options, undefined) + } + + async constructVectorStore( + embeddings: Embeddings, + collection: Collection, + indexName: string, + textKey: string, + embeddingKey: string, + _: Document>[] | undefined + ): Promise { + return new MongoDBAtlasVectorSearch(embeddings, { + collection: collection, + indexName: indexName, + textKey: textKey, + embeddingKey: embeddingKey + }) + } +} + +module.exports = { nodeClass: MongoDBExisting_VectorStores } diff --git a/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts new file mode 100644 index 00000000..7d22f035 --- /dev/null +++ b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts @@ -0,0 +1,59 @@ +import { flatten } from 'lodash' +import { Collection } from 'mongodb' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { VectorStore } from 'langchain/vectorstores/base' +import { MongoDBAtlasVectorSearch } from 'langchain/vectorstores/mongodb_atlas' +import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { MongoDBSearchBase } from './MongoDBSearchBase' + +class MongoDBUpsert_VectorStores extends MongoDBSearchBase implements INode { + constructor() { + super() + this.label = 'MongoDB Upsert Document' + this.name = 'MongoDBUpsert' + this.version = 1.0 + this.description = 'Upsert documents to MongoDB Atlas' + this.inputs.unshift({ + label: 'Document', + name: 'document', + type: 'Document', + list: true + }) + } + + async constructVectorStore( + embeddings: Embeddings, + collection: Collection, + indexName: string, + textKey: string, + embeddingKey: string, + docs: Document>[] + ): Promise { + const mongoDBAtlasVectorSearch = new MongoDBAtlasVectorSearch(embeddings, { + collection: collection, + indexName: indexName, + textKey: textKey, + embeddingKey: embeddingKey + }) + await mongoDBAtlasVectorSearch.addDocuments(docs) + return mongoDBAtlasVectorSearch + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const docs = nodeData.inputs?.document as Document[] + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + if (flattenDocs[i] && flattenDocs[i].pageContent) { + const document = new Document(flattenDocs[i]) + finalDocs.push(document) + } + } + + return super.init(nodeData, _, options, finalDocs) + } +} + +module.exports = { nodeClass: MongoDBUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/MongoDB/mongodb.png b/packages/components/nodes/vectorstores/MongoDB/mongodb.png new file mode 100644 index 00000000..5586fe0a Binary files /dev/null and b/packages/components/nodes/vectorstores/MongoDB/mongodb.png differ diff --git a/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts index 9d1a4f45..4da58eaf 100644 --- a/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts +++ b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts @@ -56,7 +56,7 @@ class RedisUpsert_VectorStores extends RedisSearchBase implements INode { } } - return super.init(nodeData, _, options, flattenDocs) + return super.init(nodeData, _, options, finalDocs) } } diff --git a/packages/components/package.json b/packages/components/package.json index 0df99a11..c7a29a9f 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -55,6 +55,7 @@ "llmonitor": "^0.5.5", "mammoth": "^1.5.1", "moment": "^2.29.3", + "mongodb": "^6.2.0", "mysql2": "^3.5.1", "node-fetch": "^2.6.11", "node-html-markdown": "^1.3.0",