From 3481d7decab88b08f224fa26ab5376fe8fa539e3 Mon Sep 17 00:00:00 2001 From: Govind Kumar Date: Sat, 24 Jun 2023 11:56:27 +0530 Subject: [PATCH 1/2] added qdrant vector store integration --- .../Qdrant_Existing/Qdrant_Existing.ts | 119 ++++++++++++++++++ .../Qdrant_Existing/qdrant_logo.svg | 27 ++++ .../Qdrant_Upsert/Qdrant_Upsert.ts | 119 ++++++++++++++++++ .../Qdrant_Upsert/qdrant_logo.svg | 27 ++++ packages/components/package.json | 1 + 5 files changed, 293 insertions(+) create mode 100644 packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts create mode 100644 packages/components/nodes/vectorstores/Qdrant_Existing/qdrant_logo.svg create mode 100644 packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts create mode 100644 packages/components/nodes/vectorstores/Qdrant_Upsert/qdrant_logo.svg diff --git a/packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts b/packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts new file mode 100644 index 00000000..031917f5 --- /dev/null +++ b/packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts @@ -0,0 +1,119 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { QdrantClient } from '@qdrant/js-client-rest' +import { QdrantVectorStore, QdrantLibArgs } from 'langchain/vectorstores/qdrant' +import { Embeddings } from 'langchain/embeddings/base' +import { getBaseClasses } from '../../../src/utils' + +class Qdrant_Existing_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'Qdrant Load Existing Index' + this.name = 'qdrantExistingIndex' + this.type = 'Qdrant' + this.icon = 'qdrant_logo.svg' + this.category = 'Vector Stores' + this.description = 'Load existing index from Qdrant (i.e., documents have been upserted)' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Qdrant Server URL', + name: 'qdrantServerUrl', + type: 'string' + }, + { + label: 'Qdrant Collection Name', + name: 'qdrantCollection', + type: 'string' + }, + { + label: 'Qdrant API Key', + name: 'qdrantApiKey', + type: 'password' + }, + { + label: 'Qdrant Collection Cofiguration', + name: 'qdrantCollectionCofiguration', + type: 'json', + optional: true, + + additionalParams: true + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Qdrant Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Qdrant Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(QdrantVectorStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string + const collectionName = nodeData.inputs?.qdrantCollection as string + const qdrantApiKey = nodeData.inputs?.qdrantApiKey as string + let qdrantCollectionCofiguration = nodeData.inputs?.qdrantCollectionCofiguration + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseInt(topK, 10) : 4 + + // connect to Qdrant Cloud + const client = new QdrantClient({ + url: qdrantServerUrl, + apiKey: qdrantApiKey + }) + + const dbConfig: QdrantLibArgs = { + client, + collectionName + } + + if (qdrantCollectionCofiguration) { + qdrantCollectionCofiguration = + typeof qdrantCollectionCofiguration === 'object' ? qdrantCollectionCofiguration : JSON.parse(qdrantCollectionCofiguration) + dbConfig.collectionConfig = qdrantCollectionCofiguration + } + + const vectorStore = await QdrantVectorStore.fromExistingCollection(embeddings, dbConfig) + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: Qdrant_Existing_VectorStores } diff --git a/packages/components/nodes/vectorstores/Qdrant_Existing/qdrant_logo.svg b/packages/components/nodes/vectorstores/Qdrant_Existing/qdrant_logo.svg new file mode 100644 index 00000000..82fb8b39 --- /dev/null +++ b/packages/components/nodes/vectorstores/Qdrant_Existing/qdrant_logo.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts b/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts new file mode 100644 index 00000000..111fc5c3 --- /dev/null +++ b/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts @@ -0,0 +1,119 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { QdrantClient } from '@qdrant/js-client-rest' +import { QdrantVectorStore, QdrantLibArgs } from 'langchain/vectorstores/qdrant' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { getBaseClasses } from '../../../src/utils' +import { flatten } from 'lodash' + +class QdrantUpsert_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'Qdrant Upsert Document' + this.name = 'qdrantUpsert' + this.type = 'Qdrant' + this.icon = 'qdrant_logo.svg' + this.category = 'Vector Stores' + this.description = 'Upsert documents to Qdrant' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Document', + name: 'document', + type: 'Document', + list: true + }, + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Qdrant Server URL', + name: 'qdrantServerUrl', + type: 'string' + }, + { + label: 'Qdrant Collection Name', + name: 'qdrantCollection', + type: 'string' + }, + { + label: 'Qdrant API Key', + name: 'qdrantApiKey', + type: 'password' + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Qdrant Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Qdrant Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(QdrantVectorStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string + const collectionName = nodeData.inputs?.qdrantCollection as string + const qdrantApiKey = nodeData.inputs?.qdrantApiKey as string + const docs = nodeData.inputs?.document as Document[] + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseInt(topK, 10) : 4 + + // connect to Qdrant Cloud + const client = new QdrantClient({ + url: qdrantServerUrl, + apiKey: qdrantApiKey + }) + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + finalDocs.push(new Document(flattenDocs[i])) + } + + const dbConfig: QdrantLibArgs = { + client, + url: qdrantServerUrl, + collectionName + } + const vectorStore = await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig) + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: QdrantUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/Qdrant_Upsert/qdrant_logo.svg b/packages/components/nodes/vectorstores/Qdrant_Upsert/qdrant_logo.svg new file mode 100644 index 00000000..82fb8b39 --- /dev/null +++ b/packages/components/nodes/vectorstores/Qdrant_Upsert/qdrant_logo.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/components/package.json b/packages/components/package.json index 738c7752..fd7e1b14 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -20,6 +20,7 @@ "@getzep/zep-js": "^0.3.1", "@huggingface/inference": "1", "@pinecone-database/pinecone": "^0.0.12", + "@qdrant/js-client-rest": "^1.2.2", "@supabase/supabase-js": "^2.21.0", "@types/js-yaml": "^4.0.5", "axios": "^0.27.2", From ab029a845e3a5a0ac4a20a6173194334eeed8f9c Mon Sep 17 00:00:00 2001 From: Govind Kumar Date: Mon, 26 Jun 2023 20:46:37 +0530 Subject: [PATCH 2/2] added revised changes --- .../nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts | 7 ++++--- .../nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts b/packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts index 031917f5..f1eef8f9 100644 --- a/packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts +++ b/packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts @@ -32,7 +32,8 @@ class Qdrant_Existing_VectorStores implements INode { { label: 'Qdrant Server URL', name: 'qdrantServerUrl', - type: 'string' + type: 'string', + placeholder: 'http://localhost:6333' }, { label: 'Qdrant Collection Name', @@ -42,14 +43,14 @@ class Qdrant_Existing_VectorStores implements INode { { label: 'Qdrant API Key', name: 'qdrantApiKey', - type: 'password' + type: 'password', + optional: true }, { label: 'Qdrant Collection Cofiguration', name: 'qdrantCollectionCofiguration', type: 'json', optional: true, - additionalParams: true }, { diff --git a/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts b/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts index 111fc5c3..dae1d31d 100644 --- a/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts +++ b/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts @@ -40,7 +40,8 @@ class QdrantUpsert_VectorStores implements INode { { label: 'Qdrant Server URL', name: 'qdrantServerUrl', - type: 'string' + type: 'string', + placeholder: 'http://localhost:6333' }, { label: 'Qdrant Collection Name', @@ -50,7 +51,8 @@ class QdrantUpsert_VectorStores implements INode { { label: 'Qdrant API Key', name: 'qdrantApiKey', - type: 'password' + type: 'password', + optional: true }, { label: 'Top K',