diff --git a/packages/components/nodes/vectorstores/Weaviate_Existing/Weaviate_Existing.ts b/packages/components/nodes/vectorstores/Weaviate_Existing/Weaviate_Existing.ts new file mode 100644 index 00000000..1f573eff --- /dev/null +++ b/packages/components/nodes/vectorstores/Weaviate_Existing/Weaviate_Existing.ts @@ -0,0 +1,125 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { getBaseClasses } from '../../../src/utils' +import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client' +import { WeaviateLibArgs, WeaviateStore } from 'langchain/vectorstores/weaviate' + +class Weaviate_Existing_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'Weaviate Load Existing Index' + this.name = 'weaviateExistingIndex' + this.type = 'Weaviate' + this.icon = 'weaviate.png' + this.category = 'Vector Stores' + this.description = 'Load existing index from Weaviate (i.e: Document has been upserted)' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Weaviate Scheme', + name: 'weaviateScheme', + type: 'string', + default: 'https' + }, + { + label: 'Weaviate Host', + name: 'weaviateHost', + type: 'string', + default: 'localhost' + }, + { + label: 'Weaviate Index', + name: 'weaviateIndex', + type: 'string', + placeholder: 'Test' + }, + { + label: 'Weaviate API Key', + name: 'weaviateApiKey', + type: 'password', + optional: true + }, + { + label: 'Weaviate Text Key', + name: 'weaviateTextKey', + type: 'string', + placeholder: 'text', + optional: true, + additionalParams: true + }, + { + label: 'Weaviate Metadata Keys', + name: 'weaviateMetadataKeys', + type: 'string', + rows: 4, + placeholder: `["foo"]`, + optional: true, + additionalParams: true + } + ] + this.outputs = [ + { + label: 'Weaviate Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Weaviate Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(WeaviateStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const weaviateScheme = nodeData.inputs?.weaviateScheme as string + const weaviateHost = nodeData.inputs?.weaviateHost as string + const weaviateIndex = nodeData.inputs?.weaviateIndex as string + const weaviateApiKey = nodeData.inputs?.weaviateApiKey as string + const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string + const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string + + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + + const client: WeaviateClient = weaviate.client({ + scheme: weaviateScheme || 'https', + host: weaviateHost || 'localhost', + apiKey: new ApiKey(weaviateApiKey || 'default') + }) + + const obj: WeaviateLibArgs = { + client, + indexName: weaviateIndex + } + + if (weaviateTextKey) obj.textKey = weaviateTextKey + if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, '')) + + const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj) + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever() + return retriever + } else if (output === 'vectorStore') { + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: Weaviate_Existing_VectorStores } diff --git a/packages/components/nodes/vectorstores/Weaviate_Existing/weaviate.png b/packages/components/nodes/vectorstores/Weaviate_Existing/weaviate.png new file mode 100644 index 00000000..25a39e33 Binary files /dev/null and b/packages/components/nodes/vectorstores/Weaviate_Existing/weaviate.png differ diff --git a/packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts b/packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts new file mode 100644 index 00000000..1477e51e --- /dev/null +++ b/packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts @@ -0,0 +1,137 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { getBaseClasses } from '../../../src/utils' +import { WeaviateLibArgs, WeaviateStore } from 'langchain/vectorstores/weaviate' +import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client' + +class WeaviateUpsert_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'Weaviate Upsert Document' + this.name = 'weaviateUpsert' + this.type = 'Weaviate' + this.icon = 'weaviate.png' + this.category = 'Vector Stores' + this.description = 'Upsert documents to Weaviate' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Document', + name: 'document', + type: 'Document' + }, + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Weaviate Scheme', + name: 'weaviateScheme', + type: 'string', + default: 'https' + }, + { + label: 'Weaviate Host', + name: 'weaviateHost', + type: 'string', + default: 'localhost' + }, + { + label: 'Weaviate Index', + name: 'weaviateIndex', + type: 'string', + placeholder: 'Test' + }, + { + label: 'Weaviate API Key', + name: 'weaviateApiKey', + type: 'password', + optional: true + }, + { + label: 'Weaviate Text Key', + name: 'weaviateTextKey', + type: 'string', + placeholder: 'text', + optional: true, + additionalParams: true + }, + { + label: 'Weaviate Metadata Keys', + name: 'weaviateMetadataKeys', + type: 'string', + rows: 4, + placeholder: `["foo"]`, + optional: true, + additionalParams: true + } + ] + this.outputs = [ + { + label: 'Weaviate Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Weaviate Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(WeaviateStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const weaviateScheme = nodeData.inputs?.weaviateScheme as string + const weaviateHost = nodeData.inputs?.weaviateHost as string + const weaviateIndex = nodeData.inputs?.weaviateIndex as string + const weaviateApiKey = nodeData.inputs?.weaviateApiKey as string + const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string + const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string + + const docs = nodeData.inputs?.document as Document[] + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + + const client: WeaviateClient = weaviate.client({ + scheme: weaviateScheme || 'https', + host: weaviateHost || 'localhost', + apiKey: new ApiKey(weaviateApiKey || 'default') + }) + + const finalDocs = [] + for (let i = 0; i < docs.length; i += 1) { + finalDocs.push(new Document(docs[i])) + } + + const obj: WeaviateLibArgs = { + client, + indexName: weaviateIndex + } + + if (weaviateTextKey) obj.textKey = weaviateTextKey + if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, '')) + + const vectorStore = await WeaviateStore.fromDocuments(finalDocs, embeddings, obj) + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever() + return retriever + } else if (output === 'vectorStore') { + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: WeaviateUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/Weaviate_Upsert/weaviate.png b/packages/components/nodes/vectorstores/Weaviate_Upsert/weaviate.png new file mode 100644 index 00000000..25a39e33 Binary files /dev/null and b/packages/components/nodes/vectorstores/Weaviate_Upsert/weaviate.png differ diff --git a/packages/components/package.json b/packages/components/package.json index 9a4eeab4..1fc01047 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -28,11 +28,13 @@ "dotenv": "^16.0.0", "express": "^4.17.3", "form-data": "^4.0.0", + "graphql": "^16.6.0", "langchain": "^0.0.63", "mammoth": "^1.5.1", "moment": "^2.29.3", "node-fetch": "2", "pdf-parse": "^1.1.1", + "weaviate-ts-client": "^1.1.0", "ws": "^8.9.0" }, "devDependencies": {