farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/Indexing (#1802)

Browse Source 
* indexing

* fix for multiple files upsert

* fix default Postgres port

* fix SQLite node description

* add MySQLRecordManager node

* fix MySQL unique index

* add upsert history

* update jsx ui

* lint-fix

* update dialog details

* update llamaindex pinecone

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>
This commit is contained in:
Henry Heng
2024-04-02 23:47:19 +01:00
committed by GitHub
parent 957694a912
commit e422ce287b
67 changed files with 3006 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/vectorstores/Astra/Astra.ts
+3 -2
View File
@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
import { Embeddings } from '@langchain/core/embeddings'
import { Document } from '@langchain/core/documents'
import { AstraDBVectorStore, AstraLibArgs } from '@langchain/community/vectorstores/astradb'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData } from '../../../src/utils'
import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
@@ -101,7 +101,7 @@ class Astra_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const vectorDimension = nodeData.inputs?.vectorDimension as number
@@ -142,6 +142,7 @@ class Astra_VectorStores implements INode {
try {
await AstraDBVectorStore.fromDocuments(finalDocs, embeddings, astraConfig)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Chroma/Chroma.ts
+30 -4
View File
@@ -2,9 +2,10 @@ import { flatten } from 'lodash'
import { Chroma } from '@langchain/community/vectorstores/chroma'
import { Embeddings } from '@langchain/core/embeddings'
import { Document } from '@langchain/core/documents'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { ChromaExtended } from './core'
import { index } from '../../../src/indexing'
class Chroma_VectorStores implements INode {
label: string
@@ -23,7 +24,7 @@ class Chroma_VectorStores implements INode {
constructor() {
this.label = 'Chroma'
this.name = 'chroma'
this.version = 1.0
this.version = 2.0
this.type = 'Chroma'
this.icon = 'chroma.svg'
this.category = 'Vector Stores'
@@ -51,6 +52,13 @@ class Chroma_VectorStores implements INode {
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Record Manager',
name: 'recordManager',
type: 'RecordManager',
description: 'Keep track of the record to prevent duplication',
optional: true
},
{
label: 'Collection Name',
name: 'collectionName',
@@ -95,11 +103,12 @@ class Chroma_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const collectionName = nodeData.inputs?.collectionName as string
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const chromaURL = nodeData.inputs?.chromaURL as string
const recordManager = nodeData.inputs?.recordManager
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
@@ -121,7 +130,24 @@ class Chroma_VectorStores implements INode {
if (chromaApiKey) obj.chromaApiKey = chromaApiKey
try {
await ChromaExtended.fromDocuments(finalDocs, embeddings, obj)
if (recordManager) {
const vectorStore = await ChromaExtended.fromExistingCollection(embeddings, obj)
await recordManager.createSchema()
const res = await index({
docsSource: finalDocs,
recordManager,
vectorStore,
options: {
cleanup: recordManager?.cleanup,
sourceIdKey: recordManager?.sourceIdKey ?? 'source',
vectorStoreName: collectionName
}
})
return res
} else {
await ChromaExtended.fromDocuments(finalDocs, embeddings, obj)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts
+30 -4
View File
@@ -3,8 +3,9 @@ import { Client, ClientOptions } from '@elastic/elasticsearch'
import { Document } from '@langchain/core/documents'
import { Embeddings } from '@langchain/core/embeddings'
import { ElasticClientArgs, ElasticVectorSearch } from '@langchain/community/vectorstores/elasticsearch'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { index } from '../../../src/indexing'
class Elasticsearch_VectorStores implements INode {
label: string
@@ -23,7 +24,7 @@ class Elasticsearch_VectorStores implements INode {
constructor() {
this.label = 'Elasticsearch'
this.name = 'elasticsearch'
this.version = 1.0
this.version = 2.0
this.description =
'Upsert embedded data and perform similarity search upon query using Elasticsearch, a distributed search and analytics engine'
this.type = 'Elasticsearch'
@@ -50,6 +51,13 @@ class Elasticsearch_VectorStores implements INode {
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Record Manager',
name: 'recordManager',
type: 'RecordManager',
description: 'Keep track of the record to prevent duplication',
optional: true
},
{
label: 'Index Name',
name: 'indexName',
@@ -105,13 +113,14 @@ class Elasticsearch_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
const cloudId = getCredentialParam('cloudId', credentialData, nodeData)
const indexName = nodeData.inputs?.indexName as string
const embeddings = nodeData.inputs?.embeddings as Embeddings
const similarityMeasure = nodeData.inputs?.similarityMeasure as string
const recordManager = nodeData.inputs?.recordManager
const docs = nodeData.inputs?.document as Document[]
const flattenDocs = docs && docs.length ? flatten(docs) : []
@@ -134,7 +143,24 @@ class Elasticsearch_VectorStores implements INode {
const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
try {
await vectorStore.addDocuments(finalDocs)
if (recordManager) {
const vectorStore = await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs)
await recordManager.createSchema()
const res = await index({
docsSource: finalDocs,
recordManager,
vectorStore,
options: {
cleanup: recordManager?.cleanup,
sourceIdKey: recordManager?.sourceIdKey ?? 'source',
vectorStoreName: indexName
}
})
return res
} else {
await vectorStore.addDocuments(finalDocs)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Faiss/Faiss.ts
+4 -2
View File
@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
import { Document } from '@langchain/core/documents'
import { FaissStore } from '@langchain/community/vectorstores/faiss'
import { Embeddings } from '@langchain/core/embeddings'
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class Faiss_VectorStores implements INode {
@@ -74,7 +74,7 @@ class Faiss_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData): Promise<void> {
async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const basePath = nodeData.inputs?.basePath as string
@@ -95,6 +95,8 @@ class Faiss_VectorStores implements INode {
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number) => {
return await similaritySearchVectorWithScore(query, k, vectorStore)
}
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts
+3 -2
View File
@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
import { MemoryVectorStore } from 'langchain/vectorstores/memory'
import { Embeddings } from '@langchain/core/embeddings'
import { Document } from '@langchain/core/documents'
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class InMemoryVectorStore_VectorStores implements INode {
@@ -64,7 +64,7 @@ class InMemoryVectorStore_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData): Promise<void> {
async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
@@ -78,6 +78,7 @@ class InMemoryVectorStore_VectorStores implements INode {
try {
await MemoryVectorStore.fromDocuments(finalDocs, embeddings)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Milvus/Milvus.ts
+4 -2
View File
@@ -3,7 +3,7 @@ import { DataType, ErrorCode, MetricType, IndexType } from '@zilliz/milvus2-sdk-
import { Document } from '@langchain/core/documents'
import { MilvusLibArgs, Milvus } from '@langchain/community/vectorstores/milvus'
import { Embeddings } from '@langchain/core/embeddings'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
interface InsertRow {
@@ -109,7 +109,7 @@ class Milvus_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
// server setup
const address = nodeData.inputs?.milvusServerUrl as string
const collectionName = nodeData.inputs?.milvusCollection as string
@@ -147,6 +147,8 @@ class Milvus_VectorStores implements INode {
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: string) => {
return await similaritySearchVectorWithScore(query, k, vectorStore, undefined, filter)
}
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/MongoDBAtlas/MongoDBAtlas.ts
+3 -2
View File
@@ -3,7 +3,7 @@ import { MongoClient } from 'mongodb'
import { MongoDBAtlasVectorSearch } from '@langchain/mongodb'
import { Embeddings } from '@langchain/core/embeddings'
import { Document } from '@langchain/core/documents'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
@@ -113,7 +113,7 @@ class MongoDBAtlas_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const databaseName = nodeData.inputs?.databaseName as string
const collectionName = nodeData.inputs?.collectionName as string
@@ -149,6 +149,7 @@ class MongoDBAtlas_VectorStores implements INode {
embeddingKey
})
await mongoDBAtlasVectorSearch.addDocuments(finalDocs)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/OpenSearch/OpenSearch.ts
+3 -2
View File
@@ -3,7 +3,7 @@ import { Client } from '@opensearch-project/opensearch'
import { Document } from '@langchain/core/documents'
import { OpenSearchVectorStore } from '@langchain/community/vectorstores/opensearch'
import { Embeddings } from '@langchain/core/embeddings'
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class OpenSearch_VectorStores implements INode {
@@ -79,7 +79,7 @@ class OpenSearch_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData): Promise<void> {
async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const opensearchURL = nodeData.inputs?.opensearchURL as string
@@ -102,6 +102,7 @@ class OpenSearch_VectorStores implements INode {
client,
indexName: indexName
})
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Pinecone/Pinecone.ts
+33 -6
View File
@@ -3,9 +3,10 @@ import { Pinecone } from '@pinecone-database/pinecone'
import { PineconeStoreParams, PineconeStore } from '@langchain/pinecone'
import { Embeddings } from '@langchain/core/embeddings'
import { Document } from '@langchain/core/documents'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
import { index } from '../../../src/indexing'
class Pinecone_VectorStores implements INode {
label: string
@@ -24,7 +25,7 @@ class Pinecone_VectorStores implements INode {
constructor() {
this.label = 'Pinecone'
this.name = 'pinecone'
this.version = 2.0
this.version = 3.0
this.type = 'Pinecone'
this.icon = 'pinecone.svg'
this.category = 'Vector Stores'
@@ -50,6 +51,13 @@ class Pinecone_VectorStores implements INode {
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Record Manager',
name: 'recordManager',
type: 'RecordManager',
description: 'Keep track of the record to prevent duplication',
optional: true
},
{
label: 'Pinecone Index',
name: 'pineconeIndex',
@@ -97,11 +105,12 @@ class Pinecone_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
const index = nodeData.inputs?.pineconeIndex as string
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const _index = nodeData.inputs?.pineconeIndex as string
const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const recordManager = nodeData.inputs?.recordManager
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
@@ -110,7 +119,7 @@ class Pinecone_VectorStores implements INode {
apiKey: pineconeApiKey
})
const pineconeIndex = client.Index(index)
const pineconeIndex = client.Index(_index)
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
@@ -127,7 +136,25 @@ class Pinecone_VectorStores implements INode {
if (pineconeNamespace) obj.namespace = pineconeNamespace
try {
await PineconeStore.fromDocuments(finalDocs, embeddings, obj)
if (recordManager) {
const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)
await recordManager.createSchema()
const res = await index({
docsSource: finalDocs,
recordManager,
vectorStore,
options: {
cleanup: recordManager?.cleanup,
sourceIdKey: recordManager?.sourceIdKey ?? 'source',
vectorStoreName: pineconeNamespace
}
})
return res
} else {
await PineconeStore.fromDocuments(finalDocs, embeddings, obj)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Pinecone/Pinecone_LlamaIndex.ts
+3 -2
View File
@@ -13,7 +13,7 @@ import {
import { FetchResponse, Index, Pinecone, ScoredPineconeRecord } from '@pinecone-database/pinecone'
import { flatten } from 'lodash'
import { Document as LCDocument } from 'langchain/document'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { flattenObject, getCredentialData, getCredentialParam } from '../../../src/utils'
class PineconeLlamaIndex_VectorStores implements INode {
@@ -110,7 +110,7 @@ class PineconeLlamaIndex_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const indexName = nodeData.inputs?.pineconeIndex as string
const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
const docs = nodeData.inputs?.document as LCDocument[]
@@ -144,6 +144,7 @@ class PineconeLlamaIndex_VectorStores implements INode {
try {
await VectorStoreIndex.fromDocuments(llamadocs, { serviceContext, storageContext })
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Postgres/Postgres.ts
+42 -7
View File
@@ -4,8 +4,9 @@ import { DataSourceOptions } from 'typeorm'
import { Embeddings } from '@langchain/core/embeddings'
import { Document } from '@langchain/core/documents'
import { TypeORMVectorStore, TypeORMVectorStoreDocument } from '@langchain/community/vectorstores/typeorm'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { index } from '../../../src/indexing'
class Postgres_VectorStores implements INode {
label: string
@@ -24,7 +25,7 @@ class Postgres_VectorStores implements INode {
constructor() {
this.label = 'Postgres'
this.name = 'postgres'
this.version = 3.0
this.version = 4.0
this.type = 'Postgres'
this.icon = 'postgres.svg'
this.category = 'Vector Stores'
@@ -50,6 +51,13 @@ class Postgres_VectorStores implements INode {
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Record Manager',
name: 'recordManager',
type: 'RecordManager',
description: 'Keep track of the record to prevent duplication',
optional: true
},
{
label: 'Host',
name: 'host',
@@ -108,7 +116,7 @@ class Postgres_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const user = getCredentialParam('user', credentialData, nodeData)
const password = getCredentialParam('password', credentialData, nodeData)
@@ -117,6 +125,7 @@ class Postgres_VectorStores implements INode {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const additionalConfig = nodeData.inputs?.additionalConfig as string
const recordManager = nodeData.inputs?.recordManager
let additionalConfiguration = {}
if (additionalConfig) {
@@ -151,11 +160,37 @@ class Postgres_VectorStores implements INode {
}
try {
const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)
if (recordManager) {
const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args)
// Avoid Illegal invocation error
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
// Avoid Illegal invocation error
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
}
await recordManager.createSchema()
const res = await index({
docsSource: finalDocs,
recordManager,
vectorStore,
options: {
cleanup: recordManager?.cleanup,
sourceIdKey: recordManager?.sourceIdKey ?? 'source',
vectorStoreName: tableName
}
})
return res
} else {
const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)
// Avoid Illegal invocation error
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
}
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {
throw new Error(e)
packages/components/nodes/vectorstores/Qdrant/Qdrant.ts
+85 -4
View File
@@ -1,13 +1,19 @@
import { flatten } from 'lodash'
import { v4 as uuid } from 'uuid'
import { QdrantClient } from '@qdrant/js-client-rest'
import { VectorStoreRetrieverInput } from '@langchain/core/vectorstores'
import { Document } from '@langchain/core/documents'
import { QdrantVectorStore, QdrantLibArgs } from '@langchain/community/vectorstores/qdrant'
import { Embeddings } from '@langchain/core/embeddings'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { index } from '../../../src/indexing'
type RetrieverConfig = Partial<VectorStoreRetrieverInput<QdrantVectorStore>>
type QdrantAddDocumentOptions = {
customPayload?: Record<string, any>[]
ids?: string[]
}
class Qdrant_VectorStores implements INode {
label: string
@@ -26,7 +32,7 @@ class Qdrant_VectorStores implements INode {
constructor() {
this.label = 'Qdrant'
this.name = 'qdrant'
this.version = 1.0
this.version = 2.0
this.type = 'Qdrant'
this.icon = 'qdrant.png'
this.category = 'Vector Stores'
@@ -55,6 +61,13 @@ class Qdrant_VectorStores implements INode {
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Record Manager',
name: 'recordManager',
type: 'RecordManager',
description: 'Keep track of the record to prevent duplication',
optional: true
},
{
label: 'Qdrant Server URL',
name: 'qdrantServerUrl',
@@ -138,13 +151,14 @@ class Qdrant_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string
const collectionName = nodeData.inputs?.qdrantCollection as string
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
const recordManager = nodeData.inputs?.recordManager
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
@@ -178,7 +192,74 @@ class Qdrant_VectorStores implements INode {
}
try {
await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
if (recordManager) {
const vectorStore = new QdrantVectorStore(embeddings, dbConfig)
await vectorStore.ensureCollection()
vectorStore.addVectors = async (
vectors: number[][],
documents: Document[],
documentOptions?: QdrantAddDocumentOptions
): Promise<void> => {
if (vectors.length === 0) {
return
}
await vectorStore.ensureCollection()
const points = vectors.map((embedding, idx) => ({
id: documentOptions?.ids?.length ? documentOptions?.ids[idx] : uuid(),
vector: embedding,
payload: {
content: documents[idx].pageContent,
metadata: documents[idx].metadata,
customPayload: documentOptions?.customPayload?.length ? documentOptions?.customPayload[idx] : undefined
}
}))
try {
await client.upsert(collectionName, {
wait: true,
points
})
} catch (e: any) {
const error = new Error(`${e?.status ?? 'Undefined error code'} ${e?.message}: ${e?.data?.status?.error}`)
throw error
}
}
vectorStore.delete = async (params: { ids: string[] }): Promise<void> => {
const { ids } = params
if (ids?.length) {
try {
client.delete(collectionName, {
points: ids
})
} catch (e) {
console.error('Failed to delete')
}
}
}
await recordManager.createSchema()
const res = await index({
docsSource: finalDocs,
recordManager,
vectorStore,
options: {
cleanup: recordManager?.cleanup,
sourceIdKey: recordManager?.sourceIdKey ?? 'source',
vectorStoreName: collectionName
}
})
return res
} else {
await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Redis/Redis.ts
+4 -2
View File
@@ -3,7 +3,7 @@ import { createClient, SearchOptions, RedisClientOptions } from 'redis'
import { Embeddings } from '@langchain/core/embeddings'
import { RedisVectorStore, RedisVectorStoreConfig } from '@langchain/community/vectorstores/redis'
import { Document } from '@langchain/core/documents'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { escapeAllStrings, escapeSpecialChars, unEscapeSpecialChars } from './utils'
@@ -138,7 +138,7 @@ class Redis_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const indexName = nodeData.inputs?.indexName as string
let contentKey = nodeData.inputs?.contentKey as string
@@ -203,6 +203,8 @@ class Redis_VectorStores implements INode {
filter
)
}
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/SimpleStore/SimpleStore.ts
+3 -2
View File
@@ -2,7 +2,7 @@ import path from 'path'
import { flatten } from 'lodash'
import { storageContextFromDefaults, serviceContextFromDefaults, VectorStoreIndex, Document } from 'llamaindex'
import { Document as LCDocument } from 'langchain/document'
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getUserHome } from '../../../src'
class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
@@ -79,7 +79,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData): Promise<void> {
async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
const basePath = nodeData.inputs?.basePath as string
const docs = nodeData.inputs?.document as LCDocument[]
const embeddings = nodeData.inputs?.embeddings
@@ -105,6 +105,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
try {
await VectorStoreIndex.fromDocuments(llamadocs, { serviceContext, storageContext })
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Singlestore/Singlestore.ts
+3 -2
View File
@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
import { Embeddings } from '@langchain/core/embeddings'
import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from '@langchain/community/vectorstores/singlestore'
import { Document } from '@langchain/core/documents'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
class SingleStore_VectorStores implements INode {
@@ -118,7 +118,7 @@ class SingleStore_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const user = getCredentialParam('user', credentialData, nodeData)
const password = getCredentialParam('password', credentialData, nodeData)
@@ -151,6 +151,7 @@ class SingleStore_VectorStores implements INode {
try {
const vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
vectorStore.addDocuments.bind(vectorStore)(finalDocs)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Supabase/Supabase.ts
+38 -8
View File
@@ -3,9 +3,10 @@ import { createClient } from '@supabase/supabase-js'
import { Document } from '@langchain/core/documents'
import { Embeddings } from '@langchain/core/embeddings'
import { SupabaseVectorStore, SupabaseLibArgs } from '@langchain/community/vectorstores/supabase'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
import { index } from '../../../src/indexing'
class Supabase_VectorStores implements INode {
label: string
@@ -24,7 +25,7 @@ class Supabase_VectorStores implements INode {
constructor() {
this.label = 'Supabase'
this.name = 'supabase'
this.version = 2.0
this.version = 3.0
this.type = 'Supabase'
this.icon = 'supabase.svg'
this.category = 'Vector Stores'
@@ -50,6 +51,13 @@ class Supabase_VectorStores implements INode {
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Record Manager',
name: 'recordManager',
type: 'RecordManager',
description: 'Keep track of the record to prevent duplication',
optional: true
},
{
label: 'Supabase Project URL',
name: 'supabaseProjUrl',
@@ -99,12 +107,13 @@ class Supabase_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string
const tableName = nodeData.inputs?.tableName as string
const queryName = nodeData.inputs?.queryName as string
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const recordManager = nodeData.inputs?.recordManager
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData)
@@ -120,11 +129,32 @@ class Supabase_VectorStores implements INode {
}
try {
await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, {
client,
tableName: tableName,
queryName: queryName
})
if (recordManager) {
const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, {
client,
tableName: tableName,
queryName: queryName
})
await recordManager.createSchema()
const res = await index({
docsSource: finalDocs,
recordManager,
vectorStore,
options: {
cleanup: recordManager?.cleanup,
sourceIdKey: recordManager?.sourceIdKey ?? 'source',
vectorStoreName: tableName + '_' + queryName
}
})
return res
} else {
await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, {
client,
tableName: tableName,
queryName: queryName
})
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Vectara/Vectara.ts
+3 -2
View File
@@ -9,7 +9,7 @@ import {
} from '@langchain/community/vectorstores/vectara'
import { Document } from '@langchain/core/documents'
import { Embeddings } from '@langchain/core/embeddings'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
class Vectara_VectorStores implements INode {
@@ -144,7 +144,7 @@ class Vectara_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
const customerId = getCredentialParam('customerID', credentialData, nodeData)
@@ -204,6 +204,7 @@ class Vectara_VectorStores implements INode {
const vectorStore = new VectaraStore(vectaraArgs)
await vectorStore.addFiles(vectaraFiles)
}
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/Weaviate/Weaviate.ts
+33 -5
View File
@@ -1,11 +1,12 @@
import { flatten } from 'lodash'
import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client'
import { WeaviateLibArgs, WeaviateStore } from '@langchain/community/vectorstores/weaviate'
import { WeaviateLibArgs, WeaviateStore } from '@langchain/weaviate'
import { Document } from '@langchain/core/documents'
import { Embeddings } from '@langchain/core/embeddings'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
import { index } from '../../../src/indexing'
class Weaviate_VectorStores implements INode {
label: string
@@ -24,7 +25,7 @@ class Weaviate_VectorStores implements INode {
constructor() {
this.label = 'Weaviate'
this.name = 'weaviate'
this.version = 2.0
this.version = 3.0
this.type = 'Weaviate'
this.icon = 'weaviate.png'
this.category = 'Vector Stores'
@@ -53,6 +54,13 @@ class Weaviate_VectorStores implements INode {
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Record Manager',
name: 'recordManager',
type: 'RecordManager',
description: 'Keep track of the record to prevent duplication',
optional: true
},
{
label: 'Weaviate Scheme',
name: 'weaviateScheme',
@@ -125,7 +133,7 @@ class Weaviate_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const weaviateScheme = nodeData.inputs?.weaviateScheme as string
const weaviateHost = nodeData.inputs?.weaviateHost as string
const weaviateIndex = nodeData.inputs?.weaviateIndex as string
@@ -133,6 +141,7 @@ class Weaviate_VectorStores implements INode {
const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const recordManager = nodeData.inputs?.recordManager
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
@@ -154,6 +163,7 @@ class Weaviate_VectorStores implements INode {
}
const obj: WeaviateLibArgs = {
//@ts-ignore
client,
indexName: weaviateIndex
}
@@ -162,7 +172,24 @@ class Weaviate_VectorStores implements INode {
if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
try {
await WeaviateStore.fromDocuments(finalDocs, embeddings, obj)
if (recordManager) {
const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj)
await recordManager.createSchema()
const res = await index({
docsSource: finalDocs,
recordManager,
vectorStore,
options: {
cleanup: recordManager?.cleanup,
sourceIdKey: recordManager?.sourceIdKey ?? 'source',
vectorStoreName: weaviateTextKey ? weaviateIndex + '_' + weaviateTextKey : weaviateIndex
}
})
return res
} else {
await WeaviateStore.fromDocuments(finalDocs, embeddings, obj)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {
throw new Error(e)
}
@@ -189,6 +216,7 @@ class Weaviate_VectorStores implements INode {
const client: WeaviateClient = weaviate.client(clientConfig)
const obj: WeaviateLibArgs = {
//@ts-ignore
client,
indexName: weaviateIndex
}
packages/components/nodes/vectorstores/Zep/Zep.ts
+3 -2
View File
@@ -3,7 +3,7 @@ import { IDocument, ZepClient } from '@getzep/zep-js'
import { ZepVectorStore, IZepConfig } from '@langchain/community/vectorstores/zep'
import { Embeddings } from '@langchain/core/embeddings'
import { Document } from '@langchain/core/documents'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
@@ -106,7 +106,7 @@ class Zep_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const baseURL = nodeData.inputs?.baseURL as string
const zepCollection = nodeData.inputs?.zepCollection as string
const dimension = (nodeData.inputs?.dimension as number) ?? 1536
@@ -134,6 +134,7 @@ class Zep_VectorStores implements INode {
try {
await ZepVectorStore.fromDocuments(finalDocs, embeddings, zepConfig)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}
packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts
+3 -2
View File
@@ -3,7 +3,7 @@ import { IDocument, ZepClient } from '@getzep/zep-cloud'
import { IZepConfig, ZepVectorStore } from '@getzep/zep-cloud/langchain'
import { Embeddings } from 'langchain/embeddings/base'
import { Document } from 'langchain/document'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
import { FakeEmbeddings } from 'langchain/embeddings/fake'
@@ -89,7 +89,7 @@ class Zep_CloudVectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
const zepCollection = nodeData.inputs?.zepCollection as string
const docs = nodeData.inputs?.document as Document[]
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
@@ -109,6 +109,7 @@ class Zep_CloudVectorStores implements INode {
}
try {
await ZepVectorStore.fromDocuments(finalDocs, new FakeEmbeddings(), zepConfig)
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}