mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 13:00:56 +03:00
Chore/update deprecating nodes (#2540)
* update deprecating nodes * add filters use cases to marketplace * update log level
This commit is contained in:
Henry Heng
@@ -30,7 +30,6 @@ class Chroma_VectorStores implements INode {
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert embedded data and perform similarity search upon query using Chroma, an open-source embedding database'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
import { Chroma } from '@langchain/community/vectorstores/chroma'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ChromaExtended } from './core'
|
||||
|
||||
class Chroma_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Chroma Load Existing Index'
|
||||
this.name = 'chromaExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Chroma'
|
||||
this.icon = 'chroma.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Chroma (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed if you have chroma on cloud services with X-Api-key',
|
||||
optional: true,
|
||||
credentialNames: ['chromaApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Collection Name',
|
||||
name: 'collectionName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Chroma URL',
|
||||
name: 'chromaURL',
|
||||
type: 'string',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Chroma Metadata Filter',
|
||||
name: 'chromaMetadataFilter',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Chroma Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Chroma Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(Chroma)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const collectionName = nodeData.inputs?.collectionName as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const chromaURL = nodeData.inputs?.chromaURL as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
|
||||
|
||||
const chromaMetadataFilter = nodeData.inputs?.chromaMetadataFilter
|
||||
|
||||
const obj: {
|
||||
collectionName: string
|
||||
url?: string
|
||||
chromaApiKey?: string
|
||||
filter?: object | undefined
|
||||
} = { collectionName }
|
||||
if (chromaURL) obj.url = chromaURL
|
||||
if (chromaApiKey) obj.chromaApiKey = chromaApiKey
|
||||
if (chromaMetadataFilter) {
|
||||
const metadatafilter = typeof chromaMetadataFilter === 'object' ? chromaMetadataFilter : JSON.parse(chromaMetadataFilter)
|
||||
obj.filter = metadatafilter
|
||||
}
|
||||
|
||||
const vectorStore = await ChromaExtended.fromExistingCollection(embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
if (chromaMetadataFilter) {
|
||||
;(vectorStore as any).filter = obj.filter
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Chroma_Existing_VectorStores }
|
||||
@@ -1,129 +0,0 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { Chroma } from '@langchain/community/vectorstores/chroma'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ChromaExtended } from './core'
|
||||
|
||||
class ChromaUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Chroma Upsert Document'
|
||||
this.name = 'chromaUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Chroma'
|
||||
this.icon = 'chroma.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Chroma'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed if you have chroma on cloud services with X-Api-key',
|
||||
optional: true,
|
||||
credentialNames: ['chromaApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Collection Name',
|
||||
name: 'collectionName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Chroma URL',
|
||||
name: 'chromaURL',
|
||||
type: 'string',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Chroma Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Chroma Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(Chroma)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const collectionName = nodeData.inputs?.collectionName as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const chromaURL = nodeData.inputs?.chromaURL as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const obj: {
|
||||
collectionName: string
|
||||
url?: string
|
||||
chromaApiKey?: string
|
||||
} = { collectionName }
|
||||
if (chromaURL) obj.url = chromaURL
|
||||
if (chromaApiKey) obj.chromaApiKey = chromaApiKey
|
||||
|
||||
const vectorStore = await ChromaExtended.fromDocuments(finalDocs, embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: ChromaUpsert_VectorStores }
|
||||
@@ -1,208 +0,0 @@
|
||||
import {
|
||||
getBaseClasses,
|
||||
getCredentialData,
|
||||
getCredentialParam,
|
||||
ICommonObject,
|
||||
INodeData,
|
||||
INodeOutputsValue,
|
||||
INodeParams
|
||||
} from '../../../src'
|
||||
import { Client, ClientOptions } from '@elastic/elasticsearch'
|
||||
import { ElasticClientArgs, ElasticVectorSearch } from '@langchain/community/vectorstores/elasticsearch'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
|
||||
export abstract class ElasticSearchBase {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
protected constructor() {
|
||||
this.type = 'Elasticsearch'
|
||||
this.icon = 'elasticsearch.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.badge = 'DEPRECATING'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['elasticsearchApi', 'elasticSearchUserPassword']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Index Name',
|
||||
name: 'indexName',
|
||||
placeholder: '<INDEX_NAME>',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Similarity',
|
||||
name: 'similarity',
|
||||
description: 'Similarity measure used in Elasticsearch.',
|
||||
type: 'options',
|
||||
default: 'l2_norm',
|
||||
options: [
|
||||
{
|
||||
label: 'l2_norm',
|
||||
name: 'l2_norm'
|
||||
},
|
||||
{
|
||||
label: 'dot_product',
|
||||
name: 'dot_product'
|
||||
},
|
||||
{
|
||||
label: 'cosine',
|
||||
name: 'cosine'
|
||||
}
|
||||
],
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Elasticsearch Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Elasticsearch Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(ElasticVectorSearch)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
abstract constructVectorStore(
|
||||
embeddings: Embeddings,
|
||||
elasticSearchClientArgs: ElasticClientArgs,
|
||||
docs: Document<Record<string, any>>[] | undefined
|
||||
): Promise<VectorStore>
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document<Record<string, any>>[] | undefined): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
|
||||
const cloudId = getCredentialParam('cloudId', credentialData, nodeData)
|
||||
const indexName = nodeData.inputs?.indexName as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const similarityMeasure = nodeData.inputs?.similarityMeasure as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
const elasticSearchClientArgs = this.prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
|
||||
|
||||
const vectorStore = await this.constructVectorStore(embeddings, elasticSearchClientArgs, docs)
|
||||
|
||||
if (output === 'retriever') {
|
||||
return vectorStore.asRetriever(k)
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
|
||||
protected prepareConnectionOptions(
|
||||
endPoint: string | undefined,
|
||||
cloudId: string | undefined,
|
||||
credentialData: ICommonObject,
|
||||
nodeData: INodeData
|
||||
) {
|
||||
let elasticSearchClientOptions: ClientOptions = {}
|
||||
if (endPoint) {
|
||||
let apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
elasticSearchClientOptions = {
|
||||
node: endPoint,
|
||||
auth: {
|
||||
apiKey: apiKey
|
||||
}
|
||||
}
|
||||
} else if (cloudId) {
|
||||
let username = getCredentialParam('username', credentialData, nodeData)
|
||||
let password = getCredentialParam('password', credentialData, nodeData)
|
||||
if (cloudId.startsWith('http')) {
|
||||
elasticSearchClientOptions = {
|
||||
node: cloudId,
|
||||
auth: {
|
||||
username: username,
|
||||
password: password
|
||||
},
|
||||
tls: {
|
||||
rejectUnauthorized: false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
elasticSearchClientOptions = {
|
||||
cloud: {
|
||||
id: cloudId
|
||||
},
|
||||
auth: {
|
||||
username: username,
|
||||
password: password
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return elasticSearchClientOptions
|
||||
}
|
||||
|
||||
protected prepareClientArgs(
|
||||
endPoint: string | undefined,
|
||||
cloudId: string | undefined,
|
||||
credentialData: ICommonObject,
|
||||
nodeData: INodeData,
|
||||
similarityMeasure: string,
|
||||
indexName: string
|
||||
) {
|
||||
let elasticSearchClientOptions = this.prepareConnectionOptions(endPoint, cloudId, credentialData, nodeData)
|
||||
let vectorSearchOptions = {}
|
||||
switch (similarityMeasure) {
|
||||
case 'dot_product':
|
||||
vectorSearchOptions = {
|
||||
similarity: 'dot_product'
|
||||
}
|
||||
break
|
||||
case 'cosine':
|
||||
vectorSearchOptions = {
|
||||
similarity: 'cosine'
|
||||
}
|
||||
break
|
||||
default:
|
||||
vectorSearchOptions = {
|
||||
similarity: 'l2_norm'
|
||||
}
|
||||
}
|
||||
const elasticSearchClientArgs: ElasticClientArgs = {
|
||||
client: new Client(elasticSearchClientOptions),
|
||||
indexName: indexName,
|
||||
vectorSearchOptions: vectorSearchOptions
|
||||
}
|
||||
return elasticSearchClientArgs
|
||||
}
|
||||
}
|
||||
@@ -31,7 +31,6 @@ class Elasticsearch_VectorStores implements INode {
|
||||
this.icon = 'elasticsearch.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { ElasticClientArgs, ElasticVectorSearch } from '@langchain/community/vectorstores/elasticsearch'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { ElasticSearchBase } from './ElasticSearchBase'
|
||||
import { ICommonObject, INode, INodeData } from '../../../src/Interface'
|
||||
|
||||
class ElasicsearchExisting_VectorStores extends ElasticSearchBase implements INode {
|
||||
constructor() {
|
||||
super()
|
||||
this.label = 'Elasticsearch Load Existing Index'
|
||||
this.name = 'ElasticsearchIndex'
|
||||
this.version = 1.0
|
||||
this.description = 'Load existing index from Elasticsearch (i.e: Document has been upserted)'
|
||||
}
|
||||
|
||||
async constructVectorStore(
|
||||
embeddings: Embeddings,
|
||||
elasticSearchClientArgs: ElasticClientArgs,
|
||||
_: Document<Record<string, any>>[] | undefined
|
||||
): Promise<VectorStore> {
|
||||
return await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs)
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
return super.init(nodeData, _, options, undefined)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: ElasicsearchExisting_VectorStores }
|
||||
@@ -1,56 +0,0 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { ElasticClientArgs, ElasticVectorSearch } from '@langchain/community/vectorstores/elasticsearch'
|
||||
import { ICommonObject, INode, INodeData } from '../../../src/Interface'
|
||||
import { ElasticSearchBase } from './ElasticSearchBase'
|
||||
|
||||
class ElasicsearchUpsert_VectorStores extends ElasticSearchBase implements INode {
|
||||
constructor() {
|
||||
super()
|
||||
this.label = 'Elasticsearch Upsert Document'
|
||||
this.name = 'ElasticsearchUpsert'
|
||||
this.version = 1.0
|
||||
this.description = 'Upsert documents to Elasticsearch'
|
||||
this.inputs.unshift({
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
})
|
||||
}
|
||||
|
||||
async constructVectorStore(
|
||||
embeddings: Embeddings,
|
||||
elasticSearchClientArgs: ElasticClientArgs,
|
||||
docs: Document<Record<string, any>>[]
|
||||
): Promise<VectorStore> {
|
||||
const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
|
||||
await vectorStore.addDocuments(docs)
|
||||
return vectorStore
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
// The following code is a workaround for a bug (Langchain Issue #1589) in the underlying library.
|
||||
// Store does not support object in metadata and fail silently
|
||||
finalDocs.forEach((d) => {
|
||||
delete d.metadata.pdf
|
||||
delete d.metadata.loc
|
||||
})
|
||||
// end of workaround
|
||||
return super.init(nodeData, _, options, finalDocs)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: ElasicsearchUpsert_VectorStores }
|
||||
@@ -27,7 +27,6 @@ class Faiss_VectorStores implements INode {
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert embedded data and perform similarity search upon query using Faiss library from Meta'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
import { FaissStore } from '@langchain/community/vectorstores/faiss'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
|
||||
class Faiss_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Faiss Load Existing Index'
|
||||
this.name = 'faissExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Faiss'
|
||||
this.icon = 'faiss.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Faiss (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Base Path to load',
|
||||
name: 'basePath',
|
||||
description: 'Path to load faiss.index file',
|
||||
placeholder: `C:\\Users\\User\\Desktop`,
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Faiss Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Faiss Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(FaissStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const basePath = nodeData.inputs?.basePath as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const vectorStore = await FaissStore.load(basePath, embeddings)
|
||||
|
||||
// Avoid illegal invocation error
|
||||
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number) => {
|
||||
const index = vectorStore.index
|
||||
|
||||
if (k > index.ntotal()) {
|
||||
const total = index.ntotal()
|
||||
console.warn(`k (${k}) is greater than the number of elements in the index (${total}), setting k to ${total}`)
|
||||
k = total
|
||||
}
|
||||
|
||||
const result = index.search(query, k)
|
||||
return result.labels.map((id, index) => {
|
||||
const uuid = vectorStore._mapping[id]
|
||||
return [vectorStore.docstore.search(uuid), result.distances[index]] as [Document, number]
|
||||
})
|
||||
}
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Faiss_Existing_VectorStores }
|
||||
@@ -1,121 +0,0 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { FaissStore } from '@langchain/community/vectorstores/faiss'
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
|
||||
class FaissUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Faiss Upsert Document'
|
||||
this.name = 'faissUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Faiss'
|
||||
this.icon = 'faiss.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Faiss'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Base Path to store',
|
||||
name: 'basePath',
|
||||
description: 'Path to store faiss.index file',
|
||||
placeholder: `C:\\Users\\User\\Desktop`,
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Faiss Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Faiss Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(FaissStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const basePath = nodeData.inputs?.basePath as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const vectorStore = await FaissStore.fromDocuments(finalDocs, embeddings)
|
||||
await vectorStore.save(basePath)
|
||||
|
||||
// Avoid illegal invocation error
|
||||
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number) => {
|
||||
const index = vectorStore.index
|
||||
|
||||
if (k > index.ntotal()) {
|
||||
const total = index.ntotal()
|
||||
console.warn(`k (${k}) is greater than the number of elements in the index (${total}), setting k to ${total}`)
|
||||
k = total
|
||||
}
|
||||
|
||||
const result = index.search(query, k)
|
||||
return result.labels.map((id, index) => {
|
||||
const uuid = vectorStore._mapping[id]
|
||||
return [vectorStore.docstore.search(uuid), result.distances[index]] as [Document, number]
|
||||
})
|
||||
}
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: FaissUpsert_VectorStores }
|
||||
@@ -33,7 +33,6 @@ class Milvus_VectorStores implements INode {
|
||||
this.category = 'Vector Stores'
|
||||
this.description = `Upsert embedded data and perform similarity search upon query using Milvus, world's most advanced open-source vector database`
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
import { DataType, ErrorCode } from '@zilliz/milvus2-sdk-node'
|
||||
import { MilvusLibArgs, Milvus } from '@langchain/community/vectorstores/milvus'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class Milvus_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Milvus Load Existing collection'
|
||||
this.name = 'milvusExistingCollection'
|
||||
this.version = 2.0
|
||||
this.type = 'Milvus'
|
||||
this.icon = 'milvus.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing collection from Milvus (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
optional: true,
|
||||
credentialNames: ['milvusAuth']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Milvus Server URL',
|
||||
name: 'milvusServerUrl',
|
||||
type: 'string',
|
||||
placeholder: 'http://localhost:19530'
|
||||
},
|
||||
{
|
||||
label: 'Milvus Collection Name',
|
||||
name: 'milvusCollection',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Milvus Filter',
|
||||
name: 'milvusFilter',
|
||||
type: 'string',
|
||||
optional: true,
|
||||
description:
|
||||
'Filter data with a simple string query. Refer Milvus <a target="_blank" href="https://milvus.io/blog/2022-08-08-How-to-use-string-data-to-empower-your-similarity-search-applications.md#Hybrid-search">docs</a> for more details.',
|
||||
placeholder: 'doc=="a"',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Milvus Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Milvus Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(Milvus)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
// server setup
|
||||
const address = nodeData.inputs?.milvusServerUrl as string
|
||||
const collectionName = nodeData.inputs?.milvusCollection as string
|
||||
const milvusFilter = nodeData.inputs?.milvusFilter as string
|
||||
|
||||
// embeddings
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
|
||||
// output
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
// format data
|
||||
const k = topK ? parseInt(topK, 10) : 4
|
||||
|
||||
// credential
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const milvusUser = getCredentialParam('milvusUser', credentialData, nodeData)
|
||||
const milvusPassword = getCredentialParam('milvusPassword', credentialData, nodeData)
|
||||
|
||||
// init MilvusLibArgs
|
||||
const milVusArgs: MilvusLibArgs = {
|
||||
url: address,
|
||||
collectionName: collectionName
|
||||
}
|
||||
|
||||
if (milvusUser) milVusArgs.username = milvusUser
|
||||
if (milvusPassword) milVusArgs.password = milvusPassword
|
||||
|
||||
const vectorStore = await Milvus.fromExistingCollection(embeddings, milVusArgs)
|
||||
|
||||
// Avoid Illegal Invocation
|
||||
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: string) => {
|
||||
const hasColResp = await vectorStore.client.hasCollection({
|
||||
collection_name: vectorStore.collectionName
|
||||
})
|
||||
if (hasColResp.status.error_code !== ErrorCode.SUCCESS) {
|
||||
throw new Error(`Error checking collection: ${hasColResp}`)
|
||||
}
|
||||
if (hasColResp.value === false) {
|
||||
throw new Error(`Collection not found: ${vectorStore.collectionName}, please create collection before search.`)
|
||||
}
|
||||
|
||||
const filterStr = milvusFilter ?? filter ?? ''
|
||||
|
||||
await vectorStore.grabCollectionFields()
|
||||
|
||||
const loadResp = await vectorStore.client.loadCollectionSync({
|
||||
collection_name: vectorStore.collectionName
|
||||
})
|
||||
|
||||
if (loadResp.error_code !== ErrorCode.SUCCESS) {
|
||||
throw new Error(`Error loading collection: ${loadResp}`)
|
||||
}
|
||||
|
||||
const outputFields = vectorStore.fields.filter((field) => field !== vectorStore.vectorField)
|
||||
|
||||
const searchResp = await vectorStore.client.search({
|
||||
collection_name: vectorStore.collectionName,
|
||||
search_params: {
|
||||
anns_field: vectorStore.vectorField,
|
||||
topk: k.toString(),
|
||||
metric_type: vectorStore.indexCreateParams.metric_type,
|
||||
params: vectorStore.indexSearchParams
|
||||
},
|
||||
output_fields: outputFields,
|
||||
vector_type: DataType.FloatVector,
|
||||
vectors: [query],
|
||||
filter: filterStr
|
||||
})
|
||||
if (searchResp.status.error_code !== ErrorCode.SUCCESS) {
|
||||
throw new Error(`Error searching data: ${JSON.stringify(searchResp)}`)
|
||||
}
|
||||
const results: [Document, number][] = []
|
||||
searchResp.results.forEach((result) => {
|
||||
const fields = {
|
||||
pageContent: '',
|
||||
metadata: {} as Record<string, any>
|
||||
}
|
||||
Object.keys(result).forEach((key) => {
|
||||
if (key === vectorStore.textField) {
|
||||
fields.pageContent = result[key]
|
||||
} else if (vectorStore.fields.includes(key) || key === vectorStore.primaryField) {
|
||||
if (typeof result[key] === 'string') {
|
||||
const { isJson, obj } = checkJsonString(result[key])
|
||||
fields.metadata[key] = isJson ? obj : result[key]
|
||||
} else {
|
||||
fields.metadata[key] = result[key]
|
||||
}
|
||||
}
|
||||
})
|
||||
results.push([new Document(fields), result.score])
|
||||
})
|
||||
return results
|
||||
}
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
if (milvusFilter) {
|
||||
;(vectorStore as any).filter = milvusFilter
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
function checkJsonString(value: string): { isJson: boolean; obj: any } {
|
||||
try {
|
||||
const result = JSON.parse(value)
|
||||
return { isJson: true, obj: result }
|
||||
} catch (e) {
|
||||
return { isJson: false, obj: null }
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Milvus_Existing_VectorStores }
|
||||
@@ -1,285 +0,0 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { DataType, ErrorCode, MetricType, IndexType } from '@zilliz/milvus2-sdk-node'
|
||||
import { MilvusLibArgs, Milvus } from '@langchain/community/vectorstores/milvus'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
interface InsertRow {
|
||||
[x: string]: string | number[]
|
||||
}
|
||||
|
||||
class Milvus_Upsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Milvus Upsert Document'
|
||||
this.name = 'milvusUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Milvus'
|
||||
this.icon = 'milvus.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Milvus'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
optional: true,
|
||||
credentialNames: ['milvusAuth']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Milvus Server URL',
|
||||
name: 'milvusServerUrl',
|
||||
type: 'string',
|
||||
placeholder: 'http://localhost:19530'
|
||||
},
|
||||
{
|
||||
label: 'Milvus Collection Name',
|
||||
name: 'milvusCollection',
|
||||
type: 'string'
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Milvus Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Milvus Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(Milvus)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
// server setup
|
||||
const address = nodeData.inputs?.milvusServerUrl as string
|
||||
const collectionName = nodeData.inputs?.milvusCollection as string
|
||||
|
||||
// embeddings
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
|
||||
// output
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
// format data
|
||||
const k = topK ? parseInt(topK, 10) : 4
|
||||
|
||||
// credential
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const milvusUser = getCredentialParam('milvusUser', credentialData, nodeData)
|
||||
const milvusPassword = getCredentialParam('milvusPassword', credentialData, nodeData)
|
||||
|
||||
// init MilvusLibArgs
|
||||
const milVusArgs: MilvusLibArgs = {
|
||||
url: address,
|
||||
collectionName: collectionName
|
||||
}
|
||||
|
||||
if (milvusUser) milVusArgs.username = milvusUser
|
||||
if (milvusPassword) milVusArgs.password = milvusPassword
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const vectorStore = await MilvusUpsert.fromDocuments(finalDocs, embeddings, milVusArgs)
|
||||
|
||||
// Avoid Illegal Invocation
|
||||
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: string) => {
|
||||
const hasColResp = await vectorStore.client.hasCollection({
|
||||
collection_name: vectorStore.collectionName
|
||||
})
|
||||
if (hasColResp.status.error_code !== ErrorCode.SUCCESS) {
|
||||
throw new Error(`Error checking collection: ${hasColResp}`)
|
||||
}
|
||||
if (hasColResp.value === false) {
|
||||
throw new Error(`Collection not found: ${vectorStore.collectionName}, please create collection before search.`)
|
||||
}
|
||||
|
||||
const filterStr = filter ?? ''
|
||||
|
||||
await vectorStore.grabCollectionFields()
|
||||
|
||||
const loadResp = await vectorStore.client.loadCollectionSync({
|
||||
collection_name: vectorStore.collectionName
|
||||
})
|
||||
if (loadResp.error_code !== ErrorCode.SUCCESS) {
|
||||
throw new Error(`Error loading collection: ${loadResp}`)
|
||||
}
|
||||
|
||||
const outputFields = vectorStore.fields.filter((field) => field !== vectorStore.vectorField)
|
||||
|
||||
const searchResp = await vectorStore.client.search({
|
||||
collection_name: vectorStore.collectionName,
|
||||
search_params: {
|
||||
anns_field: vectorStore.vectorField,
|
||||
topk: k.toString(),
|
||||
metric_type: vectorStore.indexCreateParams.metric_type,
|
||||
params: vectorStore.indexSearchParams
|
||||
},
|
||||
output_fields: outputFields,
|
||||
vector_type: DataType.FloatVector,
|
||||
vectors: [query],
|
||||
filter: filterStr
|
||||
})
|
||||
if (searchResp.status.error_code !== ErrorCode.SUCCESS) {
|
||||
throw new Error(`Error searching data: ${JSON.stringify(searchResp)}`)
|
||||
}
|
||||
const results: [Document, number][] = []
|
||||
searchResp.results.forEach((result) => {
|
||||
const fields = {
|
||||
pageContent: '',
|
||||
metadata: {} as Record<string, any>
|
||||
}
|
||||
Object.keys(result).forEach((key) => {
|
||||
if (key === vectorStore.textField) {
|
||||
fields.pageContent = result[key]
|
||||
} else if (vectorStore.fields.includes(key) || key === vectorStore.primaryField) {
|
||||
if (typeof result[key] === 'string') {
|
||||
const { isJson, obj } = checkJsonString(result[key])
|
||||
fields.metadata[key] = isJson ? obj : result[key]
|
||||
} else {
|
||||
fields.metadata[key] = result[key]
|
||||
}
|
||||
}
|
||||
})
|
||||
results.push([new Document(fields), result.score])
|
||||
})
|
||||
return results
|
||||
}
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
function checkJsonString(value: string): { isJson: boolean; obj: any } {
|
||||
try {
|
||||
const result = JSON.parse(value)
|
||||
return { isJson: true, obj: result }
|
||||
} catch (e) {
|
||||
return { isJson: false, obj: null }
|
||||
}
|
||||
}
|
||||
|
||||
class MilvusUpsert extends Milvus {
|
||||
async addVectors(vectors: number[][], documents: Document[]): Promise<void> {
|
||||
if (vectors.length === 0) {
|
||||
return
|
||||
}
|
||||
await this.ensureCollection(vectors, documents)
|
||||
|
||||
const insertDatas: InsertRow[] = []
|
||||
|
||||
for (let index = 0; index < vectors.length; index++) {
|
||||
const vec = vectors[index]
|
||||
const doc = documents[index]
|
||||
const data: InsertRow = {
|
||||
[this.textField]: doc.pageContent,
|
||||
[this.vectorField]: vec
|
||||
}
|
||||
this.fields.forEach((field) => {
|
||||
switch (field) {
|
||||
case this.primaryField:
|
||||
if (!this.autoId) {
|
||||
if (doc.metadata[this.primaryField] === undefined) {
|
||||
throw new Error(
|
||||
`The Collection's primaryField is configured with autoId=false, thus its value must be provided through metadata.`
|
||||
)
|
||||
}
|
||||
data[field] = doc.metadata[this.primaryField]
|
||||
}
|
||||
break
|
||||
case this.textField:
|
||||
data[field] = doc.pageContent
|
||||
break
|
||||
case this.vectorField:
|
||||
data[field] = vec
|
||||
break
|
||||
default: // metadata fields
|
||||
if (doc.metadata[field] === undefined) {
|
||||
throw new Error(`The field "${field}" is not provided in documents[${index}].metadata.`)
|
||||
} else if (typeof doc.metadata[field] === 'object') {
|
||||
data[field] = JSON.stringify(doc.metadata[field])
|
||||
} else {
|
||||
data[field] = doc.metadata[field]
|
||||
}
|
||||
break
|
||||
}
|
||||
})
|
||||
|
||||
insertDatas.push(data)
|
||||
}
|
||||
|
||||
const descIndexResp = await this.client.describeIndex({
|
||||
collection_name: this.collectionName
|
||||
})
|
||||
|
||||
if (descIndexResp.status.error_code === ErrorCode.IndexNotExist) {
|
||||
const resp = await this.client.createIndex({
|
||||
collection_name: this.collectionName,
|
||||
field_name: this.vectorField,
|
||||
index_name: `myindex_${Date.now().toString()}`,
|
||||
index_type: IndexType.AUTOINDEX,
|
||||
metric_type: MetricType.L2
|
||||
})
|
||||
if (resp.error_code !== ErrorCode.SUCCESS) {
|
||||
throw new Error(`Error creating index`)
|
||||
}
|
||||
}
|
||||
|
||||
const insertResp = await this.client.insert({
|
||||
collection_name: this.collectionName,
|
||||
fields_data: insertDatas
|
||||
})
|
||||
|
||||
if (insertResp.status.error_code !== ErrorCode.SUCCESS) {
|
||||
throw new Error(`Error inserting data: ${JSON.stringify(insertResp)}`)
|
||||
}
|
||||
|
||||
await this.client.flushSync({ collection_names: [this.collectionName] })
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Milvus_Upsert_VectorStores }
|
||||
@@ -30,7 +30,6 @@ class MongoDBAtlas_VectorStores implements INode {
|
||||
this.icon = 'mongodb.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
import {
|
||||
getBaseClasses,
|
||||
getCredentialData,
|
||||
getCredentialParam,
|
||||
ICommonObject,
|
||||
INodeData,
|
||||
INodeOutputsValue,
|
||||
INodeParams
|
||||
} from '../../../src'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { MongoDBAtlasVectorSearch } from '@langchain/community/vectorstores/mongodb_atlas'
|
||||
import { Collection, MongoClient } from 'mongodb'
|
||||
|
||||
export abstract class MongoDBSearchBase {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
mongoClient: MongoClient
|
||||
|
||||
protected constructor() {
|
||||
this.type = 'MongoDB Atlas'
|
||||
this.icon = 'mongodb.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['mongoDBUrlApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Database',
|
||||
name: 'databaseName',
|
||||
placeholder: '<DB_NAME>',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Collection Name',
|
||||
name: 'collectionName',
|
||||
placeholder: '<COLLECTION_NAME>',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Index Name',
|
||||
name: 'indexName',
|
||||
placeholder: '<VECTOR_INDEX_NAME>',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Content Field',
|
||||
name: 'textKey',
|
||||
description: 'Name of the field (column) that contains the actual content',
|
||||
type: 'string',
|
||||
default: 'text',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Embedded Field',
|
||||
name: 'embeddingKey',
|
||||
description: 'Name of the field (column) that contains the Embedding',
|
||||
type: 'string',
|
||||
default: 'embedding',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'MongoDB Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'MongoDB Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(MongoDBAtlasVectorSearch)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
abstract constructVectorStore(
|
||||
embeddings: Embeddings,
|
||||
collection: Collection,
|
||||
indexName: string,
|
||||
textKey: string,
|
||||
embeddingKey: string,
|
||||
docs: Document<Record<string, any>>[] | undefined
|
||||
): Promise<VectorStore>
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document<Record<string, any>>[] | undefined): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const databaseName = nodeData.inputs?.databaseName as string
|
||||
const collectionName = nodeData.inputs?.collectionName as string
|
||||
const indexName = nodeData.inputs?.indexName as string
|
||||
let textKey = nodeData.inputs?.textKey as string
|
||||
let embeddingKey = nodeData.inputs?.embeddingKey as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
let mongoDBConnectUrl = getCredentialParam('mongoDBConnectUrl', credentialData, nodeData)
|
||||
|
||||
this.mongoClient = new MongoClient(mongoDBConnectUrl)
|
||||
const collection = this.mongoClient.db(databaseName).collection(collectionName)
|
||||
if (!textKey || textKey === '') textKey = 'text'
|
||||
if (!embeddingKey || embeddingKey === '') embeddingKey = 'embedding'
|
||||
const vectorStore = await this.constructVectorStore(embeddings, collection, indexName, textKey, embeddingKey, docs)
|
||||
|
||||
if (output === 'retriever') {
|
||||
return vectorStore.asRetriever(k)
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
import { Collection } from 'mongodb'
|
||||
import { MongoDBAtlasVectorSearch } from '@langchain/community/vectorstores/mongodb_atlas'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { MongoDBSearchBase } from './MongoDBSearchBase'
|
||||
import { ICommonObject, INode, INodeData } from '../../../src/Interface'
|
||||
|
||||
class MongoDBExisting_VectorStores extends MongoDBSearchBase implements INode {
|
||||
constructor() {
|
||||
super()
|
||||
this.label = 'MongoDB Atlas Load Existing Index'
|
||||
this.name = 'MongoDBIndex'
|
||||
this.version = 1.0
|
||||
this.description = 'Load existing data from MongoDB Atlas (i.e: Document has been upserted)'
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
return super.init(nodeData, _, options, undefined)
|
||||
}
|
||||
|
||||
async constructVectorStore(
|
||||
embeddings: Embeddings,
|
||||
collection: Collection,
|
||||
indexName: string,
|
||||
textKey: string,
|
||||
embeddingKey: string,
|
||||
_: Document<Record<string, any>>[] | undefined
|
||||
): Promise<VectorStore> {
|
||||
return new MongoDBAtlasVectorSearch(embeddings, {
|
||||
collection: collection,
|
||||
indexName: indexName,
|
||||
textKey: textKey,
|
||||
embeddingKey: embeddingKey
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: MongoDBExisting_VectorStores }
|
||||
@@ -1,59 +0,0 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { Collection } from 'mongodb'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { MongoDBAtlasVectorSearch } from '@langchain/community/vectorstores/mongodb_atlas'
|
||||
import { ICommonObject, INode, INodeData } from '../../../src/Interface'
|
||||
import { MongoDBSearchBase } from './MongoDBSearchBase'
|
||||
|
||||
class MongoDBUpsert_VectorStores extends MongoDBSearchBase implements INode {
|
||||
constructor() {
|
||||
super()
|
||||
this.label = 'MongoDB Atlas Upsert Document'
|
||||
this.name = 'MongoDBUpsert'
|
||||
this.version = 1.0
|
||||
this.description = 'Upsert documents to MongoDB Atlas'
|
||||
this.inputs.unshift({
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
})
|
||||
}
|
||||
|
||||
async constructVectorStore(
|
||||
embeddings: Embeddings,
|
||||
collection: Collection,
|
||||
indexName: string,
|
||||
textKey: string,
|
||||
embeddingKey: string,
|
||||
docs: Document<Record<string, any>>[]
|
||||
): Promise<VectorStore> {
|
||||
const mongoDBAtlasVectorSearch = new MongoDBAtlasVectorSearch(embeddings, {
|
||||
collection: collection,
|
||||
indexName: indexName,
|
||||
textKey: textKey,
|
||||
embeddingKey: embeddingKey
|
||||
})
|
||||
await mongoDBAtlasVectorSearch.addDocuments(docs)
|
||||
return mongoDBAtlasVectorSearch
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
const document = new Document(flattenDocs[i])
|
||||
finalDocs.push(document)
|
||||
}
|
||||
}
|
||||
|
||||
return super.init(nodeData, _, options, finalDocs)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: MongoDBUpsert_VectorStores }
|
||||
@@ -29,7 +29,6 @@ class OpenSearch_VectorStores implements INode {
|
||||
this.category = 'Vector Stores'
|
||||
this.description = `Upsert embedded data and perform similarity search upon query using OpenSearch, an open-source, all-in-one vector database`
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
import { OpenSearchVectorStore } from '@langchain/community/vectorstores/opensearch'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { Client } from '@opensearch-project/opensearch'
|
||||
import { flatten } from 'lodash'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class OpenSearchUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'OpenSearch Upsert Document'
|
||||
this.name = 'openSearchUpsertDocument'
|
||||
this.version = 1.0
|
||||
this.type = 'OpenSearch'
|
||||
this.icon = 'opensearch.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to OpenSearch'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'OpenSearch URL',
|
||||
name: 'opensearchURL',
|
||||
type: 'string',
|
||||
placeholder: 'http://127.0.0.1:9200'
|
||||
},
|
||||
{
|
||||
label: 'Index Name',
|
||||
name: 'indexName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'OpenSearch Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'OpenSearch Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(OpenSearchVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const opensearchURL = nodeData.inputs?.opensearchURL as string
|
||||
const indexName = nodeData.inputs?.indexName as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const client = new Client({
|
||||
nodes: [opensearchURL]
|
||||
})
|
||||
|
||||
const vectorStore = await OpenSearchVectorStore.fromDocuments(finalDocs, embeddings, {
|
||||
client,
|
||||
indexName: indexName
|
||||
})
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: OpenSearchUpsert_VectorStores }
|
||||
@@ -1,99 +0,0 @@
|
||||
import { OpenSearchVectorStore } from '@langchain/community/vectorstores/opensearch'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Client } from '@opensearch-project/opensearch'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class OpenSearch_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'OpenSearch Load Existing Index'
|
||||
this.name = 'openSearchExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'OpenSearch'
|
||||
this.icon = 'opensearch.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from OpenSearch (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'OpenSearch URL',
|
||||
name: 'opensearchURL',
|
||||
type: 'string',
|
||||
placeholder: 'http://127.0.0.1:9200'
|
||||
},
|
||||
{
|
||||
label: 'Index Name',
|
||||
name: 'indexName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'OpenSearch Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'OpenSearch Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(OpenSearchVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const opensearchURL = nodeData.inputs?.opensearchURL as string
|
||||
const indexName = nodeData.inputs?.indexName as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const client = new Client({
|
||||
nodes: [opensearchURL]
|
||||
})
|
||||
|
||||
const vectorStore = new OpenSearchVectorStore(embeddings, {
|
||||
client,
|
||||
indexName
|
||||
})
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: OpenSearch_Existing_VectorStores }
|
||||
@@ -48,7 +48,6 @@ class Pinecone_VectorStores implements INode {
|
||||
this.category = 'Vector Stores'
|
||||
this.description = `Upsert embedded data and perform similarity or mmr search using Pinecone, a leading fully managed hosted vector database`
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
import { Pinecone } from '@pinecone-database/pinecone'
|
||||
import { PineconeStoreParams, PineconeStore } from '@langchain/pinecone'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class Pinecone_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Pinecone Load Existing Index'
|
||||
this.name = 'pineconeExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Pinecone'
|
||||
this.icon = 'pinecone.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Pinecone (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['pineconeApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Index',
|
||||
name: 'pineconeIndex',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Namespace',
|
||||
name: 'pineconeNamespace',
|
||||
type: 'string',
|
||||
placeholder: 'my-first-namespace',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Metadata Filter',
|
||||
name: 'pineconeMetadataFilter',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Pinecone Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(PineconeStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const index = nodeData.inputs?.pineconeIndex as string
|
||||
const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
|
||||
const pineconeMetadataFilter = nodeData.inputs?.pineconeMetadataFilter
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
|
||||
|
||||
const client = new Pinecone({
|
||||
apiKey: pineconeApiKey
|
||||
})
|
||||
|
||||
const pineconeIndex = client.Index(index)
|
||||
|
||||
const obj: PineconeStoreParams = {
|
||||
pineconeIndex
|
||||
}
|
||||
|
||||
if (pineconeNamespace) obj.namespace = pineconeNamespace
|
||||
if (pineconeMetadataFilter) {
|
||||
const metadatafilter = typeof pineconeMetadataFilter === 'object' ? pineconeMetadataFilter : JSON.parse(pineconeMetadataFilter)
|
||||
obj.filter = metadatafilter
|
||||
}
|
||||
|
||||
const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Pinecone_Existing_VectorStores }
|
||||
@@ -1,133 +0,0 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { Pinecone } from '@pinecone-database/pinecone'
|
||||
import { PineconeStoreParams, PineconeStore } from '@langchain/pinecone'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
|
||||
class PineconeUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Pinecone Upsert Document'
|
||||
this.name = 'pineconeUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Pinecone'
|
||||
this.icon = 'pinecone.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Pinecone'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['pineconeApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Index',
|
||||
name: 'pineconeIndex',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Namespace',
|
||||
name: 'pineconeNamespace',
|
||||
type: 'string',
|
||||
placeholder: 'my-first-namespace',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Pinecone Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(PineconeStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const index = nodeData.inputs?.pineconeIndex as string
|
||||
const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
|
||||
|
||||
const client = new Pinecone({
|
||||
apiKey: pineconeApiKey
|
||||
})
|
||||
|
||||
const pineconeIndex = client.Index(index)
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const obj: PineconeStoreParams = {
|
||||
pineconeIndex
|
||||
}
|
||||
|
||||
if (pineconeNamespace) obj.namespace = pineconeNamespace
|
||||
|
||||
const vectorStore = await PineconeStore.fromDocuments(finalDocs, embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: PineconeUpsert_VectorStores }
|
||||
@@ -31,7 +31,6 @@ class Postgres_VectorStores implements INode {
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert embedded data and perform similarity search upon query using pgvector on Postgres'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,202 +0,0 @@
|
||||
import { DataSourceOptions } from 'typeorm'
|
||||
import { Pool } from 'pg'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { TypeORMVectorStore, TypeORMVectorStoreDocument } from '@langchain/community/vectorstores/typeorm'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
|
||||
class Postgres_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Postgres Load Existing Index'
|
||||
this.name = 'postgresExistingIndex'
|
||||
this.version = 2.0
|
||||
this.type = 'Postgres'
|
||||
this.icon = 'postgres.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Postgres using pgvector (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['PostgresApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Host',
|
||||
name: 'host',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Database',
|
||||
name: 'database',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'SSL Connection',
|
||||
name: 'sslConnection',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
optional: false
|
||||
},
|
||||
{
|
||||
label: 'Port',
|
||||
name: 'port',
|
||||
type: 'number',
|
||||
placeholder: '6432',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string',
|
||||
placeholder: 'documents',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Additional Configuration',
|
||||
name: 'additionalConfig',
|
||||
type: 'json',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Postgres Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Postgres Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(TypeORMVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const user = getCredentialParam('user', credentialData, nodeData)
|
||||
const password = getCredentialParam('password', credentialData, nodeData)
|
||||
const _tableName = nodeData.inputs?.tableName as string
|
||||
const tableName = _tableName ? _tableName : 'documents'
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const additionalConfig = nodeData.inputs?.additionalConfig as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
const sslConnection = nodeData.inputs?.sslConnection as boolean
|
||||
|
||||
let additionalConfiguration = {}
|
||||
if (additionalConfig) {
|
||||
try {
|
||||
additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
|
||||
} catch (exception) {
|
||||
throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
|
||||
}
|
||||
}
|
||||
|
||||
const postgresConnectionOptions = {
|
||||
...additionalConfiguration,
|
||||
type: 'postgres',
|
||||
host: nodeData.inputs?.host as string,
|
||||
port: nodeData.inputs?.port as number,
|
||||
username: user,
|
||||
password: password,
|
||||
database: nodeData.inputs?.database as string,
|
||||
ssl: sslConnection
|
||||
}
|
||||
|
||||
const args = {
|
||||
postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions,
|
||||
tableName: tableName
|
||||
}
|
||||
|
||||
const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args)
|
||||
|
||||
// Rewrite the method to use pg pool connection instead of the default connection
|
||||
/* Otherwise a connection error is displayed when the chain tries to execute the function
|
||||
[chain/start] [1:chain:ConversationalRetrievalQAChain] Entering Chain run with input: { "question": "what the document is about", "chat_history": [] }
|
||||
[retriever/start] [1:chain:ConversationalRetrievalQAChain > 2:retriever:VectorStoreRetriever] Entering Retriever run with input: { "query": "what the document is about" }
|
||||
[ERROR]: uncaughtException: Illegal invocation TypeError: Illegal invocation at Socket.ref (node:net:1524:18) at Connection.ref (.../node_modules/pg/lib/connection.js:183:17) at Client.ref (.../node_modules/pg/lib/client.js:591:21) at BoundPool._pulseQueue (/node_modules/pg-pool/index.js:148:28) at .../node_modules/pg-pool/index.js:184:37 at process.processTicksAndRejections (node:internal/process/task_queues:77:11)
|
||||
*/
|
||||
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
|
||||
const embeddingString = `[${query.join(',')}]`
|
||||
const _filter = filter ?? '{}'
|
||||
|
||||
const queryString = `
|
||||
SELECT *, embedding <=> $1 as "_distance"
|
||||
FROM ${tableName}
|
||||
WHERE metadata @> $2
|
||||
ORDER BY "_distance" ASC
|
||||
LIMIT $3;`
|
||||
|
||||
const poolOptions = {
|
||||
host: postgresConnectionOptions.host,
|
||||
port: postgresConnectionOptions.port,
|
||||
user: postgresConnectionOptions.username,
|
||||
password: postgresConnectionOptions.password,
|
||||
database: postgresConnectionOptions.database
|
||||
}
|
||||
const pool = new Pool(poolOptions)
|
||||
const conn = await pool.connect()
|
||||
|
||||
const documents = await conn.query(queryString, [embeddingString, _filter, k])
|
||||
|
||||
conn.release()
|
||||
|
||||
const results = [] as [TypeORMVectorStoreDocument, number][]
|
||||
for (const doc of documents.rows) {
|
||||
if (doc._distance != null && doc.pageContent != null) {
|
||||
const document = new Document(doc) as TypeORMVectorStoreDocument
|
||||
document.id = doc.id
|
||||
results.push([document, doc._distance])
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Postgres_Existing_VectorStores }
|
||||
@@ -1,218 +0,0 @@
|
||||
import { DataSourceOptions } from 'typeorm'
|
||||
import { flatten } from 'lodash'
|
||||
import { Pool } from 'pg'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { TypeORMVectorStore, TypeORMVectorStoreDocument } from '@langchain/community/vectorstores/typeorm'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class PostgresUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Postgres Upsert Document'
|
||||
this.name = 'postgresUpsert'
|
||||
this.version = 2.0
|
||||
this.type = 'Postgres'
|
||||
this.icon = 'postgres.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Postgres using pgvector'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['PostgresApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Host',
|
||||
name: 'host',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Database',
|
||||
name: 'database',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'SSL Connection',
|
||||
name: 'sslConnection',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
optional: false
|
||||
},
|
||||
{
|
||||
label: 'Port',
|
||||
name: 'port',
|
||||
type: 'number',
|
||||
placeholder: '6432',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string',
|
||||
placeholder: 'documents',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Additional Configuration',
|
||||
name: 'additionalConfig',
|
||||
type: 'json',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Postgres Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Postgres Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(TypeORMVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const user = getCredentialParam('user', credentialData, nodeData)
|
||||
const password = getCredentialParam('password', credentialData, nodeData)
|
||||
const _tableName = nodeData.inputs?.tableName as string
|
||||
const tableName = _tableName ? _tableName : 'documents'
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const additionalConfig = nodeData.inputs?.additionalConfig as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
const sslConnection = nodeData.inputs?.sslConnection as boolean
|
||||
|
||||
let additionalConfiguration = {}
|
||||
if (additionalConfig) {
|
||||
try {
|
||||
additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
|
||||
} catch (exception) {
|
||||
throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
|
||||
}
|
||||
}
|
||||
|
||||
const postgresConnectionOptions = {
|
||||
...additionalConfiguration,
|
||||
type: 'postgres',
|
||||
host: nodeData.inputs?.host as string,
|
||||
port: nodeData.inputs?.port as number,
|
||||
username: user,
|
||||
password: password,
|
||||
database: nodeData.inputs?.database as string,
|
||||
ssl: sslConnection
|
||||
}
|
||||
|
||||
const args = {
|
||||
postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions,
|
||||
tableName: tableName
|
||||
}
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)
|
||||
|
||||
// Rewrite the method to use pg pool connection instead of the default connection
|
||||
/* Otherwise a connection error is displayed when the chain tries to execute the function
|
||||
[chain/start] [1:chain:ConversationalRetrievalQAChain] Entering Chain run with input: { "question": "what the document is about", "chat_history": [] }
|
||||
[retriever/start] [1:chain:ConversationalRetrievalQAChain > 2:retriever:VectorStoreRetriever] Entering Retriever run with input: { "query": "what the document is about" }
|
||||
[ERROR]: uncaughtException: Illegal invocation TypeError: Illegal invocation at Socket.ref (node:net:1524:18) at Connection.ref (.../node_modules/pg/lib/connection.js:183:17) at Client.ref (.../node_modules/pg/lib/client.js:591:21) at BoundPool._pulseQueue (/node_modules/pg-pool/index.js:148:28) at .../node_modules/pg-pool/index.js:184:37 at process.processTicksAndRejections (node:internal/process/task_queues:77:11)
|
||||
*/
|
||||
vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
|
||||
const embeddingString = `[${query.join(',')}]`
|
||||
const _filter = filter ?? '{}'
|
||||
|
||||
const queryString = `
|
||||
SELECT *, embedding <=> $1 as "_distance"
|
||||
FROM ${tableName}
|
||||
WHERE metadata @> $2
|
||||
ORDER BY "_distance" ASC
|
||||
LIMIT $3;`
|
||||
|
||||
const poolOptions = {
|
||||
host: postgresConnectionOptions.host,
|
||||
port: postgresConnectionOptions.port,
|
||||
user: postgresConnectionOptions.username,
|
||||
password: postgresConnectionOptions.password,
|
||||
database: postgresConnectionOptions.database
|
||||
}
|
||||
const pool = new Pool(poolOptions)
|
||||
const conn = await pool.connect()
|
||||
|
||||
const documents = await conn.query(queryString, [embeddingString, _filter, k])
|
||||
|
||||
conn.release()
|
||||
|
||||
const results = [] as [TypeORMVectorStoreDocument, number][]
|
||||
for (const doc of documents.rows) {
|
||||
if (doc._distance != null && doc.pageContent != null) {
|
||||
const document = new Document(doc) as TypeORMVectorStoreDocument
|
||||
document.id = doc.id
|
||||
results.push([document, doc._distance])
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: PostgresUpsert_VectorStores }
|
||||
@@ -39,7 +39,6 @@ class Qdrant_VectorStores implements INode {
|
||||
this.description =
|
||||
'Upsert embedded data and perform similarity search upon query using Qdrant, a scalable open source vector database written in Rust'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
import { QdrantClient } from '@qdrant/js-client-rest'
|
||||
import { QdrantVectorStore, QdrantLibArgs } from '@langchain/community/vectorstores/qdrant'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { VectorStoreRetrieverInput } from '@langchain/core/vectorstores'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
type RetrieverConfig = Partial<VectorStoreRetrieverInput<QdrantVectorStore>>
|
||||
|
||||
class Qdrant_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Qdrant Load Existing Index'
|
||||
this.name = 'qdrantExistingIndex'
|
||||
this.version = 2.0
|
||||
this.type = 'Qdrant'
|
||||
this.icon = 'qdrant.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Qdrant (i.e., documents have been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed when using Qdrant cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['qdrantApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Server URL',
|
||||
name: 'qdrantServerUrl',
|
||||
type: 'string',
|
||||
placeholder: 'http://localhost:6333'
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Collection Name',
|
||||
name: 'qdrantCollection',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Vector Dimension',
|
||||
name: 'qdrantVectorDimension',
|
||||
type: 'number',
|
||||
default: 1536,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Similarity',
|
||||
name: 'qdrantSimilarity',
|
||||
description: 'Similarity measure used in Qdrant.',
|
||||
type: 'options',
|
||||
default: 'Cosine',
|
||||
options: [
|
||||
{
|
||||
label: 'Cosine',
|
||||
name: 'Cosine'
|
||||
},
|
||||
{
|
||||
label: 'Euclid',
|
||||
name: 'Euclid'
|
||||
},
|
||||
{
|
||||
label: 'Dot',
|
||||
name: 'Dot'
|
||||
}
|
||||
],
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Additional Collection Cofiguration',
|
||||
name: 'qdrantCollectionConfiguration',
|
||||
description:
|
||||
'Refer to <a target="_blank" href="https://qdrant.tech/documentation/concepts/collections">collection docs</a> for more reference',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Search Filter',
|
||||
name: 'qdrantFilter',
|
||||
description: 'Only return points which satisfy the conditions',
|
||||
type: 'json',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Qdrant Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(QdrantVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string
|
||||
const collectionName = nodeData.inputs?.qdrantCollection as string
|
||||
let qdrantCollectionConfiguration = nodeData.inputs?.qdrantCollectionConfiguration
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
|
||||
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
let queryFilter = nodeData.inputs?.qdrantFilter
|
||||
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
|
||||
|
||||
const client = new QdrantClient({
|
||||
url: qdrantServerUrl,
|
||||
apiKey: qdrantApiKey
|
||||
})
|
||||
|
||||
const dbConfig: QdrantLibArgs = {
|
||||
client,
|
||||
collectionName
|
||||
}
|
||||
|
||||
const retrieverConfig: RetrieverConfig = {
|
||||
k
|
||||
}
|
||||
|
||||
if (qdrantCollectionConfiguration) {
|
||||
qdrantCollectionConfiguration =
|
||||
typeof qdrantCollectionConfiguration === 'object'
|
||||
? qdrantCollectionConfiguration
|
||||
: JSON.parse(qdrantCollectionConfiguration)
|
||||
dbConfig.collectionConfig = {
|
||||
...qdrantCollectionConfiguration,
|
||||
vectors: {
|
||||
...qdrantCollectionConfiguration.vectors,
|
||||
size: qdrantVectorDimension ? parseInt(qdrantVectorDimension, 10) : 1536,
|
||||
distance: qdrantSimilarity ?? 'Cosine'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (queryFilter) {
|
||||
retrieverConfig.filter = typeof queryFilter === 'object' ? queryFilter : JSON.parse(queryFilter)
|
||||
}
|
||||
|
||||
const vectorStore = await QdrantVectorStore.fromExistingCollection(embeddings, dbConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(retrieverConfig)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
if (queryFilter) {
|
||||
;(vectorStore as any).filter = retrieverConfig.filter
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Qdrant_Existing_VectorStores }
|
||||
@@ -1,213 +0,0 @@
|
||||
import { QdrantClient } from '@qdrant/js-client-rest'
|
||||
import { QdrantVectorStore, QdrantLibArgs } from '@langchain/community/vectorstores/qdrant'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { flatten } from 'lodash'
|
||||
import { VectorStoreRetrieverInput } from '@langchain/core/vectorstores'
|
||||
|
||||
type RetrieverConfig = Partial<VectorStoreRetrieverInput<QdrantVectorStore>>
|
||||
|
||||
class QdrantUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Qdrant Upsert Document'
|
||||
this.name = 'qdrantUpsert'
|
||||
this.version = 3.0
|
||||
this.type = 'Qdrant'
|
||||
this.icon = 'qdrant.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Qdrant'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed when using Qdrant cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['qdrantApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Server URL',
|
||||
name: 'qdrantServerUrl',
|
||||
type: 'string',
|
||||
placeholder: 'http://localhost:6333'
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Collection Name',
|
||||
name: 'qdrantCollection',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Vector Dimension',
|
||||
name: 'qdrantVectorDimension',
|
||||
type: 'number',
|
||||
default: 1536,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Upsert Batch Size',
|
||||
name: 'batchSize',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
description: 'Upsert in batches of size N',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Similarity',
|
||||
name: 'qdrantSimilarity',
|
||||
description: 'Similarity measure used in Qdrant.',
|
||||
type: 'options',
|
||||
default: 'Cosine',
|
||||
options: [
|
||||
{
|
||||
label: 'Cosine',
|
||||
name: 'Cosine'
|
||||
},
|
||||
{
|
||||
label: 'Euclid',
|
||||
name: 'Euclid'
|
||||
},
|
||||
{
|
||||
label: 'Dot',
|
||||
name: 'Dot'
|
||||
}
|
||||
],
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Search Filter',
|
||||
name: 'qdrantFilter',
|
||||
description: 'Only return points which satisfy the conditions',
|
||||
type: 'json',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Qdrant Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(QdrantVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string
|
||||
const collectionName = nodeData.inputs?.qdrantCollection as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
|
||||
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
|
||||
const _batchSize = nodeData.inputs?.batchSize
|
||||
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
let queryFilter = nodeData.inputs?.qdrantFilter
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
|
||||
|
||||
const client = new QdrantClient({
|
||||
url: qdrantServerUrl,
|
||||
apiKey: qdrantApiKey
|
||||
})
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const dbConfig: QdrantLibArgs = {
|
||||
client,
|
||||
url: qdrantServerUrl,
|
||||
collectionName,
|
||||
collectionConfig: {
|
||||
vectors: {
|
||||
size: qdrantVectorDimension ? parseInt(qdrantVectorDimension, 10) : 1536,
|
||||
distance: qdrantSimilarity ?? 'Cosine'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const retrieverConfig: RetrieverConfig = {
|
||||
k
|
||||
}
|
||||
|
||||
if (queryFilter) {
|
||||
retrieverConfig.filter = typeof queryFilter === 'object' ? queryFilter : JSON.parse(queryFilter)
|
||||
}
|
||||
|
||||
let vectorStore: QdrantVectorStore | undefined = undefined
|
||||
if (_batchSize) {
|
||||
const batchSize = parseInt(_batchSize, 10)
|
||||
for (let i = 0; i < finalDocs.length; i += batchSize) {
|
||||
const batch = finalDocs.slice(i, i + batchSize)
|
||||
vectorStore = await QdrantVectorStore.fromDocuments(batch, embeddings, dbConfig)
|
||||
}
|
||||
} else {
|
||||
vectorStore = await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
|
||||
}
|
||||
|
||||
if (vectorStore === undefined) {
|
||||
throw new Error('No documents to upsert')
|
||||
} else {
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(retrieverConfig)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: QdrantUpsert_VectorStores }
|
||||
@@ -52,7 +52,6 @@ class Redis_VectorStores implements INode {
|
||||
this.icon = 'redis.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,237 +0,0 @@
|
||||
import { createClient, SearchOptions, RedisClientOptions } from 'redis'
|
||||
import { isEqual } from 'lodash'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { RedisVectorStore } from '@langchain/community/vectorstores/redis'
|
||||
import { escapeSpecialChars, unEscapeSpecialChars } from './utils'
|
||||
import {
|
||||
getBaseClasses,
|
||||
getCredentialData,
|
||||
getCredentialParam,
|
||||
ICommonObject,
|
||||
INodeData,
|
||||
INodeOutputsValue,
|
||||
INodeParams
|
||||
} from '../../../src'
|
||||
|
||||
let redisClientSingleton: ReturnType<typeof createClient>
|
||||
let redisClientOption: RedisClientOptions
|
||||
|
||||
const getRedisClient = async (option: RedisClientOptions) => {
|
||||
if (!redisClientSingleton) {
|
||||
// if client doesn't exists
|
||||
redisClientSingleton = createClient(option)
|
||||
await redisClientSingleton.connect()
|
||||
redisClientOption = option
|
||||
return redisClientSingleton
|
||||
} else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
|
||||
// if client exists but option changed
|
||||
redisClientSingleton.quit()
|
||||
redisClientSingleton = createClient(option)
|
||||
await redisClientSingleton.connect()
|
||||
redisClientOption = option
|
||||
return redisClientSingleton
|
||||
}
|
||||
return redisClientSingleton
|
||||
}
|
||||
|
||||
export abstract class RedisSearchBase {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
redisClient: ReturnType<typeof createClient>
|
||||
|
||||
protected constructor() {
|
||||
this.type = 'Redis'
|
||||
this.icon = 'redis.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['redisCacheUrlApi', 'redisCacheApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Index Name',
|
||||
name: 'indexName',
|
||||
placeholder: '<VECTOR_INDEX_NAME>',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Replace Index?',
|
||||
name: 'replaceIndex',
|
||||
description: 'Selecting this option will delete the existing index and recreate a new one',
|
||||
default: false,
|
||||
type: 'boolean'
|
||||
},
|
||||
{
|
||||
label: 'Content Field',
|
||||
name: 'contentKey',
|
||||
description: 'Name of the field (column) that contains the actual content',
|
||||
type: 'string',
|
||||
default: 'content',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Metadata Field',
|
||||
name: 'metadataKey',
|
||||
description: 'Name of the field (column) that contains the metadata of the document',
|
||||
type: 'string',
|
||||
default: 'metadata',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Vector Field',
|
||||
name: 'vectorKey',
|
||||
description: 'Name of the field (column) that contains the vector',
|
||||
type: 'string',
|
||||
default: 'content_vector',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Redis Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Redis Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(RedisVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
abstract constructVectorStore(
|
||||
embeddings: Embeddings,
|
||||
indexName: string,
|
||||
replaceIndex: boolean,
|
||||
docs: Document<Record<string, any>>[] | undefined
|
||||
): Promise<VectorStore>
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document<Record<string, any>>[] | undefined): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const indexName = nodeData.inputs?.indexName as string
|
||||
let contentKey = nodeData.inputs?.contentKey as string
|
||||
let metadataKey = nodeData.inputs?.metadataKey as string
|
||||
let vectorKey = nodeData.inputs?.vectorKey as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const replaceIndex = nodeData.inputs?.replaceIndex as boolean
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
let redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
|
||||
if (!redisUrl || redisUrl === '') {
|
||||
const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
|
||||
const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
|
||||
const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
|
||||
const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
|
||||
|
||||
redisUrl = 'redis://' + username + ':' + password + '@' + host + ':' + portStr
|
||||
}
|
||||
|
||||
this.redisClient = await getRedisClient({ url: redisUrl })
|
||||
|
||||
const vectorStore = await this.constructVectorStore(embeddings, indexName, replaceIndex, docs)
|
||||
if (!contentKey || contentKey === '') contentKey = 'content'
|
||||
if (!metadataKey || metadataKey === '') metadataKey = 'metadata'
|
||||
if (!vectorKey || vectorKey === '') vectorKey = 'content_vector'
|
||||
|
||||
const buildQuery = (query: number[], k: number, filter?: string[]): [string, SearchOptions] => {
|
||||
const vectorScoreField = 'vector_score'
|
||||
|
||||
let hybridFields = '*'
|
||||
// if a filter is set, modify the hybrid query
|
||||
if (filter && filter.length) {
|
||||
// `filter` is a list of strings, then it's applied using the OR operator in the metadata key
|
||||
hybridFields = `@${metadataKey}:(${filter.map(escapeSpecialChars).join('|')})`
|
||||
}
|
||||
|
||||
const baseQuery = `${hybridFields} => [KNN ${k} @${vectorKey} $vector AS ${vectorScoreField}]`
|
||||
const returnFields = [metadataKey, contentKey, vectorScoreField]
|
||||
|
||||
const options: SearchOptions = {
|
||||
PARAMS: {
|
||||
vector: Buffer.from(new Float32Array(query).buffer)
|
||||
},
|
||||
RETURN: returnFields,
|
||||
SORTBY: vectorScoreField,
|
||||
DIALECT: 2,
|
||||
LIMIT: {
|
||||
from: 0,
|
||||
size: k
|
||||
}
|
||||
}
|
||||
|
||||
return [baseQuery, options]
|
||||
}
|
||||
|
||||
vectorStore.similaritySearchVectorWithScore = async (
|
||||
query: number[],
|
||||
k: number,
|
||||
filter?: string[]
|
||||
): Promise<[Document, number][]> => {
|
||||
const results = await this.redisClient.ft.search(indexName, ...buildQuery(query, k, filter))
|
||||
const result: [Document, number][] = []
|
||||
|
||||
if (results.total) {
|
||||
for (const res of results.documents) {
|
||||
if (res.value) {
|
||||
const document = res.value
|
||||
if (document.vector_score) {
|
||||
const metadataString = unEscapeSpecialChars(document[metadataKey] as string)
|
||||
result.push([
|
||||
new Document({
|
||||
pageContent: document[contentKey] as string,
|
||||
metadata: JSON.parse(metadataString)
|
||||
}),
|
||||
Number(document.vector_score)
|
||||
])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
if (output === 'retriever') {
|
||||
return vectorStore.asRetriever(k)
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { RedisVectorStore, RedisVectorStoreConfig } from '@langchain/community/vectorstores/redis'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { ICommonObject, INode, INodeData } from '../../../src/Interface'
|
||||
import { RedisSearchBase } from './RedisSearchBase'
|
||||
|
||||
class RedisExisting_VectorStores extends RedisSearchBase implements INode {
|
||||
constructor() {
|
||||
super()
|
||||
this.label = 'Redis Load Existing Index'
|
||||
this.name = 'RedisIndex'
|
||||
this.version = 1.0
|
||||
this.description = 'Load existing index from Redis (i.e: Document has been upserted)'
|
||||
|
||||
// Remove replaceIndex from inputs as it is not applicable while fetching data from Redis
|
||||
let input = this.inputs.find((i) => i.name === 'replaceIndex')
|
||||
if (input) this.inputs.splice(this.inputs.indexOf(input), 1)
|
||||
}
|
||||
|
||||
async constructVectorStore(
|
||||
embeddings: Embeddings,
|
||||
indexName: string,
|
||||
// eslint-disable-next-line unused-imports/no-unused-vars
|
||||
replaceIndex: boolean,
|
||||
_: Document<Record<string, any>>[]
|
||||
): Promise<VectorStore> {
|
||||
const storeConfig: RedisVectorStoreConfig = {
|
||||
redisClient: this.redisClient,
|
||||
indexName: indexName
|
||||
}
|
||||
|
||||
return new RedisVectorStore(embeddings, storeConfig)
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
return super.init(nodeData, _, options, undefined)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: RedisExisting_VectorStores }
|
||||
@@ -1,62 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { flatten } from 'lodash'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { RedisVectorStore, RedisVectorStoreConfig } from '@langchain/community/vectorstores/redis'
|
||||
import { RedisSearchBase } from './RedisSearchBase'
|
||||
import { ICommonObject, INode, INodeData } from '../../../src/Interface'
|
||||
import { escapeAllStrings } from './utils'
|
||||
|
||||
class RedisUpsert_VectorStores extends RedisSearchBase implements INode {
|
||||
constructor() {
|
||||
super()
|
||||
this.label = 'Redis Upsert Document'
|
||||
this.name = 'RedisUpsert'
|
||||
this.version = 1.0
|
||||
this.description = 'Upsert documents to Redis'
|
||||
this.inputs.unshift({
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
})
|
||||
}
|
||||
|
||||
async constructVectorStore(
|
||||
embeddings: Embeddings,
|
||||
indexName: string,
|
||||
replaceIndex: boolean,
|
||||
docs: Document<Record<string, any>>[]
|
||||
): Promise<VectorStore> {
|
||||
const storeConfig: RedisVectorStoreConfig = {
|
||||
redisClient: this.redisClient,
|
||||
indexName: indexName
|
||||
}
|
||||
if (replaceIndex) {
|
||||
let response = await this.redisClient.ft.dropIndex(indexName)
|
||||
if (process.env.DEBUG === 'true') {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Redis Vector Store :: Dropping index [${indexName}], Received Response [${response}]`)
|
||||
}
|
||||
}
|
||||
return await RedisVectorStore.fromDocuments(docs, embeddings, storeConfig)
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
const document = new Document(flattenDocs[i])
|
||||
escapeAllStrings(document.metadata)
|
||||
finalDocs.push(document)
|
||||
}
|
||||
}
|
||||
|
||||
return super.init(nodeData, _, options, finalDocs)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: RedisUpsert_VectorStores }
|
||||
@@ -29,7 +29,6 @@ class SingleStore_VectorStores implements INode {
|
||||
this.description =
|
||||
'Upsert embedded data and perform similarity search upon query using SingleStore, a fast and distributed cloud relational database'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from '@langchain/community/vectorstores/singlestore'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class SingleStoreExisting_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'SingleStore Load Existing Table'
|
||||
this.name = 'singlestoreExisting'
|
||||
this.version = 1.0
|
||||
this.type = 'SingleStore'
|
||||
this.icon = 'singlestore.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing document from SingleStore'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Needed when using SingleStore cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['singleStoreApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Host',
|
||||
name: 'host',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Database',
|
||||
name: 'database',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string',
|
||||
placeholder: 'embeddings',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Content Column Name',
|
||||
name: 'contentColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'content',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Vector Column Name',
|
||||
name: 'vectorColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'vector',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Metadata Column Name',
|
||||
name: 'metadataColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'metadata',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'SingleStore Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'SingleStore Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const user = getCredentialParam('user', credentialData, nodeData)
|
||||
const password = getCredentialParam('password', credentialData, nodeData)
|
||||
|
||||
const singleStoreConnectionConfig = {
|
||||
connectionOptions: {
|
||||
host: nodeData.inputs?.host as string,
|
||||
port: 3306,
|
||||
user,
|
||||
password,
|
||||
database: nodeData.inputs?.database as string
|
||||
},
|
||||
...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
|
||||
...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
|
||||
...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
|
||||
...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
|
||||
} as SingleStoreVectorStoreConfig
|
||||
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
let vectorStore: SingleStoreVectorStore
|
||||
|
||||
vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: SingleStoreExisting_VectorStores }
|
||||
@@ -1,166 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from '@langchain/community/vectorstores/singlestore'
|
||||
import { flatten } from 'lodash'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class SingleStoreUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'SingleStore Upsert Document'
|
||||
this.name = 'singlestoreUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'SingleStore'
|
||||
this.icon = 'singlestore.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to SingleStore'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Needed when using SingleStore cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['singleStoreApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Host',
|
||||
name: 'host',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Database',
|
||||
name: 'database',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string',
|
||||
placeholder: 'embeddings',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Content Column Name',
|
||||
name: 'contentColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'content',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Vector Column Name',
|
||||
name: 'vectorColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'vector',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Metadata Column Name',
|
||||
name: 'metadataColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'metadata',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'SingleStore Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'SingleStore Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const user = getCredentialParam('user', credentialData, nodeData)
|
||||
const password = getCredentialParam('password', credentialData, nodeData)
|
||||
|
||||
const singleStoreConnectionConfig = {
|
||||
connectionOptions: {
|
||||
host: nodeData.inputs?.host as string,
|
||||
port: 3306,
|
||||
user,
|
||||
password,
|
||||
database: nodeData.inputs?.database as string
|
||||
},
|
||||
...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
|
||||
...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
|
||||
...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
|
||||
...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
|
||||
} as SingleStoreVectorStoreConfig
|
||||
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
let vectorStore: SingleStoreVectorStore
|
||||
|
||||
vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
|
||||
vectorStore.addDocuments.bind(vectorStore)(finalDocs)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: SingleStoreUpsert_VectorStores }
|
||||
@@ -32,7 +32,6 @@ class Supabase_VectorStores implements INode {
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert embedded data and perform similarity or mmr search upon query using Supabase via pgvector extension'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { SupabaseLibArgs, SupabaseVectorStore } from '@langchain/community/vectorstores/supabase'
|
||||
import { createClient } from '@supabase/supabase-js'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class Supabase_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Supabase Load Existing Index'
|
||||
this.name = 'supabaseExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Supabase'
|
||||
this.icon = 'supabase.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Supabase (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['supabaseApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Supabase Project URL',
|
||||
name: 'supabaseProjUrl',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Query Name',
|
||||
name: 'queryName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Supabase Metadata Filter',
|
||||
name: 'supabaseMetadataFilter',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Supabase Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Supabase Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(SupabaseVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string
|
||||
const tableName = nodeData.inputs?.tableName as string
|
||||
const queryName = nodeData.inputs?.queryName as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const supabaseMetadataFilter = nodeData.inputs?.supabaseMetadataFilter
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData)
|
||||
|
||||
const client = createClient(supabaseProjUrl, supabaseApiKey)
|
||||
|
||||
const obj: SupabaseLibArgs = {
|
||||
client,
|
||||
tableName,
|
||||
queryName
|
||||
}
|
||||
|
||||
if (supabaseMetadataFilter) {
|
||||
const metadatafilter = typeof supabaseMetadataFilter === 'object' ? supabaseMetadataFilter : JSON.parse(supabaseMetadataFilter)
|
||||
obj.filter = metadatafilter
|
||||
}
|
||||
|
||||
const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
if (supabaseMetadataFilter) {
|
||||
;(vectorStore as any).filter = obj.filter
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Supabase_Existing_VectorStores }
|
||||
@@ -1,157 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { SupabaseVectorStore } from '@langchain/community/vectorstores/supabase'
|
||||
import { createClient } from '@supabase/supabase-js'
|
||||
import { flatten } from 'lodash'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
|
||||
class SupabaseUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Supabase Upsert Document'
|
||||
this.name = 'supabaseUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Supabase'
|
||||
this.icon = 'supabase.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Supabase'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['supabaseApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Supabase Project URL',
|
||||
name: 'supabaseProjUrl',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Query Name',
|
||||
name: 'queryName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Supabase Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Supabase Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(SupabaseVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string
|
||||
const tableName = nodeData.inputs?.tableName as string
|
||||
const queryName = nodeData.inputs?.queryName as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData)
|
||||
|
||||
const client = createClient(supabaseProjUrl, supabaseApiKey)
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
|
||||
const vectorStore = await SupabaseUpsertVectorStore.fromDocuments(finalDocs, embeddings, {
|
||||
client,
|
||||
tableName: tableName,
|
||||
queryName: queryName
|
||||
})
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
class SupabaseUpsertVectorStore extends SupabaseVectorStore {
|
||||
async addVectors(vectors: number[][], documents: Document[]): Promise<string[]> {
|
||||
if (vectors.length === 0) {
|
||||
return []
|
||||
}
|
||||
const rows = vectors.map((embedding, idx) => ({
|
||||
content: documents[idx].pageContent,
|
||||
embedding,
|
||||
metadata: documents[idx].metadata
|
||||
}))
|
||||
|
||||
let returnedIds: string[] = []
|
||||
for (let i = 0; i < rows.length; i += this.upsertBatchSize) {
|
||||
const chunk = rows.slice(i, i + this.upsertBatchSize).map((row, index) => {
|
||||
return { id: index, ...row }
|
||||
})
|
||||
|
||||
const res = await this.client.from(this.tableName).upsert(chunk).select()
|
||||
if (res.error) {
|
||||
throw new Error(`Error inserting: ${res.error.message} ${res.status} ${res.statusText}`)
|
||||
}
|
||||
if (res.data) {
|
||||
returnedIds = returnedIds.concat(res.data.map((row) => row.id))
|
||||
}
|
||||
}
|
||||
return returnedIds
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: SupabaseUpsert_VectorStores }
|
||||
@@ -36,7 +36,6 @@ class Upstash_VectorStores implements INode {
|
||||
this.description =
|
||||
'Upsert data as embedding or string and perform similarity search with Upstash, the leading serverless data platform'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -36,7 +36,6 @@ class Vectara_VectorStores implements INode {
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert embedded data and perform similarity search upon query using Vectara, a LLM-powered search-as-a-service'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from '@langchain/community/vectorstores/vectara'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
|
||||
class VectaraExisting_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Vectara Load Existing Index'
|
||||
this.name = 'vectaraExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Vectara'
|
||||
this.icon = 'vectara.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Vectara (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['vectaraApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Metadata Filter',
|
||||
name: 'filter',
|
||||
description:
|
||||
'Filter to apply to Vectara metadata. Refer to the <a target="_blank" href="https://docs.flowiseai.com/vector-stores/vectara">documentation</a> on how to use Vectara filters with Flowise.',
|
||||
type: 'string',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences Before',
|
||||
name: 'sentencesBefore',
|
||||
description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences After',
|
||||
name: 'sentencesAfter',
|
||||
description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Lambda',
|
||||
name: 'lambda',
|
||||
description:
|
||||
'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Defaults to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Vectara Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Vectara Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
const customerId = getCredentialParam('customerID', credentialData, nodeData)
|
||||
const corpusId = getCredentialParam('corpusID', credentialData, nodeData).split(',')
|
||||
|
||||
const vectaraMetadataFilter = nodeData.inputs?.filter as string
|
||||
const sentencesBefore = nodeData.inputs?.sentencesBefore as number
|
||||
const sentencesAfter = nodeData.inputs?.sentencesAfter as number
|
||||
const lambda = nodeData.inputs?.lambda as number
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseInt(topK, 10) : 4
|
||||
|
||||
const vectaraArgs: VectaraLibArgs = {
|
||||
apiKey: apiKey,
|
||||
customerId: customerId,
|
||||
corpusId: corpusId,
|
||||
source: 'flowise'
|
||||
}
|
||||
|
||||
const vectaraFilter: VectaraFilter = {}
|
||||
if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
|
||||
if (lambda) vectaraFilter.lambda = lambda
|
||||
|
||||
const vectaraContextConfig: VectaraContextConfig = {}
|
||||
if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
|
||||
if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
|
||||
vectaraFilter.contextConfig = vectaraContextConfig
|
||||
|
||||
const vectorStore = new VectaraStore(vectaraArgs)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k, vectaraFilter)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
if (vectaraMetadataFilter) {
|
||||
;(vectorStore as any).filter = vectaraFilter.filter
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: VectaraExisting_VectorStores }
|
||||
@@ -1,196 +0,0 @@
|
||||
import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig, VectaraFile } from '@langchain/community/vectorstores/vectara'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { getFileFromStorage } from '../../../src'
|
||||
|
||||
class VectaraUpload_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Vectara Upload File'
|
||||
this.name = 'vectaraUpload'
|
||||
this.version = 1.0
|
||||
this.type = 'Vectara'
|
||||
this.icon = 'vectara.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upload files to Vectara'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['vectaraApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'File',
|
||||
name: 'file',
|
||||
description:
|
||||
'File to upload to Vectara. Supported file types: https://docs.vectara.com/docs/api-reference/indexing-apis/file-upload/file-upload-filetypes',
|
||||
type: 'file'
|
||||
},
|
||||
{
|
||||
label: 'Metadata Filter',
|
||||
name: 'filter',
|
||||
description:
|
||||
'Filter to apply to Vectara metadata. Refer to the <a target="_blank" href="https://docs.flowiseai.com/vector-stores/vectara">documentation</a> on how to use Vectara filters with Flowise.',
|
||||
type: 'string',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences Before',
|
||||
name: 'sentencesBefore',
|
||||
description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences After',
|
||||
name: 'sentencesAfter',
|
||||
description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Lambda',
|
||||
name: 'lambda',
|
||||
description:
|
||||
'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Defaults to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Vectara Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Vectara Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
const customerId = getCredentialParam('customerID', credentialData, nodeData)
|
||||
const corpusId = getCredentialParam('corpusID', credentialData, nodeData).split(',')
|
||||
|
||||
const fileBase64 = nodeData.inputs?.file
|
||||
const vectaraMetadataFilter = nodeData.inputs?.filter as string
|
||||
const sentencesBefore = nodeData.inputs?.sentencesBefore as number
|
||||
const sentencesAfter = nodeData.inputs?.sentencesAfter as number
|
||||
const lambda = nodeData.inputs?.lambda as number
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseInt(topK, 10) : 4
|
||||
|
||||
const vectaraArgs: VectaraLibArgs = {
|
||||
apiKey: apiKey,
|
||||
customerId: customerId,
|
||||
corpusId: corpusId,
|
||||
source: 'flowise'
|
||||
}
|
||||
|
||||
const vectaraFilter: VectaraFilter = {}
|
||||
if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
|
||||
if (lambda) vectaraFilter.lambda = lambda
|
||||
|
||||
const vectaraContextConfig: VectaraContextConfig = {}
|
||||
if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
|
||||
if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
|
||||
vectaraFilter.contextConfig = vectaraContextConfig
|
||||
|
||||
let files: string[] = []
|
||||
const vectaraFiles: VectaraFile[] = []
|
||||
|
||||
if (fileBase64.startsWith('FILE-STORAGE::')) {
|
||||
const fileName = fileBase64.replace('FILE-STORAGE::', '')
|
||||
if (fileName.startsWith('[') && fileName.endsWith(']')) {
|
||||
files = JSON.parse(fileName)
|
||||
} else {
|
||||
files = [fileName]
|
||||
}
|
||||
const chatflowid = options.chatflowid
|
||||
|
||||
for (const file of files) {
|
||||
const fileData = await getFileFromStorage(file, chatflowid)
|
||||
const blob = new Blob([fileData])
|
||||
vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
|
||||
}
|
||||
} else {
|
||||
if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
|
||||
files = JSON.parse(fileBase64)
|
||||
} else {
|
||||
files = [fileBase64]
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const splitDataURI = file.split(',')
|
||||
splitDataURI.pop()
|
||||
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
|
||||
const blob = new Blob([bf])
|
||||
vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
|
||||
}
|
||||
}
|
||||
|
||||
const vectorStore = new VectaraStore(vectaraArgs)
|
||||
await vectorStore.addFiles(vectaraFiles)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k, vectaraFilter)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
const getFileName = (fileBase64: string) => {
|
||||
let fileNames = []
|
||||
if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
|
||||
const files = JSON.parse(fileBase64)
|
||||
for (const file of files) {
|
||||
const splitDataURI = file.split(',')
|
||||
const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
|
||||
fileNames.push(filename)
|
||||
}
|
||||
return fileNames.join(', ')
|
||||
} else {
|
||||
const splitDataURI = fileBase64.split(',')
|
||||
const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
|
||||
return filename
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: VectaraUpload_VectorStores }
|
||||
@@ -1,155 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from '@langchain/community/vectorstores/vectara'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { flatten } from 'lodash'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
|
||||
class VectaraUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Vectara Upsert Document'
|
||||
this.name = 'vectaraUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Vectara'
|
||||
this.icon = 'vectara.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Vectara'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['vectaraApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Metadata Filter',
|
||||
name: 'filter',
|
||||
description:
|
||||
'Filter to apply to Vectara metadata. Refer to the <a target="_blank" href="https://docs.flowiseai.com/vector-stores/vectara">documentation</a> on how to use Vectara filters with Flowise.',
|
||||
type: 'string',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences Before',
|
||||
name: 'sentencesBefore',
|
||||
description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences After',
|
||||
name: 'sentencesAfter',
|
||||
description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Lambda',
|
||||
name: 'lambda',
|
||||
description:
|
||||
'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Defaults to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Vectara Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Vectara Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
const customerId = getCredentialParam('customerID', credentialData, nodeData)
|
||||
const corpusId = getCredentialParam('corpusID', credentialData, nodeData).split(',')
|
||||
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = {} as Embeddings
|
||||
const vectaraMetadataFilter = nodeData.inputs?.filter as string
|
||||
const sentencesBefore = nodeData.inputs?.sentencesBefore as number
|
||||
const sentencesAfter = nodeData.inputs?.sentencesAfter as number
|
||||
const lambda = nodeData.inputs?.lambda as number
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseInt(topK, 10) : 4
|
||||
|
||||
const vectaraArgs: VectaraLibArgs = {
|
||||
apiKey: apiKey,
|
||||
customerId: customerId,
|
||||
corpusId: corpusId,
|
||||
source: 'flowise'
|
||||
}
|
||||
|
||||
const vectaraFilter: VectaraFilter = {}
|
||||
if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
|
||||
if (lambda) vectaraFilter.lambda = lambda
|
||||
|
||||
const vectaraContextConfig: VectaraContextConfig = {}
|
||||
if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
|
||||
if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
|
||||
vectaraFilter.contextConfig = vectaraContextConfig
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const vectorStore = await VectaraStore.fromDocuments(finalDocs, embeddings, vectaraArgs)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k, vectaraFilter)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: VectaraUpsert_VectorStores }
|
||||
@@ -32,7 +32,6 @@ class Weaviate_VectorStores implements INode {
|
||||
this.description =
|
||||
'Upsert embedded data and perform similarity or mmr search using Weaviate, a scalable open-source vector database'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client'
|
||||
import { WeaviateLibArgs, WeaviateStore } from '@langchain/community/vectorstores/weaviate'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
|
||||
class Weaviate_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Weaviate Load Existing Index'
|
||||
this.name = 'weaviateExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Weaviate'
|
||||
this.icon = 'weaviate.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Weaviate (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed when using Weaviate cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['weaviateApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Scheme',
|
||||
name: 'weaviateScheme',
|
||||
type: 'options',
|
||||
default: 'https',
|
||||
options: [
|
||||
{
|
||||
label: 'https',
|
||||
name: 'https'
|
||||
},
|
||||
{
|
||||
label: 'http',
|
||||
name: 'http'
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Host',
|
||||
name: 'weaviateHost',
|
||||
type: 'string',
|
||||
placeholder: 'localhost:8080'
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Index',
|
||||
name: 'weaviateIndex',
|
||||
type: 'string',
|
||||
placeholder: 'Test'
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Text Key',
|
||||
name: 'weaviateTextKey',
|
||||
type: 'string',
|
||||
placeholder: 'text',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Metadata Keys',
|
||||
name: 'weaviateMetadataKeys',
|
||||
type: 'string',
|
||||
rows: 4,
|
||||
placeholder: `["foo"]`,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Weaviate Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(WeaviateStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const weaviateScheme = nodeData.inputs?.weaviateScheme as string
|
||||
const weaviateHost = nodeData.inputs?.weaviateHost as string
|
||||
const weaviateIndex = nodeData.inputs?.weaviateIndex as string
|
||||
const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string
|
||||
const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
|
||||
|
||||
const clientConfig: any = {
|
||||
scheme: weaviateScheme,
|
||||
host: weaviateHost
|
||||
}
|
||||
if (weaviateApiKey) clientConfig.apiKey = new ApiKey(weaviateApiKey)
|
||||
|
||||
const client: WeaviateClient = weaviate.client(clientConfig)
|
||||
|
||||
const obj: WeaviateLibArgs = {
|
||||
client,
|
||||
indexName: weaviateIndex
|
||||
}
|
||||
|
||||
if (weaviateTextKey) obj.textKey = weaviateTextKey
|
||||
if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
|
||||
|
||||
const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Weaviate_Existing_VectorStores }
|
||||
@@ -1,174 +0,0 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { WeaviateLibArgs, WeaviateStore } from '@langchain/community/vectorstores/weaviate'
|
||||
import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client'
|
||||
import { flatten } from 'lodash'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class WeaviateUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Weaviate Upsert Document'
|
||||
this.name = 'weaviateUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Weaviate'
|
||||
this.icon = 'weaviate.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Weaviate'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed when using Weaviate cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['weaviateApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Scheme',
|
||||
name: 'weaviateScheme',
|
||||
type: 'options',
|
||||
default: 'https',
|
||||
options: [
|
||||
{
|
||||
label: 'https',
|
||||
name: 'https'
|
||||
},
|
||||
{
|
||||
label: 'http',
|
||||
name: 'http'
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Host',
|
||||
name: 'weaviateHost',
|
||||
type: 'string',
|
||||
placeholder: 'localhost:8080'
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Index',
|
||||
name: 'weaviateIndex',
|
||||
type: 'string',
|
||||
placeholder: 'Test'
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Text Key',
|
||||
name: 'weaviateTextKey',
|
||||
type: 'string',
|
||||
placeholder: 'text',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Metadata Keys',
|
||||
name: 'weaviateMetadataKeys',
|
||||
type: 'string',
|
||||
rows: 4,
|
||||
placeholder: `["foo"]`,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Weaviate Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(WeaviateStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const weaviateScheme = nodeData.inputs?.weaviateScheme as string
|
||||
const weaviateHost = nodeData.inputs?.weaviateHost as string
|
||||
const weaviateIndex = nodeData.inputs?.weaviateIndex as string
|
||||
const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string
|
||||
const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
|
||||
|
||||
const clientConfig: any = {
|
||||
scheme: weaviateScheme,
|
||||
host: weaviateHost
|
||||
}
|
||||
if (weaviateApiKey) clientConfig.apiKey = new ApiKey(weaviateApiKey)
|
||||
|
||||
const client: WeaviateClient = weaviate.client(clientConfig)
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const obj: WeaviateLibArgs = {
|
||||
client,
|
||||
indexName: weaviateIndex
|
||||
}
|
||||
|
||||
if (weaviateTextKey) obj.textKey = weaviateTextKey
|
||||
if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
|
||||
|
||||
const vectorStore = await WeaviateStore.fromDocuments(finalDocs, embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: WeaviateUpsert_VectorStores }
|
||||
@@ -31,7 +31,6 @@ class Zep_VectorStores implements INode {
|
||||
this.description =
|
||||
'Upsert embedded data and perform similarity or mmr search upon query using Zep, a fast and scalable building block for LLM apps'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
@@ -1,240 +0,0 @@
|
||||
import { IDocument, ZepClient } from '@getzep/zep-js'
|
||||
import { ZepVectorStore, IZepConfig } from '@langchain/community/vectorstores/zep'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class Zep_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Zep Load Existing Index - Open Source'
|
||||
this.name = 'zepExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Zep'
|
||||
this.icon = 'zep.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Zep (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
optional: true,
|
||||
description: 'Configure JWT authentication on your Zep instance (Optional)',
|
||||
credentialNames: ['zepMemoryApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Base URL',
|
||||
name: 'baseURL',
|
||||
type: 'string',
|
||||
default: 'http://127.0.0.1:8000'
|
||||
},
|
||||
{
|
||||
label: 'Zep Collection',
|
||||
name: 'zepCollection',
|
||||
type: 'string',
|
||||
placeholder: 'my-first-collection'
|
||||
},
|
||||
{
|
||||
label: 'Zep Metadata Filter',
|
||||
name: 'zepMetadataFilter',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Embedding Dimension',
|
||||
name: 'dimension',
|
||||
type: 'number',
|
||||
default: 1536,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Zep Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Zep Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(ZepVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const baseURL = nodeData.inputs?.baseURL as string
|
||||
const zepCollection = nodeData.inputs?.zepCollection as string
|
||||
const zepMetadataFilter = nodeData.inputs?.zepMetadataFilter
|
||||
const dimension = nodeData.inputs?.dimension as number
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
|
||||
const zepConfig: IZepConfig & Partial<ZepFilter> = {
|
||||
apiUrl: baseURL,
|
||||
collectionName: zepCollection,
|
||||
embeddingDimensions: dimension,
|
||||
isAutoEmbedded: false
|
||||
}
|
||||
if (apiKey) zepConfig.apiKey = apiKey
|
||||
if (zepMetadataFilter) {
|
||||
const metadatafilter = typeof zepMetadataFilter === 'object' ? zepMetadataFilter : JSON.parse(zepMetadataFilter)
|
||||
zepConfig.filter = metadatafilter
|
||||
}
|
||||
|
||||
const vectorStore = await ZepExistingVS.fromExistingIndex(embeddings, zepConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
if (zepMetadataFilter) {
|
||||
;(vectorStore as any).filter = zepConfig.filter
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
interface ZepFilter {
|
||||
filter: Record<string, any>
|
||||
}
|
||||
|
||||
function zepDocsToDocumentsAndScore(results: IDocument[]): [Document, number][] {
|
||||
return results.map((d) => [
|
||||
new Document({
|
||||
pageContent: d.content,
|
||||
metadata: d.metadata
|
||||
}),
|
||||
d.score ? d.score : 0
|
||||
])
|
||||
}
|
||||
|
||||
function assignMetadata(value: string | Record<string, unknown> | object | undefined): Record<string, unknown> | undefined {
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
return value as Record<string, unknown>
|
||||
}
|
||||
if (value !== undefined) {
|
||||
console.warn('Metadata filters must be an object, Record, or undefined.')
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
class ZepExistingVS extends ZepVectorStore {
|
||||
filter?: Record<string, any>
|
||||
args?: IZepConfig & Partial<ZepFilter>
|
||||
|
||||
constructor(embeddings: Embeddings, args: IZepConfig & Partial<ZepFilter>) {
|
||||
super(embeddings, args)
|
||||
this.filter = args.filter
|
||||
this.args = args
|
||||
}
|
||||
|
||||
async initalizeCollection(args: IZepConfig & Partial<ZepFilter>) {
|
||||
this.client = await ZepClient.init(args.apiUrl, args.apiKey)
|
||||
try {
|
||||
this.collection = await this.client.document.getCollection(args.collectionName)
|
||||
} catch (err) {
|
||||
if (err instanceof Error) {
|
||||
if (err.name === 'NotFoundError') {
|
||||
await this.createNewCollection(args)
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async createNewCollection(args: IZepConfig & Partial<ZepFilter>) {
|
||||
if (!args.embeddingDimensions) {
|
||||
throw new Error(
|
||||
`Collection ${args.collectionName} not found. You can create a new Collection by providing embeddingDimensions.`
|
||||
)
|
||||
}
|
||||
|
||||
this.collection = await this.client.document.addCollection({
|
||||
name: args.collectionName,
|
||||
description: args.description,
|
||||
metadata: args.metadata,
|
||||
embeddingDimensions: args.embeddingDimensions,
|
||||
isAutoEmbedded: false
|
||||
})
|
||||
}
|
||||
|
||||
async similaritySearchVectorWithScore(
|
||||
query: number[],
|
||||
k: number,
|
||||
filter?: Record<string, unknown> | undefined
|
||||
): Promise<[Document, number][]> {
|
||||
if (filter && this.filter) {
|
||||
throw new Error('cannot provide both `filter` and `this.filter`')
|
||||
}
|
||||
const _filters = filter ?? this.filter
|
||||
const ANDFilters = []
|
||||
for (const filterKey in _filters) {
|
||||
let filterVal = _filters[filterKey]
|
||||
if (typeof filterVal === 'string') filterVal = `"${filterVal}"`
|
||||
ANDFilters.push({ jsonpath: `$[*] ? (@.${filterKey} == ${filterVal})` })
|
||||
}
|
||||
const newfilter = {
|
||||
where: { and: ANDFilters }
|
||||
}
|
||||
await this.initalizeCollection(this.args!).catch((err) => {
|
||||
console.error('Error initializing collection:', err)
|
||||
throw err
|
||||
})
|
||||
const results = await this.collection.search(
|
||||
{
|
||||
embedding: new Float32Array(query),
|
||||
metadata: assignMetadata(newfilter)
|
||||
},
|
||||
k
|
||||
)
|
||||
return zepDocsToDocumentsAndScore(results)
|
||||
}
|
||||
|
||||
static async fromExistingIndex(embeddings: Embeddings, dbConfig: IZepConfig & Partial<ZepFilter>): Promise<ZepVectorStore> {
|
||||
const instance = new this(embeddings, dbConfig)
|
||||
return instance
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Zep_Existing_VectorStores }
|
||||
@@ -1,137 +0,0 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { ZepVectorStore, IZepConfig } from '@langchain/community/vectorstores/zep'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
class Zep_Upsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Zep Upsert Document - Open Source'
|
||||
this.name = 'zepUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Zep'
|
||||
this.icon = 'zep.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Zep'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'DEPRECATING'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
optional: true,
|
||||
description: 'Configure JWT authentication on your Zep instance (Optional)',
|
||||
credentialNames: ['zepMemoryApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Base URL',
|
||||
name: 'baseURL',
|
||||
type: 'string',
|
||||
default: 'http://127.0.0.1:8000'
|
||||
},
|
||||
{
|
||||
label: 'Zep Collection',
|
||||
name: 'zepCollection',
|
||||
type: 'string',
|
||||
placeholder: 'my-first-collection'
|
||||
},
|
||||
{
|
||||
label: 'Embedding Dimension',
|
||||
name: 'dimension',
|
||||
type: 'number',
|
||||
default: 1536,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Zep Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Zep Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(ZepVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const baseURL = nodeData.inputs?.baseURL as string
|
||||
const zepCollection = nodeData.inputs?.zepCollection as string
|
||||
const dimension = (nodeData.inputs?.dimension as number) ?? 1536
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
}
|
||||
|
||||
const zepConfig: IZepConfig = {
|
||||
apiUrl: baseURL,
|
||||
collectionName: zepCollection,
|
||||
embeddingDimensions: dimension,
|
||||
isAutoEmbedded: false
|
||||
}
|
||||
if (apiKey) zepConfig.apiKey = apiKey
|
||||
|
||||
const vectorStore = await ZepVectorStore.fromDocuments(finalDocs, embeddings, zepConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Zep_Upsert_VectorStores }
|
||||
@@ -32,7 +32,6 @@ class Zep_CloudVectorStores implements INode {
|
||||
this.description =
|
||||
'Upsert embedded data and perform similarity or mmr search upon query using Zep, a fast and scalable building block for LLM apps'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.badge = 'NEW'
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
|
||||
Reference in New Issue
Block a user