mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 15:00:57 +03:00
Merge branch 'main' into feature/Milvus
# Conflicts: # packages/components/package.json
This commit is contained in:
Henry
+34
-5
@@ -1,27 +1,39 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Chroma } from 'langchain/vectorstores/chroma'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ChromaExtended } from './core'
|
||||
|
||||
class Chroma_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Chroma Load Existing Index'
|
||||
this.name = 'chromaExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Chroma'
|
||||
this.icon = 'chroma.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Chroma (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed if you have chroma on cloud services with X-Api-key',
|
||||
optional: true,
|
||||
credentialNames: ['chromaApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
@@ -38,6 +50,15 @@ class Chroma_Existing_VectorStores implements INode {
|
||||
name: 'chromaURL',
|
||||
type: 'string',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
@@ -54,24 +75,32 @@ class Chroma_Existing_VectorStores implements INode {
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const collectionName = nodeData.inputs?.collectionName as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const chromaURL = nodeData.inputs?.chromaURL as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
|
||||
|
||||
const obj: {
|
||||
collectionName: string
|
||||
url?: string
|
||||
chromaApiKey?: string
|
||||
} = { collectionName }
|
||||
if (chromaURL) obj.url = chromaURL
|
||||
if (chromaApiKey) obj.chromaApiKey = chromaApiKey
|
||||
|
||||
const vectorStore = await Chroma.fromExistingCollection(embeddings, obj)
|
||||
const vectorStore = await ChromaExtended.fromExistingCollection(embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever()
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
+36
-6
@@ -1,28 +1,41 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Chroma } from 'langchain/vectorstores/chroma'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { flatten } from 'lodash'
|
||||
import { ChromaExtended } from './core'
|
||||
|
||||
class ChromaUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Chroma Upsert Document'
|
||||
this.name = 'chromaUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Chroma'
|
||||
this.icon = 'chroma.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Chroma'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed if you have chroma on cloud services with X-Api-key',
|
||||
optional: true,
|
||||
credentialNames: ['chromaApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
@@ -45,6 +58,15 @@ class ChromaUpsert_VectorStores implements INode {
|
||||
name: 'chromaURL',
|
||||
type: 'string',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
@@ -61,14 +83,19 @@ class ChromaUpsert_VectorStores implements INode {
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const collectionName = nodeData.inputs?.collectionName as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const chromaURL = nodeData.inputs?.chromaURL as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const flattenDocs = docs && docs.length ? docs.flat() : []
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
@@ -77,15 +104,18 @@ class ChromaUpsert_VectorStores implements INode {
|
||||
const obj: {
|
||||
collectionName: string
|
||||
url?: string
|
||||
chromaApiKey?: string
|
||||
} = { collectionName }
|
||||
if (chromaURL) obj.url = chromaURL
|
||||
if (chromaApiKey) obj.chromaApiKey = chromaApiKey
|
||||
|
||||
const vectorStore = await Chroma.fromDocuments(finalDocs, embeddings, obj)
|
||||
const vectorStore = await ChromaExtended.fromDocuments(finalDocs, embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever()
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
|
Before Width: | Height: | Size: 622 B After Width: | Height: | Size: 622 B |
@@ -0,0 +1,49 @@
|
||||
import { Chroma, ChromaLibArgs } from 'langchain/vectorstores/chroma'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import type { Collection } from 'chromadb'
|
||||
|
||||
interface ChromaAuth {
|
||||
chromaApiKey?: string
|
||||
}
|
||||
|
||||
export class ChromaExtended extends Chroma {
|
||||
chromaApiKey?: string
|
||||
|
||||
constructor(embeddings: Embeddings, args: ChromaLibArgs & Partial<ChromaAuth>) {
|
||||
super(embeddings, args)
|
||||
this.chromaApiKey = args.chromaApiKey
|
||||
}
|
||||
|
||||
static async fromExistingCollection(embeddings: Embeddings, dbConfig: ChromaLibArgs & Partial<ChromaAuth>): Promise<Chroma> {
|
||||
const instance = new this(embeddings, dbConfig)
|
||||
await instance.ensureCollection()
|
||||
return instance
|
||||
}
|
||||
|
||||
async ensureCollection(): Promise<Collection> {
|
||||
if (!this.collection) {
|
||||
if (!this.index) {
|
||||
const { ChromaClient } = await Chroma.imports()
|
||||
const obj: any = {
|
||||
path: this.url
|
||||
}
|
||||
if (this.chromaApiKey) {
|
||||
obj.fetchOptions = {
|
||||
headers: {
|
||||
'X-Api-Key': this.chromaApiKey
|
||||
}
|
||||
}
|
||||
}
|
||||
this.index = new ChromaClient(obj)
|
||||
}
|
||||
try {
|
||||
this.collection = await this.index.getOrCreateCollection({
|
||||
name: this.collectionName
|
||||
})
|
||||
} catch (err) {
|
||||
throw new Error(`Chroma getOrCreateCollection error: ${err}`)
|
||||
}
|
||||
}
|
||||
return this.collection
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
<svg width="209" height="135" viewBox="0 0 209 135" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<ellipse cx="136.019" cy="67.2304" rx="66.6667" ry="64" fill="#FFDE2D"/>
|
||||
<ellipse cx="69.352" cy="67.2304" rx="66.6667" ry="64" fill="#327EFF"/>
|
||||
<path d="M2.68528 67.2304C2.68527 31.8842 32.5329 3.23047 69.3519 3.23047L69.3519 67.2304L2.68528 67.2304Z" fill="#327EFF"/>
|
||||
<path d="M136.019 67.2305C136.019 102.577 106.171 131.23 69.3519 131.23L69.3519 67.2305L136.019 67.2305Z" fill="#FF6446"/>
|
||||
<path d="M69.352 67.2304C69.352 31.8842 99.1997 3.23047 136.019 3.23047L136.019 67.2304L69.352 67.2304Z" fill="#FF6446"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 622 B |
@@ -0,0 +1,84 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { FaissStore } from 'langchain/vectorstores/faiss'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
|
||||
class Faiss_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Faiss Load Existing Index'
|
||||
this.name = 'faissExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Faiss'
|
||||
this.icon = 'faiss.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Faiss (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Base Path to load',
|
||||
name: 'basePath',
|
||||
description: 'Path to load faiss.index file',
|
||||
placeholder: `C:\\Users\\User\\Desktop`,
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Faiss Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Faiss Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(FaissStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const basePath = nodeData.inputs?.basePath as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const vectorStore = await FaissStore.load(basePath, embeddings)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Faiss_Existing_VectorStores }
|
||||
@@ -0,0 +1,10 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-chart-dots-3" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||||
<path d="M5 7m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0"></path>
|
||||
<path d="M16 15m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0"></path>
|
||||
<path d="M18 6m-3 0a3 3 0 1 0 6 0a3 3 0 1 0 -6 0"></path>
|
||||
<path d="M6 18m-3 0a3 3 0 1 0 6 0a3 3 0 1 0 -6 0"></path>
|
||||
<path d="M9 17l5 -1.5"></path>
|
||||
<path d="M6.5 8.5l7.81 5.37"></path>
|
||||
<path d="M7 7l8 -1"></path>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 648 B |
@@ -0,0 +1,100 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { FaissStore } from 'langchain/vectorstores/faiss'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class FaissUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Faiss Upsert Document'
|
||||
this.name = 'faissUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Faiss'
|
||||
this.icon = 'faiss.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Faiss'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Base Path to store',
|
||||
name: 'basePath',
|
||||
description: 'Path to store faiss.index file',
|
||||
placeholder: `C:\\Users\\User\\Desktop`,
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Faiss Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Faiss Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(FaissStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const basePath = nodeData.inputs?.basePath as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
|
||||
const vectorStore = await FaissStore.fromDocuments(finalDocs, embeddings)
|
||||
await vectorStore.save(basePath)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: FaissUpsert_VectorStores }
|
||||
@@ -0,0 +1,10 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-chart-dots-3" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||||
<path d="M5 7m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0"></path>
|
||||
<path d="M16 15m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0"></path>
|
||||
<path d="M18 6m-3 0a3 3 0 1 0 6 0a3 3 0 1 0 -6 0"></path>
|
||||
<path d="M6 18m-3 0a3 3 0 1 0 6 0a3 3 0 1 0 -6 0"></path>
|
||||
<path d="M9 17l5 -1.5"></path>
|
||||
<path d="M6.5 8.5l7.81 5.37"></path>
|
||||
<path d="M7 7l8 -1"></path>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 648 B |
@@ -3,10 +3,12 @@ import { MemoryVectorStore } from 'langchain/vectorstores/memory'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class InMemoryVectorStore_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
@@ -18,6 +20,7 @@ class InMemoryVectorStore_VectorStores implements INode {
|
||||
constructor() {
|
||||
this.label = 'In-Memory Vector Store'
|
||||
this.name = 'memoryVectorStore'
|
||||
this.version = 1.0
|
||||
this.type = 'Memory'
|
||||
this.icon = 'memory.svg'
|
||||
this.category = 'Vector Stores'
|
||||
@@ -34,6 +37,14 @@ class InMemoryVectorStore_VectorStores implements INode {
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
@@ -54,8 +65,10 @@ class InMemoryVectorStore_VectorStores implements INode {
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const flattenDocs = docs && docs.length ? docs.flat() : []
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
@@ -64,9 +77,10 @@ class InMemoryVectorStore_VectorStores implements INode {
|
||||
const vectorStore = await MemoryVectorStore.fromDocuments(finalDocs, embeddings)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever()
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Client } from '@opensearch-project/opensearch'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
|
||||
class OpenSearch_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'OpenSearch Load Existing Index'
|
||||
this.name = 'openSearchExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'OpenSearch'
|
||||
this.icon = 'opensearch.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from OpenSearch (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'OpenSearch URL',
|
||||
name: 'opensearchURL',
|
||||
type: 'string',
|
||||
placeholder: 'http://127.0.0.1:9200'
|
||||
},
|
||||
{
|
||||
label: 'Index Name',
|
||||
name: 'indexName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'OpenSearch Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'OpenSearch Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(OpenSearchVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const opensearchURL = nodeData.inputs?.opensearchURL as string
|
||||
const indexName = nodeData.inputs?.indexName as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const client = new Client({
|
||||
nodes: [opensearchURL]
|
||||
})
|
||||
|
||||
const vectorStore = new OpenSearchVectorStore(embeddings, {
|
||||
client,
|
||||
indexName
|
||||
})
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: OpenSearch_Existing_VectorStores }
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 5.1 KiB |
@@ -0,0 +1,112 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { Client } from '@opensearch-project/opensearch'
|
||||
import { flatten } from 'lodash'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
|
||||
class OpenSearchUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'OpenSearch Upsert Document'
|
||||
this.name = 'openSearchUpsertDocument'
|
||||
this.version = 1.0
|
||||
this.type = 'OpenSearch'
|
||||
this.icon = 'opensearch.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to OpenSearch'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'OpenSearch URL',
|
||||
name: 'opensearchURL',
|
||||
type: 'string',
|
||||
placeholder: 'http://127.0.0.1:9200'
|
||||
},
|
||||
{
|
||||
label: 'Index Name',
|
||||
name: 'indexName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'OpenSearch Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'OpenSearch Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(OpenSearchVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const opensearchURL = nodeData.inputs?.opensearchURL as string
|
||||
const indexName = nodeData.inputs?.indexName as string
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
|
||||
const client = new Client({
|
||||
nodes: [opensearchURL]
|
||||
})
|
||||
|
||||
const vectorStore = await OpenSearchVectorStore.fromDocuments(finalDocs, embeddings, {
|
||||
client,
|
||||
indexName: indexName
|
||||
})
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: OpenSearchUpsert_VectorStores }
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 5.1 KiB |
@@ -1,44 +1,43 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { PineconeClient } from '@pinecone-database/pinecone'
|
||||
import { PineconeLibArgs, PineconeStore } from 'langchain/vectorstores/pinecone'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
|
||||
class Pinecone_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Pinecone Load Existing Index'
|
||||
this.name = 'pineconeExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Pinecone'
|
||||
this.icon = 'pinecone.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Pinecone (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['pineconeApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Api Key',
|
||||
name: 'pineconeApiKey',
|
||||
type: 'password'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Environment',
|
||||
name: 'pineconeEnv',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Index',
|
||||
name: 'pineconeIndex',
|
||||
@@ -49,6 +48,7 @@ class Pinecone_Existing_VectorStores implements INode {
|
||||
name: 'pineconeNamespace',
|
||||
type: 'string',
|
||||
placeholder: 'my-first-namespace',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
@@ -57,6 +57,15 @@ class Pinecone_Existing_VectorStores implements INode {
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
@@ -73,15 +82,18 @@ class Pinecone_Existing_VectorStores implements INode {
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const pineconeApiKey = nodeData.inputs?.pineconeApiKey as string
|
||||
const pineconeEnv = nodeData.inputs?.pineconeEnv as string
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const index = nodeData.inputs?.pineconeIndex as string
|
||||
const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
|
||||
const pineconeMetadataFilter = nodeData.inputs?.pineconeMetadataFilter
|
||||
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
|
||||
const pineconeEnv = getCredentialParam('pineconeEnv', credentialData, nodeData)
|
||||
|
||||
const client = new PineconeClient()
|
||||
await client.init({
|
||||
@@ -104,9 +116,10 @@ class Pinecone_Existing_VectorStores implements INode {
|
||||
const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever()
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
|
||||
@@ -1,29 +1,39 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { PineconeClient } from '@pinecone-database/pinecone'
|
||||
import { PineconeLibArgs, PineconeStore } from 'langchain/vectorstores/pinecone'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class PineconeUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Pinecone Upsert Document'
|
||||
this.name = 'pineconeUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Pinecone'
|
||||
this.icon = 'pinecone.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Pinecone'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['pineconeApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
@@ -36,16 +46,6 @@ class PineconeUpsert_VectorStores implements INode {
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Api Key',
|
||||
name: 'pineconeApiKey',
|
||||
type: 'password'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Environment',
|
||||
name: 'pineconeEnv',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Index',
|
||||
name: 'pineconeIndex',
|
||||
@@ -56,6 +56,16 @@ class PineconeUpsert_VectorStores implements INode {
|
||||
name: 'pineconeNamespace',
|
||||
type: 'string',
|
||||
placeholder: 'my-first-namespace',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
@@ -73,14 +83,18 @@ class PineconeUpsert_VectorStores implements INode {
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const pineconeApiKey = nodeData.inputs?.pineconeApiKey as string
|
||||
const pineconeEnv = nodeData.inputs?.pineconeEnv as string
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const index = nodeData.inputs?.pineconeIndex as string
|
||||
const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
|
||||
const pineconeEnv = getCredentialParam('pineconeEnv', credentialData, nodeData)
|
||||
|
||||
const client = new PineconeClient()
|
||||
await client.init({
|
||||
@@ -90,7 +104,7 @@ class PineconeUpsert_VectorStores implements INode {
|
||||
|
||||
const pineconeIndex = client.Index(index)
|
||||
|
||||
const flattenDocs = docs && docs.length ? docs.flat() : []
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
@@ -105,9 +119,10 @@ class PineconeUpsert_VectorStores implements INode {
|
||||
const vectorStore = await PineconeStore.fromDocuments(finalDocs, embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever()
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { QdrantClient } from '@qdrant/js-client-rest'
|
||||
import { QdrantVectorStore, QdrantLibArgs } from 'langchain/vectorstores/qdrant'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
|
||||
class Qdrant_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Qdrant Load Existing Index'
|
||||
this.name = 'qdrantExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Qdrant'
|
||||
this.icon = 'qdrant.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Qdrant (i.e., documents have been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed when using Qdrant cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['qdrantApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Server URL',
|
||||
name: 'qdrantServerUrl',
|
||||
type: 'string',
|
||||
placeholder: 'http://localhost:6333'
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Collection Name',
|
||||
name: 'qdrantCollection',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Collection Cofiguration',
|
||||
name: 'qdrantCollectionCofiguration',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Qdrant Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(QdrantVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string
|
||||
const collectionName = nodeData.inputs?.qdrantCollection as string
|
||||
let qdrantCollectionCofiguration = nodeData.inputs?.qdrantCollectionCofiguration
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
|
||||
|
||||
const client = new QdrantClient({
|
||||
url: qdrantServerUrl,
|
||||
apiKey: qdrantApiKey
|
||||
})
|
||||
|
||||
const dbConfig: QdrantLibArgs = {
|
||||
client,
|
||||
collectionName
|
||||
}
|
||||
|
||||
if (qdrantCollectionCofiguration) {
|
||||
qdrantCollectionCofiguration =
|
||||
typeof qdrantCollectionCofiguration === 'object' ? qdrantCollectionCofiguration : JSON.parse(qdrantCollectionCofiguration)
|
||||
dbConfig.collectionConfig = qdrantCollectionCofiguration
|
||||
}
|
||||
|
||||
const vectorStore = await QdrantVectorStore.fromExistingCollection(embeddings, dbConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Qdrant_Existing_VectorStores }
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
@@ -0,0 +1,127 @@
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { QdrantClient } from '@qdrant/js-client-rest'
|
||||
import { QdrantVectorStore, QdrantLibArgs } from 'langchain/vectorstores/qdrant'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class QdrantUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Qdrant Upsert Document'
|
||||
this.name = 'qdrantUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Qdrant'
|
||||
this.icon = 'qdrant.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Qdrant'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed when using Qdrant cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['qdrantApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Server URL',
|
||||
name: 'qdrantServerUrl',
|
||||
type: 'string',
|
||||
placeholder: 'http://localhost:6333'
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Collection Name',
|
||||
name: 'qdrantCollection',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Qdrant Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Qdrant Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(QdrantVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string
|
||||
const collectionName = nodeData.inputs?.qdrantCollection as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
|
||||
|
||||
const client = new QdrantClient({
|
||||
url: qdrantServerUrl,
|
||||
apiKey: qdrantApiKey
|
||||
})
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
|
||||
const dbConfig: QdrantLibArgs = {
|
||||
client,
|
||||
url: qdrantServerUrl,
|
||||
collectionName
|
||||
}
|
||||
const vectorStore = await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: QdrantUpsert_VectorStores }
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
@@ -0,0 +1,146 @@
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore'
|
||||
|
||||
class SingleStoreExisting_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'SingleStore Load Existing Table'
|
||||
this.name = 'singlestoreExisting'
|
||||
this.version = 1.0
|
||||
this.type = 'SingleStore'
|
||||
this.icon = 'singlestore.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing document from SingleStore'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Needed when using SingleStore cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['singleStoreApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Host',
|
||||
name: 'host',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Database',
|
||||
name: 'database',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string',
|
||||
placeholder: 'embeddings',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Content Column Name',
|
||||
name: 'contentColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'content',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Vector Column Name',
|
||||
name: 'vectorColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'vector',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Metadata Column Name',
|
||||
name: 'metadataColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'metadata',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'SingleStore Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'SingleStore Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const user = getCredentialParam('user', credentialData, nodeData)
|
||||
const password = getCredentialParam('password', credentialData, nodeData)
|
||||
|
||||
const singleStoreConnectionConfig = {
|
||||
connectionOptions: {
|
||||
host: nodeData.inputs?.host as string,
|
||||
port: 3306,
|
||||
user,
|
||||
password,
|
||||
database: nodeData.inputs?.database as string
|
||||
},
|
||||
...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
|
||||
...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
|
||||
...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
|
||||
...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
|
||||
} as SingleStoreVectorStoreConfig
|
||||
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
let vectorStore: SingleStoreVectorStore
|
||||
|
||||
vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: SingleStoreExisting_VectorStores }
|
||||
@@ -0,0 +1,20 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg width="256px" height="256px" viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" preserveAspectRatio="xMidYMid">
|
||||
<title>SingleStore</title>
|
||||
<defs>
|
||||
<linearGradient x1="67.3449258%" y1="-26.0044686%" x2="-18.5227789%" y2="22.9877555%" id="singleStoreLinearGradient-1">
|
||||
<stop stop-color="#FF7BFF" offset="0%"></stop>
|
||||
<stop stop-color="#AA00FF" offset="35.0158%"></stop>
|
||||
<stop stop-color="#8800CC" offset="100%"></stop>
|
||||
</linearGradient>
|
||||
<linearGradient x1="36.2591509%" y1="-19.3628763%" x2="111.72205%" y2="44.9975357%" id="singleStoreLinearGradient-2">
|
||||
<stop stop-color="#FF7BFF" offset="3.54358%"></stop>
|
||||
<stop stop-color="#8800CC" offset="57.6537%"></stop>
|
||||
<stop stop-color="#311B92" offset="100%"></stop>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<g>
|
||||
<path d="M133.793438,0 C161.220114,7.62806846 186.208847,26.8506986 196.569923,50.3452712 C212.416431,88.4856134 208.759637,136.695058 191.389506,165.376569 C176.761849,188.8709 154.211058,201.381085 128.308006,201.075829 C88.0823106,200.770814 55.4752171,168.732936 55.1704441,128.456768 C55.1704441,88.1803574 86.8634599,54.9221798 128.308006,54.9221798 C135.012288,54.9221798 144.679052,55.851955 155.649674,60.4286222 C155.649674,60.4286222 147.762766,55.757287 127.50695,52.6192355 C69.3015772,44.9912153 0.621898574,89.095884 16.4683968,190.701711 C38.409617,229.757339 80.4639504,256.303989 128.308006,255.997284 C198.703093,255.692994 256.299161,198.024717 255.996071,127.236226 C255.996071,59.498847 200.836263,1.83073691 133.793438,0 Z" fill="url(#singleStoreLinearGradient-1)"></path>
|
||||
<path d="M181.635561,54.0037552 C171.884031,33.5605356 151.771183,17.3889203 127.087223,10.9813448 C121.601791,9.45574074 115.811828,8.84547014 109.412318,8.540359 C99.9653199,8.540359 90.8230945,9.76087603 81.376096,12.2018618 C57.9109865,19.2196838 41.455174,32.950265 31.7034025,43.6293966 C19.20906,57.9701518 10.9810571,72.9211776 6.1052197,87.8722034 C6.1052197,88.1774594 5.8004708,88.4824739 5.8004708,89.0927445 C5.4957219,90.3132857 4.27677462,93.9746678 4.27677462,94.8901944 C3.97202572,95.500465 3.97202572,96.4157502 3.66730098,97.0260207 C3.36255208,98.2465619 3.05780318,99.4668616 2.75307844,100.687403 C2.75307844,100.992659 2.75307844,101.297673 2.44832954,101.602688 C-5.47492441,140.963571 7.68750379,176.286091 15.6107577,189.406305 C17.5925312,192.688049 19.2199033,195.425935 20.8508738,197.938019 C2.87119611,100.298588 54.558966,53.6984992 113.373885,52.477958 C144.152582,51.8679289 174.931278,64.3778723 189.863709,89.3980006 C188.949389,75.6672745 188.035312,68.0392543 181.635561,54.0037552 Z" fill="url(#singleStoreLinearGradient-2)"></path>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.8 KiB |
@@ -0,0 +1,162 @@
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class SingleStoreUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'SingleStore Upsert Document'
|
||||
this.name = 'singlestoreUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'SingleStore'
|
||||
this.icon = 'singlestore.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to SingleStore'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Needed when using SingleStore cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['singleStoreApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Host',
|
||||
name: 'host',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Database',
|
||||
name: 'database',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string',
|
||||
placeholder: 'embeddings',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Content Column Name',
|
||||
name: 'contentColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'content',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Vector Column Name',
|
||||
name: 'vectorColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'vector',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Metadata Column Name',
|
||||
name: 'metadataColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'metadata',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'SingleStore Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'SingleStore Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const user = getCredentialParam('user', credentialData, nodeData)
|
||||
const password = getCredentialParam('password', credentialData, nodeData)
|
||||
|
||||
const singleStoreConnectionConfig = {
|
||||
connectionOptions: {
|
||||
host: nodeData.inputs?.host as string,
|
||||
port: 3306,
|
||||
user,
|
||||
password,
|
||||
database: nodeData.inputs?.database as string
|
||||
},
|
||||
...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
|
||||
...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
|
||||
...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
|
||||
...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
|
||||
} as SingleStoreVectorStoreConfig
|
||||
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
|
||||
let vectorStore: SingleStoreVectorStore
|
||||
|
||||
vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
|
||||
vectorStore.addDocuments.bind(vectorStore)(finalDocs)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: SingleStoreUpsert_VectorStores }
|
||||
@@ -0,0 +1,20 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg width="256px" height="256px" viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" preserveAspectRatio="xMidYMid">
|
||||
<title>SingleStore</title>
|
||||
<defs>
|
||||
<linearGradient x1="67.3449258%" y1="-26.0044686%" x2="-18.5227789%" y2="22.9877555%" id="singleStoreLinearGradient-1">
|
||||
<stop stop-color="#FF7BFF" offset="0%"></stop>
|
||||
<stop stop-color="#AA00FF" offset="35.0158%"></stop>
|
||||
<stop stop-color="#8800CC" offset="100%"></stop>
|
||||
</linearGradient>
|
||||
<linearGradient x1="36.2591509%" y1="-19.3628763%" x2="111.72205%" y2="44.9975357%" id="singleStoreLinearGradient-2">
|
||||
<stop stop-color="#FF7BFF" offset="3.54358%"></stop>
|
||||
<stop stop-color="#8800CC" offset="57.6537%"></stop>
|
||||
<stop stop-color="#311B92" offset="100%"></stop>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<g>
|
||||
<path d="M133.793438,0 C161.220114,7.62806846 186.208847,26.8506986 196.569923,50.3452712 C212.416431,88.4856134 208.759637,136.695058 191.389506,165.376569 C176.761849,188.8709 154.211058,201.381085 128.308006,201.075829 C88.0823106,200.770814 55.4752171,168.732936 55.1704441,128.456768 C55.1704441,88.1803574 86.8634599,54.9221798 128.308006,54.9221798 C135.012288,54.9221798 144.679052,55.851955 155.649674,60.4286222 C155.649674,60.4286222 147.762766,55.757287 127.50695,52.6192355 C69.3015772,44.9912153 0.621898574,89.095884 16.4683968,190.701711 C38.409617,229.757339 80.4639504,256.303989 128.308006,255.997284 C198.703093,255.692994 256.299161,198.024717 255.996071,127.236226 C255.996071,59.498847 200.836263,1.83073691 133.793438,0 Z" fill="url(#singleStoreLinearGradient-1)"></path>
|
||||
<path d="M181.635561,54.0037552 C171.884031,33.5605356 151.771183,17.3889203 127.087223,10.9813448 C121.601791,9.45574074 115.811828,8.84547014 109.412318,8.540359 C99.9653199,8.540359 90.8230945,9.76087603 81.376096,12.2018618 C57.9109865,19.2196838 41.455174,32.950265 31.7034025,43.6293966 C19.20906,57.9701518 10.9810571,72.9211776 6.1052197,87.8722034 C6.1052197,88.1774594 5.8004708,88.4824739 5.8004708,89.0927445 C5.4957219,90.3132857 4.27677462,93.9746678 4.27677462,94.8901944 C3.97202572,95.500465 3.97202572,96.4157502 3.66730098,97.0260207 C3.36255208,98.2465619 3.05780318,99.4668616 2.75307844,100.687403 C2.75307844,100.992659 2.75307844,101.297673 2.44832954,101.602688 C-5.47492441,140.963571 7.68750379,176.286091 15.6107577,189.406305 C17.5925312,192.688049 19.2199033,195.425935 20.8508738,197.938019 C2.87119611,100.298588 54.558966,53.6984992 113.373885,52.477958 C144.152582,51.8679289 174.931278,64.3778723 189.863709,89.3980006 C188.949389,75.6672745 188.035312,68.0392543 181.635561,54.0037552 Z" fill="url(#singleStoreLinearGradient-2)"></path>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.8 KiB |
@@ -1,39 +1,43 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { SupabaseLibArgs, SupabaseVectorStore } from 'langchain/vectorstores/supabase'
|
||||
import { createClient } from '@supabase/supabase-js'
|
||||
|
||||
class Supabase_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Supabase Load Existing Index'
|
||||
this.name = 'supabaseExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Supabase'
|
||||
this.icon = 'supabase.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Supabase (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['supabaseApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Supabase API Key',
|
||||
name: 'supabaseApiKey',
|
||||
type: 'password'
|
||||
},
|
||||
{
|
||||
label: 'Supabase Project URL',
|
||||
name: 'supabaseProjUrl',
|
||||
@@ -55,6 +59,15 @@ class Supabase_Existing_VectorStores implements INode {
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
@@ -71,14 +84,18 @@ class Supabase_Existing_VectorStores implements INode {
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const supabaseApiKey = nodeData.inputs?.supabaseApiKey as string
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string
|
||||
const tableName = nodeData.inputs?.tableName as string
|
||||
const queryName = nodeData.inputs?.queryName as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const supabaseMetadataFilter = nodeData.inputs?.supabaseMetadataFilter
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData)
|
||||
|
||||
const client = createClient(supabaseProjUrl, supabaseApiKey)
|
||||
|
||||
@@ -96,9 +113,10 @@ class Supabase_Existing_VectorStores implements INode {
|
||||
const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever()
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
|
||||
@@ -1,29 +1,39 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { SupabaseVectorStore } from 'langchain/vectorstores/supabase'
|
||||
import { createClient } from '@supabase/supabase-js'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class SupabaseUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Supabase Upsert Document'
|
||||
this.name = 'supabaseUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Supabase'
|
||||
this.icon = 'supabase.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Supabase'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['supabaseApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
@@ -36,11 +46,6 @@ class SupabaseUpsert_VectorStores implements INode {
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Supabase API Key',
|
||||
name: 'supabaseApiKey',
|
||||
type: 'password'
|
||||
},
|
||||
{
|
||||
label: 'Supabase Project URL',
|
||||
name: 'supabaseProjUrl',
|
||||
@@ -55,6 +60,15 @@ class SupabaseUpsert_VectorStores implements INode {
|
||||
label: 'Query Name',
|
||||
name: 'queryName',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
@@ -71,18 +85,22 @@ class SupabaseUpsert_VectorStores implements INode {
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const supabaseApiKey = nodeData.inputs?.supabaseApiKey as string
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string
|
||||
const tableName = nodeData.inputs?.tableName as string
|
||||
const queryName = nodeData.inputs?.queryName as string
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData)
|
||||
|
||||
const client = createClient(supabaseProjUrl, supabaseApiKey)
|
||||
|
||||
const flattenDocs = docs && docs.length ? docs.flat() : []
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
@@ -95,9 +113,10 @@ class SupabaseUpsert_VectorStores implements INode {
|
||||
})
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever()
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from 'langchain/vectorstores/vectara'
|
||||
|
||||
class VectaraExisting_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Vectara Load Existing Index'
|
||||
this.name = 'vectaraExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Vectara'
|
||||
this.icon = 'vectara.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Vectara (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['vectaraApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Vectara Metadata Filter',
|
||||
name: 'filter',
|
||||
description:
|
||||
'Filter to apply to Vectara metadata. Refer to the <a target="_blank" href="https://docs.flowiseai.com/vector-stores/vectara">documentation</a> on how to use Vectara filters with Flowise.',
|
||||
type: 'string',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences Before',
|
||||
name: 'sentencesBefore',
|
||||
description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences After',
|
||||
name: 'sentencesAfter',
|
||||
description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Lambda',
|
||||
name: 'lambda',
|
||||
description:
|
||||
'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Defaults to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Vectara Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Vectara Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
const customerId = getCredentialParam('customerID', credentialData, nodeData)
|
||||
const corpusId = getCredentialParam('corpusID', credentialData, nodeData)
|
||||
|
||||
const vectaraMetadataFilter = nodeData.inputs?.filter as string
|
||||
const sentencesBefore = nodeData.inputs?.sentencesBefore as number
|
||||
const sentencesAfter = nodeData.inputs?.sentencesAfter as number
|
||||
const lambda = nodeData.inputs?.lambda as number
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseInt(topK, 10) : 4
|
||||
|
||||
const vectaraArgs: VectaraLibArgs = {
|
||||
apiKey: apiKey,
|
||||
customerId: customerId,
|
||||
corpusId: corpusId
|
||||
}
|
||||
|
||||
const vectaraFilter: VectaraFilter = {}
|
||||
if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
|
||||
if (lambda) vectaraFilter.lambda = lambda
|
||||
|
||||
const vectaraContextConfig: VectaraContextConfig = {}
|
||||
if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
|
||||
if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
|
||||
vectaraFilter.contextConfig = vectaraContextConfig
|
||||
|
||||
const vectorStore = new VectaraStore(vectaraArgs)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k, vectaraFilter)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: VectaraExisting_VectorStores }
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
@@ -0,0 +1,150 @@
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from 'langchain/vectorstores/vectara'
|
||||
import { Document } from 'langchain/document'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class VectaraUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Vectara Upsert Document'
|
||||
this.name = 'vectaraUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Vectara'
|
||||
this.icon = 'vectara.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Vectara'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['vectaraApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Vectara Metadata Filter',
|
||||
name: 'filter',
|
||||
description:
|
||||
'Filter to apply to Vectara metadata. Refer to the <a target="_blank" href="https://docs.flowiseai.com/vector-stores/vectara">documentation</a> on how to use Vectara filters with Flowise.',
|
||||
type: 'string',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences Before',
|
||||
name: 'sentencesBefore',
|
||||
description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Sentences After',
|
||||
name: 'sentencesAfter',
|
||||
description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Lambda',
|
||||
name: 'lambda',
|
||||
description:
|
||||
'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Defaults to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Vectara Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Vectara Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
const customerId = getCredentialParam('customerID', credentialData, nodeData)
|
||||
const corpusId = getCredentialParam('corpusID', credentialData, nodeData)
|
||||
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = {} as Embeddings
|
||||
const vectaraMetadataFilter = nodeData.inputs?.filter as string
|
||||
const sentencesBefore = nodeData.inputs?.sentencesBefore as number
|
||||
const sentencesAfter = nodeData.inputs?.sentencesAfter as number
|
||||
const lambda = nodeData.inputs?.lambda as number
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseInt(topK, 10) : 4
|
||||
|
||||
const vectaraArgs: VectaraLibArgs = {
|
||||
apiKey: apiKey,
|
||||
customerId: customerId,
|
||||
corpusId: corpusId
|
||||
}
|
||||
|
||||
const vectaraFilter: VectaraFilter = {}
|
||||
if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
|
||||
if (lambda) vectaraFilter.lambda = lambda
|
||||
|
||||
const vectaraContextConfig: VectaraContextConfig = {}
|
||||
if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
|
||||
if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
|
||||
vectaraFilter.contextConfig = vectaraContextConfig
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
|
||||
const vectorStore = await VectaraStore.fromDocuments(finalDocs, embeddings, vectaraArgs)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k, vectaraFilter)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: VectaraUpsert_VectorStores }
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
@@ -1,28 +1,39 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client'
|
||||
import { WeaviateLibArgs, WeaviateStore } from 'langchain/vectorstores/weaviate'
|
||||
|
||||
class Weaviate_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Weaviate Load Existing Index'
|
||||
this.name = 'weaviateExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Weaviate'
|
||||
this.icon = 'weaviate.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Weaviate (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed when using Weaviate cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['weaviateApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
@@ -57,12 +68,6 @@ class Weaviate_Existing_VectorStores implements INode {
|
||||
type: 'string',
|
||||
placeholder: 'Test'
|
||||
},
|
||||
{
|
||||
label: 'Weaviate API Key',
|
||||
name: 'weaviateApiKey',
|
||||
type: 'password',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Text Key',
|
||||
name: 'weaviateTextKey',
|
||||
@@ -79,6 +84,15 @@ class Weaviate_Existing_VectorStores implements INode {
|
||||
placeholder: `["foo"]`,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
@@ -95,16 +109,19 @@ class Weaviate_Existing_VectorStores implements INode {
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const weaviateScheme = nodeData.inputs?.weaviateScheme as string
|
||||
const weaviateHost = nodeData.inputs?.weaviateHost as string
|
||||
const weaviateIndex = nodeData.inputs?.weaviateIndex as string
|
||||
const weaviateApiKey = nodeData.inputs?.weaviateApiKey as string
|
||||
const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string
|
||||
const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
|
||||
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
|
||||
|
||||
const clientConfig: any = {
|
||||
scheme: weaviateScheme,
|
||||
@@ -125,9 +142,10 @@ class Weaviate_Existing_VectorStores implements INode {
|
||||
const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever()
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
|
||||
@@ -1,29 +1,41 @@
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { WeaviateLibArgs, WeaviateStore } from 'langchain/vectorstores/weaviate'
|
||||
import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class WeaviateUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Weaviate Upsert Document'
|
||||
this.name = 'weaviateUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Weaviate'
|
||||
this.icon = 'weaviate.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Weaviate'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
description: 'Only needed when using Weaviate cloud hosted',
|
||||
optional: true,
|
||||
credentialNames: ['weaviateApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
@@ -64,12 +76,6 @@ class WeaviateUpsert_VectorStores implements INode {
|
||||
type: 'string',
|
||||
placeholder: 'Test'
|
||||
},
|
||||
{
|
||||
label: 'Weaviate API Key',
|
||||
name: 'weaviateApiKey',
|
||||
type: 'password',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Weaviate Text Key',
|
||||
name: 'weaviateTextKey',
|
||||
@@ -86,6 +92,15 @@ class WeaviateUpsert_VectorStores implements INode {
|
||||
placeholder: `["foo"]`,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
@@ -102,17 +117,20 @@ class WeaviateUpsert_VectorStores implements INode {
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const weaviateScheme = nodeData.inputs?.weaviateScheme as string
|
||||
const weaviateHost = nodeData.inputs?.weaviateHost as string
|
||||
const weaviateIndex = nodeData.inputs?.weaviateIndex as string
|
||||
const weaviateApiKey = nodeData.inputs?.weaviateApiKey as string
|
||||
const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string
|
||||
const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
|
||||
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
|
||||
|
||||
const clientConfig: any = {
|
||||
scheme: weaviateScheme,
|
||||
@@ -122,7 +140,7 @@ class WeaviateUpsert_VectorStores implements INode {
|
||||
|
||||
const client: WeaviateClient = weaviate.client(clientConfig)
|
||||
|
||||
const flattenDocs = docs && docs.length ? docs.flat() : []
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
@@ -139,9 +157,10 @@ class WeaviateUpsert_VectorStores implements INode {
|
||||
const vectorStore = await WeaviateStore.fromDocuments(finalDocs, embeddings, obj)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever()
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ZepVectorStore, IZepConfig } from 'langchain/vectorstores/zep'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { IDocument, ZepClient } from '@getzep/zep-js'
|
||||
|
||||
class Zep_Existing_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Zep Load Existing Index'
|
||||
this.name = 'zepExistingIndex'
|
||||
this.version = 1.0
|
||||
this.type = 'Zep'
|
||||
this.icon = 'zep.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing index from Zep (i.e: Document has been upserted)'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
optional: true,
|
||||
description: 'Configure JWT authentication on your Zep instance (Optional)',
|
||||
credentialNames: ['zepMemoryApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Base URL',
|
||||
name: 'baseURL',
|
||||
type: 'string',
|
||||
default: 'http://127.0.0.1:8000'
|
||||
},
|
||||
{
|
||||
label: 'Zep Collection',
|
||||
name: 'zepCollection',
|
||||
type: 'string',
|
||||
placeholder: 'my-first-collection'
|
||||
},
|
||||
{
|
||||
label: 'Zep Metadata Filter',
|
||||
name: 'zepMetadataFilter',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Embedding Dimension',
|
||||
name: 'dimension',
|
||||
type: 'number',
|
||||
default: 1536,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Pinecone Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Pinecone Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(ZepVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const baseURL = nodeData.inputs?.baseURL as string
|
||||
const zepCollection = nodeData.inputs?.zepCollection as string
|
||||
const zepMetadataFilter = nodeData.inputs?.zepMetadataFilter
|
||||
const dimension = nodeData.inputs?.dimension as number
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
|
||||
const zepConfig: IZepConfig & Partial<ZepFilter> = {
|
||||
apiUrl: baseURL,
|
||||
collectionName: zepCollection,
|
||||
embeddingDimensions: dimension,
|
||||
isAutoEmbedded: false
|
||||
}
|
||||
if (apiKey) zepConfig.apiKey = apiKey
|
||||
if (zepMetadataFilter) {
|
||||
const metadatafilter = typeof zepMetadataFilter === 'object' ? zepMetadataFilter : JSON.parse(zepMetadataFilter)
|
||||
zepConfig.filter = metadatafilter
|
||||
}
|
||||
|
||||
const vectorStore = await ZepExistingVS.fromExistingIndex(embeddings, zepConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
interface ZepFilter {
|
||||
filter: Record<string, any>
|
||||
}
|
||||
|
||||
function zepDocsToDocumentsAndScore(results: IDocument[]): [Document, number][] {
|
||||
return results.map((d) => [
|
||||
new Document({
|
||||
pageContent: d.content,
|
||||
metadata: d.metadata
|
||||
}),
|
||||
d.score ? d.score : 0
|
||||
])
|
||||
}
|
||||
|
||||
function assignMetadata(value: string | Record<string, unknown> | object | undefined): Record<string, unknown> | undefined {
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
return value as Record<string, unknown>
|
||||
}
|
||||
if (value !== undefined) {
|
||||
console.warn('Metadata filters must be an object, Record, or undefined.')
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
class ZepExistingVS extends ZepVectorStore {
|
||||
filter?: Record<string, any>
|
||||
args?: IZepConfig & Partial<ZepFilter>
|
||||
|
||||
constructor(embeddings: Embeddings, args: IZepConfig & Partial<ZepFilter>) {
|
||||
super(embeddings, args)
|
||||
this.filter = args.filter
|
||||
this.args = args
|
||||
}
|
||||
|
||||
async initalizeCollection(args: IZepConfig & Partial<ZepFilter>) {
|
||||
this.client = await ZepClient.init(args.apiUrl, args.apiKey)
|
||||
try {
|
||||
this.collection = await this.client.document.getCollection(args.collectionName)
|
||||
} catch (err) {
|
||||
if (err instanceof Error) {
|
||||
if (err.name === 'NotFoundError') {
|
||||
await this.createNewCollection(args)
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async createNewCollection(args: IZepConfig & Partial<ZepFilter>) {
|
||||
if (!args.embeddingDimensions) {
|
||||
throw new Error(
|
||||
`Collection ${args.collectionName} not found. You can create a new Collection by providing embeddingDimensions.`
|
||||
)
|
||||
}
|
||||
|
||||
this.collection = await this.client.document.addCollection({
|
||||
name: args.collectionName,
|
||||
description: args.description,
|
||||
metadata: args.metadata,
|
||||
embeddingDimensions: args.embeddingDimensions,
|
||||
isAutoEmbedded: false
|
||||
})
|
||||
}
|
||||
|
||||
async similaritySearchVectorWithScore(
|
||||
query: number[],
|
||||
k: number,
|
||||
filter?: Record<string, unknown> | undefined
|
||||
): Promise<[Document, number][]> {
|
||||
if (filter && this.filter) {
|
||||
throw new Error('cannot provide both `filter` and `this.filter`')
|
||||
}
|
||||
const _filters = filter ?? this.filter
|
||||
const ANDFilters = []
|
||||
for (const filterKey in _filters) {
|
||||
let filterVal = _filters[filterKey]
|
||||
if (typeof filterVal === 'string') filterVal = `"${filterVal}"`
|
||||
ANDFilters.push({ jsonpath: `$[*] ? (@.${filterKey} == ${filterVal})` })
|
||||
}
|
||||
const newfilter = {
|
||||
where: { and: ANDFilters }
|
||||
}
|
||||
await this.initalizeCollection(this.args!).catch((err) => {
|
||||
console.error('Error initializing collection:', err)
|
||||
throw err
|
||||
})
|
||||
const results = await this.collection.search(
|
||||
{
|
||||
embedding: new Float32Array(query),
|
||||
metadata: assignMetadata(newfilter)
|
||||
},
|
||||
k
|
||||
)
|
||||
return zepDocsToDocumentsAndScore(results)
|
||||
}
|
||||
|
||||
static async fromExistingIndex(embeddings: Embeddings, dbConfig: IZepConfig & Partial<ZepFilter>): Promise<ZepVectorStore> {
|
||||
const instance = new this(embeddings, dbConfig)
|
||||
return instance
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Zep_Existing_VectorStores }
|
||||
@@ -0,0 +1,133 @@
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ZepVectorStore, IZepConfig } from 'langchain/vectorstores/zep'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class Zep_Upsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Zep Upsert Document'
|
||||
this.name = 'zepUpsert'
|
||||
this.version = 1.0
|
||||
this.type = 'Zep'
|
||||
this.icon = 'zep.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to Zep'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
optional: true,
|
||||
description: 'Configure JWT authentication on your Zep instance (Optional)',
|
||||
credentialNames: ['zepMemoryApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Base URL',
|
||||
name: 'baseURL',
|
||||
type: 'string',
|
||||
default: 'http://127.0.0.1:8000'
|
||||
},
|
||||
{
|
||||
label: 'Zep Collection',
|
||||
name: 'zepCollection',
|
||||
type: 'string',
|
||||
placeholder: 'my-first-collection'
|
||||
},
|
||||
{
|
||||
label: 'Embedding Dimension',
|
||||
name: 'dimension',
|
||||
type: 'number',
|
||||
default: 1536,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
description: 'Number of top results to fetch. Default to 4',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Zep Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'Zep Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(ZepVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const baseURL = nodeData.inputs?.baseURL as string
|
||||
const zepCollection = nodeData.inputs?.zepCollection as string
|
||||
const dimension = (nodeData.inputs?.dimension as number) ?? 1536
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseFloat(topK) : 4
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
|
||||
const zepConfig: IZepConfig = {
|
||||
apiUrl: baseURL,
|
||||
collectionName: zepCollection,
|
||||
embeddingDimensions: dimension,
|
||||
isAutoEmbedded: false
|
||||
}
|
||||
if (apiKey) zepConfig.apiKey = apiKey
|
||||
|
||||
const vectorStore = await ZepVectorStore.fromDocuments(finalDocs, embeddings, zepConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Zep_Upsert_VectorStores }
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
Reference in New Issue
Block a user