farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 17:01:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into FEATURE/RAG-VectorStores-Updates

Browse Source 
# Conflicts:
#	packages/components/package.json
This commit is contained in:
vinodkiran
2024-01-16 11:12:52 +05:30
parent b3f362630a 24d227598b
commit 0d19dc5be4
66 changed files with 2346 additions and 1540 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/vectorstores/Astra/Astra.ts
+190
View File
@@ -0,0 +1,190 @@
import { flatten } from 'lodash'
import { Embeddings } from 'langchain/embeddings/base'
import { Document } from 'langchain/document'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getCredentialData } from '../../../src/utils'
import { AstraDBVectorStore, AstraLibArgs } from '@langchain/community/vectorstores/astradb'
class Astra_VectorStores implements INode {
label: string
name: string
version: number
description: string
type: string
icon: string
category: string
badge: string
baseClasses: string[]
inputs: INodeParams[]
credential: INodeParams
outputs: INodeOutputsValue[]
constructor() {
this.label = 'Astra'
this.name = 'Astra'
this.version = 1.0
this.type = 'Astra'
this.icon = 'astra.svg'
this.category = 'Vector Stores'
this.description = `Upsert embedded data and perform similarity search upon query using DataStax Astra DB, a serverless vector database that’s perfect for managing mission-critical AI workloads`
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
this.badge = 'NEW'
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['AstraDBApi']
}
this.inputs = [
{
label: 'Document',
name: 'document',
type: 'Document',
list: true,
optional: true
},
{
label: 'Embeddings',
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Vector Dimension',
name: 'vectorDimension',
type: 'number',
placeholder: '1536',
optional: true,
description: 'Dimension used for storing vector embedding'
},
{
label: 'Similarity Metric',
name: 'similarityMetric',
type: 'string',
placeholder: 'cosine',
optional: true,
description: 'cosine | euclidean | dot_product'
},
{
label: 'Top K',
name: 'topK',
description: 'Number of top results to fetch. Default to 4',
placeholder: '4',
type: 'number',
additionalParams: true,
optional: true
}
]
this.outputs = [
{
label: 'Astra Retriever',
name: 'retriever',
baseClasses: this.baseClasses
},
{
label: 'Astra Vector Store',
name: 'vectorStore',
baseClasses: [this.type, ...getBaseClasses(AstraDBVectorStore)]
}
]
}
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const vectorDimension = nodeData.inputs?.vectorDimension as number
const similarityMetric = nodeData.inputs?.similarityMetric as 'cosine' | 'euclidean' | 'dot_product' | undefined
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const expectedSimilarityMetric = ['cosine', 'euclidean', 'dot_product']
if (similarityMetric && !expectedSimilarityMetric.includes(similarityMetric)) {
throw new Error(`Invalid Similarity Metric should be one of 'cosine' | 'euclidean' | 'dot_product'`)
}
const clientConfig = {
token: credentialData?.applicationToken,
endpoint: credentialData?.dbEndPoint
}
const astraConfig: AstraLibArgs = {
...clientConfig,
collection: credentialData.collectionName ?? 'flowise_test',
collectionOptions: {
vector: {
dimension: vectorDimension ?? 1536,
metric: similarityMetric ?? 'cosine'
}
}
}
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
try {
await AstraDBVectorStore.fromDocuments(finalDocs, embeddings, astraConfig)
} catch (e) {
throw new Error(e)
}
}
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const vectorDimension = nodeData.inputs?.vectorDimension as number
const similarityMetric = nodeData.inputs?.similarityMetric as 'cosine' | 'euclidean' | 'dot_product' | undefined
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
const k = topK ? parseFloat(topK) : 4
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const expectedSimilarityMetric = ['cosine', 'euclidean', 'dot_product']
if (similarityMetric && !expectedSimilarityMetric.includes(similarityMetric)) {
throw new Error(`Invalid Similarity Metric should be one of 'cosine' | 'euclidean' | 'dot_product'`)
}
const clientConfig = {
token: credentialData?.applicationToken,
endpoint: credentialData?.dbEndPoint
}
const astraConfig: AstraLibArgs = {
...clientConfig,
collection: credentialData.collectionName ?? 'flowise_test',
collectionOptions: {
vector: {
dimension: vectorDimension ?? 1536,
metric: similarityMetric ?? 'cosine'
}
}
}
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const vectorStore = await AstraDBVectorStore.fromExistingIndex(embeddings, astraConfig)
if (output === 'retriever') {
const retriever = vectorStore.asRetriever(k)
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
return vectorStore
}
return vectorStore
}
}
module.exports = { nodeClass: Astra_VectorStores }
packages/components/nodes/vectorstores/Astra/astra.svg
+12
View File
@@ -0,0 +1,12 @@
<svg width="1200" height="1200" viewBox="0 0 1200 1200" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="1200" height="1200" fill="black"/>
<g clip-path="url(#clip0_102_1968)">
<path d="M508.819 464.97H267.001V737.697H508.819L569.566 690.526V512.14L508.819 464.97ZM313.864 512.14H522.703V690.575H313.864V512.14Z" fill="white"/>
<path d="M917.531 514.121V468H696.425L636.389 514.121V577.447L696.425 623.568H889.124V688.545H648.348V734.667H875.409L935.444 688.545V623.568L875.409 577.447H682.709V514.121H917.531Z" fill="white"/>
</g>
<defs>
<clipPath id="clip0_102_1968">
<rect width="668.444" height="266.667" fill="white" transform="translate(267 468)"/>
</clipPath>
</defs>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 694 B

packages/components/nodes/vectorstores/Milvus/Milvus.ts
+11 -1
View File
@@ -65,6 +65,14 @@ class Milvus_VectorStores implements INode {
name: 'milvusCollection',
type: 'string'
},
{
label: 'Milvus Text Field',
name: 'milvusTextField',
type: 'string',
placeholder: 'langchain_text',
optional: true,
additionalParams: true
},
{
label: 'Milvus Filter',
name: 'milvusFilter',
@@ -150,6 +158,7 @@ class Milvus_VectorStores implements INode {
const address = nodeData.inputs?.milvusServerUrl as string
const collectionName = nodeData.inputs?.milvusCollection as string
const milvusFilter = nodeData.inputs?.milvusFilter as string
const textField = nodeData.inputs?.milvusTextField as string
// embeddings
const embeddings = nodeData.inputs?.embeddings as Embeddings
@@ -169,7 +178,8 @@ class Milvus_VectorStores implements INode {
// init MilvusLibArgs
const milVusArgs: MilvusLibArgs = {
url: address,
collectionName: collectionName
collectionName: collectionName,
textField: textField
}
if (milvusUser) milVusArgs.username = milvusUser
packages/components/nodes/vectorstores/Postgres/Postgres.ts
+11 -2
View File
@@ -24,7 +24,7 @@ class Postgres_VectorStores implements INode {
constructor() {
this.label = 'Postgres'
this.name = 'postgres'
this.version = 1.0
this.version = 2.0
this.type = 'Postgres'
this.icon = 'postgres.svg'
this.category = 'Vector Stores'
@@ -60,6 +60,13 @@ class Postgres_VectorStores implements INode {
name: 'database',
type: 'string'
},
{
label: 'SSL Connection',
name: 'sslConnection',
type: 'boolean',
default: false,
optional: false
},
{
label: 'Port',
name: 'port',
@@ -117,6 +124,7 @@ class Postgres_VectorStores implements INode {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const additionalConfig = nodeData.inputs?.additionalConfig as string
const sslConnection = nodeData.inputs?.sslConnection as boolean
let additionalConfiguration = {}
if (additionalConfig) {
@@ -134,7 +142,8 @@ class Postgres_VectorStores implements INode {
port: nodeData.inputs?.port as number,
username: user,
password: password,
database: nodeData.inputs?.database as string
database: nodeData.inputs?.database as string,
ssl: sslConnection
}
const args = {
packages/components/nodes/vectorstores/Postgres/Postgres_Exisiting.ts
+11 -2
View File
@@ -23,7 +23,7 @@ class Postgres_Existing_VectorStores implements INode {
constructor() {
this.label = 'Postgres Load Existing Index'
this.name = 'postgresExistingIndex'
this.version = 1.0
this.version = 2.0
this.type = 'Postgres'
this.icon = 'postgres.svg'
this.category = 'Vector Stores'
@@ -52,6 +52,13 @@ class Postgres_Existing_VectorStores implements INode {
name: 'database',
type: 'string'
},
{
label: 'SSL Connection',
name: 'sslConnection',
type: 'boolean',
default: false,
optional: false
},
{
label: 'Port',
name: 'port',
@@ -109,6 +116,7 @@ class Postgres_Existing_VectorStores implements INode {
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
const k = topK ? parseFloat(topK) : 4
const sslConnection = nodeData.inputs?.sslConnection as boolean
let additionalConfiguration = {}
if (additionalConfig) {
@@ -126,7 +134,8 @@ class Postgres_Existing_VectorStores implements INode {
port: nodeData.inputs?.port as number,
username: user,
password: password,
database: nodeData.inputs?.database as string
database: nodeData.inputs?.database as string,
ssl: sslConnection
}
const args = {
packages/components/nodes/vectorstores/Postgres/Postgres_Upsert.ts
+11 -2
View File
@@ -24,7 +24,7 @@ class PostgresUpsert_VectorStores implements INode {
constructor() {
this.label = 'Postgres Upsert Document'
this.name = 'postgresUpsert'
this.version = 1.0
this.version = 2.0
this.type = 'Postgres'
this.icon = 'postgres.svg'
this.category = 'Vector Stores'
@@ -59,6 +59,13 @@ class PostgresUpsert_VectorStores implements INode {
name: 'database',
type: 'string'
},
{
label: 'SSL Connection',
name: 'sslConnection',
type: 'boolean',
default: false,
optional: false
},
{
label: 'Port',
name: 'port',
@@ -117,6 +124,7 @@ class PostgresUpsert_VectorStores implements INode {
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
const k = topK ? parseFloat(topK) : 4
const sslConnection = nodeData.inputs?.sslConnection as boolean
let additionalConfiguration = {}
if (additionalConfig) {
@@ -134,7 +142,8 @@ class PostgresUpsert_VectorStores implements INode {
port: nodeData.inputs?.port as number,
username: user,
password: password,
database: nodeData.inputs?.database as string
database: nodeData.inputs?.database as string,
ssl: sslConnection
}
const args = {
packages/components/nodes/vectorstores/Qdrant/Qdrant.ts
+30 -2
View File
@@ -149,9 +149,12 @@ class Qdrant_VectorStores implements INode {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
const port = Qdrant_VectorStores.determinePortByUrl(qdrantServerUrl)
const client = new QdrantClient({
url: qdrantServerUrl,
apiKey: qdrantApiKey
apiKey: qdrantApiKey,
port: port
})
const flattenDocs = docs && docs.length ? flatten(docs) : []
@@ -198,9 +201,12 @@ class Qdrant_VectorStores implements INode {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
const port = Qdrant_VectorStores.determinePortByUrl(qdrantServerUrl)
const client = new QdrantClient({
url: qdrantServerUrl,
apiKey: qdrantApiKey
apiKey: qdrantApiKey,
port: port
})
const dbConfig: QdrantLibArgs = {
@@ -242,6 +248,28 @@ class Qdrant_VectorStores implements INode {
}
return vectorStore
}
/**
* Determine the port number from the given URL.
*
* The problem is when not doing this the qdrant-client.js will fall back on 6663 when you enter a port 443 and 80.
* See: https://stackoverflow.com/questions/59104197/nodejs-new-url-urlhttps-myurl-com80-lists-the-port-as-empty
* @param qdrantServerUrl the url to get the port from
*/
static determinePortByUrl(qdrantServerUrl: string): number {
const parsedUrl = new URL(qdrantServerUrl)
let port = parsedUrl.port ? parseInt(parsedUrl.port) : 6663
if (parsedUrl.protocol === 'https:' && parsedUrl.port === '') {
port = 443
}
if (parsedUrl.protocol === 'http:' && parsedUrl.port === '') {
port = 80
}
return port
}
}
module.exports = { nodeClass: Qdrant_VectorStores }
packages/components/nodes/vectorstores/Vectara/Vectara.ts
+37 -6
View File
@@ -1,5 +1,5 @@
import { flatten } from 'lodash'
import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig, VectaraFile } from 'langchain/vectorstores/vectara'
import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig, VectaraFile, MMRConfig } from 'langchain/vectorstores/vectara'
import { Document } from 'langchain/document'
import { Embeddings } from 'langchain/embeddings/base'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
@@ -22,7 +22,7 @@ class Vectara_VectorStores implements INode {
constructor() {
this.label = 'Vectara'
this.name = 'vectara'
this.version = 1.0
this.version = 2.0
this.type = 'Vectara'
this.icon = 'vectara.png'
this.category = 'Vector Stores'
@@ -82,7 +82,9 @@ class Vectara_VectorStores implements INode {
label: 'Lambda',
name: 'lambda',
description:
'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
'Enable hybrid search to improve retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.' +
'A value of 0.0 means that only neural search is used, while a value of 1.0 means that only keyword-based search is used. Defaults to 0.0 (neural only).',
default: 0.0,
type: 'number',
additionalParams: true,
optional: true
@@ -90,8 +92,30 @@ class Vectara_VectorStores implements INode {
{
label: 'Top K',
name: 'topK',
description: 'Number of top results to fetch. Defaults to 4',
placeholder: '4',
description: 'Number of top results to fetch. Defaults to 5',
placeholder: '5',
type: 'number',
additionalParams: true,
optional: true
},
{
label: 'MMR K',
name: 'mmrK',
description: 'Number of top results to fetch for MMR. Defaults to 50',
placeholder: '50',
type: 'number',
additionalParams: true,
optional: true
},
{
label: 'MMR diversity bias',
name: 'mmrDiversityBias',
step: 0.1,
description:
'The diversity bias to use for MMR. This is a value between 0.0 and 1.0' +
'Values closer to 1.0 optimize for the most diverse results.' +
'Defaults to 0 (MMR disabled)',
placeholder: '0.0',
type: 'number',
additionalParams: true,
optional: true
@@ -191,7 +215,9 @@ class Vectara_VectorStores implements INode {
const lambda = nodeData.inputs?.lambda as number
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
const k = topK ? parseFloat(topK) : 4
const k = topK ? parseFloat(topK) : 5
const mmrK = nodeData.inputs?.mmrK as number
const mmrDiversityBias = nodeData.inputs?.mmrDiversityBias as number
const vectaraArgs: VectaraLibArgs = {
apiKey: apiKey,
@@ -208,6 +234,11 @@ class Vectara_VectorStores implements INode {
if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
vectaraFilter.contextConfig = vectaraContextConfig
const mmrConfig: MMRConfig = {}
mmrConfig.enabled = mmrDiversityBias > 0
mmrConfig.mmrTopK = mmrK
mmrConfig.diversityBias = mmrDiversityBias
vectaraFilter.mmrConfig = mmrConfig
const vectorStore = new VectaraStore(vectaraArgs)