[Prostgres Vector Store] Add PGVector Driver option + Fix null character issue w/ TypeORM Driver (#3367)

* Add PGVector Driver option + Fix null character issue w/ TypeORM Driver

* Handle file upload case with PGVector

* Cleanup

* Fix data filtering for chatflow uploaded files

* Add distanceStrategy parameter

* Fix query to improve chatflow uploaded files filtering

* Ensure PGVector release connections

* Await client connected

* Make Postgres credentials optionnal when set on env variables

* Document env variables in nodes directories

* Prevent reuse client

* Fix empty metadataFilter

* Update CONTRIBUTING.md

* Update Postgres.ts

---------

Co-authored-by: Henry Heng <henryheng@flowiseai.com>

packages/components/nodes/vectorstores/Postgres/Postgres.ts

@@ -1,13 +1,16 @@
-import { Pool } from 'pg'
 import { flatten } from 'lodash'
-import { DataSourceOptions } from 'typeorm'
-import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { TypeORMVectorStore, TypeORMVectorStoreDocument } from '@langchain/community/vectorstores/typeorm'
 import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
-import { FLOWISE_CHATID, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { FLOWISE_CHATID, getBaseClasses } from '../../../src/utils'
 import { index } from '../../../src/indexing'
 import { howToUseFileUpload } from '../VectorStoreUtils'
+import { VectorStore } from '@langchain/core/vectorstores'
+import { VectorStoreDriver } from './driver/Base'
+import { TypeORMDriver } from './driver/TypeORM'
+import { PGVectorDriver } from './driver/PGVector'
+import { getContentColumnName, getDatabase, getHost, getPort, getTableName } from './utils'
+
+const serverCredentialsExists = !!process.env.POSTGRES_VECTORSTORE_USER && !!process.env.POSTGRES_VECTORSTORE_PASSWORD
 
 class Postgres_VectorStores implements INode {
     label: string
@@ -26,7 +29,7 @@ class Postgres_VectorStores implements INode {
     constructor() {
         this.label = 'Postgres'
         this.name = 'postgres'
-        this.version = 6.0
+        this.version = 7.0
         this.type = 'Postgres'
         this.icon = 'postgres.svg'
         this.category = 'Vector Stores'
@@ -36,7 +39,8 @@ class Postgres_VectorStores implements INode {
             label: 'Connect Credential',
             name: 'credential',
             type: 'credential',
-            credentialNames: ['PostgresApi']
+            credentialNames: ['PostgresApi'],
+            optional: serverCredentialsExists
         }
         this.inputs = [
             {
@@ -61,28 +65,74 @@ class Postgres_VectorStores implements INode {
             {
                 label: 'Host',
                 name: 'host',
-                type: 'string'
+                type: 'string',
+                placeholder: getHost(),
+                optional: !!getHost()
             },
             {
                 label: 'Database',
                 name: 'database',
-                type: 'string'
+                type: 'string',
+                placeholder: getDatabase(),
+                optional: !!getDatabase()
             },
             {
                 label: 'Port',
                 name: 'port',
                 type: 'number',
-                placeholder: '5432',
+                placeholder: getPort(),
                 optional: true
             },
             {
                 label: 'Table Name',
                 name: 'tableName',
                 type: 'string',
-                placeholder: 'documents',
+                placeholder: getTableName(),
                 additionalParams: true,
                 optional: true
             },
+            {
+                label: 'Driver',
+                name: 'driver',
+                type: 'options',
+                default: 'typeorm',
+                description: 'Different option to connect to Postgres',
+                options: [
+                    {
+                        label: 'TypeORM',
+                        name: 'typeorm'
+                    },
+                    {
+                        label: 'PGVector',
+                        name: 'pgvector'
+                    }
+                ],
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Distance Strategy',
+                name: 'distanceStrategy',
+                description: 'Strategy for calculating distances between vectors',
+                type: 'options',
+                options: [
+                    {
+                        label: 'Cosine',
+                        name: 'cosine'
+                    },
+                    {
+                        label: 'Euclidean',
+                        name: 'euclidean'
+                    },
+                    {
+                        label: 'Inner Product',
+                        name: 'innerProduct'
+                    }
+                ],
+                additionalParams: true,
+                default: 'cosine',
+                optional: true
+            },
             {
                 label: 'File Upload',
                 name: 'fileUpload',
@@ -117,6 +167,15 @@ class Postgres_VectorStores implements INode {
                 type: 'json',
                 additionalParams: true,
                 optional: true
+            },
+            {
+                label: 'Content Column Name',
+                name: 'contentColumnName',
+                description: 'Column name to store the text content (PGVector Driver only, others use pageContent)',
+                type: 'string',
+                placeholder: getContentColumnName(),
+                additionalParams: true,
+                optional: true
             }
         ]
         this.outputs = [
@@ -128,7 +187,7 @@ class Postgres_VectorStores implements INode {
             {
                 label: 'Postgres Vector Store',
                 name: 'vectorStore',
-                baseClasses: [this.type, ...getBaseClasses(TypeORMVectorStore)]
+                baseClasses: [this.type, ...getBaseClasses(VectorStore)]
             }
         ]
     }
@@ -136,43 +195,15 @@ class Postgres_VectorStores implements INode {
     //@ts-ignore
     vectorStoreMethods = {
         async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
-            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-            const user = getCredentialParam('user', credentialData, nodeData)
-            const password = getCredentialParam('password', credentialData, nodeData)
-            const _tableName = nodeData.inputs?.tableName as string
-            const tableName = _tableName ? _tableName : 'documents'
+            const tableName = getTableName(nodeData)
             const docs = nodeData.inputs?.document as Document[]
-            const embeddings = nodeData.inputs?.embeddings as Embeddings
-            const additionalConfig = nodeData.inputs?.additionalConfig as string
             const recordManager = nodeData.inputs?.recordManager
             const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean
-
-            let additionalConfiguration = {}
-            if (additionalConfig) {
-                try {
-                    additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
-                } catch (exception) {
-                    throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
-                }
-            }
-
-            const postgresConnectionOptions = {
-                ...additionalConfiguration,
-                type: 'postgres',
-                host: nodeData.inputs?.host as string,
-                port: nodeData.inputs?.port as number,
-                username: user,
-                password: password,
-                database: nodeData.inputs?.database as string
-            }
-
-            const args = {
-                postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions,
-                tableName: tableName
-            }
+            const vectorStoreDriver: VectorStoreDriver = Postgres_VectorStores.getDriverFromConfig(nodeData, options)
 
             const flattenDocs = docs && docs.length ? flatten(docs) : []
             const finalDocs = []
+
             for (let i = 0; i < flattenDocs.length; i += 1) {
                 if (flattenDocs[i] && flattenDocs[i].pageContent) {
                     if (isFileUploadEnabled && options.chatId) {
@@ -184,24 +215,7 @@ class Postgres_VectorStores implements INode {
 
             try {
                 if (recordManager) {
-                    const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args)
-
-                    // Avoid Illegal invocation error
-                    vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
-                        return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
-                    }
-
-                    vectorStore.delete = async (params: { ids: string[] }): Promise<void> => {
-                        const { ids } = params
-
-                        if (ids?.length) {
-                            try {
-                                vectorStore.appDataSource.getRepository(vectorStore.documentEntity).delete(ids)
-                            } catch (e) {
-                                console.error('Failed to delete')
-                            }
-                        }
-                    }
+                    const vectorStore = await vectorStoreDriver.instanciate()
 
                     await recordManager.createSchema()
 
@@ -218,12 +232,7 @@ class Postgres_VectorStores implements INode {
 
                     return res
                 } else {
-                    const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)
-
-                    // Avoid Illegal invocation error
-                    vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
-                        return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
-                    }
+                    await vectorStoreDriver.fromDocuments(finalDocs)
 
                     return { numAdded: finalDocs.length, addedDocs: finalDocs }
                 }
@@ -232,40 +241,11 @@ class Postgres_VectorStores implements INode {
             }
         },
         async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
-            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-            const user = getCredentialParam('user', credentialData, nodeData)
-            const password = getCredentialParam('password', credentialData, nodeData)
-            const _tableName = nodeData.inputs?.tableName as string
-            const tableName = _tableName ? _tableName : 'documents'
-            const embeddings = nodeData.inputs?.embeddings as Embeddings
-            const additionalConfig = nodeData.inputs?.additionalConfig as string
+            const vectorStoreDriver: VectorStoreDriver = Postgres_VectorStores.getDriverFromConfig(nodeData, options)
+            const tableName = getTableName(nodeData)
             const recordManager = nodeData.inputs?.recordManager
 
-            let additionalConfiguration = {}
-            if (additionalConfig) {
-                try {
-                    additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
-                } catch (exception) {
-                    throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
-                }
-            }
-
-            const postgresConnectionOptions = {
-                ...additionalConfiguration,
-                type: 'postgres',
-                host: nodeData.inputs?.host as string,
-                port: nodeData.inputs?.port as number,
-                username: user,
-                password: password,
-                database: nodeData.inputs?.database as string
-            }
-
-            const args = {
-                postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions,
-                tableName: tableName
-            }
-
-            const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args)
+            const vectorStore = await vectorStoreDriver.instanciate()
 
             try {
                 if (recordManager) {
@@ -286,13 +266,7 @@ class Postgres_VectorStores implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const user = getCredentialParam('user', credentialData, nodeData)
-        const password = getCredentialParam('password', credentialData, nodeData)
-        const _tableName = nodeData.inputs?.tableName as string
-        const tableName = _tableName ? _tableName : 'documents'
-        const embeddings = nodeData.inputs?.embeddings as Embeddings
-        const additionalConfig = nodeData.inputs?.additionalConfig as string
+        const vectorStoreDriver: VectorStoreDriver = Postgres_VectorStores.getDriverFromConfig(nodeData, options)
         const output = nodeData.outputs?.output as string
         const topK = nodeData.inputs?.topK as string
         const k = topK ? parseFloat(topK) : 4
@@ -304,50 +278,13 @@ class Postgres_VectorStores implements INode {
             pgMetadataFilter = typeof _pgMetadataFilter === 'object' ? _pgMetadataFilter : JSON.parse(_pgMetadataFilter)
         }
         if (isFileUploadEnabled && options.chatId) {
-            pgMetadataFilter = pgMetadataFilter || {}
             pgMetadataFilter = {
-                ...pgMetadataFilter,
-                [FLOWISE_CHATID]: options.chatId,
-                $notexists: FLOWISE_CHATID // special filter to check if the field does not exist
+                ...(pgMetadataFilter || {}),
+                [FLOWISE_CHATID]: options.chatId
             }
         }
 
-        let additionalConfiguration = {}
-        if (additionalConfig) {
-            try {
-                additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
-            } catch (exception) {
-                throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
-            }
-        }
-
-        const postgresConnectionOptions = {
-            ...additionalConfiguration,
-            type: 'postgres',
-            host: nodeData.inputs?.host as string,
-            port: nodeData.inputs?.port as number,
-            username: user, // Required by TypeORMVectorStore
-            user: user, // Required by Pool in similaritySearchVectorWithScore
-            password: password,
-            database: nodeData.inputs?.database as string
-        }
-
-        const args = {
-            postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions,
-            tableName: tableName
-        }
-
-        const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args)
-
-        // Rewrite the method to use pg pool connection instead of the default connection
-        /* Otherwise a connection error is displayed when the chain tries to execute the function
-            [chain/start] [1:chain:ConversationalRetrievalQAChain] Entering Chain run with input: { "question": "what the document is about", "chat_history": [] }
-            [retriever/start] [1:chain:ConversationalRetrievalQAChain > 2:retriever:VectorStoreRetriever] Entering Retriever run with input: { "query": "what the document is about" }
-            [ERROR]: uncaughtException:  Illegal invocation TypeError: Illegal invocation at Socket.ref (node:net:1524:18) at Connection.ref (.../node_modules/pg/lib/connection.js:183:17) at Client.ref (.../node_modules/pg/lib/client.js:591:21) at BoundPool._pulseQueue (/node_modules/pg-pool/index.js:148:28) at .../node_modules/pg-pool/index.js:184:37 at process.processTicksAndRejections (node:internal/process/task_queues:77:11)
-        */
-        vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
-            return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter ?? pgMetadataFilter)
-        }
+        const vectorStore = await vectorStoreDriver.instanciate(pgMetadataFilter)
 
         if (output === 'retriever') {
             const retriever = vectorStore.asRetriever(k)
@@ -361,51 +298,17 @@ class Postgres_VectorStores implements INode {
         }
         return vectorStore
     }
-}
 
-const similaritySearchVectorWithScore = async (
-    query: number[],
-    k: number,
-    tableName: string,
-    postgresConnectionOptions: ICommonObject,
-    filter?: any
-) => {
-    const embeddingString = `[${query.join(',')}]`
-    let _filter = '{}'
-    let notExists = ''
-    if (filter && typeof filter === 'object') {
-        if (filter.$notexists) {
-            notExists = `OR NOT (metadata ? '${filter.$notexists}')`
-            delete filter.$notexists
-        }
-        _filter = JSON.stringify(filter)
-    }
-
-    const queryString = `
-        SELECT *, embedding <=> $1 as "_distance"
-        FROM ${tableName}
-        WHERE metadata @> $2
-        ${notExists}
-        ORDER BY "_distance" ASC
-        LIMIT $3;`
-
-    const pool = new Pool(postgresConnectionOptions)
-    const conn = await pool.connect()
-
-    const documents = await conn.query(queryString, [embeddingString, _filter, k])
-
-    conn.release()
-
-    const results = [] as [TypeORMVectorStoreDocument, number][]
-    for (const doc of documents.rows) {
-        if (doc._distance != null && doc.pageContent != null) {
-            const document = new Document(doc) as TypeORMVectorStoreDocument
-            document.id = doc.id
-            results.push([document, doc._distance])
+    static getDriverFromConfig(nodeData: INodeData, options: ICommonObject): VectorStoreDriver {
+        switch (nodeData.inputs?.driver) {
+            case 'typeorm':
+                return new TypeORMDriver(nodeData, options)
+            case 'pgvector':
+                return new PGVectorDriver(nodeData, options)
+            default:
+                return new TypeORMDriver(nodeData, options)
         }
     }
-
-    return results
 }
 
 module.exports = { nodeClass: Postgres_VectorStores }