Feature/Indexing (#1802)

* indexing

* fix for multiple files upsert

* fix default Postgres port

* fix SQLite node description

* add MySQLRecordManager node

* fix MySQL unique index

* add upsert history

* update jsx ui

* lint-fix

* update dialog details

* update llamaindex pinecone

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>

packages/components/credentials/MySQLApi.credential.ts

@@ -0,0 +1,31 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class MySQLApi implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    description: string
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'MySQL API'
+        this.name = 'MySQLApi'
+        this.version = 1.0
+        this.inputs = [
+            {
+                label: 'User',
+                name: 'user',
+                type: 'string',
+                placeholder: '<MYSQL_USERNAME>'
+            },
+            {
+                label: 'Password',
+                name: 'password',
+                type: 'password',
+                placeholder: '<MYSQL_PASSWORD>'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: MySQLApi }

packages/components/nodes/recordmanager/MySQLRecordManager/MySQLrecordManager.ts

@@ -0,0 +1,361 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
+import { DataSource, QueryRunner } from 'typeorm'
+
+class MySQLRecordManager_RecordManager implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    badge: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'MySQL Record Manager'
+        this.name = 'MySQLRecordManager'
+        this.version = 1.0
+        this.type = 'MySQL RecordManager'
+        this.icon = 'mysql.png'
+        this.category = 'Record Manager'
+        this.description = 'Use MySQL to keep track of document writes into the vector databases'
+        this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(MySQLRecordManager)]
+        this.badge = 'NEW'
+        this.inputs = [
+            {
+                label: 'Host',
+                name: 'host',
+                type: 'string'
+            },
+            {
+                label: 'Database',
+                name: 'database',
+                type: 'string'
+            },
+            {
+                label: 'Port',
+                name: 'port',
+                type: 'number',
+                placeholder: '3306',
+                optional: true
+            },
+            {
+                label: 'Additional Connection Configuration',
+                name: 'additionalConfig',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Table Name',
+                name: 'tableName',
+                type: 'string',
+                placeholder: 'upsertion_records',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Namespace',
+                name: 'namespace',
+                type: 'string',
+                description: 'If not specified, chatflowid will be used',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Cleanup',
+                name: 'cleanup',
+                type: 'options',
+                description:
+                    'Read more on the difference between different cleanup methods <a target="_blank" href="https://js.langchain.com/docs/modules/data_connection/indexing/#deletion-modes">here</a>',
+                options: [
+                    {
+                        label: 'None',
+                        name: 'none',
+                        description: 'No clean up of old content'
+                    },
+                    {
+                        label: 'Incremental',
+                        name: 'incremental',
+                        description:
+                            'Delete previous versions of the content if content of the source document has changed. Important!! SourceId Key must be specified and document metadata must contains the specified key'
+                    },
+                    {
+                        label: 'Full',
+                        name: 'full',
+                        description:
+                            'Same as incremental, but if the source document has been deleted, it will be deleted from vector store as well, incremental mode will not.'
+                    }
+                ],
+                additionalParams: true,
+                default: 'none'
+            },
+            {
+                label: 'SourceId Key',
+                name: 'sourceIdKey',
+                type: 'string',
+                description:
+                    'Key used to get the true source of document, to be compared against the record. Document metadata must contains SourceId Key',
+                default: 'source',
+                placeholder: 'source',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['MySQLApi']
+        }
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const user = getCredentialParam('user', credentialData, nodeData)
+        const password = getCredentialParam('password', credentialData, nodeData)
+        const _tableName = nodeData.inputs?.tableName as string
+        const tableName = _tableName ? _tableName : 'upsertion_records'
+        const additionalConfig = nodeData.inputs?.additionalConfig as string
+        const _namespace = nodeData.inputs?.namespace as string
+        const namespace = _namespace ? _namespace : options.chatflowid
+        const cleanup = nodeData.inputs?.cleanup as string
+        const _sourceIdKey = nodeData.inputs?.sourceIdKey as string
+        const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source'
+
+        let additionalConfiguration = {}
+        if (additionalConfig) {
+            try {
+                additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
+            } catch (exception) {
+                throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
+            }
+        }
+
+        const mysqlOptions = {
+            ...additionalConfiguration,
+            type: 'mysql',
+            host: nodeData.inputs?.host as string,
+            port: nodeData.inputs?.port as number,
+            username: user,
+            password: password,
+            database: nodeData.inputs?.database as string
+        }
+
+        const args = {
+            mysqlOptions,
+            tableName: tableName
+        }
+
+        const recordManager = new MySQLRecordManager(namespace, args)
+
+        ;(recordManager as any).cleanup = cleanup
+        ;(recordManager as any).sourceIdKey = sourceIdKey
+
+        return recordManager
+    }
+}
+
+type MySQLRecordManagerOptions = {
+    mysqlOptions: any
+    tableName?: string
+}
+
+class MySQLRecordManager implements RecordManagerInterface {
+    lc_namespace = ['langchain', 'recordmanagers', 'mysql']
+
+    datasource: DataSource
+
+    queryRunner: QueryRunner
+
+    tableName: string
+
+    namespace: string
+
+    constructor(namespace: string, config: MySQLRecordManagerOptions) {
+        const { mysqlOptions, tableName } = config
+        this.namespace = namespace
+        this.tableName = tableName || 'upsertion_records'
+        this.datasource = new DataSource(mysqlOptions)
+    }
+
+    async createSchema(): Promise<void> {
+        try {
+            const appDataSource = await this.datasource.initialize()
+
+            this.queryRunner = appDataSource.createQueryRunner()
+
+            await this.queryRunner.manager.query(`create table if not exists \`${this.tableName}\` (
+                \`uuid\` varchar(36) primary key default (UUID()),
+                \`key\` varchar(36) not null,
+                \`namespace\` varchar(36) not null,
+                \`updated_at\` DOUBLE precision not null,
+                \`group_id\` varchar(36),
+                unique key \`unique_key_namespace\` (\`key\`,
+\`namespace\`));`)
+            const columns = [`updated_at`, `key`, `namespace`, `group_id`]
+            for (const column of columns) {
+                // MySQL does not support 'IF NOT EXISTS' function for Index
+                const Check = await this.queryRunner.manager.query(
+                    `SELECT COUNT(1) IndexIsThere FROM INFORMATION_SCHEMA.STATISTICS 
+                        WHERE table_schema=DATABASE() AND table_name='${this.tableName}' AND index_name='${column}_index';`
+                )
+                if (Check[0].IndexIsThere === 0)
+                    await this.queryRunner.manager.query(`CREATE INDEX \`${column}_index\`
+        ON \`${this.tableName}\` (\`${column}\`);`)
+            }
+        } catch (e: any) {
+            // This error indicates that the table already exists
+            // Due to asynchronous nature of the code, it is possible that
+            // the table is created between the time we check if it exists
+            // and the time we try to create it. It can be safely ignored.
+            if ('code' in e && e.code === '23505') {
+                return
+            }
+            throw e
+        }
+    }
+
+    async getTime(): Promise<number> {
+        try {
+            const res = await this.queryRunner.manager.query(`SELECT UNIX_TIMESTAMP(NOW()) AS epoch`)
+            return Number.parseFloat(res[0].epoch)
+        } catch (error) {
+            console.error('Error getting time in MySQLRecordManager:')
+            throw error
+        }
+    }
+
+    async update(keys: string[], updateOptions?: UpdateOptions): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const updatedAt = await this.getTime()
+        const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
+
+        if (timeAtLeast && updatedAt < timeAtLeast) {
+            throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`)
+        }
+
+        const groupIds = _groupIds ?? keys.map(() => null)
+
+        if (groupIds.length !== keys.length) {
+            throw new Error(`Number of keys (${keys.length}) does not match number of group_ids (${groupIds.length})`)
+        }
+
+        const recordsToUpsert = keys.map((key, i) => [
+            key,
+            this.namespace,
+            updatedAt,
+            groupIds[i] ?? null // Ensure groupIds[i] is null if undefined
+        ])
+
+        const query = `
+            INSERT INTO \`${this.tableName}\` (\`key\`, \`namespace\`, \`updated_at\`, \`group_id\`)
+            VALUES (?, ?, ?, ?)
+            ON DUPLICATE KEY UPDATE updated_at = updated_at;`
+
+        // To handle multiple files upsert
+        for (const record of recordsToUpsert) {
+            // Consider using a transaction for batch operations
+            await this.queryRunner.manager.query(query, record.flat())
+        }
+    }
+
+    async exists(keys: string[]): Promise<boolean[]> {
+        if (keys.length === 0) {
+            return []
+        }
+
+        // Prepare the placeholders and the query
+        const placeholders = keys.map(() => `?`).join(', ')
+        const query = `
+    SELECT \`key\`
+    FROM \`${this.tableName}\`
+    WHERE \`namespace\` = ? AND \`key\` IN (${placeholders})`
+
+        // Initialize an array to fill with the existence checks
+        const existsArray = new Array(keys.length).fill(false)
+
+        try {
+            // Execute the query
+            const rows = await this.queryRunner.manager.query(query, [this.namespace, ...keys.flat()])
+            // Create a set of existing keys for faster lookup
+            const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key))
+            // Map the input keys to booleans indicating if they exist
+            keys.forEach((key, index) => {
+                existsArray[index] = existingKeysSet.has(key)
+            })
+            return existsArray
+        } catch (error) {
+            console.error('Error checking existence of keys')
+            throw error // Allow the caller to handle the error
+        }
+    }
+
+    async listKeys(options?: ListKeyOptions): Promise<string[]> {
+        try {
+            const { before, after, limit, groupIds } = options ?? {}
+            let query = `SELECT \`key\` FROM \`${this.tableName}\` WHERE \`namespace\` = ?`
+            const values: (string | number | string[])[] = [this.namespace]
+
+            if (before) {
+                query += ` AND \`updated_at\` < ?`
+                values.push(before)
+            }
+
+            if (after) {
+                query += ` AND \`updated_at\` > ?`
+                values.push(after)
+            }
+
+            if (limit) {
+                query += ` LIMIT ?`
+                values.push(limit)
+            }
+
+            if (groupIds && Array.isArray(groupIds)) {
+                query += ` AND \`group_id\` IN (${groupIds
+                    .filter((gid) => gid !== null)
+                    .map(() => '?')
+                    .join(', ')})`
+                values.push(...groupIds.filter((gid): gid is string => gid !== null))
+            }
+
+            query += ';'
+
+            // Directly using try/catch with async/await for cleaner flow
+            const result = await this.queryRunner.manager.query(query, values)
+            return result.map((row: { key: string }) => row.key)
+        } catch (error) {
+            console.error('MySQLRecordManager listKeys Error: ')
+            throw error // Re-throw the error to be handled by the caller
+        }
+    }
+
+    async deleteKeys(keys: string[]): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const placeholders = keys.map(() => '?').join(', ')
+        const query = `DELETE FROM \`${this.tableName}\` WHERE \`namespace\` = ? AND \`key\` IN (${placeholders});`
+        const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v))
+
+        // Directly using try/catch with async/await for cleaner flow
+        try {
+            await this.queryRunner.manager.query(query, values)
+        } catch (error) {
+            console.error('Error deleting keys')
+            throw error // Re-throw the error to be handled by the caller
+        }
+    }
+}
+
+module.exports = { nodeClass: MySQLRecordManager_RecordManager }

packages/components/nodes/recordmanager/PostgresRecordManager/PostgresRecordManager.ts

@@ -0,0 +1,332 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
+import { DataSource, QueryRunner } from 'typeorm'
+
+class PostgresRecordManager_RecordManager implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    badge: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Postgres Record Manager'
+        this.name = 'postgresRecordManager'
+        this.version = 1.0
+        this.type = 'Postgres RecordManager'
+        this.icon = 'postgres.svg'
+        this.category = 'Record Manager'
+        this.description = 'Use Postgres to keep track of document writes into the vector databases'
+        this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(PostgresRecordManager)]
+        this.badge = 'NEW'
+        this.inputs = [
+            {
+                label: 'Host',
+                name: 'host',
+                type: 'string'
+            },
+            {
+                label: 'Database',
+                name: 'database',
+                type: 'string'
+            },
+            {
+                label: 'Port',
+                name: 'port',
+                type: 'number',
+                placeholder: '5432',
+                optional: true
+            },
+            {
+                label: 'Additional Connection Configuration',
+                name: 'additionalConfig',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Table Name',
+                name: 'tableName',
+                type: 'string',
+                placeholder: 'upsertion_records',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Namespace',
+                name: 'namespace',
+                type: 'string',
+                description: 'If not specified, chatflowid will be used',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Cleanup',
+                name: 'cleanup',
+                type: 'options',
+                description:
+                    'Read more on the difference between different cleanup methods <a target="_blank" href="https://js.langchain.com/docs/modules/data_connection/indexing/#deletion-modes">here</a>',
+                options: [
+                    {
+                        label: 'None',
+                        name: 'none',
+                        description: 'No clean up of old content'
+                    },
+                    {
+                        label: 'Incremental',
+                        name: 'incremental',
+                        description:
+                            'Delete previous versions of the content if content of the source document has changed. Important!! SourceId Key must be specified and document metadata must contains the specified key'
+                    },
+                    {
+                        label: 'Full',
+                        name: 'full',
+                        description:
+                            'Same as incremental, but if the source document has been deleted, it will be deleted from vector store as well, incremental mode will not.'
+                    }
+                ],
+                additionalParams: true,
+                default: 'none'
+            },
+            {
+                label: 'SourceId Key',
+                name: 'sourceIdKey',
+                type: 'string',
+                description:
+                    'Key used to get the true source of document, to be compared against the record. Document metadata must contains SourceId Key',
+                default: 'source',
+                placeholder: 'source',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['PostgresApi']
+        }
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const user = getCredentialParam('user', credentialData, nodeData)
+        const password = getCredentialParam('password', credentialData, nodeData)
+        const _tableName = nodeData.inputs?.tableName as string
+        const tableName = _tableName ? _tableName : 'upsertion_records'
+        const additionalConfig = nodeData.inputs?.additionalConfig as string
+        const _namespace = nodeData.inputs?.namespace as string
+        const namespace = _namespace ? _namespace : options.chatflowid
+        const cleanup = nodeData.inputs?.cleanup as string
+        const _sourceIdKey = nodeData.inputs?.sourceIdKey as string
+        const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source'
+
+        let additionalConfiguration = {}
+        if (additionalConfig) {
+            try {
+                additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
+            } catch (exception) {
+                throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
+            }
+        }
+
+        const postgresConnectionOptions = {
+            ...additionalConfiguration,
+            type: 'postgres',
+            host: nodeData.inputs?.host as string,
+            port: nodeData.inputs?.port as number,
+            username: user,
+            password: password,
+            database: nodeData.inputs?.database as string
+        }
+
+        const args = {
+            postgresConnectionOptions: postgresConnectionOptions,
+            tableName: tableName
+        }
+
+        const recordManager = new PostgresRecordManager(namespace, args)
+
+        ;(recordManager as any).cleanup = cleanup
+        ;(recordManager as any).sourceIdKey = sourceIdKey
+
+        return recordManager
+    }
+}
+
+type PostgresRecordManagerOptions = {
+    postgresConnectionOptions: any
+    tableName?: string
+}
+
+class PostgresRecordManager implements RecordManagerInterface {
+    lc_namespace = ['langchain', 'recordmanagers', 'postgres']
+
+    datasource: DataSource
+
+    queryRunner: QueryRunner
+
+    tableName: string
+
+    namespace: string
+
+    constructor(namespace: string, config: PostgresRecordManagerOptions) {
+        const { postgresConnectionOptions, tableName } = config
+        this.namespace = namespace
+        this.datasource = new DataSource(postgresConnectionOptions)
+        this.tableName = tableName || 'upsertion_records'
+    }
+
+    async createSchema(): Promise<void> {
+        try {
+            const appDataSource = await this.datasource.initialize()
+
+            this.queryRunner = appDataSource.createQueryRunner()
+
+            await this.queryRunner.manager.query(`
+  CREATE TABLE IF NOT EXISTS "${this.tableName}" (
+    uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    key TEXT NOT NULL,
+    namespace TEXT NOT NULL,
+    updated_at Double PRECISION NOT NULL,
+    group_id TEXT,
+    UNIQUE (key, namespace)
+  );
+  CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at);
+  CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key);
+  CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace);
+  CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
+        } catch (e: any) {
+            // This error indicates that the table already exists
+            // Due to asynchronous nature of the code, it is possible that
+            // the table is created between the time we check if it exists
+            // and the time we try to create it. It can be safely ignored.
+            if ('code' in e && e.code === '23505') {
+                return
+            }
+            throw e
+        }
+    }
+
+    async getTime(): Promise<number> {
+        const res = await this.queryRunner.manager.query('SELECT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)')
+        return Number.parseFloat(res[0].extract)
+    }
+
+    /**
+     * Generates the SQL placeholders for a specific row at the provided index.
+     *
+     * @param index - The index of the row for which placeholders need to be generated.
+     * @param numOfColumns - The number of columns we are inserting data into.
+     * @returns The SQL placeholders for the row values.
+     */
+    private generatePlaceholderForRowAt(index: number, numOfColumns: number): string {
+        const placeholders = []
+        for (let i = 0; i < numOfColumns; i += 1) {
+            placeholders.push(`$${index * numOfColumns + i + 1}`)
+        }
+        return `(${placeholders.join(', ')})`
+    }
+
+    async update(keys: string[], updateOptions?: UpdateOptions): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const updatedAt = await this.getTime()
+        const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
+
+        if (timeAtLeast && updatedAt < timeAtLeast) {
+            throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`)
+        }
+
+        const groupIds = _groupIds ?? keys.map(() => null)
+
+        if (groupIds.length !== keys.length) {
+            throw new Error(`Number of keys (${keys.length}) does not match number of group_ids ${groupIds.length})`)
+        }
+
+        const recordsToUpsert = keys.map((key, i) => [key, this.namespace, updatedAt, groupIds[i]])
+
+        const valuesPlaceholders = recordsToUpsert.map((_, j) => this.generatePlaceholderForRowAt(j, recordsToUpsert[0].length)).join(', ')
+
+        const query = `INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) VALUES ${valuesPlaceholders} ON CONFLICT (key, namespace) DO UPDATE SET updated_at = EXCLUDED.updated_at;`
+        await this.queryRunner.manager.query(query, recordsToUpsert.flat())
+    }
+
+    async exists(keys: string[]): Promise<boolean[]> {
+        if (keys.length === 0) {
+            return []
+        }
+
+        const startIndex = 2
+        const arrayPlaceholders = keys.map((_, i) => `$${i + startIndex}`).join(', ')
+
+        const query = `
+        SELECT k, (key is not null) ex from unnest(ARRAY[${arrayPlaceholders}]) k left join "${this.tableName}" on k=key and namespace = $1;
+        `
+        const res = await this.queryRunner.manager.query(query, [this.namespace, ...keys.flat()])
+        return res.map((row: { ex: boolean }) => row.ex)
+    }
+
+    async listKeys(options?: ListKeyOptions): Promise<string[]> {
+        const { before, after, limit, groupIds } = options ?? {}
+        let query = `SELECT key FROM "${this.tableName}" WHERE namespace = $1`
+        const values: (string | number | (string | null)[])[] = [this.namespace]
+
+        let index = 2
+        if (before) {
+            values.push(before)
+            query += ` AND updated_at < $${index}`
+            index += 1
+        }
+
+        if (after) {
+            values.push(after)
+            query += ` AND updated_at > $${index}`
+            index += 1
+        }
+
+        if (limit) {
+            values.push(limit)
+            query += ` LIMIT $${index}`
+            index += 1
+        }
+
+        if (groupIds) {
+            values.push(groupIds)
+            query += ` AND group_id = ANY($${index})`
+            index += 1
+        }
+
+        query += ';'
+        const res = await this.queryRunner.manager.query(query, values)
+        return res.map((row: { key: string }) => row.key)
+    }
+
+    async deleteKeys(keys: string[]): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const query = `DELETE FROM "${this.tableName}" WHERE namespace = $1 AND key = ANY($2);`
+        await this.queryRunner.manager.query(query, [this.namespace, keys])
+    }
+
+    /**
+     * Terminates the connection pool.
+     * @returns {Promise<void>}
+     */
+    async end(): Promise<void> {
+        if (this.datasource && this.datasource.isInitialized) await this.datasource.destroy()
+    }
+}
+
+module.exports = { nodeClass: PostgresRecordManager_RecordManager }

packages/components/nodes/recordmanager/SQLiteRecordManager/SQLiteRecordManager.ts

@@ -0,0 +1,332 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses } from '../../../src/utils'
+import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
+import { DataSource, QueryRunner } from 'typeorm'
+import path from 'path'
+
+class SQLiteRecordManager_RecordManager implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    badge: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'SQLite Record Manager'
+        this.name = 'SQLiteRecordManager'
+        this.version = 1.0
+        this.type = 'SQLite RecordManager'
+        this.icon = 'sqlite.png'
+        this.category = 'Record Manager'
+        this.description = 'Use SQLite to keep track of document writes into the vector databases'
+        this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(SQLiteRecordManager)]
+        this.badge = 'NEW'
+        this.inputs = [
+            {
+                label: 'Database File Path',
+                name: 'databaseFilePath',
+                type: 'string',
+                placeholder: 'C:\\Users\\User\\.flowise\\database.sqlite'
+            },
+            {
+                label: 'Additional Connection Configuration',
+                name: 'additionalConfig',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Table Name',
+                name: 'tableName',
+                type: 'string',
+                placeholder: 'upsertion_records',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Namespace',
+                name: 'namespace',
+                type: 'string',
+                description: 'If not specified, chatflowid will be used',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Cleanup',
+                name: 'cleanup',
+                type: 'options',
+                description:
+                    'Read more on the difference between different cleanup methods <a target="_blank" href="https://js.langchain.com/docs/modules/data_connection/indexing/#deletion-modes">here</a>',
+                options: [
+                    {
+                        label: 'None',
+                        name: 'none',
+                        description: 'No clean up of old content'
+                    },
+                    {
+                        label: 'Incremental',
+                        name: 'incremental',
+                        description:
+                            'Delete previous versions of the content if content of the source document has changed. Important!! SourceId Key must be specified and document metadata must contains the specified key'
+                    },
+                    {
+                        label: 'Full',
+                        name: 'full',
+                        description:
+                            'Same as incremental, but if the source document has been deleted, it will be deleted from vector store as well, incremental mode will not.'
+                    }
+                ],
+                additionalParams: true,
+                default: 'none'
+            },
+            {
+                label: 'SourceId Key',
+                name: 'sourceIdKey',
+                type: 'string',
+                description:
+                    'Key used to get the true source of document, to be compared against the record. Document metadata must contains SourceId Key',
+                default: 'source',
+                placeholder: 'source',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const _tableName = nodeData.inputs?.tableName as string
+        const tableName = _tableName ? _tableName : 'upsertion_records'
+        const additionalConfig = nodeData.inputs?.additionalConfig as string
+        const _namespace = nodeData.inputs?.namespace as string
+        const namespace = _namespace ? _namespace : options.chatflowid
+        const cleanup = nodeData.inputs?.cleanup as string
+        const _sourceIdKey = nodeData.inputs?.sourceIdKey as string
+        const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source'
+        const databaseFilePath = nodeData.inputs?.databaseFilePath as string
+
+        let additionalConfiguration = {}
+        if (additionalConfig) {
+            try {
+                additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
+            } catch (exception) {
+                throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
+            }
+        }
+
+        const sqliteOptions = {
+            ...additionalConfiguration,
+            type: 'sqlite',
+            database: path.resolve(databaseFilePath)
+        }
+
+        const args = {
+            sqliteOptions,
+            tableName: tableName
+        }
+
+        const recordManager = new SQLiteRecordManager(namespace, args)
+
+        ;(recordManager as any).cleanup = cleanup
+        ;(recordManager as any).sourceIdKey = sourceIdKey
+
+        return recordManager
+    }
+}
+
+type SQLiteRecordManagerOptions = {
+    sqliteOptions: any
+    tableName?: string
+}
+
+class SQLiteRecordManager implements RecordManagerInterface {
+    lc_namespace = ['langchain', 'recordmanagers', 'sqlite']
+
+    datasource: DataSource
+
+    queryRunner: QueryRunner
+
+    tableName: string
+
+    namespace: string
+
+    constructor(namespace: string, config: SQLiteRecordManagerOptions) {
+        const { sqliteOptions, tableName } = config
+        this.namespace = namespace
+        this.tableName = tableName || 'upsertion_records'
+        this.datasource = new DataSource(sqliteOptions)
+    }
+
+    async createSchema(): Promise<void> {
+        try {
+            const appDataSource = await this.datasource.initialize()
+
+            this.queryRunner = appDataSource.createQueryRunner()
+
+            await this.queryRunner.manager.query(`
+CREATE TABLE IF NOT EXISTS "${this.tableName}" (
+  uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
+  key TEXT NOT NULL,
+  namespace TEXT NOT NULL,
+  updated_at REAL NOT NULL,
+  group_id TEXT,
+  UNIQUE (key, namespace)
+);
+CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at);
+CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key);
+CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace);
+CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
+        } catch (e: any) {
+            // This error indicates that the table already exists
+            // Due to asynchronous nature of the code, it is possible that
+            // the table is created between the time we check if it exists
+            // and the time we try to create it. It can be safely ignored.
+            if ('code' in e && e.code === '23505') {
+                return
+            }
+            throw e
+        }
+    }
+
+    async getTime(): Promise<number> {
+        try {
+            const res = await this.queryRunner.manager.query(`SELECT strftime('%s', 'now') AS epoch`)
+            return Number.parseFloat(res[0].epoch)
+        } catch (error) {
+            console.error('Error getting time in SQLiteRecordManager:')
+            throw error
+        }
+    }
+
+    async update(keys: string[], updateOptions?: UpdateOptions): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const updatedAt = await this.getTime()
+        const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
+
+        if (timeAtLeast && updatedAt < timeAtLeast) {
+            throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`)
+        }
+
+        const groupIds = _groupIds ?? keys.map(() => null)
+
+        if (groupIds.length !== keys.length) {
+            throw new Error(`Number of keys (${keys.length}) does not match number of group_ids (${groupIds.length})`)
+        }
+
+        const recordsToUpsert = keys.map((key, i) => [
+            key,
+            this.namespace,
+            updatedAt,
+            groupIds[i] ?? null // Ensure groupIds[i] is null if undefined
+        ])
+
+        const query = `
+        INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id)
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at`
+
+        // To handle multiple files upsert
+        for (const record of recordsToUpsert) {
+            // Consider using a transaction for batch operations
+            await this.queryRunner.manager.query(query, record.flat())
+        }
+    }
+
+    async exists(keys: string[]): Promise<boolean[]> {
+        if (keys.length === 0) {
+            return []
+        }
+
+        // Prepare the placeholders and the query
+        const placeholders = keys.map(() => `?`).join(', ')
+        const sql = `
+    SELECT key
+    FROM "${this.tableName}"
+    WHERE namespace = ? AND key IN (${placeholders})`
+
+        // Initialize an array to fill with the existence checks
+        const existsArray = new Array(keys.length).fill(false)
+
+        try {
+            // Execute the query
+            const rows = await this.queryRunner.manager.query(sql, [this.namespace, ...keys.flat()])
+            // Create a set of existing keys for faster lookup
+            const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key))
+            // Map the input keys to booleans indicating if they exist
+            keys.forEach((key, index) => {
+                existsArray[index] = existingKeysSet.has(key)
+            })
+            return existsArray
+        } catch (error) {
+            console.error('Error checking existence of keys')
+            throw error // Allow the caller to handle the error
+        }
+    }
+
+    async listKeys(options?: ListKeyOptions): Promise<string[]> {
+        const { before, after, limit, groupIds } = options ?? {}
+        let query = `SELECT key FROM "${this.tableName}" WHERE namespace = ?`
+        const values: (string | number | string[])[] = [this.namespace]
+
+        if (before) {
+            query += ` AND updated_at < ?`
+            values.push(before)
+        }
+
+        if (after) {
+            query += ` AND updated_at > ?`
+            values.push(after)
+        }
+
+        if (limit) {
+            query += ` LIMIT ?`
+            values.push(limit)
+        }
+
+        if (groupIds && Array.isArray(groupIds)) {
+            query += ` AND group_id IN (${groupIds
+                .filter((gid) => gid !== null)
+                .map(() => '?')
+                .join(', ')})`
+            values.push(...groupIds.filter((gid): gid is string => gid !== null))
+        }
+
+        query += ';'
+
+        // Directly using try/catch with async/await for cleaner flow
+        try {
+            const result = await this.queryRunner.manager.query(query, values)
+            return result.map((row: { key: string }) => row.key)
+        } catch (error) {
+            console.error('Error listing keys.')
+            throw error // Re-throw the error to be handled by the caller
+        }
+    }
+
+    async deleteKeys(keys: string[]): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const placeholders = keys.map(() => '?').join(', ')
+        const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${placeholders});`
+        const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v))
+
+        // Directly using try/catch with async/await for cleaner flow
+        try {
+            await this.queryRunner.manager.query(query, values)
+        } catch (error) {
+            console.error('Error deleting keys')
+            throw error // Re-throw the error to be handled by the caller
+        }
+    }
+}
+
+module.exports = { nodeClass: SQLiteRecordManager_RecordManager }

packages/components/nodes/vectorstores/Astra/Astra.ts

@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
 import { AstraDBVectorStore, AstraLibArgs } from '@langchain/community/vectorstores/astradb'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
 
@@ -101,7 +101,7 @@ class Astra_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const vectorDimension = nodeData.inputs?.vectorDimension as number
@@ -142,6 +142,7 @@ class Astra_VectorStores implements INode {
 
             try {
                 await AstraDBVectorStore.fromDocuments(finalDocs, embeddings, astraConfig)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Chroma/Chroma.ts

@@ -2,9 +2,10 @@ import { flatten } from 'lodash'
 import { Chroma } from '@langchain/community/vectorstores/chroma'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { ChromaExtended } from './core'
+import { index } from '../../../src/indexing'
 
 class Chroma_VectorStores implements INode {
     label: string
@@ -23,7 +24,7 @@ class Chroma_VectorStores implements INode {
     constructor() {
         this.label = 'Chroma'
         this.name = 'chroma'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'Chroma'
         this.icon = 'chroma.svg'
         this.category = 'Vector Stores'
@@ -51,6 +52,13 @@ class Chroma_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Collection Name',
                 name: 'collectionName',
@@ -95,11 +103,12 @@ class Chroma_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const collectionName = nodeData.inputs?.collectionName as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const chromaURL = nodeData.inputs?.chromaURL as string
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
@@ -121,7 +130,24 @@ class Chroma_VectorStores implements INode {
             if (chromaApiKey) obj.chromaApiKey = chromaApiKey
 
             try {
-                await ChromaExtended.fromDocuments(finalDocs, embeddings, obj)
+                if (recordManager) {
+                    const vectorStore = await ChromaExtended.fromExistingCollection(embeddings, obj)
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: collectionName
+                        }
+                    })
+                    return res
+                } else {
+                    await ChromaExtended.fromDocuments(finalDocs, embeddings, obj)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts

@@ -3,8 +3,9 @@ import { Client, ClientOptions } from '@elastic/elasticsearch'
 import { Document } from '@langchain/core/documents'
 import { Embeddings } from '@langchain/core/embeddings'
 import { ElasticClientArgs, ElasticVectorSearch } from '@langchain/community/vectorstores/elasticsearch'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { index } from '../../../src/indexing'
 
 class Elasticsearch_VectorStores implements INode {
     label: string
@@ -23,7 +24,7 @@ class Elasticsearch_VectorStores implements INode {
     constructor() {
         this.label = 'Elasticsearch'
         this.name = 'elasticsearch'
-        this.version = 1.0
+        this.version = 2.0
         this.description =
             'Upsert embedded data and perform similarity search upon query using Elasticsearch, a distributed search and analytics engine'
         this.type = 'Elasticsearch'
@@ -50,6 +51,13 @@ class Elasticsearch_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Index Name',
                 name: 'indexName',
@@ -105,13 +113,14 @@ class Elasticsearch_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
             const cloudId = getCredentialParam('cloudId', credentialData, nodeData)
             const indexName = nodeData.inputs?.indexName as string
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const similarityMeasure = nodeData.inputs?.similarityMeasure as string
+            const recordManager = nodeData.inputs?.recordManager
 
             const docs = nodeData.inputs?.document as Document[]
             const flattenDocs = docs && docs.length ? flatten(docs) : []
@@ -134,7 +143,24 @@ class Elasticsearch_VectorStores implements INode {
             const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
 
             try {
-                await vectorStore.addDocuments(finalDocs)
+                if (recordManager) {
+                    const vectorStore = await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs)
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: indexName
+                        }
+                    })
+                    return res
+                } else {
+                    await vectorStore.addDocuments(finalDocs)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Faiss/Faiss.ts

@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
 import { Document } from '@langchain/core/documents'
 import { FaissStore } from '@langchain/community/vectorstores/faiss'
 import { Embeddings } from '@langchain/core/embeddings'
-import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
 
 class Faiss_VectorStores implements INode {
@@ -74,7 +74,7 @@ class Faiss_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData): Promise<void> {
+        async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const basePath = nodeData.inputs?.basePath as string
@@ -95,6 +95,8 @@ class Faiss_VectorStores implements INode {
                 vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number) => {
                     return await similaritySearchVectorWithScore(query, k, vectorStore)
                 }
+
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts

@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
 import { MemoryVectorStore } from 'langchain/vectorstores/memory'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
 
 class InMemoryVectorStore_VectorStores implements INode {
@@ -64,7 +64,7 @@ class InMemoryVectorStore_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData): Promise<void> {
+        async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
 
@@ -78,6 +78,7 @@ class InMemoryVectorStore_VectorStores implements INode {
 
             try {
                 await MemoryVectorStore.fromDocuments(finalDocs, embeddings)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Milvus/Milvus.ts

@@ -3,7 +3,7 @@ import { DataType, ErrorCode, MetricType, IndexType } from '@zilliz/milvus2-sdk-
 import { Document } from '@langchain/core/documents'
 import { MilvusLibArgs, Milvus } from '@langchain/community/vectorstores/milvus'
 import { Embeddings } from '@langchain/core/embeddings'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 
 interface InsertRow {
@@ -109,7 +109,7 @@ class Milvus_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             // server setup
             const address = nodeData.inputs?.milvusServerUrl as string
             const collectionName = nodeData.inputs?.milvusCollection as string
@@ -147,6 +147,8 @@ class Milvus_VectorStores implements INode {
                 vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: string) => {
                     return await similaritySearchVectorWithScore(query, k, vectorStore, undefined, filter)
                 }
+
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/MongoDBAtlas/MongoDBAtlas.ts

@@ -3,7 +3,7 @@ import { MongoClient } from 'mongodb'
 import { MongoDBAtlasVectorSearch } from '@langchain/mongodb'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
 
@@ -113,7 +113,7 @@ class MongoDBAtlas_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const databaseName = nodeData.inputs?.databaseName as string
             const collectionName = nodeData.inputs?.collectionName as string
@@ -149,6 +149,7 @@ class MongoDBAtlas_VectorStores implements INode {
                     embeddingKey
                 })
                 await mongoDBAtlasVectorSearch.addDocuments(finalDocs)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/OpenSearch/OpenSearch.ts

@@ -3,7 +3,7 @@ import { Client } from '@opensearch-project/opensearch'
 import { Document } from '@langchain/core/documents'
 import { OpenSearchVectorStore } from '@langchain/community/vectorstores/opensearch'
 import { Embeddings } from '@langchain/core/embeddings'
-import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
 
 class OpenSearch_VectorStores implements INode {
@@ -79,7 +79,7 @@ class OpenSearch_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData): Promise<void> {
+        async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const opensearchURL = nodeData.inputs?.opensearchURL as string
@@ -102,6 +102,7 @@ class OpenSearch_VectorStores implements INode {
                     client,
                     indexName: indexName
                 })
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Pinecone/Pinecone.ts

@@ -3,9 +3,10 @@ import { Pinecone } from '@pinecone-database/pinecone'
 import { PineconeStoreParams, PineconeStore } from '@langchain/pinecone'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
+import { index } from '../../../src/indexing'
 
 class Pinecone_VectorStores implements INode {
     label: string
@@ -24,7 +25,7 @@ class Pinecone_VectorStores implements INode {
     constructor() {
         this.label = 'Pinecone'
         this.name = 'pinecone'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Pinecone'
         this.icon = 'pinecone.svg'
         this.category = 'Vector Stores'
@@ -50,6 +51,13 @@ class Pinecone_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Pinecone Index',
                 name: 'pineconeIndex',
@@ -97,11 +105,12 @@ class Pinecone_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
-            const index = nodeData.inputs?.pineconeIndex as string
+            const _index = nodeData.inputs?.pineconeIndex as string
             const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
@@ -110,7 +119,7 @@ class Pinecone_VectorStores implements INode {
                 apiKey: pineconeApiKey
             })
 
-            const pineconeIndex = client.Index(index)
+            const pineconeIndex = client.Index(_index)
 
             const flattenDocs = docs && docs.length ? flatten(docs) : []
             const finalDocs = []
@@ -127,7 +136,25 @@ class Pinecone_VectorStores implements INode {
             if (pineconeNamespace) obj.namespace = pineconeNamespace
 
             try {
-                await PineconeStore.fromDocuments(finalDocs, embeddings, obj)
+                if (recordManager) {
+                    const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: pineconeNamespace
+                        }
+                    })
+
+                    return res
+                } else {
+                    await PineconeStore.fromDocuments(finalDocs, embeddings, obj)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Pinecone/Pinecone_LlamaIndex.ts

@@ -13,7 +13,7 @@ import {
 import { FetchResponse, Index, Pinecone, ScoredPineconeRecord } from '@pinecone-database/pinecone'
 import { flatten } from 'lodash'
 import { Document as LCDocument } from 'langchain/document'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { flattenObject, getCredentialData, getCredentialParam } from '../../../src/utils'
 
 class PineconeLlamaIndex_VectorStores implements INode {
@@ -110,7 +110,7 @@ class PineconeLlamaIndex_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const indexName = nodeData.inputs?.pineconeIndex as string
             const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
             const docs = nodeData.inputs?.document as LCDocument[]
@@ -144,6 +144,7 @@ class PineconeLlamaIndex_VectorStores implements INode {
 
             try {
                 await VectorStoreIndex.fromDocuments(llamadocs, { serviceContext, storageContext })
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Postgres/Postgres.ts

@@ -4,8 +4,9 @@ import { DataSourceOptions } from 'typeorm'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
 import { TypeORMVectorStore, TypeORMVectorStoreDocument } from '@langchain/community/vectorstores/typeorm'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { index } from '../../../src/indexing'
 
 class Postgres_VectorStores implements INode {
     label: string
@@ -24,7 +25,7 @@ class Postgres_VectorStores implements INode {
     constructor() {
         this.label = 'Postgres'
         this.name = 'postgres'
-        this.version = 3.0
+        this.version = 4.0
         this.type = 'Postgres'
         this.icon = 'postgres.svg'
         this.category = 'Vector Stores'
@@ -50,6 +51,13 @@ class Postgres_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Host',
                 name: 'host',
@@ -108,7 +116,7 @@ class Postgres_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const user = getCredentialParam('user', credentialData, nodeData)
             const password = getCredentialParam('password', credentialData, nodeData)
@@ -117,6 +125,7 @@ class Postgres_VectorStores implements INode {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const additionalConfig = nodeData.inputs?.additionalConfig as string
+            const recordManager = nodeData.inputs?.recordManager
 
             let additionalConfiguration = {}
             if (additionalConfig) {
@@ -151,11 +160,37 @@ class Postgres_VectorStores implements INode {
             }
 
             try {
-                const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)
+                if (recordManager) {
+                    const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args)
 
-                // Avoid Illegal invocation error
+                    // Avoid Illegal invocation error
-                vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
+                    vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
-                    return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
+                        return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
+                    }
+
+                    await recordManager.createSchema()
+
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: tableName
+                        }
+                    })
+
+                    return res
+                } else {
+                    const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)
+
+                    // Avoid Illegal invocation error
+                    vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
+                        return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
+                    }
+
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
                 }
             } catch (e) {
                 throw new Error(e)

packages/components/nodes/vectorstores/Qdrant/Qdrant.ts

@@ -1,13 +1,19 @@
 import { flatten } from 'lodash'
+import { v4 as uuid } from 'uuid'
 import { QdrantClient } from '@qdrant/js-client-rest'
 import { VectorStoreRetrieverInput } from '@langchain/core/vectorstores'
 import { Document } from '@langchain/core/documents'
 import { QdrantVectorStore, QdrantLibArgs } from '@langchain/community/vectorstores/qdrant'
 import { Embeddings } from '@langchain/core/embeddings'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { index } from '../../../src/indexing'
 
 type RetrieverConfig = Partial<VectorStoreRetrieverInput<QdrantVectorStore>>
+type QdrantAddDocumentOptions = {
+    customPayload?: Record<string, any>[]
+    ids?: string[]
+}
 
 class Qdrant_VectorStores implements INode {
     label: string
@@ -26,7 +32,7 @@ class Qdrant_VectorStores implements INode {
     constructor() {
         this.label = 'Qdrant'
         this.name = 'qdrant'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'Qdrant'
         this.icon = 'qdrant.png'
         this.category = 'Vector Stores'
@@ -55,6 +61,13 @@ class Qdrant_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Qdrant Server URL',
                 name: 'qdrantServerUrl',
@@ -138,13 +151,14 @@ class Qdrant_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string
             const collectionName = nodeData.inputs?.qdrantCollection as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
             const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
@@ -178,7 +192,74 @@ class Qdrant_VectorStores implements INode {
             }
 
             try {
-                await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
+                if (recordManager) {
+                    const vectorStore = new QdrantVectorStore(embeddings, dbConfig)
+                    await vectorStore.ensureCollection()
+
+                    vectorStore.addVectors = async (
+                        vectors: number[][],
+                        documents: Document[],
+                        documentOptions?: QdrantAddDocumentOptions
+                    ): Promise<void> => {
+                        if (vectors.length === 0) {
+                            return
+                        }
+
+                        await vectorStore.ensureCollection()
+
+                        const points = vectors.map((embedding, idx) => ({
+                            id: documentOptions?.ids?.length ? documentOptions?.ids[idx] : uuid(),
+                            vector: embedding,
+                            payload: {
+                                content: documents[idx].pageContent,
+                                metadata: documents[idx].metadata,
+                                customPayload: documentOptions?.customPayload?.length ? documentOptions?.customPayload[idx] : undefined
+                            }
+                        }))
+
+                        try {
+                            await client.upsert(collectionName, {
+                                wait: true,
+                                points
+                            })
+                        } catch (e: any) {
+                            const error = new Error(`${e?.status ?? 'Undefined error code'} ${e?.message}: ${e?.data?.status?.error}`)
+                            throw error
+                        }
+                    }
+
+                    vectorStore.delete = async (params: { ids: string[] }): Promise<void> => {
+                        const { ids } = params
+
+                        if (ids?.length) {
+                            try {
+                                client.delete(collectionName, {
+                                    points: ids
+                                })
+                            } catch (e) {
+                                console.error('Failed to delete')
+                            }
+                        }
+                    }
+
+                    await recordManager.createSchema()
+
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: collectionName
+                        }
+                    })
+
+                    return res
+                } else {
+                    await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Redis/Redis.ts

@@ -3,7 +3,7 @@ import { createClient, SearchOptions, RedisClientOptions } from 'redis'
 import { Embeddings } from '@langchain/core/embeddings'
 import { RedisVectorStore, RedisVectorStoreConfig } from '@langchain/community/vectorstores/redis'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { escapeAllStrings, escapeSpecialChars, unEscapeSpecialChars } from './utils'
 
@@ -138,7 +138,7 @@ class Redis_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const indexName = nodeData.inputs?.indexName as string
             let contentKey = nodeData.inputs?.contentKey as string
@@ -203,6 +203,8 @@ class Redis_VectorStores implements INode {
                         filter
                     )
                 }
+
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/SimpleStore/SimpleStore.ts

@@ -2,7 +2,7 @@ import path from 'path'
 import { flatten } from 'lodash'
 import { storageContextFromDefaults, serviceContextFromDefaults, VectorStoreIndex, Document } from 'llamaindex'
 import { Document as LCDocument } from 'langchain/document'
-import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getUserHome } from '../../../src'
 
 class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
@@ -79,7 +79,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData): Promise<void> {
+        async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
             const basePath = nodeData.inputs?.basePath as string
             const docs = nodeData.inputs?.document as LCDocument[]
             const embeddings = nodeData.inputs?.embeddings
@@ -105,6 +105,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
 
             try {
                 await VectorStoreIndex.fromDocuments(llamadocs, { serviceContext, storageContext })
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Singlestore/Singlestore.ts

@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
 import { Embeddings } from '@langchain/core/embeddings'
 import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from '@langchain/community/vectorstores/singlestore'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 
 class SingleStore_VectorStores implements INode {
@@ -118,7 +118,7 @@ class SingleStore_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const user = getCredentialParam('user', credentialData, nodeData)
             const password = getCredentialParam('password', credentialData, nodeData)
@@ -151,6 +151,7 @@ class SingleStore_VectorStores implements INode {
             try {
                 const vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
                 vectorStore.addDocuments.bind(vectorStore)(finalDocs)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Supabase/Supabase.ts

@@ -3,9 +3,10 @@ import { createClient } from '@supabase/supabase-js'
 import { Document } from '@langchain/core/documents'
 import { Embeddings } from '@langchain/core/embeddings'
 import { SupabaseVectorStore, SupabaseLibArgs } from '@langchain/community/vectorstores/supabase'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
+import { index } from '../../../src/indexing'
 
 class Supabase_VectorStores implements INode {
     label: string
@@ -24,7 +25,7 @@ class Supabase_VectorStores implements INode {
     constructor() {
         this.label = 'Supabase'
         this.name = 'supabase'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Supabase'
         this.icon = 'supabase.svg'
         this.category = 'Vector Stores'
@@ -50,6 +51,13 @@ class Supabase_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Supabase Project URL',
                 name: 'supabaseProjUrl',
@@ -99,12 +107,13 @@ class Supabase_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string
             const tableName = nodeData.inputs?.tableName as string
             const queryName = nodeData.inputs?.queryName as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData)
@@ -120,11 +129,32 @@ class Supabase_VectorStores implements INode {
             }
 
             try {
-                await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, {
+                if (recordManager) {
-                    client,
+                    const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, {
-                    tableName: tableName,
+                        client,
-                    queryName: queryName
+                        tableName: tableName,
-                })
+                        queryName: queryName
+                    })
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: tableName + '_' + queryName
+                        }
+                    })
+                    return res
+                } else {
+                    await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, {
+                        client,
+                        tableName: tableName,
+                        queryName: queryName
+                    })
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Vectara/Vectara.ts

@@ -9,7 +9,7 @@ import {
 } from '@langchain/community/vectorstores/vectara'
 import { Document } from '@langchain/core/documents'
 import { Embeddings } from '@langchain/core/embeddings'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 
 class Vectara_VectorStores implements INode {
@@ -144,7 +144,7 @@ class Vectara_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
             const customerId = getCredentialParam('customerID', credentialData, nodeData)
@@ -204,6 +204,7 @@ class Vectara_VectorStores implements INode {
                     const vectorStore = new VectaraStore(vectaraArgs)
                     await vectorStore.addFiles(vectaraFiles)
                 }
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Weaviate/Weaviate.ts

@@ -1,11 +1,12 @@
 import { flatten } from 'lodash'
 import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client'
-import { WeaviateLibArgs, WeaviateStore } from '@langchain/community/vectorstores/weaviate'
+import { WeaviateLibArgs, WeaviateStore } from '@langchain/weaviate'
 import { Document } from '@langchain/core/documents'
 import { Embeddings } from '@langchain/core/embeddings'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
+import { index } from '../../../src/indexing'
 
 class Weaviate_VectorStores implements INode {
     label: string
@@ -24,7 +25,7 @@ class Weaviate_VectorStores implements INode {
     constructor() {
         this.label = 'Weaviate'
         this.name = 'weaviate'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Weaviate'
         this.icon = 'weaviate.png'
         this.category = 'Vector Stores'
@@ -53,6 +54,13 @@ class Weaviate_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Weaviate Scheme',
                 name: 'weaviateScheme',
@@ -125,7 +133,7 @@ class Weaviate_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const weaviateScheme = nodeData.inputs?.weaviateScheme as string
             const weaviateHost = nodeData.inputs?.weaviateHost as string
             const weaviateIndex = nodeData.inputs?.weaviateIndex as string
@@ -133,6 +141,7 @@ class Weaviate_VectorStores implements INode {
             const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
@@ -154,6 +163,7 @@ class Weaviate_VectorStores implements INode {
             }
 
             const obj: WeaviateLibArgs = {
+                //@ts-ignore
                 client,
                 indexName: weaviateIndex
             }
@@ -162,7 +172,24 @@ class Weaviate_VectorStores implements INode {
             if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
 
             try {
-                await WeaviateStore.fromDocuments(finalDocs, embeddings, obj)
+                if (recordManager) {
+                    const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj)
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: weaviateTextKey ? weaviateIndex + '_' + weaviateTextKey : weaviateIndex
+                        }
+                    })
+                    return res
+                } else {
+                    await WeaviateStore.fromDocuments(finalDocs, embeddings, obj)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }
@@ -189,6 +216,7 @@ class Weaviate_VectorStores implements INode {
         const client: WeaviateClient = weaviate.client(clientConfig)
 
         const obj: WeaviateLibArgs = {
+            //@ts-ignore
             client,
             indexName: weaviateIndex
         }

packages/components/nodes/vectorstores/Zep/Zep.ts

@@ -3,7 +3,7 @@ import { IDocument, ZepClient } from '@getzep/zep-js'
 import { ZepVectorStore, IZepConfig } from '@langchain/community/vectorstores/zep'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
 
@@ -106,7 +106,7 @@ class Zep_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const baseURL = nodeData.inputs?.baseURL as string
             const zepCollection = nodeData.inputs?.zepCollection as string
             const dimension = (nodeData.inputs?.dimension as number) ?? 1536
@@ -134,6 +134,7 @@ class Zep_VectorStores implements INode {
 
             try {
                 await ZepVectorStore.fromDocuments(finalDocs, embeddings, zepConfig)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts

@@ -3,7 +3,7 @@ import { IDocument, ZepClient } from '@getzep/zep-cloud'
 import { IZepConfig, ZepVectorStore } from '@getzep/zep-cloud/langchain'
 import { Embeddings } from 'langchain/embeddings/base'
 import { Document } from 'langchain/document'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
 import { FakeEmbeddings } from 'langchain/embeddings/fake'
@@ -89,7 +89,7 @@ class Zep_CloudVectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const zepCollection = nodeData.inputs?.zepCollection as string
             const docs = nodeData.inputs?.document as Document[]
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
@@ -109,6 +109,7 @@ class Zep_CloudVectorStores implements INode {
             }
             try {
                 await ZepVectorStore.fromDocuments(finalDocs, new FakeEmbeddings(), zepConfig)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/package.json

@@ -42,6 +42,7 @@
         "@langchain/mongodb": "^0.0.1",
         "@langchain/openai": "^0.0.14",
         "@langchain/pinecone": "^0.0.3",
+        "@langchain/weaviate": "^0.0.1",
         "@mistralai/mistralai": "0.1.3",
         "@notionhq/client": "^2.2.8",
         "@opensearch-project/opensearch": "^1.2.0",

packages/components/src/Interface.ts

@@ -113,7 +113,7 @@ export interface INode extends INodeProperties {
         [key: string]: (nodeData: INodeData, options?: ICommonObject) => Promise<INodeOptionsValue[]>
     }
     vectorStoreMethods?: {
-        upsert: (nodeData: INodeData, options?: ICommonObject) => Promise<void>
+        upsert: (nodeData: INodeData, options?: ICommonObject) => Promise<IndexingResult | void>
         search: (nodeData: INodeData, options?: ICommonObject) => Promise<any>
         delete: (nodeData: INodeData, options?: ICommonObject) => Promise<void>
     }
@@ -181,6 +181,7 @@ export type MessageContentImageUrl = {
 
 import { PromptTemplate as LangchainPromptTemplate, PromptTemplateInput } from '@langchain/core/prompts'
 import { VectorStore } from '@langchain/core/vectorstores'
+import { Document } from '@langchain/core/documents'
 
 export class PromptTemplate extends LangchainPromptTemplate {
     promptValues: ICommonObject
@@ -271,6 +272,15 @@ export abstract class FlowiseSummaryMemory extends ConversationSummaryMemory imp
     abstract clearChatMessages(overrideSessionId?: string): Promise<void>
 }
 
+export type IndexingResult = {
+    numAdded: number
+    numDeleted: number
+    numUpdated: number
+    numSkipped: number
+    totalKeys: number
+    addedDocs: Document[]
+}
+
 export interface IVisionChatModal {
     id: string
     configuredModel: string

packages/components/src/indexing.ts

@@ -0,0 +1,355 @@
+import { VectorStore } from '@langchain/core/vectorstores'
+import { v5 as uuidv5 } from 'uuid'
+import { RecordManagerInterface, UUIDV5_NAMESPACE } from '@langchain/community/indexes/base'
+import { insecureHash } from '@langchain/core/utils/hash'
+import { Document, DocumentInterface } from '@langchain/core/documents'
+import { BaseDocumentLoader } from 'langchain/document_loaders/base.js'
+import { IndexingResult } from './Interface'
+
+type Metadata = Record<string, unknown>
+
+type StringOrDocFunc = string | ((doc: DocumentInterface) => string)
+
+export interface HashedDocumentInterface extends DocumentInterface {
+    uid: string
+    hash_?: string
+    contentHash?: string
+    metadataHash?: string
+    pageContent: string
+    metadata: Metadata
+    calculateHashes(): void
+    toDocument(): DocumentInterface
+}
+
+interface HashedDocumentArgs {
+    pageContent: string
+    metadata: Metadata
+    uid: string
+}
+
+/**
+ * HashedDocument is a Document with hashes calculated.
+ * Hashes are calculated based on page content and metadata.
+ * It is used for indexing.
+ */
+export class _HashedDocument implements HashedDocumentInterface {
+    uid: string
+
+    hash_?: string
+
+    contentHash?: string
+
+    metadataHash?: string
+
+    pageContent: string
+
+    metadata: Metadata
+
+    constructor(fields: HashedDocumentArgs) {
+        this.uid = fields.uid
+        this.pageContent = fields.pageContent
+        this.metadata = fields.metadata
+    }
+
+    calculateHashes(): void {
+        const forbiddenKeys = ['hash_', 'content_hash', 'metadata_hash']
+
+        for (const key of forbiddenKeys) {
+            if (key in this.metadata) {
+                throw new Error(
+                    `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(', ')}]`
+                )
+            }
+        }
+
+        const contentHash = this._hashStringToUUID(this.pageContent)
+
+        try {
+            const metadataHash = this._hashNestedDictToUUID(this.metadata)
+            this.contentHash = contentHash
+            this.metadataHash = metadataHash
+        } catch (e) {
+            throw new Error(`Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`)
+        }
+
+        this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash)
+
+        if (!this.uid) {
+            this.uid = this.hash_
+        }
+    }
+
+    toDocument(): DocumentInterface {
+        return new Document({
+            pageContent: this.pageContent,
+            metadata: this.metadata
+        })
+    }
+
+    static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument {
+        const doc = new this({
+            pageContent: document.pageContent,
+            metadata: document.metadata,
+            uid: uid || (document as DocumentInterface & { uid: string }).uid
+        })
+        doc.calculateHashes()
+        return doc
+    }
+
+    private _hashStringToUUID(inputString: string): string {
+        const hash_value = insecureHash(inputString)
+        return uuidv5(hash_value, UUIDV5_NAMESPACE)
+    }
+
+    private _hashNestedDictToUUID(data: Record<string, unknown>): string {
+        const serialized_data = JSON.stringify(data, Object.keys(data).sort())
+        const hash_value = insecureHash(serialized_data)
+        return uuidv5(hash_value, UUIDV5_NAMESPACE)
+    }
+}
+
+export type CleanupMode = 'full' | 'incremental'
+
+export type IndexOptions = {
+    /**
+     * The number of documents to index in one batch.
+     */
+    batchSize?: number
+    /**
+     * The cleanup mode to use. Can be "full", "incremental" or undefined.
+     * - **Incremental**: Cleans up all documents that haven't been updated AND
+     *   that are associated with source ids that were seen
+     *   during indexing.
+     *   Clean up is done continuously during indexing helping
+     *   to minimize the probability of users seeing duplicated
+     *   content.
+     * - **Full**: Delete all documents that haven to been returned by the loader.
+     *   Clean up runs after all documents have been indexed.
+     *   This means that users may see duplicated content during indexing.
+     * - **undefined**: Do not delete any documents.
+     */
+    cleanup?: CleanupMode
+    /**
+     * Optional key that helps identify the original source of the document.
+     * Must either be a string representing the key of the source in the metadata
+     * or a function that takes a document and returns a string representing the source.
+     * **Required when cleanup is incremental**.
+     */
+    sourceIdKey?: StringOrDocFunc
+    /**
+     * Batch size to use when cleaning up documents.
+     */
+    cleanupBatchSize?: number
+    /**
+     * Force update documents even if they are present in the
+     * record manager. Useful if you are re-indexing with updated embeddings.
+     */
+    forceUpdate?: boolean
+
+    vectorStoreName?: string
+}
+
+export function _batch<T>(size: number, iterable: T[]): T[][] {
+    const batches: T[][] = []
+    let currentBatch: T[] = []
+
+    iterable.forEach((item) => {
+        currentBatch.push(item)
+
+        if (currentBatch.length >= size) {
+            batches.push(currentBatch)
+            currentBatch = []
+        }
+    })
+
+    if (currentBatch.length > 0) {
+        batches.push(currentBatch)
+    }
+
+    return batches
+}
+
+export function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[] {
+    const seen = new Set<string>()
+    const deduplicated: HashedDocumentInterface[] = []
+
+    for (const hashedDoc of hashedDocuments) {
+        if (!hashedDoc.hash_) {
+            throw new Error('Hashed document does not have a hash')
+        }
+
+        if (!seen.has(hashedDoc.hash_)) {
+            seen.add(hashedDoc.hash_)
+            deduplicated.push(hashedDoc)
+        }
+    }
+    return deduplicated
+}
+
+export function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null {
+    if (sourceIdKey === null) {
+        return (_doc: DocumentInterface) => null
+    } else if (typeof sourceIdKey === 'string') {
+        return (doc: DocumentInterface) => doc.metadata[sourceIdKey]
+    } else if (typeof sourceIdKey === 'function') {
+        return sourceIdKey
+    } else {
+        throw new Error(`sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`)
+    }
+}
+
+export const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {
+    if ('load' in arg && typeof arg.load === 'function' && 'loadAndSplit' in arg && typeof arg.loadAndSplit === 'function') {
+        return true
+    }
+    return false
+}
+
+interface IndexArgs {
+    docsSource: BaseDocumentLoader | DocumentInterface[]
+    recordManager: RecordManagerInterface
+    vectorStore: VectorStore
+    options?: IndexOptions
+}
+
+/**
+ * Index data from the doc source into the vector store.
+ *
+ * Indexing functionality uses a manager to keep track of which documents
+ * are in the vector store.
+ *
+ * This allows us to keep track of which documents were updated, and which
+ * documents were deleted, which documents should be skipped.
+ *
+ * For the time being, documents are indexed using their hashes, and users
+ *  are not able to specify the uid of the document.
+ *
+ * @param {IndexArgs} args
+ * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
+ * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
+ * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
+ * @param {IndexOptions | undefined} args.options Options for indexing.
+ * @returns {Promise<IndexingResult>}
+ */
+export async function index(args: IndexArgs): Promise<IndexingResult> {
+    const { docsSource, recordManager, vectorStore, options } = args
+    const { batchSize = 100, cleanup, sourceIdKey, cleanupBatchSize = 1000, forceUpdate = false, vectorStoreName } = options ?? {}
+
+    if (cleanup === 'incremental' && !sourceIdKey) {
+        throw new Error("sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.")
+    }
+
+    if (vectorStoreName) {
+        ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
+    }
+
+    const docs = _isBaseDocumentLoader(docsSource) ? await docsSource.load() : docsSource
+
+    const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null)
+
+    const indexStartDt = await recordManager.getTime()
+    let numAdded = 0
+    let addedDocs: Document[] = []
+    let numDeleted = 0
+    let numUpdated = 0
+    let numSkipped = 0
+    let totalKeys = 0
+
+    const batches = _batch<DocumentInterface>(batchSize ?? 100, docs)
+
+    for (const batch of batches) {
+        const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc)))
+
+        const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc))
+
+        if (cleanup === 'incremental') {
+            hashedDocs.forEach((_hashedDoc, index) => {
+                const source = sourceIds[index]
+                if (source === null) {
+                    throw new Error('sourceIdKey must be provided when cleanup is incremental')
+                }
+            })
+        }
+
+        const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid))
+
+        const uids: string[] = []
+        const docsToIndex: DocumentInterface[] = []
+        const docsToUpdate: string[] = []
+        const seenDocs = new Set<string>()
+        hashedDocs.forEach((hashedDoc, i) => {
+            const docExists = batchExists[i]
+            if (docExists) {
+                if (forceUpdate) {
+                    seenDocs.add(hashedDoc.uid)
+                } else {
+                    docsToUpdate.push(hashedDoc.uid)
+                    return
+                }
+            }
+            uids.push(hashedDoc.uid)
+            docsToIndex.push(hashedDoc.toDocument())
+        })
+
+        if (docsToUpdate.length > 0) {
+            await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt })
+            numSkipped += docsToUpdate.length
+        }
+
+        if (docsToIndex.length > 0) {
+            await vectorStore.addDocuments(docsToIndex, { ids: uids })
+            const newDocs = docsToIndex.map((docs) => ({
+                pageContent: docs.pageContent,
+                metadata: docs.metadata
+            }))
+            addedDocs.push(...newDocs)
+            numAdded += docsToIndex.length - seenDocs.size
+            numUpdated += seenDocs.size
+        }
+
+        await recordManager.update(
+            hashedDocs.map((doc) => doc.uid),
+            { timeAtLeast: indexStartDt, groupIds: sourceIds }
+        )
+
+        if (cleanup === 'incremental') {
+            sourceIds.forEach((sourceId) => {
+                if (!sourceId) throw new Error('Source id cannot be null')
+            })
+            const uidsToDelete = await recordManager.listKeys({
+                before: indexStartDt,
+                groupIds: sourceIds
+            })
+            await vectorStore.delete({ ids: uidsToDelete })
+            await recordManager.deleteKeys(uidsToDelete)
+            numDeleted += uidsToDelete.length
+        }
+    }
+
+    if (cleanup === 'full') {
+        let uidsToDelete = await recordManager.listKeys({
+            before: indexStartDt,
+            limit: cleanupBatchSize
+        })
+        while (uidsToDelete.length > 0) {
+            await vectorStore.delete({ ids: uidsToDelete })
+            await recordManager.deleteKeys(uidsToDelete)
+            numDeleted += uidsToDelete.length
+            uidsToDelete = await recordManager.listKeys({
+                before: indexStartDt,
+                limit: cleanupBatchSize
+            })
+        }
+    }
+
+    totalKeys = (await recordManager.listKeys({})).length
+
+    return {
+        numAdded,
+        numDeleted,
+        numUpdated,
+        numSkipped,
+        totalKeys,
+        addedDocs
+    }
+}

packages/server/src/Interface.ts

@@ -93,6 +93,14 @@ export interface IVariable {
     createdDate: Date
 }
 
+export interface IUpsertHistory {
+    id: string
+    chatflowid: string
+    result: string
+    flowData: string
+    date: Date
+}
+
 export interface IComponentNodes {
     [key: string]: INode
 }

packages/server/src/controllers/tools/index.ts

@@ -19,7 +19,7 @@ const creatTool = async (req: Request, res: Response, next: NextFunction) => {
 const deleteTool = async (req: Request, res: Response, next: NextFunction) => {
     try {
         if (typeof req.params.id === 'undefined' || req.params.id === '') {
-            throw new Error(`Error: toolsController.updateTool - id not provided!`)
+            throw new Error(`Error: toolsController.deleteTool - id not provided!`)
         }
         const apiResponse = await toolsService.deleteTool(req.params.id)
         if (apiResponse.executionError) {

packages/server/src/controllers/upsert-history/index.ts

@@ -0,0 +1,30 @@
+import { Request, Response, NextFunction } from 'express'
+import upsertHistoryService from '../../services/upsert-history'
+
+const getAllUpsertHistory = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const sortOrder = req.query?.order as string | undefined
+        const chatflowid = req.params?.id as string | undefined
+        const startDate = req.query?.startDate as string | undefined
+        const endDate = req.query?.endDate as string | undefined
+        const apiResponse = await upsertHistoryService.getAllUpsertHistory(sortOrder, chatflowid, startDate, endDate)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const patchDeleteUpsertHistory = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const ids = req.body.ids ?? []
+        const apiResponse = await upsertHistoryService.patchDeleteUpsertHistory(ids)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+export default {
+    getAllUpsertHistory,
+    patchDeleteUpsertHistory
+}

packages/server/src/database/entities/Assistant.ts

@@ -10,7 +10,7 @@ export class Assistant implements IAssistant {
     @Column({ type: 'text' })
     details: string
 
-    @Column({ type: 'uuid'})
+    @Column({ type: 'uuid' })
     credential: string
 
     @Column({ nullable: true })

packages/server/src/database/entities/ChatFlow.ts

@@ -34,11 +34,11 @@ export class ChatFlow implements IChatFlow {
     @Column({ nullable: true, type: 'text' })
     speechToText?: string
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @CreateDateColumn()
     createdDate: Date
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @UpdateDateColumn()
     updatedDate: Date
 

packages/server/src/database/entities/ChatMessage.ts

@@ -41,7 +41,7 @@ export class ChatMessage implements IChatMessage {
     @Column({ type: 'varchar', nullable: true })
     sessionId?: string
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @CreateDateColumn()
     createdDate: Date
 }

packages/server/src/database/entities/ChatMessageFeedback.ts

@@ -25,7 +25,7 @@ export class ChatMessageFeedback implements IChatMessageFeedback {
     @Column({ nullable: true, type: 'text' })
     content?: string
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @CreateDateColumn()
     createdDate: Date
 }

packages/server/src/database/entities/Credential.ts

@@ -16,11 +16,11 @@ export class Credential implements ICredential {
     @Column({ type: 'text' })
     encryptedData: string
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @CreateDateColumn()
     createdDate: Date
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @UpdateDateColumn()
     updatedDate: Date
 }

packages/server/src/database/entities/Tool.ts

@@ -25,11 +25,11 @@ export class Tool implements ITool {
     @Column({ nullable: true, type: 'text' })
     func?: string
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @CreateDateColumn()
     createdDate: Date
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @UpdateDateColumn()
     updatedDate: Date
 }

packages/server/src/database/entities/UpsertHistory.ts

@@ -0,0 +1,22 @@
+/* eslint-disable */
+import { Entity, Column, PrimaryGeneratedColumn, Index, CreateDateColumn } from 'typeorm'
+import { IUpsertHistory } from '../../Interface'
+
+@Entity()
+export class UpsertHistory implements IUpsertHistory {
+    @PrimaryGeneratedColumn('uuid')
+    id: string
+
+    @Index()
+    @Column()
+    chatflowid: string
+
+    @Column()
+    result: string
+
+    @Column()
+    flowData: string
+
+    @CreateDateColumn()
+    date: Date
+}

packages/server/src/database/entities/Variable.ts

@@ -16,11 +16,11 @@ export class Variable implements IVariable {
     @Column({ default: 'string', type: 'text' })
     type: string
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @CreateDateColumn()
     createdDate: Date
 
-    @Column({type:'timestamp'})
+    @Column({ type: 'timestamp' })
     @UpdateDateColumn()
     updatedDate: Date
 }

packages/server/src/database/entities/index.ts

@@ -5,6 +5,7 @@ import { Credential } from './Credential'
 import { Tool } from './Tool'
 import { Assistant } from './Assistant'
 import { Variable } from './Variable'
+import { UpsertHistory } from './UpsertHistory'
 
 export const entities = {
     ChatFlow,
@@ -13,5 +14,6 @@ export const entities = {
     Credential,
     Tool,
     Assistant,
-    Variable
+    Variable,
+    UpsertHistory
 }

packages/server/src/database/migrations/mysql/1709814301358-AddUpsertHistoryEntity.ts

@@ -0,0 +1,21 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUpsertHistoryEntity1709814301358 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`upsert_history\` (
+                \`id\` varchar(36) NOT NULL,
+                \`chatflowid\` varchar(255) NOT NULL,
+                \`result\` text NOT NULL,
+                \`flowData\` text NOT NULL,
+                \`date\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`)
+                KEY \`IDX_a0b59fd66f6e48d2b198123cb6\` (\`chatflowid\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE upsert_history`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -13,6 +13,7 @@ import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-Ad
 import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
 import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
 import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
+import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213626553 } from './1707213626553-AddFeedback'
 
 export const mysqlMigrations = [
@@ -31,5 +32,6 @@ export const mysqlMigrations = [
     AddFileUploadsToChatMessage1701788586491,
     AddVariableEntity1699325775451,
     AddSpeechToText1706364937060,
+    AddUpsertHistoryEntity1709814301358,
     AddFeedback1707213626553
 ]

packages/server/src/database/migrations/postgres/1709814301358-AddUpsertHistoryEntity.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUpsertHistoryEntity1709814301358 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS upsert_history (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "chatflowid" varchar NOT NULL,
+                "result" text NOT NULL,
+                "flowData" text NOT NULL,
+                "date" timestamp NOT NULL DEFAULT now(),
+                CONSTRAINT "PK_37327b22b6e246319bd5eeb0e88" PRIMARY KEY (id)
+            );`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE upsert_history`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -13,6 +13,7 @@ import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-Ad
 import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
 import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
 import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
+import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213601923 } from './1707213601923-AddFeedback'
 import { FieldTypes1710497452584 } from './1710497452584-FieldTypes'
 
@@ -32,6 +33,7 @@ export const postgresMigrations = [
     AddFileUploadsToChatMessage1701788586491,
     AddVariableEntity1699325775451,
     AddSpeechToText1706364937060,
+    AddUpsertHistoryEntity1709814301358,
     AddFeedback1707213601923,
     FieldTypes1710497452584
 ]

packages/server/src/database/migrations/sqlite/1709814301358-AddUpsertHistoryEntity.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUpsertHistoryEntity1709814301358 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "upsert_history" ("id" varchar PRIMARY KEY NOT NULL, "chatflowid" varchar NOT NULL, "result" text NOT NULL, "flowData" text NOT NULL, "date" datetime NOT NULL DEFAULT (datetime('now')));`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE upsert_history`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -13,6 +13,7 @@ import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-Ad
 import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
 import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
 import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
+import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213619308 } from './1707213619308-AddFeedback'
 
 export const sqliteMigrations = [
@@ -31,5 +32,6 @@ export const sqliteMigrations = [
     AddFileUploadsToChatMessage1701788586491,
     AddVariableEntity1699325775451,
     AddSpeechToText1706364937060,
+    AddUpsertHistoryEntity1709814301358,
     AddFeedback1707213619308
 ]

packages/server/src/routes/index.ts

@@ -35,6 +35,7 @@ import variablesRouter from './variables'
 import vectorRouter from './vectors'
 import verifyRouter from './verify'
 import versionRouter from './versions'
+import upsertHistoryRouter from './upsert-history'
 
 const router = express.Router()
 
@@ -74,5 +75,6 @@ router.use('/variables', variablesRouter)
 router.use('/vector', vectorRouter)
 router.use('/verify', verifyRouter)
 router.use('/version', versionRouter)
+router.use('/upsert-history', upsertHistoryRouter)
 
 export default router

packages/server/src/routes/upsert-history/index.ts

@@ -0,0 +1,15 @@
+import express from 'express'
+import upsertHistoryController from '../../controllers/upsert-history'
+const router = express.Router()
+
+// CREATE
+
+// READ
+router.get('/:id', upsertHistoryController.getAllUpsertHistory)
+
+// PATCH
+router.patch('/', upsertHistoryController.patchDeleteUpsertHistory)
+
+// DELETE
+
+export default router

packages/server/src/services/chatflows/index.ts

@@ -14,6 +14,9 @@ import logger from '../../utils/logger'
 import { getStoragePath } from 'flowise-components'
 import { IReactFlowObject } from '../../Interface'
 import { utilGetUploadsConfig } from '../../utils/getUploadsConfig'
+import { ChatMessage } from '../../database/entities/ChatMessage'
+import { ChatMessageFeedback } from '../../database/entities/ChatMessageFeedback'
+import { UpsertHistory } from '../../database/entities/UpsertHistory'
 
 // Check if chatflow valid for streaming
 const checkIfChatflowIsValidForStreaming = async (chatflowId: string): Promise<any> => {
@@ -105,9 +108,18 @@ const deleteChatflow = async (chatflowId: string): Promise<any> => {
         const appServer = getRunningExpressApp()
         const dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).delete({ id: chatflowId })
         try {
-            // Delete all  uploads corresponding to this chatflow
+            // Delete all uploads corresponding to this chatflow
             const directory = path.join(getStoragePath(), chatflowId)
             deleteFolderRecursive(directory)
+
+            // Delete all chat messages
+            await appServer.AppDataSource.getRepository(ChatMessage).delete({ chatflowid: chatflowId })
+
+            // Delete all chat feedback
+            await appServer.AppDataSource.getRepository(ChatMessageFeedback).delete({ chatflowid: chatflowId })
+
+            // Delete all upsert history
+            await appServer.AppDataSource.getRepository(UpsertHistory).delete({ chatflowid: chatflowId })
         } catch (e) {
             logger.error(`[server]: Error deleting file storage for chatflow ${chatflowId}: ${e}`)
         }

packages/server/src/services/upsert-history/index.ts

@@ -0,0 +1,66 @@
+import { MoreThanOrEqual, LessThanOrEqual } from 'typeorm'
+import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
+import { UpsertHistory } from '../../database/entities/UpsertHistory'
+
+const getAllUpsertHistory = async (
+    sortOrder: string | undefined,
+    chatflowid: string | undefined,
+    startDate: string | undefined,
+    endDate: string | undefined
+) => {
+    try {
+        const appServer = getRunningExpressApp()
+
+        const setDateToStartOrEndOfDay = (dateTimeStr: string, setHours: 'start' | 'end') => {
+            const date = new Date(dateTimeStr)
+            if (isNaN(date.getTime())) {
+                return undefined
+            }
+            setHours === 'start' ? date.setHours(0, 0, 0, 0) : date.setHours(23, 59, 59, 999)
+            return date
+        }
+
+        let fromDate
+        if (startDate) fromDate = setDateToStartOrEndOfDay(startDate, 'start')
+
+        let toDate
+        if (endDate) toDate = setDateToStartOrEndOfDay(endDate, 'end')
+
+        let upsertHistory = await appServer.AppDataSource.getRepository(UpsertHistory).find({
+            where: {
+                chatflowid,
+                ...(fromDate && { date: MoreThanOrEqual(fromDate) }),
+                ...(toDate && { date: LessThanOrEqual(toDate) })
+            },
+            order: {
+                date: sortOrder === 'DESC' ? 'DESC' : 'ASC'
+            }
+        })
+        upsertHistory = upsertHistory.map((hist) => {
+            return {
+                ...hist,
+                result: hist.result ? JSON.parse(hist.result) : {},
+                flowData: hist.flowData ? JSON.parse(hist.flowData) : {}
+            }
+        })
+
+        return upsertHistory
+    } catch (error) {
+        throw new Error(`Error: upsertHistoryServices.getAllUpsertHistory - ${error}`)
+    }
+}
+
+const patchDeleteUpsertHistory = async (ids: string[] = []): Promise<any> => {
+    try {
+        const appServer = getRunningExpressApp()
+        const dbResponse = await appServer.AppDataSource.getRepository(UpsertHistory).delete(ids)
+        return dbResponse
+    } catch (error) {
+        throw new Error(`Error: upsertHistoryServices.patchUpsertHistory - ${error}`)
+    }
+}
+
+export default {
+    getAllUpsertHistory,
+    patchDeleteUpsertHistory
+}

packages/server/src/services/variables/index.ts

@@ -18,7 +18,7 @@ const deleteVariable = async (variableId: string): Promise<any> => {
         const dbResponse = await appServer.AppDataSource.getRepository(Variable).delete({ id: variableId })
         return dbResponse
     } catch (error) {
-        throw new Error(`Error: variablesServices.createVariable - ${error}`)
+        throw new Error(`Error: variablesServices.deleteVariable - ${error}`)
     }
 }
 

packages/server/src/utils/buildChatflow.ts

@@ -36,7 +36,6 @@ import { utilAddChatMessage } from './addChatMesage'
  * @param {Request} req
  * @param {Server} socketIO
  * @param {boolean} isInternal
- * @param {boolean} isUpsert
  */
 export const utilBuildChatflow = async (req: Request, socketIO?: Server, isInternal: boolean = false): Promise<any> => {
     try {

packages/server/src/utils/index.ts

@@ -237,6 +237,84 @@ export const getEndingNodes = (nodeDependencies: INodeDependencies, graph: INode
     return endingNodeIds
 }
 
+/**
+ * Get file name from base64 string
+ * @param {string} fileBase64
+ */
+export const getFileName = (fileBase64: string): string => {
+    let fileNames = []
+    if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
+        const files = JSON.parse(fileBase64)
+        for (const file of files) {
+            const splitDataURI = file.split(',')
+            const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
+            fileNames.push(filename)
+        }
+        return fileNames.join(', ')
+    } else {
+        const splitDataURI = fileBase64.split(',')
+        const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
+        return filename
+    }
+}
+
+/**
+ * Save upsert flowData
+ * @param {INodeData} nodeData
+ * @param {Record<string, any>} upsertHistory
+ */
+export const saveUpsertFlowData = (nodeData: INodeData, upsertHistory: Record<string, any>): ICommonObject[] => {
+    const existingUpsertFlowData = upsertHistory['flowData'] ?? []
+    const paramValues: ICommonObject[] = []
+
+    for (const input in nodeData.inputs) {
+        const inputParam = nodeData.inputParams.find((inp) => inp.name === input)
+        if (!inputParam) continue
+
+        let paramValue: ICommonObject = {}
+
+        if (!nodeData.inputs[input]) {
+            continue
+        }
+        if (
+            typeof nodeData.inputs[input] === 'string' &&
+            nodeData.inputs[input].startsWith('{{') &&
+            nodeData.inputs[input].endsWith('}}')
+        ) {
+            continue
+        }
+        // Get file name instead of the base64 string
+        if (nodeData.category === 'Document Loaders' && nodeData.inputParams.find((inp) => inp.name === input)?.type === 'file') {
+            paramValue = {
+                label: inputParam?.label,
+                name: inputParam?.name,
+                type: inputParam?.type,
+                value: getFileName(nodeData.inputs[input])
+            }
+            paramValues.push(paramValue)
+            continue
+        }
+
+        paramValue = {
+            label: inputParam?.label,
+            name: inputParam?.name,
+            type: inputParam?.type,
+            value: nodeData.inputs[input]
+        }
+        paramValues.push(paramValue)
+    }
+
+    const newFlowData = {
+        label: nodeData.label,
+        name: nodeData.name,
+        category: nodeData.category,
+        id: nodeData.id,
+        paramValues
+    }
+    existingUpsertFlowData.push(newFlowData)
+    return existingUpsertFlowData
+}
+
 /**
  * Build langchain from start to end
  * @param {string[]} startingNodeIds
@@ -272,6 +350,8 @@ export const buildFlow = async (
 ) => {
     const flowNodes = cloneDeep(reactFlowNodes)
 
+    let upsertHistory: Record<string, any> = {}
+
     // Create a Queue and add our initial node in it
     const nodeQueue = [] as INodeQueue[]
     const exploredNode = {} as IExploredNode
@@ -302,12 +382,15 @@ export const buildFlow = async (
 
             let flowNodeData = cloneDeep(reactFlowNode.data)
             if (overrideConfig) flowNodeData = replaceInputsWithConfig(flowNodeData, overrideConfig)
+
+            if (isUpsert) upsertHistory['flowData'] = saveUpsertFlowData(flowNodeData, upsertHistory)
+
             const reactFlowNodeData: INodeData = resolveVariables(flowNodeData, flowNodes, question, chatHistory)
 
             // TODO: Avoid processing Text Splitter + Doc Loader once Upsert & Load Existing Vector Nodes are deprecated
             if (isUpsert && stopNodeId && nodeId === stopNodeId) {
                 logger.debug(`[server]: Upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
-                await newNodeInstance.vectorStoreMethods!['upsert']!.call(newNodeInstance, reactFlowNodeData, {
+                const indexResult = await newNodeInstance.vectorStoreMethods!['upsert']!.call(newNodeInstance, reactFlowNodeData, {
                     chatId,
                     sessionId,
                     chatflowid,
@@ -319,6 +402,7 @@ export const buildFlow = async (
                     dynamicVariables,
                     uploads
                 })
+                if (indexResult) upsertHistory['result'] = indexResult
                 logger.debug(`[server]: Finished upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
                 break
             } else {
@@ -422,7 +506,7 @@ export const buildFlow = async (
             flowNodes.push(flowNodes.splice(index, 1)[0])
         }
     }
-    return flowNodes
+    return isUpsert ? (upsertHistory as any) : flowNodes
 }
 
 /**

packages/server/src/utils/upsertVector.ts

@@ -1,5 +1,6 @@
 import { Request, Response } from 'express'
 import * as fs from 'fs'
+import { cloneDeep, omit } from 'lodash'
 import { ICommonObject } from 'flowise-components'
 import telemetryService from '../services/telemetry'
 import logger from '../utils/logger'
@@ -18,7 +19,14 @@ import { utilValidateKey } from './validateKey'
 import { IncomingInput, INodeDirectedGraph, IReactFlowObject, chatType } from '../Interface'
 import { ChatFlow } from '../database/entities/ChatFlow'
 import { getRunningExpressApp } from '../utils/getRunningExpressApp'
+import { UpsertHistory } from '../database/entities/UpsertHistory'
 
+/**
+ * Upsert documents
+ * @param {Request} req
+ * @param {Response} res
+ * @param {boolean} isInternal
+ */
 export const upsertVector = async (req: Request, res: Response, isInternal: boolean = false) => {
     try {
         const appServer = getRunningExpressApp()
@@ -78,6 +86,8 @@ export const upsertVector = async (req: Request, res: Response, isInternal: bool
             (node) =>
                 node.data.category === 'Vector Stores' && !node.data.label.includes('Upsert') && !node.data.label.includes('Load Existing')
         )
+
+        // Check if multiple vector store nodes exist, and if stopNodeId is specified
         if (vsNodes.length > 1 && !stopNodeId) {
             return res.status(500).send('There are multiple vector nodes, please provide stopNodeId in body request')
         } else if (vsNodes.length === 1 && !stopNodeId) {
@@ -99,7 +109,7 @@ export const upsertVector = async (req: Request, res: Response, isInternal: bool
 
         const { startingNodeIds, depthQueue } = getStartingNodes(filteredGraph, stopNodeId)
 
-        await buildFlow(
+        const upsertedResult = await buildFlow(
             startingNodeIds,
             nodes,
             edges,
@@ -121,6 +131,19 @@ export const upsertVector = async (req: Request, res: Response, isInternal: bool
         const startingNodes = nodes.filter((nd) => startingNodeIds.includes(nd.data.id))
 
         await appServer.chatflowPool.add(chatflowid, undefined, startingNodes, incomingInput?.overrideConfig)
+
+        // Save to DB
+        if (upsertedResult['flowData'] && upsertedResult['result']) {
+            const result = cloneDeep(upsertedResult)
+            result['flowData'] = JSON.stringify(result['flowData'])
+            result['result'] = JSON.stringify(omit(result['result'], ['totalKeys', 'addedDocs']))
+            result.chatflowid = chatflowid
+            const newUpsertHistory = new UpsertHistory()
+            Object.assign(newUpsertHistory, result)
+            const upsertHistory = appServer.AppDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
+            await appServer.AppDataSource.getRepository(UpsertHistory).save(upsertHistory)
+        }
+
         await telemetryService.createEvent({
             name: `vector_upserted`,
             data: {
@@ -131,7 +154,7 @@ export const upsertVector = async (req: Request, res: Response, isInternal: bool
                 stopNodeId
             }
         })
-        return res.status(201).send('Successfully Upserted')
+        return res.status(201).json(upsertedResult['result'] ?? { result: 'Successfully Upserted' })
     } catch (e: any) {
         logger.error('[server]: Error:', e)
         return res.status(500).send(e.message)

packages/ui/src/api/vectorstore.js

@@ -1,7 +1,11 @@
 import client from './client'
 
 const upsertVectorStore = (id, input) => client.post(`/vector/internal-upsert/${id}`, input)
+const getUpsertHistory = (id, params = {}) => client.get(`/upsert-history/${id}`, { params: { order: 'DESC', ...params } })
+const deleteUpsertHistory = (ids) => client.patch(`/upsert-history`, { ids })
 
 export default {
-    upsertVectorStore
+    getUpsertHistory,
+    upsertVectorStore,
+    deleteUpsertHistory
 }

packages/ui/src/menu-items/settings.js

@@ -1,5 +1,13 @@
 // assets
-import { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconMessage, IconAdjustmentsHorizontal } from '@tabler/icons'
+import {
+    IconTrash,
+    IconFileUpload,
+    IconFileExport,
+    IconCopy,
+    IconMessage,
+    IconDatabaseExport,
+    IconAdjustmentsHorizontal
+} from '@tabler/icons'
 
 // constant
 const icons = {
@@ -8,6 +16,7 @@ const icons = {
     IconFileExport,
     IconCopy,
     IconMessage,
+    IconDatabaseExport,
     IconAdjustmentsHorizontal
 }
 
@@ -25,6 +34,13 @@ const settings = {
             url: '',
             icon: icons.IconMessage
         },
+        {
+            id: 'viewUpsertHistory',
+            title: 'Upsert History',
+            type: 'item',
+            url: '',
+            icon: icons.IconDatabaseExport
+        },
         {
             id: 'chatflowConfiguration',
             title: 'Configuration',

packages/ui/src/utils/genericHelper.js

@@ -355,6 +355,11 @@ export const getUpsertDetails = (nodes, edges) => {
                     innerNodes.push(nodes.find((node) => node.data.id === embeddingsId))
                 }
 
+                if (vsNode.data.inputs.recordManager) {
+                    const recordManagerId = vsNode.data.inputs.recordManager.replace(/{{|}}/g, '').split('.')[0]
+                    innerNodes.push(nodes.find((node) => node.data.id === recordManagerId))
+                }
+
                 for (const doc of connectedDocs) {
                     const docId = doc.replace(/{{|}}/g, '').split('.')[0]
                     const docNode = nodes.find((node) => node.data.id === docId)

packages/ui/src/views/canvas/CanvasHeader.jsx

@@ -16,6 +16,7 @@ import SaveChatflowDialog from '@/ui-component/dialog/SaveChatflowDialog'
 import APICodeDialog from '@/views/chatflows/APICodeDialog'
 import ViewMessagesDialog from '@/ui-component/dialog/ViewMessagesDialog'
 import ChatflowConfigurationDialog from '@/ui-component/dialog/ChatflowConfigurationDialog'
+import UpsertHistoryDialog from '@/views/vectorstore/UpsertHistoryDialog'
 
 // API
 import chatflowsApi from '@/api/chatflows'
@@ -45,6 +46,8 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
     const [apiDialogProps, setAPIDialogProps] = useState({})
     const [viewMessagesDialogOpen, setViewMessagesDialogOpen] = useState(false)
     const [viewMessagesDialogProps, setViewMessagesDialogProps] = useState({})
+    const [upsertHistoryDialogOpen, setUpsertHistoryDialogOpen] = useState(false)
+    const [upsertHistoryDialogProps, setUpsertHistoryDialogProps] = useState({})
     const [chatflowConfigurationDialogOpen, setChatflowConfigurationDialogOpen] = useState(false)
     const [chatflowConfigurationDialogProps, setChatflowConfigurationDialogProps] = useState({})
 
@@ -62,6 +65,12 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
                 chatflow: chatflow
             })
             setViewMessagesDialogOpen(true)
+        } else if (setting === 'viewUpsertHistory') {
+            setUpsertHistoryDialogProps({
+                title: 'View Upsert History',
+                chatflow: chatflow
+            })
+            setUpsertHistoryDialogOpen(true)
         } else if (setting === 'chatflowConfiguration') {
             setChatflowConfigurationDialogProps({
                 title: 'Chatflow Configuration',
@@ -387,6 +396,11 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
                 dialogProps={viewMessagesDialogProps}
                 onCancel={() => setViewMessagesDialogOpen(false)}
             />
+            <UpsertHistoryDialog
+                show={upsertHistoryDialogOpen}
+                dialogProps={upsertHistoryDialogProps}
+                onCancel={() => setUpsertHistoryDialogOpen(false)}
+            />
             <ChatflowConfigurationDialog
                 key='chatflowConfiguration'
                 show={chatflowConfigurationDialogOpen}

packages/ui/src/views/chatmessage/ChatMessage.jsx

@@ -406,6 +406,10 @@ export const ChatMessage = ({ open, chatflowid, isDialog, previews, setPreviews
 
             if (response.data) {
                 const data = response.data
+                if (data.executionError) {
+                    handleError(data.msg)
+                    return
+                }
 
                 setMessages((prevMessages) => {
                     let allMessages = [...cloneDeep(prevMessages)]

packages/ui/src/views/vectorstore/UpsertHistoryDialog.jsx

@@ -0,0 +1,453 @@
+import PropTypes from 'prop-types'
+import { createPortal } from 'react-dom'
+import { useDispatch, useSelector } from 'react-redux'
+import { useEffect, useState, forwardRef } from 'react'
+import DatePicker from 'react-datepicker'
+import moment from 'moment/moment'
+
+// MUI
+import {
+    Stack,
+    Box,
+    Paper,
+    Table,
+    TableBody,
+    TableContainer,
+    TableHead,
+    TableRow,
+    TableCell,
+    IconButton,
+    Button,
+    Dialog,
+    DialogActions,
+    DialogContent,
+    DialogTitle,
+    ListItemButton,
+    Collapse,
+    Accordion,
+    AccordionSummary,
+    Typography,
+    AccordionDetails,
+    Checkbox
+} from '@mui/material'
+import { useTheme } from '@mui/material/styles'
+import ExpandMoreIcon from '@mui/icons-material/ExpandMore'
+import { IconChevronsUp, IconChevronsDown, IconTrash, IconX } from '@tabler/icons'
+
+// Project imports
+import { TableViewOnly } from '@/ui-component/table/Table'
+import { TooltipWithParser } from '@/ui-component/tooltip/TooltipWithParser'
+import HistoryEmptySVG from '@/assets/images/upsert_history_empty.svg'
+
+// Api
+import vectorstoreApi from '@/api/vectorstore'
+import useApi from '@/hooks/useApi'
+
+// Store
+import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions'
+import { baseURL } from '@/store/constant'
+import useNotifier from '@/utils/useNotifier'
+import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction } from '@/store/actions'
+
+const DatePickerCustomInput = forwardRef(function DatePickerCustomInput({ value, onClick }, ref) {
+    return (
+        <ListItemButton style={{ borderRadius: 15, border: '1px solid #e0e0e0' }} onClick={onClick} ref={ref}>
+            {value}
+        </ListItemButton>
+    )
+})
+
+DatePickerCustomInput.propTypes = {
+    value: PropTypes.string,
+    onClick: PropTypes.func
+}
+
+function UpsertHistoryRow(props) {
+    const [open, setOpen] = useState(false)
+    const [nodeConfigExpanded, setNodeConfigExpanded] = useState({})
+
+    const handleAccordionChange = (nodeLabel) => (event, isExpanded) => {
+        const accordianNodes = { ...nodeConfigExpanded }
+        accordianNodes[nodeLabel] = isExpanded
+        setNodeConfigExpanded(accordianNodes)
+    }
+
+    const isItemSelected = props.selected.indexOf(props.upsertHistory.id) !== -1
+
+    return (
+        <>
+            <TableRow sx={{ '&:last-child td, &:last-child th': { border: 0 } }}>
+                <TableCell padding='checkbox'>
+                    <Checkbox
+                        color='primary'
+                        checked={isItemSelected}
+                        onChange={(event) => props.handleSelect(event, props.upsertHistory.id)}
+                        inputProps={{
+                            'aria-labelledby': props.upsertHistory.id
+                        }}
+                    />
+                </TableCell>
+                <TableCell>{moment(props.upsertHistory.date).format('MMMM Do YYYY, h:mm:ss a')}</TableCell>
+                <TableCell>{props.upsertHistory.result?.numAdded ?? '0'}</TableCell>
+                <TableCell>{props.upsertHistory.result?.numUpdated ?? '0'}</TableCell>
+                <TableCell>{props.upsertHistory.result?.numSkipped ?? '0'}</TableCell>
+                <TableCell>{props.upsertHistory.result?.numDeleted ?? '0'}</TableCell>
+                <TableCell>
+                    <IconButton aria-label='expand row' size='small' color='inherit' onClick={() => setOpen(!open)}>
+                        {open ? <IconChevronsUp /> : <IconChevronsDown />}
+                    </IconButton>
+                </TableCell>
+            </TableRow>
+            {open && (
+                <TableRow sx={{ '& td': { border: 0 } }}>
+                    <TableCell sx={{ pb: 0, pt: 0 }} colSpan={6}>
+                        <Collapse in={open} timeout='auto' unmountOnExit>
+                            <Box sx={{ mt: 1, mb: 2 }}>
+                                {(props.upsertHistory.flowData ?? []).map((node, index) => {
+                                    return (
+                                        <Accordion
+                                            expanded={nodeConfigExpanded[node.id] || false}
+                                            onChange={handleAccordionChange(node.id)}
+                                            key={index}
+                                            disableGutters
+                                        >
+                                            <AccordionSummary
+                                                expandIcon={<ExpandMoreIcon />}
+                                                aria-controls={`nodes-accordian-${node.name}`}
+                                                id={`nodes-accordian-header-${node.name}`}
+                                            >
+                                                <div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center' }}>
+                                                    <div
+                                                        style={{
+                                                            width: 40,
+                                                            height: 40,
+                                                            marginRight: 10,
+                                                            borderRadius: '50%',
+                                                            backgroundColor: 'white'
+                                                        }}
+                                                    >
+                                                        <img
+                                                            style={{
+                                                                width: '100%',
+                                                                height: '100%',
+                                                                padding: 7,
+                                                                borderRadius: '50%',
+                                                                objectFit: 'contain'
+                                                            }}
+                                                            alt={node.name}
+                                                            src={`${baseURL}/api/v1/node-icon/${node.name}`}
+                                                        />
+                                                    </div>
+                                                    <Typography variant='h5'>{node.label}</Typography>
+                                                    <div
+                                                        style={{
+                                                            display: 'flex',
+                                                            flexDirection: 'row',
+                                                            width: 'max-content',
+                                                            borderRadius: 15,
+                                                            background: 'rgb(254,252,191)',
+                                                            padding: 5,
+                                                            paddingLeft: 10,
+                                                            paddingRight: 10,
+                                                            marginLeft: 10
+                                                        }}
+                                                    >
+                                                        <span style={{ color: 'rgb(116,66,16)', fontSize: '0.825rem' }}>{node.id}</span>
+                                                    </div>
+                                                </div>
+                                            </AccordionSummary>
+                                            <AccordionDetails>
+                                                <TableViewOnly
+                                                    sx={{ minWidth: 150 }}
+                                                    rows={node.paramValues}
+                                                    columns={Object.keys(node.paramValues[0])}
+                                                />
+                                            </AccordionDetails>
+                                        </Accordion>
+                                    )
+                                })}
+                            </Box>
+                        </Collapse>
+                    </TableCell>
+                </TableRow>
+            )}
+        </>
+    )
+}
+
+UpsertHistoryRow.propTypes = {
+    upsertHistory: PropTypes.object,
+    theme: PropTypes.any,
+    isDarkMode: PropTypes.bool,
+    selected: PropTypes.array,
+    handleSelect: PropTypes.func
+}
+
+const UpsertHistoryDialog = ({ show, dialogProps, onCancel }) => {
+    const portalElement = document.getElementById('portal')
+    const dispatch = useDispatch()
+    const customization = useSelector((state) => state.customization)
+    const theme = useTheme()
+    const getUpsertHistoryApi = useApi(vectorstoreApi.getUpsertHistory)
+
+    useNotifier()
+    const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
+    const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
+
+    const [chatflowUpsertHistory, setChatflowUpsertHistory] = useState([])
+    const [startDate, setStartDate] = useState(new Date().setMonth(new Date().getMonth() - 1))
+    const [endDate, setEndDate] = useState(new Date())
+    const [selected, setSelected] = useState([])
+
+    const onSelectAllClick = (event) => {
+        if (event.target.checked) {
+            const newSelected = chatflowUpsertHistory.map((n) => n.id)
+            setSelected(newSelected)
+            return
+        }
+        setSelected([])
+    }
+
+    const onStartDateSelected = (date) => {
+        setStartDate(date)
+        getUpsertHistoryApi.request(dialogProps.chatflow.id, {
+            startDate: date,
+            endDate: endDate
+        })
+    }
+
+    const onEndDateSelected = (date) => {
+        setEndDate(date)
+        getUpsertHistoryApi.request(dialogProps.chatflow.id, {
+            endDate: date,
+            startDate: startDate
+        })
+    }
+
+    const handleSelect = (event, id) => {
+        const selectedIndex = selected.indexOf(id)
+        let newSelected = []
+
+        if (selectedIndex === -1) {
+            newSelected = newSelected.concat(selected, id)
+        } else if (selectedIndex === 0) {
+            newSelected = newSelected.concat(selected.slice(1))
+        } else if (selectedIndex === selected.length - 1) {
+            newSelected = newSelected.concat(selected.slice(0, -1))
+        } else if (selectedIndex > 0) {
+            newSelected = newSelected.concat(selected.slice(0, selectedIndex), selected.slice(selectedIndex + 1))
+        }
+        setSelected(newSelected)
+    }
+
+    const handleRemoveHistory = async () => {
+        try {
+            await vectorstoreApi.deleteUpsertHistory(selected)
+            enqueueSnackbar({
+                message: 'Succesfully deleted upsert history',
+                options: {
+                    key: new Date().getTime() + Math.random(),
+                    variant: 'success',
+                    action: (key) => (
+                        <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                            <IconX />
+                        </Button>
+                    )
+                }
+            })
+            setChatflowUpsertHistory(chatflowUpsertHistory.filter((hist) => !selected.includes(hist.id)))
+            setSelected([])
+        } catch (error) {
+            enqueueSnackbar({
+                message: 'Error deleting upsert history',
+                options: {
+                    key: new Date().getTime() + Math.random(),
+                    variant: 'error',
+                    persist: true,
+                    action: (key) => (
+                        <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                            <IconX />
+                        </Button>
+                    )
+                }
+            })
+        }
+    }
+
+    useEffect(() => {
+        if (getUpsertHistoryApi.data) {
+            setChatflowUpsertHistory(getUpsertHistoryApi.data)
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getUpsertHistoryApi.data])
+
+    useEffect(() => {
+        if (dialogProps.chatflow) {
+            getUpsertHistoryApi.request(dialogProps.chatflow.id)
+        }
+
+        return () => {
+            setChatflowUpsertHistory([])
+            setStartDate(new Date().setMonth(new Date().getMonth() - 1))
+            setEndDate(new Date())
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [dialogProps])
+
+    useEffect(() => {
+        if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
+        else dispatch({ type: HIDE_CANVAS_DIALOG })
+        return () => dispatch({ type: HIDE_CANVAS_DIALOG })
+    }, [show, dispatch])
+
+    const component = show ? (
+        <Dialog
+            onClose={onCancel}
+            open={show}
+            fullWidth
+            maxWidth='lg'
+            aria-labelledby='upsert-history-dialog-title'
+            aria-describedby='upsert-history-dialog-description'
+        >
+            <DialogTitle sx={{ fontSize: '1rem' }} id='upsert-history-dialog-title'>
+                {dialogProps.title}
+            </DialogTitle>
+            <DialogContent>
+                <>
+                    <div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center', width: '100%', marginBottom: 10 }}>
+                        <div style={{ marginRight: 10 }}>
+                            <b style={{ marginRight: 10 }}>From Date</b>
+                            <DatePicker
+                                selected={startDate}
+                                onChange={(date) => onStartDateSelected(date)}
+                                selectsStart
+                                startDate={startDate}
+                                endDate={endDate}
+                                customInput={<DatePickerCustomInput />}
+                            />
+                        </div>
+                        <div style={{ marginRight: 10 }}>
+                            <b style={{ marginRight: 10 }}>To Date</b>
+                            <DatePicker
+                                selected={endDate}
+                                onChange={(date) => onEndDateSelected(date)}
+                                selectsEnd
+                                startDate={startDate}
+                                endDate={endDate}
+                                minDate={startDate}
+                                maxDate={new Date()}
+                                customInput={<DatePickerCustomInput />}
+                            />
+                        </div>
+                    </div>
+                    {selected.length > 0 && (
+                        <Button
+                            sx={{ mt: 1, mb: 2 }}
+                            variant='outlined'
+                            onClick={handleRemoveHistory}
+                            color='error'
+                            startIcon={<IconTrash />}
+                        >
+                            Delete {selected.length} {selected.length === 1 ? 'row' : 'rows'}
+                        </Button>
+                    )}
+                    {chatflowUpsertHistory.length <= 0 && (
+                        <Stack sx={{ alignItems: 'center', justifyContent: 'center' }} flexDirection='column'>
+                            <Box sx={{ p: 7, height: 'auto' }}>
+                                <img
+                                    style={{ objectFit: 'cover', height: '20vh', width: 'auto' }}
+                                    src={HistoryEmptySVG}
+                                    alt='HistoryEmptySVG'
+                                />
+                            </Box>
+                            <div>No Upsert History Yet</div>
+                        </Stack>
+                    )}
+                    {chatflowUpsertHistory.length > 0 && (
+                        <TableContainer component={Paper}>
+                            <Table sx={{ minWidth: 650 }} aria-label='simple table'>
+                                <TableHead>
+                                    <TableRow>
+                                        <TableCell padding='checkbox'>
+                                            <Checkbox
+                                                color='primary'
+                                                checked={selected.length === chatflowUpsertHistory.length}
+                                                onChange={onSelectAllClick}
+                                                inputProps={{
+                                                    'aria-label': 'select all'
+                                                }}
+                                            />
+                                        </TableCell>
+                                        <TableCell>Date</TableCell>
+                                        <TableCell>
+                                            Added{' '}
+                                            <TooltipWithParser
+                                                style={{ marginBottom: 2, marginLeft: 10 }}
+                                                title={'Number of vector embeddings added to Vector Store'}
+                                            />
+                                        </TableCell>
+                                        <TableCell>
+                                            Updated{' '}
+                                            <TooltipWithParser
+                                                style={{ marginBottom: 2, marginLeft: 10 }}
+                                                title={
+                                                    'Updated existing vector embeddings. Only works when a Record Manager is connected to the Vector Store'
+                                                }
+                                            />
+                                        </TableCell>
+                                        <TableCell>
+                                            Skipped{' '}
+                                            <TooltipWithParser
+                                                style={{ marginBottom: 2, marginLeft: 10 }}
+                                                title={
+                                                    'Number of same vector embeddings that exists, and were skipped re-upserting again. Only works when a Record Manager is connected to the Vector Store'
+                                                }
+                                            />
+                                        </TableCell>
+                                        <TableCell>
+                                            Deleted{' '}
+                                            <TooltipWithParser
+                                                style={{ marginBottom: 2, marginLeft: 10 }}
+                                                title={
+                                                    'Deleted vector embeddings. Only works when a Record Manager with a Cleanup method is connected to the Vector Store'
+                                                }
+                                            />
+                                        </TableCell>
+                                        <TableCell>Details</TableCell>
+                                    </TableRow>
+                                </TableHead>
+                                <TableBody>
+                                    {chatflowUpsertHistory.map((upsertHistory, index) => (
+                                        <UpsertHistoryRow
+                                            key={index}
+                                            upsertHistory={upsertHistory}
+                                            theme={theme}
+                                            isDarkMode={customization.isDarkMode}
+                                            selected={selected}
+                                            handleSelect={handleSelect}
+                                        />
+                                    ))}
+                                </TableBody>
+                            </Table>
+                        </TableContainer>
+                    )}
+                </>
+            </DialogContent>
+            <DialogActions>
+                <Button onClick={onCancel}>Close</Button>
+            </DialogActions>
+        </Dialog>
+    ) : null
+
+    return createPortal(component, portalElement)
+}
+
+UpsertHistoryDialog.propTypes = {
+    show: PropTypes.bool,
+    dialogProps: PropTypes.object,
+    onCancel: PropTypes.func
+}
+
+export default UpsertHistoryDialog

packages/ui/src/views/vectorstore/UpsertResultDialog.jsx

@@ -0,0 +1,94 @@
+import PropTypes from 'prop-types'
+import { createPortal } from 'react-dom'
+import { useDispatch, useSelector } from 'react-redux'
+import { useEffect } from 'react'
+import ReactJson from 'flowise-react-json-view'
+import { Typography, Card, CardContent, Button, Dialog, DialogActions, DialogContent, DialogTitle } from '@mui/material'
+import StatsCard from '@/ui-component/cards/StatsCard'
+import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions'
+
+const UpsertResultDialog = ({ show, dialogProps, onCancel }) => {
+    const portalElement = document.getElementById('portal')
+    const dispatch = useDispatch()
+    const customization = useSelector((state) => state.customization)
+
+    useEffect(() => {
+        if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
+        else dispatch({ type: HIDE_CANVAS_DIALOG })
+        return () => dispatch({ type: HIDE_CANVAS_DIALOG })
+    }, [show, dispatch])
+
+    const component = show ? (
+        <Dialog
+            onClose={onCancel}
+            open={show}
+            fullWidth
+            maxWidth='sm'
+            aria-labelledby='upsert-result-dialog-title'
+            aria-describedby='upsert-result-dialog-description'
+        >
+            <DialogTitle sx={{ fontSize: '1rem' }} id='upsert-result-dialog-title'>
+                Upsert Record
+            </DialogTitle>
+            <DialogContent>
+                <>
+                    <div
+                        style={{
+                            display: 'grid',
+                            gridTemplateColumns: 'repeat(4, minmax(0, 1fr))',
+                            gap: 5
+                        }}
+                    >
+                        <StatsCard title='Added' stat={dialogProps.numAdded ?? 0} />
+                        <StatsCard title='Updated' stat={dialogProps.numUpdated ?? 0} />
+                        <StatsCard title='Skipped' stat={dialogProps.numSkipped ?? 0} />
+                        <StatsCard title='Deleted' stat={dialogProps.numDeleted ?? 0} />
+                    </div>
+                    {dialogProps.addedDocs && dialogProps.addedDocs.length > 0 && (
+                        <Typography sx={{ mt: 2, mb: 2, fontWeight: 500 }}>{dialogProps.numAdded} Added Documents</Typography>
+                    )}
+                    {dialogProps.addedDocs &&
+                        dialogProps.addedDocs.length > 0 &&
+                        dialogProps.addedDocs.map((docs, index) => {
+                            return (
+                                <Card
+                                    key={index}
+                                    sx={{ border: '1px solid #e0e0e0', borderRadius: `${customization.borderRadius}px`, mb: 1 }}
+                                >
+                                    <CardContent>
+                                        <Typography sx={{ fontSize: 14 }} color='text.primary' gutterBottom>
+                                            {docs.pageContent}
+                                        </Typography>
+                                        <ReactJson
+                                            theme={customization.isDarkMode ? 'ocean' : 'rjv-default'}
+                                            style={{ padding: 10, borderRadius: 10 }}
+                                            src={docs.metadata}
+                                            name={null}
+                                            quotesOnKeys={false}
+                                            enableClipboard={false}
+                                            displayDataTypes={false}
+                                            collapsed={true}
+                                        />
+                                    </CardContent>
+                                </Card>
+                            )
+                        })}
+                </>
+            </DialogContent>
+            <DialogActions>
+                <Button onClick={onCancel}>Close</Button>
+            </DialogActions>
+        </Dialog>
+    ) : null
+
+    return createPortal(component, portalElement)
+}
+
+UpsertResultDialog.propTypes = {
+    show: PropTypes.bool,
+    dialogProps: PropTypes.object,
+    onCancel: PropTypes.func,
+    onSave: PropTypes.func
+}
+
+export default UpsertResultDialog

packages/ui/src/views/vectorstore/VectorStoreDialog.jsx

@@ -78,7 +78,7 @@ function a11yProps(index) {
     }
 }
 
-const VectorStoreDialog = ({ show, dialogProps, onCancel }) => {
+const VectorStoreDialog = ({ show, dialogProps, onCancel, onIndexResult }) => {
     const portalElement = document.getElementById('portal')
     const { reactFlowInstance } = useContext(flowContext)
     const dispatch = useDispatch()
@@ -276,7 +276,7 @@ query(formData).then((response) => {
     const onUpsertClicked = async (vectorStoreNode) => {
         setLoading(true)
         try {
-            await vectorstoreApi.upsertVectorStore(dialogProps.chatflowid, { stopNodeId: vectorStoreNode.data.id })
+            const res = await vectorstoreApi.upsertVectorStore(dialogProps.chatflowid, { stopNodeId: vectorStoreNode.data.id })
             enqueueSnackbar({
                 message: 'Succesfully upserted vector store. You can start chatting now!',
                 options: {
@@ -290,6 +290,7 @@ query(formData).then((response) => {
                 }
             })
             setLoading(false)
+            if (res && res.data && typeof res.data === 'object') onIndexResult(res.data)
         } catch (error) {
             enqueueSnackbar({
                 message: error.response.data.message,
@@ -549,7 +550,8 @@ query(formData).then((response) => {
 VectorStoreDialog.propTypes = {
     show: PropTypes.bool,
     dialogProps: PropTypes.object,
-    onCancel: PropTypes.func
+    onCancel: PropTypes.func,
+    onIndexResult: PropTypes.func
 }
 
 export default VectorStoreDialog

packages/ui/src/views/vectorstore/VectorStorePopUp.jsx

@@ -1,33 +1,19 @@
 import { useState, useRef, useEffect } from 'react'
-import { useDispatch } from 'react-redux'
 import PropTypes from 'prop-types'
 
-import { Button } from '@mui/material'
 import { IconDatabaseImport, IconX } from '@tabler/icons'
 
 // project import
 import { StyledFab } from '@/ui-component/button/StyledFab'
 import VectorStoreDialog from './VectorStoreDialog'
-
+import UpsertResultDialog from './UpsertResultDialog'
-// api
-import vectorstoreApi from '@/api/vectorstore'
-
-// Hooks
-import useNotifier from '@/utils/useNotifier'
-
-// Const
-import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction } from '@/store/actions'
 
 export const VectorStorePopUp = ({ chatflowid }) => {
-    const dispatch = useDispatch()
-
-    useNotifier()
-    const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
-    const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
-
     const [open, setOpen] = useState(false)
     const [showExpandDialog, setShowExpandDialog] = useState(false)
     const [expandDialogProps, setExpandDialogProps] = useState({})
+    const [showUpsertResultDialog, setShowUpsertResultDialog] = useState(false)
+    const [upsertResultDialogProps, setUpsertResultDialogProps] = useState({})
 
     const anchorRef = useRef(null)
     const prevOpen = useRef(open)
@@ -43,38 +29,6 @@ export const VectorStorePopUp = ({ chatflowid }) => {
         setShowExpandDialog(true)
     }
 
-    const onUpsert = async () => {
-        try {
-            await vectorstoreApi.upsertVectorStore(chatflowid, {})
-            enqueueSnackbar({
-                message: 'Succesfully upserted vector store',
-                options: {
-                    key: new Date().getTime() + Math.random(),
-                    variant: 'success',
-                    action: (key) => (
-                        <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
-                            <IconX />
-                        </Button>
-                    )
-                }
-            })
-        } catch (error) {
-            enqueueSnackbar({
-                message: error.response.data.message,
-                options: {
-                    key: new Date().getTime() + Math.random(),
-                    variant: 'error',
-                    persist: true,
-                    action: (key) => (
-                        <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
-                            <IconX />
-                        </Button>
-                    )
-                }
-            })
-        }
-    }
-
     useEffect(() => {
         if (prevOpen.current === true && open === false) {
             anchorRef.current.focus()
@@ -100,12 +54,24 @@ export const VectorStorePopUp = ({ chatflowid }) => {
             <VectorStoreDialog
                 show={showExpandDialog}
                 dialogProps={expandDialogProps}
-                onUpsert={onUpsert}
                 onCancel={() => {
                     setShowExpandDialog(false)
                     setOpen((prevopen) => !prevopen)
                 }}
+                onIndexResult={(indexRes) => {
+                    setShowExpandDialog(false)
+                    setShowUpsertResultDialog(true)
+                    setUpsertResultDialogProps({ ...indexRes })
+                }}
             ></VectorStoreDialog>
+            <UpsertResultDialog
+                show={showUpsertResultDialog}
+                dialogProps={upsertResultDialogProps}
+                onCancel={() => {
+                    setShowUpsertResultDialog(false)
+                    setOpen(false)
+                }}
+            ></UpsertResultDialog>
         </>
     )
 }