Feature/Indexing (#1802)

* indexing

* fix for multiple files upsert

* fix default Postgres port

* fix SQLite node description

* add MySQLRecordManager node

* fix MySQL unique index

* add upsert history

* update jsx ui

* lint-fix

* update dialog details

* update llamaindex pinecone

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>

packages/components/nodes/recordmanager/MySQLRecordManager/MySQLrecordManager.ts

@@ -0,0 +1,361 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
+import { DataSource, QueryRunner } from 'typeorm'
+
+class MySQLRecordManager_RecordManager implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    badge: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'MySQL Record Manager'
+        this.name = 'MySQLRecordManager'
+        this.version = 1.0
+        this.type = 'MySQL RecordManager'
+        this.icon = 'mysql.png'
+        this.category = 'Record Manager'
+        this.description = 'Use MySQL to keep track of document writes into the vector databases'
+        this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(MySQLRecordManager)]
+        this.badge = 'NEW'
+        this.inputs = [
+            {
+                label: 'Host',
+                name: 'host',
+                type: 'string'
+            },
+            {
+                label: 'Database',
+                name: 'database',
+                type: 'string'
+            },
+            {
+                label: 'Port',
+                name: 'port',
+                type: 'number',
+                placeholder: '3306',
+                optional: true
+            },
+            {
+                label: 'Additional Connection Configuration',
+                name: 'additionalConfig',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Table Name',
+                name: 'tableName',
+                type: 'string',
+                placeholder: 'upsertion_records',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Namespace',
+                name: 'namespace',
+                type: 'string',
+                description: 'If not specified, chatflowid will be used',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Cleanup',
+                name: 'cleanup',
+                type: 'options',
+                description:
+                    'Read more on the difference between different cleanup methods <a target="_blank" href="https://js.langchain.com/docs/modules/data_connection/indexing/#deletion-modes">here</a>',
+                options: [
+                    {
+                        label: 'None',
+                        name: 'none',
+                        description: 'No clean up of old content'
+                    },
+                    {
+                        label: 'Incremental',
+                        name: 'incremental',
+                        description:
+                            'Delete previous versions of the content if content of the source document has changed. Important!! SourceId Key must be specified and document metadata must contains the specified key'
+                    },
+                    {
+                        label: 'Full',
+                        name: 'full',
+                        description:
+                            'Same as incremental, but if the source document has been deleted, it will be deleted from vector store as well, incremental mode will not.'
+                    }
+                ],
+                additionalParams: true,
+                default: 'none'
+            },
+            {
+                label: 'SourceId Key',
+                name: 'sourceIdKey',
+                type: 'string',
+                description:
+                    'Key used to get the true source of document, to be compared against the record. Document metadata must contains SourceId Key',
+                default: 'source',
+                placeholder: 'source',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['MySQLApi']
+        }
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const user = getCredentialParam('user', credentialData, nodeData)
+        const password = getCredentialParam('password', credentialData, nodeData)
+        const _tableName = nodeData.inputs?.tableName as string
+        const tableName = _tableName ? _tableName : 'upsertion_records'
+        const additionalConfig = nodeData.inputs?.additionalConfig as string
+        const _namespace = nodeData.inputs?.namespace as string
+        const namespace = _namespace ? _namespace : options.chatflowid
+        const cleanup = nodeData.inputs?.cleanup as string
+        const _sourceIdKey = nodeData.inputs?.sourceIdKey as string
+        const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source'
+
+        let additionalConfiguration = {}
+        if (additionalConfig) {
+            try {
+                additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
+            } catch (exception) {
+                throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
+            }
+        }
+
+        const mysqlOptions = {
+            ...additionalConfiguration,
+            type: 'mysql',
+            host: nodeData.inputs?.host as string,
+            port: nodeData.inputs?.port as number,
+            username: user,
+            password: password,
+            database: nodeData.inputs?.database as string
+        }
+
+        const args = {
+            mysqlOptions,
+            tableName: tableName
+        }
+
+        const recordManager = new MySQLRecordManager(namespace, args)
+
+        ;(recordManager as any).cleanup = cleanup
+        ;(recordManager as any).sourceIdKey = sourceIdKey
+
+        return recordManager
+    }
+}
+
+type MySQLRecordManagerOptions = {
+    mysqlOptions: any
+    tableName?: string
+}
+
+class MySQLRecordManager implements RecordManagerInterface {
+    lc_namespace = ['langchain', 'recordmanagers', 'mysql']
+
+    datasource: DataSource
+
+    queryRunner: QueryRunner
+
+    tableName: string
+
+    namespace: string
+
+    constructor(namespace: string, config: MySQLRecordManagerOptions) {
+        const { mysqlOptions, tableName } = config
+        this.namespace = namespace
+        this.tableName = tableName || 'upsertion_records'
+        this.datasource = new DataSource(mysqlOptions)
+    }
+
+    async createSchema(): Promise<void> {
+        try {
+            const appDataSource = await this.datasource.initialize()
+
+            this.queryRunner = appDataSource.createQueryRunner()
+
+            await this.queryRunner.manager.query(`create table if not exists \`${this.tableName}\` (
+                \`uuid\` varchar(36) primary key default (UUID()),
+                \`key\` varchar(36) not null,
+                \`namespace\` varchar(36) not null,
+                \`updated_at\` DOUBLE precision not null,
+                \`group_id\` varchar(36),
+                unique key \`unique_key_namespace\` (\`key\`,
+\`namespace\`));`)
+            const columns = [`updated_at`, `key`, `namespace`, `group_id`]
+            for (const column of columns) {
+                // MySQL does not support 'IF NOT EXISTS' function for Index
+                const Check = await this.queryRunner.manager.query(
+                    `SELECT COUNT(1) IndexIsThere FROM INFORMATION_SCHEMA.STATISTICS 
+                        WHERE table_schema=DATABASE() AND table_name='${this.tableName}' AND index_name='${column}_index';`
+                )
+                if (Check[0].IndexIsThere === 0)
+                    await this.queryRunner.manager.query(`CREATE INDEX \`${column}_index\`
+        ON \`${this.tableName}\` (\`${column}\`);`)
+            }
+        } catch (e: any) {
+            // This error indicates that the table already exists
+            // Due to asynchronous nature of the code, it is possible that
+            // the table is created between the time we check if it exists
+            // and the time we try to create it. It can be safely ignored.
+            if ('code' in e && e.code === '23505') {
+                return
+            }
+            throw e
+        }
+    }
+
+    async getTime(): Promise<number> {
+        try {
+            const res = await this.queryRunner.manager.query(`SELECT UNIX_TIMESTAMP(NOW()) AS epoch`)
+            return Number.parseFloat(res[0].epoch)
+        } catch (error) {
+            console.error('Error getting time in MySQLRecordManager:')
+            throw error
+        }
+    }
+
+    async update(keys: string[], updateOptions?: UpdateOptions): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const updatedAt = await this.getTime()
+        const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
+
+        if (timeAtLeast && updatedAt < timeAtLeast) {
+            throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`)
+        }
+
+        const groupIds = _groupIds ?? keys.map(() => null)
+
+        if (groupIds.length !== keys.length) {
+            throw new Error(`Number of keys (${keys.length}) does not match number of group_ids (${groupIds.length})`)
+        }
+
+        const recordsToUpsert = keys.map((key, i) => [
+            key,
+            this.namespace,
+            updatedAt,
+            groupIds[i] ?? null // Ensure groupIds[i] is null if undefined
+        ])
+
+        const query = `
+            INSERT INTO \`${this.tableName}\` (\`key\`, \`namespace\`, \`updated_at\`, \`group_id\`)
+            VALUES (?, ?, ?, ?)
+            ON DUPLICATE KEY UPDATE updated_at = updated_at;`
+
+        // To handle multiple files upsert
+        for (const record of recordsToUpsert) {
+            // Consider using a transaction for batch operations
+            await this.queryRunner.manager.query(query, record.flat())
+        }
+    }
+
+    async exists(keys: string[]): Promise<boolean[]> {
+        if (keys.length === 0) {
+            return []
+        }
+
+        // Prepare the placeholders and the query
+        const placeholders = keys.map(() => `?`).join(', ')
+        const query = `
+    SELECT \`key\`
+    FROM \`${this.tableName}\`
+    WHERE \`namespace\` = ? AND \`key\` IN (${placeholders})`
+
+        // Initialize an array to fill with the existence checks
+        const existsArray = new Array(keys.length).fill(false)
+
+        try {
+            // Execute the query
+            const rows = await this.queryRunner.manager.query(query, [this.namespace, ...keys.flat()])
+            // Create a set of existing keys for faster lookup
+            const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key))
+            // Map the input keys to booleans indicating if they exist
+            keys.forEach((key, index) => {
+                existsArray[index] = existingKeysSet.has(key)
+            })
+            return existsArray
+        } catch (error) {
+            console.error('Error checking existence of keys')
+            throw error // Allow the caller to handle the error
+        }
+    }
+
+    async listKeys(options?: ListKeyOptions): Promise<string[]> {
+        try {
+            const { before, after, limit, groupIds } = options ?? {}
+            let query = `SELECT \`key\` FROM \`${this.tableName}\` WHERE \`namespace\` = ?`
+            const values: (string | number | string[])[] = [this.namespace]
+
+            if (before) {
+                query += ` AND \`updated_at\` < ?`
+                values.push(before)
+            }
+
+            if (after) {
+                query += ` AND \`updated_at\` > ?`
+                values.push(after)
+            }
+
+            if (limit) {
+                query += ` LIMIT ?`
+                values.push(limit)
+            }
+
+            if (groupIds && Array.isArray(groupIds)) {
+                query += ` AND \`group_id\` IN (${groupIds
+                    .filter((gid) => gid !== null)
+                    .map(() => '?')
+                    .join(', ')})`
+                values.push(...groupIds.filter((gid): gid is string => gid !== null))
+            }
+
+            query += ';'
+
+            // Directly using try/catch with async/await for cleaner flow
+            const result = await this.queryRunner.manager.query(query, values)
+            return result.map((row: { key: string }) => row.key)
+        } catch (error) {
+            console.error('MySQLRecordManager listKeys Error: ')
+            throw error // Re-throw the error to be handled by the caller
+        }
+    }
+
+    async deleteKeys(keys: string[]): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const placeholders = keys.map(() => '?').join(', ')
+        const query = `DELETE FROM \`${this.tableName}\` WHERE \`namespace\` = ? AND \`key\` IN (${placeholders});`
+        const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v))
+
+        // Directly using try/catch with async/await for cleaner flow
+        try {
+            await this.queryRunner.manager.query(query, values)
+        } catch (error) {
+            console.error('Error deleting keys')
+            throw error // Re-throw the error to be handled by the caller
+        }
+    }
+}
+
+module.exports = { nodeClass: MySQLRecordManager_RecordManager }

packages/components/nodes/recordmanager/PostgresRecordManager/PostgresRecordManager.ts

@@ -0,0 +1,332 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
+import { DataSource, QueryRunner } from 'typeorm'
+
+class PostgresRecordManager_RecordManager implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    badge: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Postgres Record Manager'
+        this.name = 'postgresRecordManager'
+        this.version = 1.0
+        this.type = 'Postgres RecordManager'
+        this.icon = 'postgres.svg'
+        this.category = 'Record Manager'
+        this.description = 'Use Postgres to keep track of document writes into the vector databases'
+        this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(PostgresRecordManager)]
+        this.badge = 'NEW'
+        this.inputs = [
+            {
+                label: 'Host',
+                name: 'host',
+                type: 'string'
+            },
+            {
+                label: 'Database',
+                name: 'database',
+                type: 'string'
+            },
+            {
+                label: 'Port',
+                name: 'port',
+                type: 'number',
+                placeholder: '5432',
+                optional: true
+            },
+            {
+                label: 'Additional Connection Configuration',
+                name: 'additionalConfig',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Table Name',
+                name: 'tableName',
+                type: 'string',
+                placeholder: 'upsertion_records',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Namespace',
+                name: 'namespace',
+                type: 'string',
+                description: 'If not specified, chatflowid will be used',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Cleanup',
+                name: 'cleanup',
+                type: 'options',
+                description:
+                    'Read more on the difference between different cleanup methods <a target="_blank" href="https://js.langchain.com/docs/modules/data_connection/indexing/#deletion-modes">here</a>',
+                options: [
+                    {
+                        label: 'None',
+                        name: 'none',
+                        description: 'No clean up of old content'
+                    },
+                    {
+                        label: 'Incremental',
+                        name: 'incremental',
+                        description:
+                            'Delete previous versions of the content if content of the source document has changed. Important!! SourceId Key must be specified and document metadata must contains the specified key'
+                    },
+                    {
+                        label: 'Full',
+                        name: 'full',
+                        description:
+                            'Same as incremental, but if the source document has been deleted, it will be deleted from vector store as well, incremental mode will not.'
+                    }
+                ],
+                additionalParams: true,
+                default: 'none'
+            },
+            {
+                label: 'SourceId Key',
+                name: 'sourceIdKey',
+                type: 'string',
+                description:
+                    'Key used to get the true source of document, to be compared against the record. Document metadata must contains SourceId Key',
+                default: 'source',
+                placeholder: 'source',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['PostgresApi']
+        }
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const user = getCredentialParam('user', credentialData, nodeData)
+        const password = getCredentialParam('password', credentialData, nodeData)
+        const _tableName = nodeData.inputs?.tableName as string
+        const tableName = _tableName ? _tableName : 'upsertion_records'
+        const additionalConfig = nodeData.inputs?.additionalConfig as string
+        const _namespace = nodeData.inputs?.namespace as string
+        const namespace = _namespace ? _namespace : options.chatflowid
+        const cleanup = nodeData.inputs?.cleanup as string
+        const _sourceIdKey = nodeData.inputs?.sourceIdKey as string
+        const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source'
+
+        let additionalConfiguration = {}
+        if (additionalConfig) {
+            try {
+                additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
+            } catch (exception) {
+                throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
+            }
+        }
+
+        const postgresConnectionOptions = {
+            ...additionalConfiguration,
+            type: 'postgres',
+            host: nodeData.inputs?.host as string,
+            port: nodeData.inputs?.port as number,
+            username: user,
+            password: password,
+            database: nodeData.inputs?.database as string
+        }
+
+        const args = {
+            postgresConnectionOptions: postgresConnectionOptions,
+            tableName: tableName
+        }
+
+        const recordManager = new PostgresRecordManager(namespace, args)
+
+        ;(recordManager as any).cleanup = cleanup
+        ;(recordManager as any).sourceIdKey = sourceIdKey
+
+        return recordManager
+    }
+}
+
+type PostgresRecordManagerOptions = {
+    postgresConnectionOptions: any
+    tableName?: string
+}
+
+class PostgresRecordManager implements RecordManagerInterface {
+    lc_namespace = ['langchain', 'recordmanagers', 'postgres']
+
+    datasource: DataSource
+
+    queryRunner: QueryRunner
+
+    tableName: string
+
+    namespace: string
+
+    constructor(namespace: string, config: PostgresRecordManagerOptions) {
+        const { postgresConnectionOptions, tableName } = config
+        this.namespace = namespace
+        this.datasource = new DataSource(postgresConnectionOptions)
+        this.tableName = tableName || 'upsertion_records'
+    }
+
+    async createSchema(): Promise<void> {
+        try {
+            const appDataSource = await this.datasource.initialize()
+
+            this.queryRunner = appDataSource.createQueryRunner()
+
+            await this.queryRunner.manager.query(`
+  CREATE TABLE IF NOT EXISTS "${this.tableName}" (
+    uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    key TEXT NOT NULL,
+    namespace TEXT NOT NULL,
+    updated_at Double PRECISION NOT NULL,
+    group_id TEXT,
+    UNIQUE (key, namespace)
+  );
+  CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at);
+  CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key);
+  CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace);
+  CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
+        } catch (e: any) {
+            // This error indicates that the table already exists
+            // Due to asynchronous nature of the code, it is possible that
+            // the table is created between the time we check if it exists
+            // and the time we try to create it. It can be safely ignored.
+            if ('code' in e && e.code === '23505') {
+                return
+            }
+            throw e
+        }
+    }
+
+    async getTime(): Promise<number> {
+        const res = await this.queryRunner.manager.query('SELECT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)')
+        return Number.parseFloat(res[0].extract)
+    }
+
+    /**
+     * Generates the SQL placeholders for a specific row at the provided index.
+     *
+     * @param index - The index of the row for which placeholders need to be generated.
+     * @param numOfColumns - The number of columns we are inserting data into.
+     * @returns The SQL placeholders for the row values.
+     */
+    private generatePlaceholderForRowAt(index: number, numOfColumns: number): string {
+        const placeholders = []
+        for (let i = 0; i < numOfColumns; i += 1) {
+            placeholders.push(`$${index * numOfColumns + i + 1}`)
+        }
+        return `(${placeholders.join(', ')})`
+    }
+
+    async update(keys: string[], updateOptions?: UpdateOptions): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const updatedAt = await this.getTime()
+        const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
+
+        if (timeAtLeast && updatedAt < timeAtLeast) {
+            throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`)
+        }
+
+        const groupIds = _groupIds ?? keys.map(() => null)
+
+        if (groupIds.length !== keys.length) {
+            throw new Error(`Number of keys (${keys.length}) does not match number of group_ids ${groupIds.length})`)
+        }
+
+        const recordsToUpsert = keys.map((key, i) => [key, this.namespace, updatedAt, groupIds[i]])
+
+        const valuesPlaceholders = recordsToUpsert.map((_, j) => this.generatePlaceholderForRowAt(j, recordsToUpsert[0].length)).join(', ')
+
+        const query = `INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) VALUES ${valuesPlaceholders} ON CONFLICT (key, namespace) DO UPDATE SET updated_at = EXCLUDED.updated_at;`
+        await this.queryRunner.manager.query(query, recordsToUpsert.flat())
+    }
+
+    async exists(keys: string[]): Promise<boolean[]> {
+        if (keys.length === 0) {
+            return []
+        }
+
+        const startIndex = 2
+        const arrayPlaceholders = keys.map((_, i) => `$${i + startIndex}`).join(', ')
+
+        const query = `
+        SELECT k, (key is not null) ex from unnest(ARRAY[${arrayPlaceholders}]) k left join "${this.tableName}" on k=key and namespace = $1;
+        `
+        const res = await this.queryRunner.manager.query(query, [this.namespace, ...keys.flat()])
+        return res.map((row: { ex: boolean }) => row.ex)
+    }
+
+    async listKeys(options?: ListKeyOptions): Promise<string[]> {
+        const { before, after, limit, groupIds } = options ?? {}
+        let query = `SELECT key FROM "${this.tableName}" WHERE namespace = $1`
+        const values: (string | number | (string | null)[])[] = [this.namespace]
+
+        let index = 2
+        if (before) {
+            values.push(before)
+            query += ` AND updated_at < $${index}`
+            index += 1
+        }
+
+        if (after) {
+            values.push(after)
+            query += ` AND updated_at > $${index}`
+            index += 1
+        }
+
+        if (limit) {
+            values.push(limit)
+            query += ` LIMIT $${index}`
+            index += 1
+        }
+
+        if (groupIds) {
+            values.push(groupIds)
+            query += ` AND group_id = ANY($${index})`
+            index += 1
+        }
+
+        query += ';'
+        const res = await this.queryRunner.manager.query(query, values)
+        return res.map((row: { key: string }) => row.key)
+    }
+
+    async deleteKeys(keys: string[]): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const query = `DELETE FROM "${this.tableName}" WHERE namespace = $1 AND key = ANY($2);`
+        await this.queryRunner.manager.query(query, [this.namespace, keys])
+    }
+
+    /**
+     * Terminates the connection pool.
+     * @returns {Promise<void>}
+     */
+    async end(): Promise<void> {
+        if (this.datasource && this.datasource.isInitialized) await this.datasource.destroy()
+    }
+}
+
+module.exports = { nodeClass: PostgresRecordManager_RecordManager }

packages/components/nodes/recordmanager/SQLiteRecordManager/SQLiteRecordManager.ts

@@ -0,0 +1,332 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses } from '../../../src/utils'
+import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
+import { DataSource, QueryRunner } from 'typeorm'
+import path from 'path'
+
+class SQLiteRecordManager_RecordManager implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    badge: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'SQLite Record Manager'
+        this.name = 'SQLiteRecordManager'
+        this.version = 1.0
+        this.type = 'SQLite RecordManager'
+        this.icon = 'sqlite.png'
+        this.category = 'Record Manager'
+        this.description = 'Use SQLite to keep track of document writes into the vector databases'
+        this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(SQLiteRecordManager)]
+        this.badge = 'NEW'
+        this.inputs = [
+            {
+                label: 'Database File Path',
+                name: 'databaseFilePath',
+                type: 'string',
+                placeholder: 'C:\\Users\\User\\.flowise\\database.sqlite'
+            },
+            {
+                label: 'Additional Connection Configuration',
+                name: 'additionalConfig',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Table Name',
+                name: 'tableName',
+                type: 'string',
+                placeholder: 'upsertion_records',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Namespace',
+                name: 'namespace',
+                type: 'string',
+                description: 'If not specified, chatflowid will be used',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Cleanup',
+                name: 'cleanup',
+                type: 'options',
+                description:
+                    'Read more on the difference between different cleanup methods <a target="_blank" href="https://js.langchain.com/docs/modules/data_connection/indexing/#deletion-modes">here</a>',
+                options: [
+                    {
+                        label: 'None',
+                        name: 'none',
+                        description: 'No clean up of old content'
+                    },
+                    {
+                        label: 'Incremental',
+                        name: 'incremental',
+                        description:
+                            'Delete previous versions of the content if content of the source document has changed. Important!! SourceId Key must be specified and document metadata must contains the specified key'
+                    },
+                    {
+                        label: 'Full',
+                        name: 'full',
+                        description:
+                            'Same as incremental, but if the source document has been deleted, it will be deleted from vector store as well, incremental mode will not.'
+                    }
+                ],
+                additionalParams: true,
+                default: 'none'
+            },
+            {
+                label: 'SourceId Key',
+                name: 'sourceIdKey',
+                type: 'string',
+                description:
+                    'Key used to get the true source of document, to be compared against the record. Document metadata must contains SourceId Key',
+                default: 'source',
+                placeholder: 'source',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const _tableName = nodeData.inputs?.tableName as string
+        const tableName = _tableName ? _tableName : 'upsertion_records'
+        const additionalConfig = nodeData.inputs?.additionalConfig as string
+        const _namespace = nodeData.inputs?.namespace as string
+        const namespace = _namespace ? _namespace : options.chatflowid
+        const cleanup = nodeData.inputs?.cleanup as string
+        const _sourceIdKey = nodeData.inputs?.sourceIdKey as string
+        const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source'
+        const databaseFilePath = nodeData.inputs?.databaseFilePath as string
+
+        let additionalConfiguration = {}
+        if (additionalConfig) {
+            try {
+                additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
+            } catch (exception) {
+                throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
+            }
+        }
+
+        const sqliteOptions = {
+            ...additionalConfiguration,
+            type: 'sqlite',
+            database: path.resolve(databaseFilePath)
+        }
+
+        const args = {
+            sqliteOptions,
+            tableName: tableName
+        }
+
+        const recordManager = new SQLiteRecordManager(namespace, args)
+
+        ;(recordManager as any).cleanup = cleanup
+        ;(recordManager as any).sourceIdKey = sourceIdKey
+
+        return recordManager
+    }
+}
+
+type SQLiteRecordManagerOptions = {
+    sqliteOptions: any
+    tableName?: string
+}
+
+class SQLiteRecordManager implements RecordManagerInterface {
+    lc_namespace = ['langchain', 'recordmanagers', 'sqlite']
+
+    datasource: DataSource
+
+    queryRunner: QueryRunner
+
+    tableName: string
+
+    namespace: string
+
+    constructor(namespace: string, config: SQLiteRecordManagerOptions) {
+        const { sqliteOptions, tableName } = config
+        this.namespace = namespace
+        this.tableName = tableName || 'upsertion_records'
+        this.datasource = new DataSource(sqliteOptions)
+    }
+
+    async createSchema(): Promise<void> {
+        try {
+            const appDataSource = await this.datasource.initialize()
+
+            this.queryRunner = appDataSource.createQueryRunner()
+
+            await this.queryRunner.manager.query(`
+CREATE TABLE IF NOT EXISTS "${this.tableName}" (
+  uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
+  key TEXT NOT NULL,
+  namespace TEXT NOT NULL,
+  updated_at REAL NOT NULL,
+  group_id TEXT,
+  UNIQUE (key, namespace)
+);
+CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at);
+CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key);
+CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace);
+CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
+        } catch (e: any) {
+            // This error indicates that the table already exists
+            // Due to asynchronous nature of the code, it is possible that
+            // the table is created between the time we check if it exists
+            // and the time we try to create it. It can be safely ignored.
+            if ('code' in e && e.code === '23505') {
+                return
+            }
+            throw e
+        }
+    }
+
+    async getTime(): Promise<number> {
+        try {
+            const res = await this.queryRunner.manager.query(`SELECT strftime('%s', 'now') AS epoch`)
+            return Number.parseFloat(res[0].epoch)
+        } catch (error) {
+            console.error('Error getting time in SQLiteRecordManager:')
+            throw error
+        }
+    }
+
+    async update(keys: string[], updateOptions?: UpdateOptions): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const updatedAt = await this.getTime()
+        const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
+
+        if (timeAtLeast && updatedAt < timeAtLeast) {
+            throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`)
+        }
+
+        const groupIds = _groupIds ?? keys.map(() => null)
+
+        if (groupIds.length !== keys.length) {
+            throw new Error(`Number of keys (${keys.length}) does not match number of group_ids (${groupIds.length})`)
+        }
+
+        const recordsToUpsert = keys.map((key, i) => [
+            key,
+            this.namespace,
+            updatedAt,
+            groupIds[i] ?? null // Ensure groupIds[i] is null if undefined
+        ])
+
+        const query = `
+        INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id)
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at`
+
+        // To handle multiple files upsert
+        for (const record of recordsToUpsert) {
+            // Consider using a transaction for batch operations
+            await this.queryRunner.manager.query(query, record.flat())
+        }
+    }
+
+    async exists(keys: string[]): Promise<boolean[]> {
+        if (keys.length === 0) {
+            return []
+        }
+
+        // Prepare the placeholders and the query
+        const placeholders = keys.map(() => `?`).join(', ')
+        const sql = `
+    SELECT key
+    FROM "${this.tableName}"
+    WHERE namespace = ? AND key IN (${placeholders})`
+
+        // Initialize an array to fill with the existence checks
+        const existsArray = new Array(keys.length).fill(false)
+
+        try {
+            // Execute the query
+            const rows = await this.queryRunner.manager.query(sql, [this.namespace, ...keys.flat()])
+            // Create a set of existing keys for faster lookup
+            const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key))
+            // Map the input keys to booleans indicating if they exist
+            keys.forEach((key, index) => {
+                existsArray[index] = existingKeysSet.has(key)
+            })
+            return existsArray
+        } catch (error) {
+            console.error('Error checking existence of keys')
+            throw error // Allow the caller to handle the error
+        }
+    }
+
+    async listKeys(options?: ListKeyOptions): Promise<string[]> {
+        const { before, after, limit, groupIds } = options ?? {}
+        let query = `SELECT key FROM "${this.tableName}" WHERE namespace = ?`
+        const values: (string | number | string[])[] = [this.namespace]
+
+        if (before) {
+            query += ` AND updated_at < ?`
+            values.push(before)
+        }
+
+        if (after) {
+            query += ` AND updated_at > ?`
+            values.push(after)
+        }
+
+        if (limit) {
+            query += ` LIMIT ?`
+            values.push(limit)
+        }
+
+        if (groupIds && Array.isArray(groupIds)) {
+            query += ` AND group_id IN (${groupIds
+                .filter((gid) => gid !== null)
+                .map(() => '?')
+                .join(', ')})`
+            values.push(...groupIds.filter((gid): gid is string => gid !== null))
+        }
+
+        query += ';'
+
+        // Directly using try/catch with async/await for cleaner flow
+        try {
+            const result = await this.queryRunner.manager.query(query, values)
+            return result.map((row: { key: string }) => row.key)
+        } catch (error) {
+            console.error('Error listing keys.')
+            throw error // Re-throw the error to be handled by the caller
+        }
+    }
+
+    async deleteKeys(keys: string[]): Promise<void> {
+        if (keys.length === 0) {
+            return
+        }
+
+        const placeholders = keys.map(() => '?').join(', ')
+        const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${placeholders});`
+        const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v))
+
+        // Directly using try/catch with async/await for cleaner flow
+        try {
+            await this.queryRunner.manager.query(query, values)
+        } catch (error) {
+            console.error('Error deleting keys')
+            throw error // Re-throw the error to be handled by the caller
+        }
+    }
+}
+
+module.exports = { nodeClass: SQLiteRecordManager_RecordManager }

packages/components/nodes/vectorstores/Astra/Astra.ts

@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
 import { AstraDBVectorStore, AstraLibArgs } from '@langchain/community/vectorstores/astradb'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
 
@@ -101,7 +101,7 @@ class Astra_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const vectorDimension = nodeData.inputs?.vectorDimension as number
@@ -142,6 +142,7 @@ class Astra_VectorStores implements INode {
 
             try {
                 await AstraDBVectorStore.fromDocuments(finalDocs, embeddings, astraConfig)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Chroma/Chroma.ts

@@ -2,9 +2,10 @@ import { flatten } from 'lodash'
 import { Chroma } from '@langchain/community/vectorstores/chroma'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { ChromaExtended } from './core'
+import { index } from '../../../src/indexing'
 
 class Chroma_VectorStores implements INode {
     label: string
@@ -23,7 +24,7 @@ class Chroma_VectorStores implements INode {
     constructor() {
         this.label = 'Chroma'
         this.name = 'chroma'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'Chroma'
         this.icon = 'chroma.svg'
         this.category = 'Vector Stores'
@@ -51,6 +52,13 @@ class Chroma_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Collection Name',
                 name: 'collectionName',
@@ -95,11 +103,12 @@ class Chroma_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const collectionName = nodeData.inputs?.collectionName as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const chromaURL = nodeData.inputs?.chromaURL as string
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
@@ -121,7 +130,24 @@ class Chroma_VectorStores implements INode {
             if (chromaApiKey) obj.chromaApiKey = chromaApiKey
 
             try {
-                await ChromaExtended.fromDocuments(finalDocs, embeddings, obj)
+                if (recordManager) {
+                    const vectorStore = await ChromaExtended.fromExistingCollection(embeddings, obj)
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: collectionName
+                        }
+                    })
+                    return res
+                } else {
+                    await ChromaExtended.fromDocuments(finalDocs, embeddings, obj)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts

@@ -3,8 +3,9 @@ import { Client, ClientOptions } from '@elastic/elasticsearch'
 import { Document } from '@langchain/core/documents'
 import { Embeddings } from '@langchain/core/embeddings'
 import { ElasticClientArgs, ElasticVectorSearch } from '@langchain/community/vectorstores/elasticsearch'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { index } from '../../../src/indexing'
 
 class Elasticsearch_VectorStores implements INode {
     label: string
@@ -23,7 +24,7 @@ class Elasticsearch_VectorStores implements INode {
     constructor() {
         this.label = 'Elasticsearch'
         this.name = 'elasticsearch'
-        this.version = 1.0
+        this.version = 2.0
         this.description =
             'Upsert embedded data and perform similarity search upon query using Elasticsearch, a distributed search and analytics engine'
         this.type = 'Elasticsearch'
@@ -50,6 +51,13 @@ class Elasticsearch_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Index Name',
                 name: 'indexName',
@@ -105,13 +113,14 @@ class Elasticsearch_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
             const cloudId = getCredentialParam('cloudId', credentialData, nodeData)
             const indexName = nodeData.inputs?.indexName as string
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const similarityMeasure = nodeData.inputs?.similarityMeasure as string
+            const recordManager = nodeData.inputs?.recordManager
 
             const docs = nodeData.inputs?.document as Document[]
             const flattenDocs = docs && docs.length ? flatten(docs) : []
@@ -134,7 +143,24 @@ class Elasticsearch_VectorStores implements INode {
             const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
 
             try {
-                await vectorStore.addDocuments(finalDocs)
+                if (recordManager) {
+                    const vectorStore = await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs)
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: indexName
+                        }
+                    })
+                    return res
+                } else {
+                    await vectorStore.addDocuments(finalDocs)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Faiss/Faiss.ts

@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
 import { Document } from '@langchain/core/documents'
 import { FaissStore } from '@langchain/community/vectorstores/faiss'
 import { Embeddings } from '@langchain/core/embeddings'
-import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
 
 class Faiss_VectorStores implements INode {
@@ -74,7 +74,7 @@ class Faiss_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData): Promise<void> {
+        async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const basePath = nodeData.inputs?.basePath as string
@@ -95,6 +95,8 @@ class Faiss_VectorStores implements INode {
                 vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number) => {
                     return await similaritySearchVectorWithScore(query, k, vectorStore)
                 }
+
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts

@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
 import { MemoryVectorStore } from 'langchain/vectorstores/memory'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
 
 class InMemoryVectorStore_VectorStores implements INode {
@@ -64,7 +64,7 @@ class InMemoryVectorStore_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData): Promise<void> {
+        async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
 
@@ -78,6 +78,7 @@ class InMemoryVectorStore_VectorStores implements INode {
 
             try {
                 await MemoryVectorStore.fromDocuments(finalDocs, embeddings)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Milvus/Milvus.ts

@@ -3,7 +3,7 @@ import { DataType, ErrorCode, MetricType, IndexType } from '@zilliz/milvus2-sdk-
 import { Document } from '@langchain/core/documents'
 import { MilvusLibArgs, Milvus } from '@langchain/community/vectorstores/milvus'
 import { Embeddings } from '@langchain/core/embeddings'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 
 interface InsertRow {
@@ -109,7 +109,7 @@ class Milvus_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             // server setup
             const address = nodeData.inputs?.milvusServerUrl as string
             const collectionName = nodeData.inputs?.milvusCollection as string
@@ -147,6 +147,8 @@ class Milvus_VectorStores implements INode {
                 vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: string) => {
                     return await similaritySearchVectorWithScore(query, k, vectorStore, undefined, filter)
                 }
+
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/MongoDBAtlas/MongoDBAtlas.ts

@@ -3,7 +3,7 @@ import { MongoClient } from 'mongodb'
 import { MongoDBAtlasVectorSearch } from '@langchain/mongodb'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
 
@@ -113,7 +113,7 @@ class MongoDBAtlas_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const databaseName = nodeData.inputs?.databaseName as string
             const collectionName = nodeData.inputs?.collectionName as string
@@ -149,6 +149,7 @@ class MongoDBAtlas_VectorStores implements INode {
                     embeddingKey
                 })
                 await mongoDBAtlasVectorSearch.addDocuments(finalDocs)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/OpenSearch/OpenSearch.ts

@@ -3,7 +3,7 @@ import { Client } from '@opensearch-project/opensearch'
 import { Document } from '@langchain/core/documents'
 import { OpenSearchVectorStore } from '@langchain/community/vectorstores/opensearch'
 import { Embeddings } from '@langchain/core/embeddings'
-import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
 
 class OpenSearch_VectorStores implements INode {
@@ -79,7 +79,7 @@ class OpenSearch_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData): Promise<void> {
+        async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const opensearchURL = nodeData.inputs?.opensearchURL as string
@@ -102,6 +102,7 @@ class OpenSearch_VectorStores implements INode {
                     client,
                     indexName: indexName
                 })
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Pinecone/Pinecone.ts

@@ -3,9 +3,10 @@ import { Pinecone } from '@pinecone-database/pinecone'
 import { PineconeStoreParams, PineconeStore } from '@langchain/pinecone'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
+import { index } from '../../../src/indexing'
 
 class Pinecone_VectorStores implements INode {
     label: string
@@ -24,7 +25,7 @@ class Pinecone_VectorStores implements INode {
     constructor() {
         this.label = 'Pinecone'
         this.name = 'pinecone'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Pinecone'
         this.icon = 'pinecone.svg'
         this.category = 'Vector Stores'
@@ -50,6 +51,13 @@ class Pinecone_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Pinecone Index',
                 name: 'pineconeIndex',
@@ -97,11 +105,12 @@ class Pinecone_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
-            const index = nodeData.inputs?.pineconeIndex as string
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
+            const _index = nodeData.inputs?.pineconeIndex as string
             const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
@@ -110,7 +119,7 @@ class Pinecone_VectorStores implements INode {
                 apiKey: pineconeApiKey
             })
 
-            const pineconeIndex = client.Index(index)
+            const pineconeIndex = client.Index(_index)
 
             const flattenDocs = docs && docs.length ? flatten(docs) : []
             const finalDocs = []
@@ -127,7 +136,25 @@ class Pinecone_VectorStores implements INode {
             if (pineconeNamespace) obj.namespace = pineconeNamespace
 
             try {
-                await PineconeStore.fromDocuments(finalDocs, embeddings, obj)
+                if (recordManager) {
+                    const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: pineconeNamespace
+                        }
+                    })
+
+                    return res
+                } else {
+                    await PineconeStore.fromDocuments(finalDocs, embeddings, obj)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Pinecone/Pinecone_LlamaIndex.ts

@@ -13,7 +13,7 @@ import {
 import { FetchResponse, Index, Pinecone, ScoredPineconeRecord } from '@pinecone-database/pinecone'
 import { flatten } from 'lodash'
 import { Document as LCDocument } from 'langchain/document'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { flattenObject, getCredentialData, getCredentialParam } from '../../../src/utils'
 
 class PineconeLlamaIndex_VectorStores implements INode {
@@ -110,7 +110,7 @@ class PineconeLlamaIndex_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const indexName = nodeData.inputs?.pineconeIndex as string
             const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
             const docs = nodeData.inputs?.document as LCDocument[]
@@ -144,6 +144,7 @@ class PineconeLlamaIndex_VectorStores implements INode {
 
             try {
                 await VectorStoreIndex.fromDocuments(llamadocs, { serviceContext, storageContext })
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Postgres/Postgres.ts

@@ -4,8 +4,9 @@ import { DataSourceOptions } from 'typeorm'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
 import { TypeORMVectorStore, TypeORMVectorStoreDocument } from '@langchain/community/vectorstores/typeorm'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { index } from '../../../src/indexing'
 
 class Postgres_VectorStores implements INode {
     label: string
@@ -24,7 +25,7 @@ class Postgres_VectorStores implements INode {
     constructor() {
         this.label = 'Postgres'
         this.name = 'postgres'
-        this.version = 3.0
+        this.version = 4.0
         this.type = 'Postgres'
         this.icon = 'postgres.svg'
         this.category = 'Vector Stores'
@@ -50,6 +51,13 @@ class Postgres_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Host',
                 name: 'host',
@@ -108,7 +116,7 @@ class Postgres_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const user = getCredentialParam('user', credentialData, nodeData)
             const password = getCredentialParam('password', credentialData, nodeData)
@@ -117,6 +125,7 @@ class Postgres_VectorStores implements INode {
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const additionalConfig = nodeData.inputs?.additionalConfig as string
+            const recordManager = nodeData.inputs?.recordManager
 
             let additionalConfiguration = {}
             if (additionalConfig) {
@@ -151,11 +160,37 @@ class Postgres_VectorStores implements INode {
             }
 
             try {
-                const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)
+                if (recordManager) {
+                    const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args)
 
-                // Avoid Illegal invocation error
-                vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
-                    return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
+                    // Avoid Illegal invocation error
+                    vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
+                        return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
+                    }
+
+                    await recordManager.createSchema()
+
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: tableName
+                        }
+                    })
+
+                    return res
+                } else {
+                    const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)
+
+                    // Avoid Illegal invocation error
+                    vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => {
+                        return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter)
+                    }
+
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
                 }
             } catch (e) {
                 throw new Error(e)

packages/components/nodes/vectorstores/Qdrant/Qdrant.ts

@@ -1,13 +1,19 @@
 import { flatten } from 'lodash'
+import { v4 as uuid } from 'uuid'
 import { QdrantClient } from '@qdrant/js-client-rest'
 import { VectorStoreRetrieverInput } from '@langchain/core/vectorstores'
 import { Document } from '@langchain/core/documents'
 import { QdrantVectorStore, QdrantLibArgs } from '@langchain/community/vectorstores/qdrant'
 import { Embeddings } from '@langchain/core/embeddings'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { index } from '../../../src/indexing'
 
 type RetrieverConfig = Partial<VectorStoreRetrieverInput<QdrantVectorStore>>
+type QdrantAddDocumentOptions = {
+    customPayload?: Record<string, any>[]
+    ids?: string[]
+}
 
 class Qdrant_VectorStores implements INode {
     label: string
@@ -26,7 +32,7 @@ class Qdrant_VectorStores implements INode {
     constructor() {
         this.label = 'Qdrant'
         this.name = 'qdrant'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'Qdrant'
         this.icon = 'qdrant.png'
         this.category = 'Vector Stores'
@@ -55,6 +61,13 @@ class Qdrant_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Qdrant Server URL',
                 name: 'qdrantServerUrl',
@@ -138,13 +151,14 @@ class Qdrant_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string
             const collectionName = nodeData.inputs?.qdrantCollection as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
             const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
             const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
@@ -178,7 +192,74 @@ class Qdrant_VectorStores implements INode {
             }
 
             try {
-                await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
+                if (recordManager) {
+                    const vectorStore = new QdrantVectorStore(embeddings, dbConfig)
+                    await vectorStore.ensureCollection()
+
+                    vectorStore.addVectors = async (
+                        vectors: number[][],
+                        documents: Document[],
+                        documentOptions?: QdrantAddDocumentOptions
+                    ): Promise<void> => {
+                        if (vectors.length === 0) {
+                            return
+                        }
+
+                        await vectorStore.ensureCollection()
+
+                        const points = vectors.map((embedding, idx) => ({
+                            id: documentOptions?.ids?.length ? documentOptions?.ids[idx] : uuid(),
+                            vector: embedding,
+                            payload: {
+                                content: documents[idx].pageContent,
+                                metadata: documents[idx].metadata,
+                                customPayload: documentOptions?.customPayload?.length ? documentOptions?.customPayload[idx] : undefined
+                            }
+                        }))
+
+                        try {
+                            await client.upsert(collectionName, {
+                                wait: true,
+                                points
+                            })
+                        } catch (e: any) {
+                            const error = new Error(`${e?.status ?? 'Undefined error code'} ${e?.message}: ${e?.data?.status?.error}`)
+                            throw error
+                        }
+                    }
+
+                    vectorStore.delete = async (params: { ids: string[] }): Promise<void> => {
+                        const { ids } = params
+
+                        if (ids?.length) {
+                            try {
+                                client.delete(collectionName, {
+                                    points: ids
+                                })
+                            } catch (e) {
+                                console.error('Failed to delete')
+                            }
+                        }
+                    }
+
+                    await recordManager.createSchema()
+
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: collectionName
+                        }
+                    })
+
+                    return res
+                } else {
+                    await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Redis/Redis.ts

@@ -3,7 +3,7 @@ import { createClient, SearchOptions, RedisClientOptions } from 'redis'
 import { Embeddings } from '@langchain/core/embeddings'
 import { RedisVectorStore, RedisVectorStoreConfig } from '@langchain/community/vectorstores/redis'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { escapeAllStrings, escapeSpecialChars, unEscapeSpecialChars } from './utils'
 
@@ -138,7 +138,7 @@ class Redis_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const indexName = nodeData.inputs?.indexName as string
             let contentKey = nodeData.inputs?.contentKey as string
@@ -203,6 +203,8 @@ class Redis_VectorStores implements INode {
                         filter
                     )
                 }
+
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/SimpleStore/SimpleStore.ts

@@ -2,7 +2,7 @@ import path from 'path'
 import { flatten } from 'lodash'
 import { storageContextFromDefaults, serviceContextFromDefaults, VectorStoreIndex, Document } from 'llamaindex'
 import { Document as LCDocument } from 'langchain/document'
-import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getUserHome } from '../../../src'
 
 class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
@@ -79,7 +79,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData): Promise<void> {
+        async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
             const basePath = nodeData.inputs?.basePath as string
             const docs = nodeData.inputs?.document as LCDocument[]
             const embeddings = nodeData.inputs?.embeddings
@@ -105,6 +105,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
 
             try {
                 await VectorStoreIndex.fromDocuments(llamadocs, { serviceContext, storageContext })
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Singlestore/Singlestore.ts

@@ -2,7 +2,7 @@ import { flatten } from 'lodash'
 import { Embeddings } from '@langchain/core/embeddings'
 import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from '@langchain/community/vectorstores/singlestore'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 
 class SingleStore_VectorStores implements INode {
@@ -118,7 +118,7 @@ class SingleStore_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const user = getCredentialParam('user', credentialData, nodeData)
             const password = getCredentialParam('password', credentialData, nodeData)
@@ -151,6 +151,7 @@ class SingleStore_VectorStores implements INode {
             try {
                 const vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
                 vectorStore.addDocuments.bind(vectorStore)(finalDocs)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Supabase/Supabase.ts

@@ -3,9 +3,10 @@ import { createClient } from '@supabase/supabase-js'
 import { Document } from '@langchain/core/documents'
 import { Embeddings } from '@langchain/core/embeddings'
 import { SupabaseVectorStore, SupabaseLibArgs } from '@langchain/community/vectorstores/supabase'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
+import { index } from '../../../src/indexing'
 
 class Supabase_VectorStores implements INode {
     label: string
@@ -24,7 +25,7 @@ class Supabase_VectorStores implements INode {
     constructor() {
         this.label = 'Supabase'
         this.name = 'supabase'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Supabase'
         this.icon = 'supabase.svg'
         this.category = 'Vector Stores'
@@ -50,6 +51,13 @@ class Supabase_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Supabase Project URL',
                 name: 'supabaseProjUrl',
@@ -99,12 +107,13 @@ class Supabase_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string
             const tableName = nodeData.inputs?.tableName as string
             const queryName = nodeData.inputs?.queryName as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData)
@@ -120,11 +129,32 @@ class Supabase_VectorStores implements INode {
             }
 
             try {
-                await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, {
-                    client,
-                    tableName: tableName,
-                    queryName: queryName
-                })
+                if (recordManager) {
+                    const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, {
+                        client,
+                        tableName: tableName,
+                        queryName: queryName
+                    })
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: tableName + '_' + queryName
+                        }
+                    })
+                    return res
+                } else {
+                    await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, {
+                        client,
+                        tableName: tableName,
+                        queryName: queryName
+                    })
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Vectara/Vectara.ts

@@ -9,7 +9,7 @@ import {
 } from '@langchain/community/vectorstores/vectara'
 import { Document } from '@langchain/core/documents'
 import { Embeddings } from '@langchain/core/embeddings'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 
 class Vectara_VectorStores implements INode {
@@ -144,7 +144,7 @@ class Vectara_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
             const customerId = getCredentialParam('customerID', credentialData, nodeData)
@@ -204,6 +204,7 @@ class Vectara_VectorStores implements INode {
                     const vectorStore = new VectaraStore(vectaraArgs)
                     await vectorStore.addFiles(vectaraFiles)
                 }
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/Weaviate/Weaviate.ts

@@ -1,11 +1,12 @@
 import { flatten } from 'lodash'
 import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client'
-import { WeaviateLibArgs, WeaviateStore } from '@langchain/community/vectorstores/weaviate'
+import { WeaviateLibArgs, WeaviateStore } from '@langchain/weaviate'
 import { Document } from '@langchain/core/documents'
 import { Embeddings } from '@langchain/core/embeddings'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
+import { index } from '../../../src/indexing'
 
 class Weaviate_VectorStores implements INode {
     label: string
@@ -24,7 +25,7 @@ class Weaviate_VectorStores implements INode {
     constructor() {
         this.label = 'Weaviate'
         this.name = 'weaviate'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Weaviate'
         this.icon = 'weaviate.png'
         this.category = 'Vector Stores'
@@ -53,6 +54,13 @@ class Weaviate_VectorStores implements INode {
                 name: 'embeddings',
                 type: 'Embeddings'
             },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
             {
                 label: 'Weaviate Scheme',
                 name: 'weaviateScheme',
@@ -125,7 +133,7 @@ class Weaviate_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const weaviateScheme = nodeData.inputs?.weaviateScheme as string
             const weaviateHost = nodeData.inputs?.weaviateHost as string
             const weaviateIndex = nodeData.inputs?.weaviateIndex as string
@@ -133,6 +141,7 @@ class Weaviate_VectorStores implements INode {
             const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
             const docs = nodeData.inputs?.document as Document[]
             const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
 
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
             const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
@@ -154,6 +163,7 @@ class Weaviate_VectorStores implements INode {
             }
 
             const obj: WeaviateLibArgs = {
+                //@ts-ignore
                 client,
                 indexName: weaviateIndex
             }
@@ -162,7 +172,24 @@ class Weaviate_VectorStores implements INode {
             if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
 
             try {
-                await WeaviateStore.fromDocuments(finalDocs, embeddings, obj)
+                if (recordManager) {
+                    const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj)
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: weaviateTextKey ? weaviateIndex + '_' + weaviateTextKey : weaviateIndex
+                        }
+                    })
+                    return res
+                } else {
+                    await WeaviateStore.fromDocuments(finalDocs, embeddings, obj)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
             } catch (e) {
                 throw new Error(e)
             }
@@ -189,6 +216,7 @@ class Weaviate_VectorStores implements INode {
         const client: WeaviateClient = weaviate.client(clientConfig)
 
         const obj: WeaviateLibArgs = {
+            //@ts-ignore
             client,
             indexName: weaviateIndex
         }

packages/components/nodes/vectorstores/Zep/Zep.ts

@@ -3,7 +3,7 @@ import { IDocument, ZepClient } from '@getzep/zep-js'
 import { ZepVectorStore, IZepConfig } from '@langchain/community/vectorstores/zep'
 import { Embeddings } from '@langchain/core/embeddings'
 import { Document } from '@langchain/core/documents'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
 
@@ -106,7 +106,7 @@ class Zep_VectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const baseURL = nodeData.inputs?.baseURL as string
             const zepCollection = nodeData.inputs?.zepCollection as string
             const dimension = (nodeData.inputs?.dimension as number) ?? 1536
@@ -134,6 +134,7 @@ class Zep_VectorStores implements INode {
 
             try {
                 await ZepVectorStore.fromDocuments(finalDocs, embeddings, zepConfig)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts

@@ -3,7 +3,7 @@ import { IDocument, ZepClient } from '@getzep/zep-cloud'
 import { IZepConfig, ZepVectorStore } from '@getzep/zep-cloud/langchain'
 import { Embeddings } from 'langchain/embeddings/base'
 import { Document } from 'langchain/document'
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
 import { FakeEmbeddings } from 'langchain/embeddings/fake'
@@ -89,7 +89,7 @@ class Zep_CloudVectorStores implements INode {
 
     //@ts-ignore
     vectorStoreMethods = {
-        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
             const zepCollection = nodeData.inputs?.zepCollection as string
             const docs = nodeData.inputs?.document as Document[]
             const credentialData = await getCredentialData(nodeData.credential ?? '', options)
@@ -109,6 +109,7 @@ class Zep_CloudVectorStores implements INode {
             }
             try {
                 await ZepVectorStore.fromDocuments(finalDocs, new FakeEmbeddings(), zepConfig)
+                return { numAdded: finalDocs.length, addedDocs: finalDocs }
             } catch (e) {
                 throw new Error(e)
             }