diff --git a/packages/components/credentials/MySQLApi.credential.ts b/packages/components/credentials/MySQLApi.credential.ts new file mode 100644 index 00000000..68cb3b0f --- /dev/null +++ b/packages/components/credentials/MySQLApi.credential.ts @@ -0,0 +1,31 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class MySQLApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'MySQL API' + this.name = 'MySQLApi' + this.version = 1.0 + this.inputs = [ + { + label: 'User', + name: 'user', + type: 'string', + placeholder: '' + }, + { + label: 'Password', + name: 'password', + type: 'password', + placeholder: '' + } + ] + } +} + +module.exports = { credClass: MySQLApi } diff --git a/packages/components/nodes/recordmanager/MySQLRecordManager/MySQLrecordManager.ts b/packages/components/nodes/recordmanager/MySQLRecordManager/MySQLrecordManager.ts new file mode 100644 index 00000000..be06da0e --- /dev/null +++ b/packages/components/nodes/recordmanager/MySQLRecordManager/MySQLrecordManager.ts @@ -0,0 +1,361 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base' +import { DataSource, QueryRunner } from 'typeorm' + +class MySQLRecordManager_RecordManager implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + badge: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'MySQL Record Manager' + this.name = 'MySQLRecordManager' + this.version = 1.0 + this.type = 'MySQL RecordManager' + this.icon = 'mysql.png' + this.category = 'Record Manager' + this.description = 'Use MySQL to keep track of document writes into the vector databases' + this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(MySQLRecordManager)] + this.badge = 'NEW' + this.inputs = [ + { + label: 'Host', + name: 'host', + type: 'string' + }, + { + label: 'Database', + name: 'database', + type: 'string' + }, + { + label: 'Port', + name: 'port', + type: 'number', + placeholder: '3306', + optional: true + }, + { + label: 'Additional Connection Configuration', + name: 'additionalConfig', + type: 'json', + additionalParams: true, + optional: true + }, + { + label: 'Table Name', + name: 'tableName', + type: 'string', + placeholder: 'upsertion_records', + additionalParams: true, + optional: true + }, + { + label: 'Namespace', + name: 'namespace', + type: 'string', + description: 'If not specified, chatflowid will be used', + additionalParams: true, + optional: true + }, + { + label: 'Cleanup', + name: 'cleanup', + type: 'options', + description: + 'Read more on the difference between different cleanup methods here', + options: [ + { + label: 'None', + name: 'none', + description: 'No clean up of old content' + }, + { + label: 'Incremental', + name: 'incremental', + description: + 'Delete previous versions of the content if content of the source document has changed. Important!! SourceId Key must be specified and document metadata must contains the specified key' + }, + { + label: 'Full', + name: 'full', + description: + 'Same as incremental, but if the source document has been deleted, it will be deleted from vector store as well, incremental mode will not.' + } + ], + additionalParams: true, + default: 'none' + }, + { + label: 'SourceId Key', + name: 'sourceIdKey', + type: 'string', + description: + 'Key used to get the true source of document, to be compared against the record. Document metadata must contains SourceId Key', + default: 'source', + placeholder: 'source', + additionalParams: true, + optional: true + } + ] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['MySQLApi'] + } + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const user = getCredentialParam('user', credentialData, nodeData) + const password = getCredentialParam('password', credentialData, nodeData) + const _tableName = nodeData.inputs?.tableName as string + const tableName = _tableName ? _tableName : 'upsertion_records' + const additionalConfig = nodeData.inputs?.additionalConfig as string + const _namespace = nodeData.inputs?.namespace as string + const namespace = _namespace ? _namespace : options.chatflowid + const cleanup = nodeData.inputs?.cleanup as string + const _sourceIdKey = nodeData.inputs?.sourceIdKey as string + const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source' + + let additionalConfiguration = {} + if (additionalConfig) { + try { + additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig) + } catch (exception) { + throw new Error('Invalid JSON in the Additional Configuration: ' + exception) + } + } + + const mysqlOptions = { + ...additionalConfiguration, + type: 'mysql', + host: nodeData.inputs?.host as string, + port: nodeData.inputs?.port as number, + username: user, + password: password, + database: nodeData.inputs?.database as string + } + + const args = { + mysqlOptions, + tableName: tableName + } + + const recordManager = new MySQLRecordManager(namespace, args) + + ;(recordManager as any).cleanup = cleanup + ;(recordManager as any).sourceIdKey = sourceIdKey + + return recordManager + } +} + +type MySQLRecordManagerOptions = { + mysqlOptions: any + tableName?: string +} + +class MySQLRecordManager implements RecordManagerInterface { + lc_namespace = ['langchain', 'recordmanagers', 'mysql'] + + datasource: DataSource + + queryRunner: QueryRunner + + tableName: string + + namespace: string + + constructor(namespace: string, config: MySQLRecordManagerOptions) { + const { mysqlOptions, tableName } = config + this.namespace = namespace + this.tableName = tableName || 'upsertion_records' + this.datasource = new DataSource(mysqlOptions) + } + + async createSchema(): Promise { + try { + const appDataSource = await this.datasource.initialize() + + this.queryRunner = appDataSource.createQueryRunner() + + await this.queryRunner.manager.query(`create table if not exists \`${this.tableName}\` ( + \`uuid\` varchar(36) primary key default (UUID()), + \`key\` varchar(36) not null, + \`namespace\` varchar(36) not null, + \`updated_at\` DOUBLE precision not null, + \`group_id\` varchar(36), + unique key \`unique_key_namespace\` (\`key\`, +\`namespace\`));`) + const columns = [`updated_at`, `key`, `namespace`, `group_id`] + for (const column of columns) { + // MySQL does not support 'IF NOT EXISTS' function for Index + const Check = await this.queryRunner.manager.query( + `SELECT COUNT(1) IndexIsThere FROM INFORMATION_SCHEMA.STATISTICS + WHERE table_schema=DATABASE() AND table_name='${this.tableName}' AND index_name='${column}_index';` + ) + if (Check[0].IndexIsThere === 0) + await this.queryRunner.manager.query(`CREATE INDEX \`${column}_index\` + ON \`${this.tableName}\` (\`${column}\`);`) + } + } catch (e: any) { + // This error indicates that the table already exists + // Due to asynchronous nature of the code, it is possible that + // the table is created between the time we check if it exists + // and the time we try to create it. It can be safely ignored. + if ('code' in e && e.code === '23505') { + return + } + throw e + } + } + + async getTime(): Promise { + try { + const res = await this.queryRunner.manager.query(`SELECT UNIX_TIMESTAMP(NOW()) AS epoch`) + return Number.parseFloat(res[0].epoch) + } catch (error) { + console.error('Error getting time in MySQLRecordManager:') + throw error + } + } + + async update(keys: string[], updateOptions?: UpdateOptions): Promise { + if (keys.length === 0) { + return + } + + const updatedAt = await this.getTime() + const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {} + + if (timeAtLeast && updatedAt < timeAtLeast) { + throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`) + } + + const groupIds = _groupIds ?? keys.map(() => null) + + if (groupIds.length !== keys.length) { + throw new Error(`Number of keys (${keys.length}) does not match number of group_ids (${groupIds.length})`) + } + + const recordsToUpsert = keys.map((key, i) => [ + key, + this.namespace, + updatedAt, + groupIds[i] ?? null // Ensure groupIds[i] is null if undefined + ]) + + const query = ` + INSERT INTO \`${this.tableName}\` (\`key\`, \`namespace\`, \`updated_at\`, \`group_id\`) + VALUES (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE updated_at = updated_at;` + + // To handle multiple files upsert + for (const record of recordsToUpsert) { + // Consider using a transaction for batch operations + await this.queryRunner.manager.query(query, record.flat()) + } + } + + async exists(keys: string[]): Promise { + if (keys.length === 0) { + return [] + } + + // Prepare the placeholders and the query + const placeholders = keys.map(() => `?`).join(', ') + const query = ` + SELECT \`key\` + FROM \`${this.tableName}\` + WHERE \`namespace\` = ? AND \`key\` IN (${placeholders})` + + // Initialize an array to fill with the existence checks + const existsArray = new Array(keys.length).fill(false) + + try { + // Execute the query + const rows = await this.queryRunner.manager.query(query, [this.namespace, ...keys.flat()]) + // Create a set of existing keys for faster lookup + const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key)) + // Map the input keys to booleans indicating if they exist + keys.forEach((key, index) => { + existsArray[index] = existingKeysSet.has(key) + }) + return existsArray + } catch (error) { + console.error('Error checking existence of keys') + throw error // Allow the caller to handle the error + } + } + + async listKeys(options?: ListKeyOptions): Promise { + try { + const { before, after, limit, groupIds } = options ?? {} + let query = `SELECT \`key\` FROM \`${this.tableName}\` WHERE \`namespace\` = ?` + const values: (string | number | string[])[] = [this.namespace] + + if (before) { + query += ` AND \`updated_at\` < ?` + values.push(before) + } + + if (after) { + query += ` AND \`updated_at\` > ?` + values.push(after) + } + + if (limit) { + query += ` LIMIT ?` + values.push(limit) + } + + if (groupIds && Array.isArray(groupIds)) { + query += ` AND \`group_id\` IN (${groupIds + .filter((gid) => gid !== null) + .map(() => '?') + .join(', ')})` + values.push(...groupIds.filter((gid): gid is string => gid !== null)) + } + + query += ';' + + // Directly using try/catch with async/await for cleaner flow + const result = await this.queryRunner.manager.query(query, values) + return result.map((row: { key: string }) => row.key) + } catch (error) { + console.error('MySQLRecordManager listKeys Error: ') + throw error // Re-throw the error to be handled by the caller + } + } + + async deleteKeys(keys: string[]): Promise { + if (keys.length === 0) { + return + } + + const placeholders = keys.map(() => '?').join(', ') + const query = `DELETE FROM \`${this.tableName}\` WHERE \`namespace\` = ? AND \`key\` IN (${placeholders});` + const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v)) + + // Directly using try/catch with async/await for cleaner flow + try { + await this.queryRunner.manager.query(query, values) + } catch (error) { + console.error('Error deleting keys') + throw error // Re-throw the error to be handled by the caller + } + } +} + +module.exports = { nodeClass: MySQLRecordManager_RecordManager } diff --git a/packages/components/nodes/recordmanager/MySQLRecordManager/mysql.png b/packages/components/nodes/recordmanager/MySQLRecordManager/mysql.png new file mode 100644 index 00000000..11c2d8b2 Binary files /dev/null and b/packages/components/nodes/recordmanager/MySQLRecordManager/mysql.png differ diff --git a/packages/components/nodes/recordmanager/PostgresRecordManager/PostgresRecordManager.ts b/packages/components/nodes/recordmanager/PostgresRecordManager/PostgresRecordManager.ts new file mode 100644 index 00000000..5d2fafb3 --- /dev/null +++ b/packages/components/nodes/recordmanager/PostgresRecordManager/PostgresRecordManager.ts @@ -0,0 +1,332 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base' +import { DataSource, QueryRunner } from 'typeorm' + +class PostgresRecordManager_RecordManager implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + badge: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'Postgres Record Manager' + this.name = 'postgresRecordManager' + this.version = 1.0 + this.type = 'Postgres RecordManager' + this.icon = 'postgres.svg' + this.category = 'Record Manager' + this.description = 'Use Postgres to keep track of document writes into the vector databases' + this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(PostgresRecordManager)] + this.badge = 'NEW' + this.inputs = [ + { + label: 'Host', + name: 'host', + type: 'string' + }, + { + label: 'Database', + name: 'database', + type: 'string' + }, + { + label: 'Port', + name: 'port', + type: 'number', + placeholder: '5432', + optional: true + }, + { + label: 'Additional Connection Configuration', + name: 'additionalConfig', + type: 'json', + additionalParams: true, + optional: true + }, + { + label: 'Table Name', + name: 'tableName', + type: 'string', + placeholder: 'upsertion_records', + additionalParams: true, + optional: true + }, + { + label: 'Namespace', + name: 'namespace', + type: 'string', + description: 'If not specified, chatflowid will be used', + additionalParams: true, + optional: true + }, + { + label: 'Cleanup', + name: 'cleanup', + type: 'options', + description: + 'Read more on the difference between different cleanup methods here', + options: [ + { + label: 'None', + name: 'none', + description: 'No clean up of old content' + }, + { + label: 'Incremental', + name: 'incremental', + description: + 'Delete previous versions of the content if content of the source document has changed. Important!! SourceId Key must be specified and document metadata must contains the specified key' + }, + { + label: 'Full', + name: 'full', + description: + 'Same as incremental, but if the source document has been deleted, it will be deleted from vector store as well, incremental mode will not.' + } + ], + additionalParams: true, + default: 'none' + }, + { + label: 'SourceId Key', + name: 'sourceIdKey', + type: 'string', + description: + 'Key used to get the true source of document, to be compared against the record. Document metadata must contains SourceId Key', + default: 'source', + placeholder: 'source', + additionalParams: true, + optional: true + } + ] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['PostgresApi'] + } + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const user = getCredentialParam('user', credentialData, nodeData) + const password = getCredentialParam('password', credentialData, nodeData) + const _tableName = nodeData.inputs?.tableName as string + const tableName = _tableName ? _tableName : 'upsertion_records' + const additionalConfig = nodeData.inputs?.additionalConfig as string + const _namespace = nodeData.inputs?.namespace as string + const namespace = _namespace ? _namespace : options.chatflowid + const cleanup = nodeData.inputs?.cleanup as string + const _sourceIdKey = nodeData.inputs?.sourceIdKey as string + const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source' + + let additionalConfiguration = {} + if (additionalConfig) { + try { + additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig) + } catch (exception) { + throw new Error('Invalid JSON in the Additional Configuration: ' + exception) + } + } + + const postgresConnectionOptions = { + ...additionalConfiguration, + type: 'postgres', + host: nodeData.inputs?.host as string, + port: nodeData.inputs?.port as number, + username: user, + password: password, + database: nodeData.inputs?.database as string + } + + const args = { + postgresConnectionOptions: postgresConnectionOptions, + tableName: tableName + } + + const recordManager = new PostgresRecordManager(namespace, args) + + ;(recordManager as any).cleanup = cleanup + ;(recordManager as any).sourceIdKey = sourceIdKey + + return recordManager + } +} + +type PostgresRecordManagerOptions = { + postgresConnectionOptions: any + tableName?: string +} + +class PostgresRecordManager implements RecordManagerInterface { + lc_namespace = ['langchain', 'recordmanagers', 'postgres'] + + datasource: DataSource + + queryRunner: QueryRunner + + tableName: string + + namespace: string + + constructor(namespace: string, config: PostgresRecordManagerOptions) { + const { postgresConnectionOptions, tableName } = config + this.namespace = namespace + this.datasource = new DataSource(postgresConnectionOptions) + this.tableName = tableName || 'upsertion_records' + } + + async createSchema(): Promise { + try { + const appDataSource = await this.datasource.initialize() + + this.queryRunner = appDataSource.createQueryRunner() + + await this.queryRunner.manager.query(` + CREATE TABLE IF NOT EXISTS "${this.tableName}" ( + uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(), + key TEXT NOT NULL, + namespace TEXT NOT NULL, + updated_at Double PRECISION NOT NULL, + group_id TEXT, + UNIQUE (key, namespace) + ); + CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at); + CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key); + CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace); + CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`) + } catch (e: any) { + // This error indicates that the table already exists + // Due to asynchronous nature of the code, it is possible that + // the table is created between the time we check if it exists + // and the time we try to create it. It can be safely ignored. + if ('code' in e && e.code === '23505') { + return + } + throw e + } + } + + async getTime(): Promise { + const res = await this.queryRunner.manager.query('SELECT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)') + return Number.parseFloat(res[0].extract) + } + + /** + * Generates the SQL placeholders for a specific row at the provided index. + * + * @param index - The index of the row for which placeholders need to be generated. + * @param numOfColumns - The number of columns we are inserting data into. + * @returns The SQL placeholders for the row values. + */ + private generatePlaceholderForRowAt(index: number, numOfColumns: number): string { + const placeholders = [] + for (let i = 0; i < numOfColumns; i += 1) { + placeholders.push(`$${index * numOfColumns + i + 1}`) + } + return `(${placeholders.join(', ')})` + } + + async update(keys: string[], updateOptions?: UpdateOptions): Promise { + if (keys.length === 0) { + return + } + + const updatedAt = await this.getTime() + const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {} + + if (timeAtLeast && updatedAt < timeAtLeast) { + throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`) + } + + const groupIds = _groupIds ?? keys.map(() => null) + + if (groupIds.length !== keys.length) { + throw new Error(`Number of keys (${keys.length}) does not match number of group_ids ${groupIds.length})`) + } + + const recordsToUpsert = keys.map((key, i) => [key, this.namespace, updatedAt, groupIds[i]]) + + const valuesPlaceholders = recordsToUpsert.map((_, j) => this.generatePlaceholderForRowAt(j, recordsToUpsert[0].length)).join(', ') + + const query = `INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) VALUES ${valuesPlaceholders} ON CONFLICT (key, namespace) DO UPDATE SET updated_at = EXCLUDED.updated_at;` + await this.queryRunner.manager.query(query, recordsToUpsert.flat()) + } + + async exists(keys: string[]): Promise { + if (keys.length === 0) { + return [] + } + + const startIndex = 2 + const arrayPlaceholders = keys.map((_, i) => `$${i + startIndex}`).join(', ') + + const query = ` + SELECT k, (key is not null) ex from unnest(ARRAY[${arrayPlaceholders}]) k left join "${this.tableName}" on k=key and namespace = $1; + ` + const res = await this.queryRunner.manager.query(query, [this.namespace, ...keys.flat()]) + return res.map((row: { ex: boolean }) => row.ex) + } + + async listKeys(options?: ListKeyOptions): Promise { + const { before, after, limit, groupIds } = options ?? {} + let query = `SELECT key FROM "${this.tableName}" WHERE namespace = $1` + const values: (string | number | (string | null)[])[] = [this.namespace] + + let index = 2 + if (before) { + values.push(before) + query += ` AND updated_at < $${index}` + index += 1 + } + + if (after) { + values.push(after) + query += ` AND updated_at > $${index}` + index += 1 + } + + if (limit) { + values.push(limit) + query += ` LIMIT $${index}` + index += 1 + } + + if (groupIds) { + values.push(groupIds) + query += ` AND group_id = ANY($${index})` + index += 1 + } + + query += ';' + const res = await this.queryRunner.manager.query(query, values) + return res.map((row: { key: string }) => row.key) + } + + async deleteKeys(keys: string[]): Promise { + if (keys.length === 0) { + return + } + + const query = `DELETE FROM "${this.tableName}" WHERE namespace = $1 AND key = ANY($2);` + await this.queryRunner.manager.query(query, [this.namespace, keys]) + } + + /** + * Terminates the connection pool. + * @returns {Promise} + */ + async end(): Promise { + if (this.datasource && this.datasource.isInitialized) await this.datasource.destroy() + } +} + +module.exports = { nodeClass: PostgresRecordManager_RecordManager } diff --git a/packages/components/nodes/recordmanager/PostgresRecordManager/postgres.svg b/packages/components/nodes/recordmanager/PostgresRecordManager/postgres.svg new file mode 100644 index 00000000..f631e7a8 --- /dev/null +++ b/packages/components/nodes/recordmanager/PostgresRecordManager/postgres.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/components/nodes/recordmanager/SQLiteRecordManager/SQLiteRecordManager.ts b/packages/components/nodes/recordmanager/SQLiteRecordManager/SQLiteRecordManager.ts new file mode 100644 index 00000000..3bd95d27 --- /dev/null +++ b/packages/components/nodes/recordmanager/SQLiteRecordManager/SQLiteRecordManager.ts @@ -0,0 +1,332 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses } from '../../../src/utils' +import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base' +import { DataSource, QueryRunner } from 'typeorm' +import path from 'path' + +class SQLiteRecordManager_RecordManager implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + badge: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'SQLite Record Manager' + this.name = 'SQLiteRecordManager' + this.version = 1.0 + this.type = 'SQLite RecordManager' + this.icon = 'sqlite.png' + this.category = 'Record Manager' + this.description = 'Use SQLite to keep track of document writes into the vector databases' + this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(SQLiteRecordManager)] + this.badge = 'NEW' + this.inputs = [ + { + label: 'Database File Path', + name: 'databaseFilePath', + type: 'string', + placeholder: 'C:\\Users\\User\\.flowise\\database.sqlite' + }, + { + label: 'Additional Connection Configuration', + name: 'additionalConfig', + type: 'json', + additionalParams: true, + optional: true + }, + { + label: 'Table Name', + name: 'tableName', + type: 'string', + placeholder: 'upsertion_records', + additionalParams: true, + optional: true + }, + { + label: 'Namespace', + name: 'namespace', + type: 'string', + description: 'If not specified, chatflowid will be used', + additionalParams: true, + optional: true + }, + { + label: 'Cleanup', + name: 'cleanup', + type: 'options', + description: + 'Read more on the difference between different cleanup methods here', + options: [ + { + label: 'None', + name: 'none', + description: 'No clean up of old content' + }, + { + label: 'Incremental', + name: 'incremental', + description: + 'Delete previous versions of the content if content of the source document has changed. Important!! SourceId Key must be specified and document metadata must contains the specified key' + }, + { + label: 'Full', + name: 'full', + description: + 'Same as incremental, but if the source document has been deleted, it will be deleted from vector store as well, incremental mode will not.' + } + ], + additionalParams: true, + default: 'none' + }, + { + label: 'SourceId Key', + name: 'sourceIdKey', + type: 'string', + description: + 'Key used to get the true source of document, to be compared against the record. Document metadata must contains SourceId Key', + default: 'source', + placeholder: 'source', + additionalParams: true, + optional: true + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const _tableName = nodeData.inputs?.tableName as string + const tableName = _tableName ? _tableName : 'upsertion_records' + const additionalConfig = nodeData.inputs?.additionalConfig as string + const _namespace = nodeData.inputs?.namespace as string + const namespace = _namespace ? _namespace : options.chatflowid + const cleanup = nodeData.inputs?.cleanup as string + const _sourceIdKey = nodeData.inputs?.sourceIdKey as string + const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source' + const databaseFilePath = nodeData.inputs?.databaseFilePath as string + + let additionalConfiguration = {} + if (additionalConfig) { + try { + additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig) + } catch (exception) { + throw new Error('Invalid JSON in the Additional Configuration: ' + exception) + } + } + + const sqliteOptions = { + ...additionalConfiguration, + type: 'sqlite', + database: path.resolve(databaseFilePath) + } + + const args = { + sqliteOptions, + tableName: tableName + } + + const recordManager = new SQLiteRecordManager(namespace, args) + + ;(recordManager as any).cleanup = cleanup + ;(recordManager as any).sourceIdKey = sourceIdKey + + return recordManager + } +} + +type SQLiteRecordManagerOptions = { + sqliteOptions: any + tableName?: string +} + +class SQLiteRecordManager implements RecordManagerInterface { + lc_namespace = ['langchain', 'recordmanagers', 'sqlite'] + + datasource: DataSource + + queryRunner: QueryRunner + + tableName: string + + namespace: string + + constructor(namespace: string, config: SQLiteRecordManagerOptions) { + const { sqliteOptions, tableName } = config + this.namespace = namespace + this.tableName = tableName || 'upsertion_records' + this.datasource = new DataSource(sqliteOptions) + } + + async createSchema(): Promise { + try { + const appDataSource = await this.datasource.initialize() + + this.queryRunner = appDataSource.createQueryRunner() + + await this.queryRunner.manager.query(` +CREATE TABLE IF NOT EXISTS "${this.tableName}" ( + uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), + key TEXT NOT NULL, + namespace TEXT NOT NULL, + updated_at REAL NOT NULL, + group_id TEXT, + UNIQUE (key, namespace) +); +CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at); +CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key); +CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace); +CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`) + } catch (e: any) { + // This error indicates that the table already exists + // Due to asynchronous nature of the code, it is possible that + // the table is created between the time we check if it exists + // and the time we try to create it. It can be safely ignored. + if ('code' in e && e.code === '23505') { + return + } + throw e + } + } + + async getTime(): Promise { + try { + const res = await this.queryRunner.manager.query(`SELECT strftime('%s', 'now') AS epoch`) + return Number.parseFloat(res[0].epoch) + } catch (error) { + console.error('Error getting time in SQLiteRecordManager:') + throw error + } + } + + async update(keys: string[], updateOptions?: UpdateOptions): Promise { + if (keys.length === 0) { + return + } + + const updatedAt = await this.getTime() + const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {} + + if (timeAtLeast && updatedAt < timeAtLeast) { + throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`) + } + + const groupIds = _groupIds ?? keys.map(() => null) + + if (groupIds.length !== keys.length) { + throw new Error(`Number of keys (${keys.length}) does not match number of group_ids (${groupIds.length})`) + } + + const recordsToUpsert = keys.map((key, i) => [ + key, + this.namespace, + updatedAt, + groupIds[i] ?? null // Ensure groupIds[i] is null if undefined + ]) + + const query = ` + INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) + VALUES (?, ?, ?, ?) + ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at` + + // To handle multiple files upsert + for (const record of recordsToUpsert) { + // Consider using a transaction for batch operations + await this.queryRunner.manager.query(query, record.flat()) + } + } + + async exists(keys: string[]): Promise { + if (keys.length === 0) { + return [] + } + + // Prepare the placeholders and the query + const placeholders = keys.map(() => `?`).join(', ') + const sql = ` + SELECT key + FROM "${this.tableName}" + WHERE namespace = ? AND key IN (${placeholders})` + + // Initialize an array to fill with the existence checks + const existsArray = new Array(keys.length).fill(false) + + try { + // Execute the query + const rows = await this.queryRunner.manager.query(sql, [this.namespace, ...keys.flat()]) + // Create a set of existing keys for faster lookup + const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key)) + // Map the input keys to booleans indicating if they exist + keys.forEach((key, index) => { + existsArray[index] = existingKeysSet.has(key) + }) + return existsArray + } catch (error) { + console.error('Error checking existence of keys') + throw error // Allow the caller to handle the error + } + } + + async listKeys(options?: ListKeyOptions): Promise { + const { before, after, limit, groupIds } = options ?? {} + let query = `SELECT key FROM "${this.tableName}" WHERE namespace = ?` + const values: (string | number | string[])[] = [this.namespace] + + if (before) { + query += ` AND updated_at < ?` + values.push(before) + } + + if (after) { + query += ` AND updated_at > ?` + values.push(after) + } + + if (limit) { + query += ` LIMIT ?` + values.push(limit) + } + + if (groupIds && Array.isArray(groupIds)) { + query += ` AND group_id IN (${groupIds + .filter((gid) => gid !== null) + .map(() => '?') + .join(', ')})` + values.push(...groupIds.filter((gid): gid is string => gid !== null)) + } + + query += ';' + + // Directly using try/catch with async/await for cleaner flow + try { + const result = await this.queryRunner.manager.query(query, values) + return result.map((row: { key: string }) => row.key) + } catch (error) { + console.error('Error listing keys.') + throw error // Re-throw the error to be handled by the caller + } + } + + async deleteKeys(keys: string[]): Promise { + if (keys.length === 0) { + return + } + + const placeholders = keys.map(() => '?').join(', ') + const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${placeholders});` + const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v)) + + // Directly using try/catch with async/await for cleaner flow + try { + await this.queryRunner.manager.query(query, values) + } catch (error) { + console.error('Error deleting keys') + throw error // Re-throw the error to be handled by the caller + } + } +} + +module.exports = { nodeClass: SQLiteRecordManager_RecordManager } diff --git a/packages/components/nodes/recordmanager/SQLiteRecordManager/sqlite.png b/packages/components/nodes/recordmanager/SQLiteRecordManager/sqlite.png new file mode 100644 index 00000000..9f35f2bd Binary files /dev/null and b/packages/components/nodes/recordmanager/SQLiteRecordManager/sqlite.png differ diff --git a/packages/components/nodes/vectorstores/Astra/Astra.ts b/packages/components/nodes/vectorstores/Astra/Astra.ts index f6a583e0..961722fa 100644 --- a/packages/components/nodes/vectorstores/Astra/Astra.ts +++ b/packages/components/nodes/vectorstores/Astra/Astra.ts @@ -2,7 +2,7 @@ import { flatten } from 'lodash' import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' import { AstraDBVectorStore, AstraLibArgs } from '@langchain/community/vectorstores/astradb' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData } from '../../../src/utils' import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' @@ -101,7 +101,7 @@ class Astra_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings const vectorDimension = nodeData.inputs?.vectorDimension as number @@ -142,6 +142,7 @@ class Astra_VectorStores implements INode { try { await AstraDBVectorStore.fromDocuments(finalDocs, embeddings, astraConfig) + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma.ts b/packages/components/nodes/vectorstores/Chroma/Chroma.ts index 70ab2d22..a8722e61 100644 --- a/packages/components/nodes/vectorstores/Chroma/Chroma.ts +++ b/packages/components/nodes/vectorstores/Chroma/Chroma.ts @@ -2,9 +2,10 @@ import { flatten } from 'lodash' import { Chroma } from '@langchain/community/vectorstores/chroma' import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChromaExtended } from './core' +import { index } from '../../../src/indexing' class Chroma_VectorStores implements INode { label: string @@ -23,7 +24,7 @@ class Chroma_VectorStores implements INode { constructor() { this.label = 'Chroma' this.name = 'chroma' - this.version = 1.0 + this.version = 2.0 this.type = 'Chroma' this.icon = 'chroma.svg' this.category = 'Vector Stores' @@ -51,6 +52,13 @@ class Chroma_VectorStores implements INode { name: 'embeddings', type: 'Embeddings' }, + { + label: 'Record Manager', + name: 'recordManager', + type: 'RecordManager', + description: 'Keep track of the record to prevent duplication', + optional: true + }, { label: 'Collection Name', name: 'collectionName', @@ -95,11 +103,12 @@ class Chroma_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const collectionName = nodeData.inputs?.collectionName as string const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings const chromaURL = nodeData.inputs?.chromaURL as string + const recordManager = nodeData.inputs?.recordManager const credentialData = await getCredentialData(nodeData.credential ?? '', options) const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData) @@ -121,7 +130,24 @@ class Chroma_VectorStores implements INode { if (chromaApiKey) obj.chromaApiKey = chromaApiKey try { - await ChromaExtended.fromDocuments(finalDocs, embeddings, obj) + if (recordManager) { + const vectorStore = await ChromaExtended.fromExistingCollection(embeddings, obj) + await recordManager.createSchema() + const res = await index({ + docsSource: finalDocs, + recordManager, + vectorStore, + options: { + cleanup: recordManager?.cleanup, + sourceIdKey: recordManager?.sourceIdKey ?? 'source', + vectorStoreName: collectionName + } + }) + return res + } else { + await ChromaExtended.fromDocuments(finalDocs, embeddings, obj) + return { numAdded: finalDocs.length, addedDocs: finalDocs } + } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts index cfc721b8..a5069739 100644 --- a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts @@ -3,8 +3,9 @@ import { Client, ClientOptions } from '@elastic/elasticsearch' import { Document } from '@langchain/core/documents' import { Embeddings } from '@langchain/core/embeddings' import { ElasticClientArgs, ElasticVectorSearch } from '@langchain/community/vectorstores/elasticsearch' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { index } from '../../../src/indexing' class Elasticsearch_VectorStores implements INode { label: string @@ -23,7 +24,7 @@ class Elasticsearch_VectorStores implements INode { constructor() { this.label = 'Elasticsearch' this.name = 'elasticsearch' - this.version = 1.0 + this.version = 2.0 this.description = 'Upsert embedded data and perform similarity search upon query using Elasticsearch, a distributed search and analytics engine' this.type = 'Elasticsearch' @@ -50,6 +51,13 @@ class Elasticsearch_VectorStores implements INode { name: 'embeddings', type: 'Embeddings' }, + { + label: 'Record Manager', + name: 'recordManager', + type: 'RecordManager', + description: 'Keep track of the record to prevent duplication', + optional: true + }, { label: 'Index Name', name: 'indexName', @@ -105,13 +113,14 @@ class Elasticsearch_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const endPoint = getCredentialParam('endpoint', credentialData, nodeData) const cloudId = getCredentialParam('cloudId', credentialData, nodeData) const indexName = nodeData.inputs?.indexName as string const embeddings = nodeData.inputs?.embeddings as Embeddings const similarityMeasure = nodeData.inputs?.similarityMeasure as string + const recordManager = nodeData.inputs?.recordManager const docs = nodeData.inputs?.document as Document[] const flattenDocs = docs && docs.length ? flatten(docs) : [] @@ -134,7 +143,24 @@ class Elasticsearch_VectorStores implements INode { const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs) try { - await vectorStore.addDocuments(finalDocs) + if (recordManager) { + const vectorStore = await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs) + await recordManager.createSchema() + const res = await index({ + docsSource: finalDocs, + recordManager, + vectorStore, + options: { + cleanup: recordManager?.cleanup, + sourceIdKey: recordManager?.sourceIdKey ?? 'source', + vectorStoreName: indexName + } + }) + return res + } else { + await vectorStore.addDocuments(finalDocs) + return { numAdded: finalDocs.length, addedDocs: finalDocs } + } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Faiss/Faiss.ts b/packages/components/nodes/vectorstores/Faiss/Faiss.ts index 8a306ed9..774e049e 100644 --- a/packages/components/nodes/vectorstores/Faiss/Faiss.ts +++ b/packages/components/nodes/vectorstores/Faiss/Faiss.ts @@ -2,7 +2,7 @@ import { flatten } from 'lodash' import { Document } from '@langchain/core/documents' import { FaissStore } from '@langchain/community/vectorstores/faiss' import { Embeddings } from '@langchain/core/embeddings' -import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' class Faiss_VectorStores implements INode { @@ -74,7 +74,7 @@ class Faiss_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData): Promise { + async upsert(nodeData: INodeData): Promise> { const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings const basePath = nodeData.inputs?.basePath as string @@ -95,6 +95,8 @@ class Faiss_VectorStores implements INode { vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number) => { return await similaritySearchVectorWithScore(query, k, vectorStore) } + + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts b/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts index aff7f637..8c54fe27 100644 --- a/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts +++ b/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts @@ -2,7 +2,7 @@ import { flatten } from 'lodash' import { MemoryVectorStore } from 'langchain/vectorstores/memory' import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' -import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' class InMemoryVectorStore_VectorStores implements INode { @@ -64,7 +64,7 @@ class InMemoryVectorStore_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData): Promise { + async upsert(nodeData: INodeData): Promise> { const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings @@ -78,6 +78,7 @@ class InMemoryVectorStore_VectorStores implements INode { try { await MemoryVectorStore.fromDocuments(finalDocs, embeddings) + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Milvus/Milvus.ts b/packages/components/nodes/vectorstores/Milvus/Milvus.ts index d6cfaea1..c0818df4 100644 --- a/packages/components/nodes/vectorstores/Milvus/Milvus.ts +++ b/packages/components/nodes/vectorstores/Milvus/Milvus.ts @@ -3,7 +3,7 @@ import { DataType, ErrorCode, MetricType, IndexType } from '@zilliz/milvus2-sdk- import { Document } from '@langchain/core/documents' import { MilvusLibArgs, Milvus } from '@langchain/community/vectorstores/milvus' import { Embeddings } from '@langchain/core/embeddings' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' interface InsertRow { @@ -109,7 +109,7 @@ class Milvus_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { // server setup const address = nodeData.inputs?.milvusServerUrl as string const collectionName = nodeData.inputs?.milvusCollection as string @@ -147,6 +147,8 @@ class Milvus_VectorStores implements INode { vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: string) => { return await similaritySearchVectorWithScore(query, k, vectorStore, undefined, filter) } + + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/MongoDBAtlas/MongoDBAtlas.ts b/packages/components/nodes/vectorstores/MongoDBAtlas/MongoDBAtlas.ts index e55b84c7..3444d372 100644 --- a/packages/components/nodes/vectorstores/MongoDBAtlas/MongoDBAtlas.ts +++ b/packages/components/nodes/vectorstores/MongoDBAtlas/MongoDBAtlas.ts @@ -3,7 +3,7 @@ import { MongoClient } from 'mongodb' import { MongoDBAtlasVectorSearch } from '@langchain/mongodb' import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' @@ -113,7 +113,7 @@ class MongoDBAtlas_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const databaseName = nodeData.inputs?.databaseName as string const collectionName = nodeData.inputs?.collectionName as string @@ -149,6 +149,7 @@ class MongoDBAtlas_VectorStores implements INode { embeddingKey }) await mongoDBAtlasVectorSearch.addDocuments(finalDocs) + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/OpenSearch/OpenSearch.ts b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch.ts index b67c8e4f..c60b4de5 100644 --- a/packages/components/nodes/vectorstores/OpenSearch/OpenSearch.ts +++ b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch.ts @@ -3,7 +3,7 @@ import { Client } from '@opensearch-project/opensearch' import { Document } from '@langchain/core/documents' import { OpenSearchVectorStore } from '@langchain/community/vectorstores/opensearch' import { Embeddings } from '@langchain/core/embeddings' -import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' class OpenSearch_VectorStores implements INode { @@ -79,7 +79,7 @@ class OpenSearch_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData): Promise { + async upsert(nodeData: INodeData): Promise> { const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings const opensearchURL = nodeData.inputs?.opensearchURL as string @@ -102,6 +102,7 @@ class OpenSearch_VectorStores implements INode { client, indexName: indexName }) + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts index 3ef8a8e8..5c4fd084 100644 --- a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts +++ b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts @@ -3,9 +3,10 @@ import { Pinecone } from '@pinecone-database/pinecone' import { PineconeStoreParams, PineconeStore } from '@langchain/pinecone' import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' +import { index } from '../../../src/indexing' class Pinecone_VectorStores implements INode { label: string @@ -24,7 +25,7 @@ class Pinecone_VectorStores implements INode { constructor() { this.label = 'Pinecone' this.name = 'pinecone' - this.version = 2.0 + this.version = 3.0 this.type = 'Pinecone' this.icon = 'pinecone.svg' this.category = 'Vector Stores' @@ -50,6 +51,13 @@ class Pinecone_VectorStores implements INode { name: 'embeddings', type: 'Embeddings' }, + { + label: 'Record Manager', + name: 'recordManager', + type: 'RecordManager', + description: 'Keep track of the record to prevent duplication', + optional: true + }, { label: 'Pinecone Index', name: 'pineconeIndex', @@ -97,11 +105,12 @@ class Pinecone_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { - const index = nodeData.inputs?.pineconeIndex as string + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { + const _index = nodeData.inputs?.pineconeIndex as string const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings + const recordManager = nodeData.inputs?.recordManager const credentialData = await getCredentialData(nodeData.credential ?? '', options) const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData) @@ -110,7 +119,7 @@ class Pinecone_VectorStores implements INode { apiKey: pineconeApiKey }) - const pineconeIndex = client.Index(index) + const pineconeIndex = client.Index(_index) const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] @@ -127,7 +136,25 @@ class Pinecone_VectorStores implements INode { if (pineconeNamespace) obj.namespace = pineconeNamespace try { - await PineconeStore.fromDocuments(finalDocs, embeddings, obj) + if (recordManager) { + const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj) + await recordManager.createSchema() + const res = await index({ + docsSource: finalDocs, + recordManager, + vectorStore, + options: { + cleanup: recordManager?.cleanup, + sourceIdKey: recordManager?.sourceIdKey ?? 'source', + vectorStoreName: pineconeNamespace + } + }) + + return res + } else { + await PineconeStore.fromDocuments(finalDocs, embeddings, obj) + return { numAdded: finalDocs.length, addedDocs: finalDocs } + } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Pinecone/Pinecone_LlamaIndex.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone_LlamaIndex.ts index c0b2e5c1..4045012e 100644 --- a/packages/components/nodes/vectorstores/Pinecone/Pinecone_LlamaIndex.ts +++ b/packages/components/nodes/vectorstores/Pinecone/Pinecone_LlamaIndex.ts @@ -13,7 +13,7 @@ import { import { FetchResponse, Index, Pinecone, ScoredPineconeRecord } from '@pinecone-database/pinecone' import { flatten } from 'lodash' import { Document as LCDocument } from 'langchain/document' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { flattenObject, getCredentialData, getCredentialParam } from '../../../src/utils' class PineconeLlamaIndex_VectorStores implements INode { @@ -110,7 +110,7 @@ class PineconeLlamaIndex_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const indexName = nodeData.inputs?.pineconeIndex as string const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string const docs = nodeData.inputs?.document as LCDocument[] @@ -144,6 +144,7 @@ class PineconeLlamaIndex_VectorStores implements INode { try { await VectorStoreIndex.fromDocuments(llamadocs, { serviceContext, storageContext }) + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Postgres/Postgres.ts b/packages/components/nodes/vectorstores/Postgres/Postgres.ts index 67a12a14..090b550d 100644 --- a/packages/components/nodes/vectorstores/Postgres/Postgres.ts +++ b/packages/components/nodes/vectorstores/Postgres/Postgres.ts @@ -4,8 +4,9 @@ import { DataSourceOptions } from 'typeorm' import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' import { TypeORMVectorStore, TypeORMVectorStoreDocument } from '@langchain/community/vectorstores/typeorm' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { index } from '../../../src/indexing' class Postgres_VectorStores implements INode { label: string @@ -24,7 +25,7 @@ class Postgres_VectorStores implements INode { constructor() { this.label = 'Postgres' this.name = 'postgres' - this.version = 3.0 + this.version = 4.0 this.type = 'Postgres' this.icon = 'postgres.svg' this.category = 'Vector Stores' @@ -50,6 +51,13 @@ class Postgres_VectorStores implements INode { name: 'embeddings', type: 'Embeddings' }, + { + label: 'Record Manager', + name: 'recordManager', + type: 'RecordManager', + description: 'Keep track of the record to prevent duplication', + optional: true + }, { label: 'Host', name: 'host', @@ -108,7 +116,7 @@ class Postgres_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const user = getCredentialParam('user', credentialData, nodeData) const password = getCredentialParam('password', credentialData, nodeData) @@ -117,6 +125,7 @@ class Postgres_VectorStores implements INode { const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings const additionalConfig = nodeData.inputs?.additionalConfig as string + const recordManager = nodeData.inputs?.recordManager let additionalConfiguration = {} if (additionalConfig) { @@ -151,11 +160,37 @@ class Postgres_VectorStores implements INode { } try { - const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args) + if (recordManager) { + const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args) - // Avoid Illegal invocation error - vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => { - return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter) + // Avoid Illegal invocation error + vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => { + return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter) + } + + await recordManager.createSchema() + + const res = await index({ + docsSource: finalDocs, + recordManager, + vectorStore, + options: { + cleanup: recordManager?.cleanup, + sourceIdKey: recordManager?.sourceIdKey ?? 'source', + vectorStoreName: tableName + } + }) + + return res + } else { + const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args) + + // Avoid Illegal invocation error + vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => { + return await similaritySearchVectorWithScore(query, k, tableName, postgresConnectionOptions, filter) + } + + return { numAdded: finalDocs.length, addedDocs: finalDocs } } } catch (e) { throw new Error(e) diff --git a/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts b/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts index 7fd2559f..c9e77507 100644 --- a/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts +++ b/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts @@ -1,13 +1,19 @@ import { flatten } from 'lodash' +import { v4 as uuid } from 'uuid' import { QdrantClient } from '@qdrant/js-client-rest' import { VectorStoreRetrieverInput } from '@langchain/core/vectorstores' import { Document } from '@langchain/core/documents' import { QdrantVectorStore, QdrantLibArgs } from '@langchain/community/vectorstores/qdrant' import { Embeddings } from '@langchain/core/embeddings' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { index } from '../../../src/indexing' type RetrieverConfig = Partial> +type QdrantAddDocumentOptions = { + customPayload?: Record[] + ids?: string[] +} class Qdrant_VectorStores implements INode { label: string @@ -26,7 +32,7 @@ class Qdrant_VectorStores implements INode { constructor() { this.label = 'Qdrant' this.name = 'qdrant' - this.version = 1.0 + this.version = 2.0 this.type = 'Qdrant' this.icon = 'qdrant.png' this.category = 'Vector Stores' @@ -55,6 +61,13 @@ class Qdrant_VectorStores implements INode { name: 'embeddings', type: 'Embeddings' }, + { + label: 'Record Manager', + name: 'recordManager', + type: 'RecordManager', + description: 'Keep track of the record to prevent duplication', + optional: true + }, { label: 'Qdrant Server URL', name: 'qdrantServerUrl', @@ -138,13 +151,14 @@ class Qdrant_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string const collectionName = nodeData.inputs?.qdrantCollection as string const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension + const recordManager = nodeData.inputs?.recordManager const credentialData = await getCredentialData(nodeData.credential ?? '', options) const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData) @@ -178,7 +192,74 @@ class Qdrant_VectorStores implements INode { } try { - await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig) + if (recordManager) { + const vectorStore = new QdrantVectorStore(embeddings, dbConfig) + await vectorStore.ensureCollection() + + vectorStore.addVectors = async ( + vectors: number[][], + documents: Document[], + documentOptions?: QdrantAddDocumentOptions + ): Promise => { + if (vectors.length === 0) { + return + } + + await vectorStore.ensureCollection() + + const points = vectors.map((embedding, idx) => ({ + id: documentOptions?.ids?.length ? documentOptions?.ids[idx] : uuid(), + vector: embedding, + payload: { + content: documents[idx].pageContent, + metadata: documents[idx].metadata, + customPayload: documentOptions?.customPayload?.length ? documentOptions?.customPayload[idx] : undefined + } + })) + + try { + await client.upsert(collectionName, { + wait: true, + points + }) + } catch (e: any) { + const error = new Error(`${e?.status ?? 'Undefined error code'} ${e?.message}: ${e?.data?.status?.error}`) + throw error + } + } + + vectorStore.delete = async (params: { ids: string[] }): Promise => { + const { ids } = params + + if (ids?.length) { + try { + client.delete(collectionName, { + points: ids + }) + } catch (e) { + console.error('Failed to delete') + } + } + } + + await recordManager.createSchema() + + const res = await index({ + docsSource: finalDocs, + recordManager, + vectorStore, + options: { + cleanup: recordManager?.cleanup, + sourceIdKey: recordManager?.sourceIdKey ?? 'source', + vectorStoreName: collectionName + } + }) + + return res + } else { + await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig) + return { numAdded: finalDocs.length, addedDocs: finalDocs } + } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Redis/Redis.ts b/packages/components/nodes/vectorstores/Redis/Redis.ts index c1b9c7d4..562e928b 100644 --- a/packages/components/nodes/vectorstores/Redis/Redis.ts +++ b/packages/components/nodes/vectorstores/Redis/Redis.ts @@ -3,7 +3,7 @@ import { createClient, SearchOptions, RedisClientOptions } from 'redis' import { Embeddings } from '@langchain/core/embeddings' import { RedisVectorStore, RedisVectorStoreConfig } from '@langchain/community/vectorstores/redis' import { Document } from '@langchain/core/documents' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { escapeAllStrings, escapeSpecialChars, unEscapeSpecialChars } from './utils' @@ -138,7 +138,7 @@ class Redis_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const indexName = nodeData.inputs?.indexName as string let contentKey = nodeData.inputs?.contentKey as string @@ -203,6 +203,8 @@ class Redis_VectorStores implements INode { filter ) } + + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/SimpleStore/SimpleStore.ts b/packages/components/nodes/vectorstores/SimpleStore/SimpleStore.ts index 36c383e9..09ec2f24 100644 --- a/packages/components/nodes/vectorstores/SimpleStore/SimpleStore.ts +++ b/packages/components/nodes/vectorstores/SimpleStore/SimpleStore.ts @@ -2,7 +2,7 @@ import path from 'path' import { flatten } from 'lodash' import { storageContextFromDefaults, serviceContextFromDefaults, VectorStoreIndex, Document } from 'llamaindex' import { Document as LCDocument } from 'langchain/document' -import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getUserHome } from '../../../src' class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode { @@ -79,7 +79,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData): Promise { + async upsert(nodeData: INodeData): Promise> { const basePath = nodeData.inputs?.basePath as string const docs = nodeData.inputs?.document as LCDocument[] const embeddings = nodeData.inputs?.embeddings @@ -105,6 +105,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode { try { await VectorStoreIndex.fromDocuments(llamadocs, { serviceContext, storageContext }) + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Singlestore/Singlestore.ts b/packages/components/nodes/vectorstores/Singlestore/Singlestore.ts index 797b7467..6fb55ae7 100644 --- a/packages/components/nodes/vectorstores/Singlestore/Singlestore.ts +++ b/packages/components/nodes/vectorstores/Singlestore/Singlestore.ts @@ -2,7 +2,7 @@ import { flatten } from 'lodash' import { Embeddings } from '@langchain/core/embeddings' import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from '@langchain/community/vectorstores/singlestore' import { Document } from '@langchain/core/documents' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' class SingleStore_VectorStores implements INode { @@ -118,7 +118,7 @@ class SingleStore_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const user = getCredentialParam('user', credentialData, nodeData) const password = getCredentialParam('password', credentialData, nodeData) @@ -151,6 +151,7 @@ class SingleStore_VectorStores implements INode { try { const vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig) vectorStore.addDocuments.bind(vectorStore)(finalDocs) + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Supabase/Supabase.ts b/packages/components/nodes/vectorstores/Supabase/Supabase.ts index 74849911..fd24f373 100644 --- a/packages/components/nodes/vectorstores/Supabase/Supabase.ts +++ b/packages/components/nodes/vectorstores/Supabase/Supabase.ts @@ -3,9 +3,10 @@ import { createClient } from '@supabase/supabase-js' import { Document } from '@langchain/core/documents' import { Embeddings } from '@langchain/core/embeddings' import { SupabaseVectorStore, SupabaseLibArgs } from '@langchain/community/vectorstores/supabase' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' +import { index } from '../../../src/indexing' class Supabase_VectorStores implements INode { label: string @@ -24,7 +25,7 @@ class Supabase_VectorStores implements INode { constructor() { this.label = 'Supabase' this.name = 'supabase' - this.version = 2.0 + this.version = 3.0 this.type = 'Supabase' this.icon = 'supabase.svg' this.category = 'Vector Stores' @@ -50,6 +51,13 @@ class Supabase_VectorStores implements INode { name: 'embeddings', type: 'Embeddings' }, + { + label: 'Record Manager', + name: 'recordManager', + type: 'RecordManager', + description: 'Keep track of the record to prevent duplication', + optional: true + }, { label: 'Supabase Project URL', name: 'supabaseProjUrl', @@ -99,12 +107,13 @@ class Supabase_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string const tableName = nodeData.inputs?.tableName as string const queryName = nodeData.inputs?.queryName as string const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings + const recordManager = nodeData.inputs?.recordManager const credentialData = await getCredentialData(nodeData.credential ?? '', options) const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData) @@ -120,11 +129,32 @@ class Supabase_VectorStores implements INode { } try { - await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, { - client, - tableName: tableName, - queryName: queryName - }) + if (recordManager) { + const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, { + client, + tableName: tableName, + queryName: queryName + }) + await recordManager.createSchema() + const res = await index({ + docsSource: finalDocs, + recordManager, + vectorStore, + options: { + cleanup: recordManager?.cleanup, + sourceIdKey: recordManager?.sourceIdKey ?? 'source', + vectorStoreName: tableName + '_' + queryName + } + }) + return res + } else { + await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, { + client, + tableName: tableName, + queryName: queryName + }) + return { numAdded: finalDocs.length, addedDocs: finalDocs } + } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Vectara/Vectara.ts b/packages/components/nodes/vectorstores/Vectara/Vectara.ts index a661b18f..db62e85c 100644 --- a/packages/components/nodes/vectorstores/Vectara/Vectara.ts +++ b/packages/components/nodes/vectorstores/Vectara/Vectara.ts @@ -9,7 +9,7 @@ import { } from '@langchain/community/vectorstores/vectara' import { Document } from '@langchain/core/documents' import { Embeddings } from '@langchain/core/embeddings' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' class Vectara_VectorStores implements INode { @@ -144,7 +144,7 @@ class Vectara_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const apiKey = getCredentialParam('apiKey', credentialData, nodeData) const customerId = getCredentialParam('customerID', credentialData, nodeData) @@ -204,6 +204,7 @@ class Vectara_VectorStores implements INode { const vectorStore = new VectaraStore(vectaraArgs) await vectorStore.addFiles(vectaraFiles) } + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts b/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts index e7cd074a..139a02de 100644 --- a/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts +++ b/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts @@ -1,11 +1,12 @@ import { flatten } from 'lodash' import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client' -import { WeaviateLibArgs, WeaviateStore } from '@langchain/community/vectorstores/weaviate' +import { WeaviateLibArgs, WeaviateStore } from '@langchain/weaviate' import { Document } from '@langchain/core/documents' import { Embeddings } from '@langchain/core/embeddings' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' +import { index } from '../../../src/indexing' class Weaviate_VectorStores implements INode { label: string @@ -24,7 +25,7 @@ class Weaviate_VectorStores implements INode { constructor() { this.label = 'Weaviate' this.name = 'weaviate' - this.version = 2.0 + this.version = 3.0 this.type = 'Weaviate' this.icon = 'weaviate.png' this.category = 'Vector Stores' @@ -53,6 +54,13 @@ class Weaviate_VectorStores implements INode { name: 'embeddings', type: 'Embeddings' }, + { + label: 'Record Manager', + name: 'recordManager', + type: 'RecordManager', + description: 'Keep track of the record to prevent duplication', + optional: true + }, { label: 'Weaviate Scheme', name: 'weaviateScheme', @@ -125,7 +133,7 @@ class Weaviate_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const weaviateScheme = nodeData.inputs?.weaviateScheme as string const weaviateHost = nodeData.inputs?.weaviateHost as string const weaviateIndex = nodeData.inputs?.weaviateIndex as string @@ -133,6 +141,7 @@ class Weaviate_VectorStores implements INode { const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings + const recordManager = nodeData.inputs?.recordManager const credentialData = await getCredentialData(nodeData.credential ?? '', options) const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData) @@ -154,6 +163,7 @@ class Weaviate_VectorStores implements INode { } const obj: WeaviateLibArgs = { + //@ts-ignore client, indexName: weaviateIndex } @@ -162,7 +172,24 @@ class Weaviate_VectorStores implements INode { if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, '')) try { - await WeaviateStore.fromDocuments(finalDocs, embeddings, obj) + if (recordManager) { + const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj) + await recordManager.createSchema() + const res = await index({ + docsSource: finalDocs, + recordManager, + vectorStore, + options: { + cleanup: recordManager?.cleanup, + sourceIdKey: recordManager?.sourceIdKey ?? 'source', + vectorStoreName: weaviateTextKey ? weaviateIndex + '_' + weaviateTextKey : weaviateIndex + } + }) + return res + } else { + await WeaviateStore.fromDocuments(finalDocs, embeddings, obj) + return { numAdded: finalDocs.length, addedDocs: finalDocs } + } } catch (e) { throw new Error(e) } @@ -189,6 +216,7 @@ class Weaviate_VectorStores implements INode { const client: WeaviateClient = weaviate.client(clientConfig) const obj: WeaviateLibArgs = { + //@ts-ignore client, indexName: weaviateIndex } diff --git a/packages/components/nodes/vectorstores/Zep/Zep.ts b/packages/components/nodes/vectorstores/Zep/Zep.ts index c8a78eac..0cddf4d1 100644 --- a/packages/components/nodes/vectorstores/Zep/Zep.ts +++ b/packages/components/nodes/vectorstores/Zep/Zep.ts @@ -3,7 +3,7 @@ import { IDocument, ZepClient } from '@getzep/zep-js' import { ZepVectorStore, IZepConfig } from '@langchain/community/vectorstores/zep' import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' @@ -106,7 +106,7 @@ class Zep_VectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const baseURL = nodeData.inputs?.baseURL as string const zepCollection = nodeData.inputs?.zepCollection as string const dimension = (nodeData.inputs?.dimension as number) ?? 1536 @@ -134,6 +134,7 @@ class Zep_VectorStores implements INode { try { await ZepVectorStore.fromDocuments(finalDocs, embeddings, zepConfig) + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts b/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts index 49da7c0d..85c32ca7 100644 --- a/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts +++ b/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts @@ -3,7 +3,7 @@ import { IDocument, ZepClient } from '@getzep/zep-cloud' import { IZepConfig, ZepVectorStore } from '@getzep/zep-cloud/langchain' import { Embeddings } from 'langchain/embeddings/base' import { Document } from 'langchain/document' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' import { FakeEmbeddings } from 'langchain/embeddings/fake' @@ -89,7 +89,7 @@ class Zep_CloudVectorStores implements INode { //@ts-ignore vectorStoreMethods = { - async upsert(nodeData: INodeData, options: ICommonObject): Promise { + async upsert(nodeData: INodeData, options: ICommonObject): Promise> { const zepCollection = nodeData.inputs?.zepCollection as string const docs = nodeData.inputs?.document as Document[] const credentialData = await getCredentialData(nodeData.credential ?? '', options) @@ -109,6 +109,7 @@ class Zep_CloudVectorStores implements INode { } try { await ZepVectorStore.fromDocuments(finalDocs, new FakeEmbeddings(), zepConfig) + return { numAdded: finalDocs.length, addedDocs: finalDocs } } catch (e) { throw new Error(e) } diff --git a/packages/components/package.json b/packages/components/package.json index e47377a7..7e4f4a89 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -42,6 +42,7 @@ "@langchain/mongodb": "^0.0.1", "@langchain/openai": "^0.0.14", "@langchain/pinecone": "^0.0.3", + "@langchain/weaviate": "^0.0.1", "@mistralai/mistralai": "0.1.3", "@notionhq/client": "^2.2.8", "@opensearch-project/opensearch": "^1.2.0", diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 725f64c6..33f50d4f 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -113,7 +113,7 @@ export interface INode extends INodeProperties { [key: string]: (nodeData: INodeData, options?: ICommonObject) => Promise } vectorStoreMethods?: { - upsert: (nodeData: INodeData, options?: ICommonObject) => Promise + upsert: (nodeData: INodeData, options?: ICommonObject) => Promise search: (nodeData: INodeData, options?: ICommonObject) => Promise delete: (nodeData: INodeData, options?: ICommonObject) => Promise } @@ -181,6 +181,7 @@ export type MessageContentImageUrl = { import { PromptTemplate as LangchainPromptTemplate, PromptTemplateInput } from '@langchain/core/prompts' import { VectorStore } from '@langchain/core/vectorstores' +import { Document } from '@langchain/core/documents' export class PromptTemplate extends LangchainPromptTemplate { promptValues: ICommonObject @@ -271,6 +272,15 @@ export abstract class FlowiseSummaryMemory extends ConversationSummaryMemory imp abstract clearChatMessages(overrideSessionId?: string): Promise } +export type IndexingResult = { + numAdded: number + numDeleted: number + numUpdated: number + numSkipped: number + totalKeys: number + addedDocs: Document[] +} + export interface IVisionChatModal { id: string configuredModel: string diff --git a/packages/components/src/indexing.ts b/packages/components/src/indexing.ts new file mode 100644 index 00000000..f9cacfc5 --- /dev/null +++ b/packages/components/src/indexing.ts @@ -0,0 +1,355 @@ +import { VectorStore } from '@langchain/core/vectorstores' +import { v5 as uuidv5 } from 'uuid' +import { RecordManagerInterface, UUIDV5_NAMESPACE } from '@langchain/community/indexes/base' +import { insecureHash } from '@langchain/core/utils/hash' +import { Document, DocumentInterface } from '@langchain/core/documents' +import { BaseDocumentLoader } from 'langchain/document_loaders/base.js' +import { IndexingResult } from './Interface' + +type Metadata = Record + +type StringOrDocFunc = string | ((doc: DocumentInterface) => string) + +export interface HashedDocumentInterface extends DocumentInterface { + uid: string + hash_?: string + contentHash?: string + metadataHash?: string + pageContent: string + metadata: Metadata + calculateHashes(): void + toDocument(): DocumentInterface +} + +interface HashedDocumentArgs { + pageContent: string + metadata: Metadata + uid: string +} + +/** + * HashedDocument is a Document with hashes calculated. + * Hashes are calculated based on page content and metadata. + * It is used for indexing. + */ +export class _HashedDocument implements HashedDocumentInterface { + uid: string + + hash_?: string + + contentHash?: string + + metadataHash?: string + + pageContent: string + + metadata: Metadata + + constructor(fields: HashedDocumentArgs) { + this.uid = fields.uid + this.pageContent = fields.pageContent + this.metadata = fields.metadata + } + + calculateHashes(): void { + const forbiddenKeys = ['hash_', 'content_hash', 'metadata_hash'] + + for (const key of forbiddenKeys) { + if (key in this.metadata) { + throw new Error( + `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(', ')}]` + ) + } + } + + const contentHash = this._hashStringToUUID(this.pageContent) + + try { + const metadataHash = this._hashNestedDictToUUID(this.metadata) + this.contentHash = contentHash + this.metadataHash = metadataHash + } catch (e) { + throw new Error(`Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`) + } + + this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash) + + if (!this.uid) { + this.uid = this.hash_ + } + } + + toDocument(): DocumentInterface { + return new Document({ + pageContent: this.pageContent, + metadata: this.metadata + }) + } + + static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument { + const doc = new this({ + pageContent: document.pageContent, + metadata: document.metadata, + uid: uid || (document as DocumentInterface & { uid: string }).uid + }) + doc.calculateHashes() + return doc + } + + private _hashStringToUUID(inputString: string): string { + const hash_value = insecureHash(inputString) + return uuidv5(hash_value, UUIDV5_NAMESPACE) + } + + private _hashNestedDictToUUID(data: Record): string { + const serialized_data = JSON.stringify(data, Object.keys(data).sort()) + const hash_value = insecureHash(serialized_data) + return uuidv5(hash_value, UUIDV5_NAMESPACE) + } +} + +export type CleanupMode = 'full' | 'incremental' + +export type IndexOptions = { + /** + * The number of documents to index in one batch. + */ + batchSize?: number + /** + * The cleanup mode to use. Can be "full", "incremental" or undefined. + * - **Incremental**: Cleans up all documents that haven't been updated AND + * that are associated with source ids that were seen + * during indexing. + * Clean up is done continuously during indexing helping + * to minimize the probability of users seeing duplicated + * content. + * - **Full**: Delete all documents that haven to been returned by the loader. + * Clean up runs after all documents have been indexed. + * This means that users may see duplicated content during indexing. + * - **undefined**: Do not delete any documents. + */ + cleanup?: CleanupMode + /** + * Optional key that helps identify the original source of the document. + * Must either be a string representing the key of the source in the metadata + * or a function that takes a document and returns a string representing the source. + * **Required when cleanup is incremental**. + */ + sourceIdKey?: StringOrDocFunc + /** + * Batch size to use when cleaning up documents. + */ + cleanupBatchSize?: number + /** + * Force update documents even if they are present in the + * record manager. Useful if you are re-indexing with updated embeddings. + */ + forceUpdate?: boolean + + vectorStoreName?: string +} + +export function _batch(size: number, iterable: T[]): T[][] { + const batches: T[][] = [] + let currentBatch: T[] = [] + + iterable.forEach((item) => { + currentBatch.push(item) + + if (currentBatch.length >= size) { + batches.push(currentBatch) + currentBatch = [] + } + }) + + if (currentBatch.length > 0) { + batches.push(currentBatch) + } + + return batches +} + +export function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[] { + const seen = new Set() + const deduplicated: HashedDocumentInterface[] = [] + + for (const hashedDoc of hashedDocuments) { + if (!hashedDoc.hash_) { + throw new Error('Hashed document does not have a hash') + } + + if (!seen.has(hashedDoc.hash_)) { + seen.add(hashedDoc.hash_) + deduplicated.push(hashedDoc) + } + } + return deduplicated +} + +export function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null { + if (sourceIdKey === null) { + return (_doc: DocumentInterface) => null + } else if (typeof sourceIdKey === 'string') { + return (doc: DocumentInterface) => doc.metadata[sourceIdKey] + } else if (typeof sourceIdKey === 'function') { + return sourceIdKey + } else { + throw new Error(`sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`) + } +} + +export const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => { + if ('load' in arg && typeof arg.load === 'function' && 'loadAndSplit' in arg && typeof arg.loadAndSplit === 'function') { + return true + } + return false +} + +interface IndexArgs { + docsSource: BaseDocumentLoader | DocumentInterface[] + recordManager: RecordManagerInterface + vectorStore: VectorStore + options?: IndexOptions +} + +/** + * Index data from the doc source into the vector store. + * + * Indexing functionality uses a manager to keep track of which documents + * are in the vector store. + * + * This allows us to keep track of which documents were updated, and which + * documents were deleted, which documents should be skipped. + * + * For the time being, documents are indexed using their hashes, and users + * are not able to specify the uid of the document. + * + * @param {IndexArgs} args + * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents. + * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents. + * @param {VectorStore} args.vectorStore The vector store to use for storing the documents. + * @param {IndexOptions | undefined} args.options Options for indexing. + * @returns {Promise} + */ +export async function index(args: IndexArgs): Promise { + const { docsSource, recordManager, vectorStore, options } = args + const { batchSize = 100, cleanup, sourceIdKey, cleanupBatchSize = 1000, forceUpdate = false, vectorStoreName } = options ?? {} + + if (cleanup === 'incremental' && !sourceIdKey) { + throw new Error("sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.") + } + + if (vectorStoreName) { + ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName + } + + const docs = _isBaseDocumentLoader(docsSource) ? await docsSource.load() : docsSource + + const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null) + + const indexStartDt = await recordManager.getTime() + let numAdded = 0 + let addedDocs: Document[] = [] + let numDeleted = 0 + let numUpdated = 0 + let numSkipped = 0 + let totalKeys = 0 + + const batches = _batch(batchSize ?? 100, docs) + + for (const batch of batches) { + const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc))) + + const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc)) + + if (cleanup === 'incremental') { + hashedDocs.forEach((_hashedDoc, index) => { + const source = sourceIds[index] + if (source === null) { + throw new Error('sourceIdKey must be provided when cleanup is incremental') + } + }) + } + + const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid)) + + const uids: string[] = [] + const docsToIndex: DocumentInterface[] = [] + const docsToUpdate: string[] = [] + const seenDocs = new Set() + hashedDocs.forEach((hashedDoc, i) => { + const docExists = batchExists[i] + if (docExists) { + if (forceUpdate) { + seenDocs.add(hashedDoc.uid) + } else { + docsToUpdate.push(hashedDoc.uid) + return + } + } + uids.push(hashedDoc.uid) + docsToIndex.push(hashedDoc.toDocument()) + }) + + if (docsToUpdate.length > 0) { + await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt }) + numSkipped += docsToUpdate.length + } + + if (docsToIndex.length > 0) { + await vectorStore.addDocuments(docsToIndex, { ids: uids }) + const newDocs = docsToIndex.map((docs) => ({ + pageContent: docs.pageContent, + metadata: docs.metadata + })) + addedDocs.push(...newDocs) + numAdded += docsToIndex.length - seenDocs.size + numUpdated += seenDocs.size + } + + await recordManager.update( + hashedDocs.map((doc) => doc.uid), + { timeAtLeast: indexStartDt, groupIds: sourceIds } + ) + + if (cleanup === 'incremental') { + sourceIds.forEach((sourceId) => { + if (!sourceId) throw new Error('Source id cannot be null') + }) + const uidsToDelete = await recordManager.listKeys({ + before: indexStartDt, + groupIds: sourceIds + }) + await vectorStore.delete({ ids: uidsToDelete }) + await recordManager.deleteKeys(uidsToDelete) + numDeleted += uidsToDelete.length + } + } + + if (cleanup === 'full') { + let uidsToDelete = await recordManager.listKeys({ + before: indexStartDt, + limit: cleanupBatchSize + }) + while (uidsToDelete.length > 0) { + await vectorStore.delete({ ids: uidsToDelete }) + await recordManager.deleteKeys(uidsToDelete) + numDeleted += uidsToDelete.length + uidsToDelete = await recordManager.listKeys({ + before: indexStartDt, + limit: cleanupBatchSize + }) + } + } + + totalKeys = (await recordManager.listKeys({})).length + + return { + numAdded, + numDeleted, + numUpdated, + numSkipped, + totalKeys, + addedDocs + } +} diff --git a/packages/server/src/Interface.ts b/packages/server/src/Interface.ts index 91fe2a93..7b405a90 100644 --- a/packages/server/src/Interface.ts +++ b/packages/server/src/Interface.ts @@ -93,6 +93,14 @@ export interface IVariable { createdDate: Date } +export interface IUpsertHistory { + id: string + chatflowid: string + result: string + flowData: string + date: Date +} + export interface IComponentNodes { [key: string]: INode } diff --git a/packages/server/src/controllers/tools/index.ts b/packages/server/src/controllers/tools/index.ts index 49cea269..196032f6 100644 --- a/packages/server/src/controllers/tools/index.ts +++ b/packages/server/src/controllers/tools/index.ts @@ -19,7 +19,7 @@ const creatTool = async (req: Request, res: Response, next: NextFunction) => { const deleteTool = async (req: Request, res: Response, next: NextFunction) => { try { if (typeof req.params.id === 'undefined' || req.params.id === '') { - throw new Error(`Error: toolsController.updateTool - id not provided!`) + throw new Error(`Error: toolsController.deleteTool - id not provided!`) } const apiResponse = await toolsService.deleteTool(req.params.id) if (apiResponse.executionError) { diff --git a/packages/server/src/controllers/upsert-history/index.ts b/packages/server/src/controllers/upsert-history/index.ts new file mode 100644 index 00000000..43f36d5b --- /dev/null +++ b/packages/server/src/controllers/upsert-history/index.ts @@ -0,0 +1,30 @@ +import { Request, Response, NextFunction } from 'express' +import upsertHistoryService from '../../services/upsert-history' + +const getAllUpsertHistory = async (req: Request, res: Response, next: NextFunction) => { + try { + const sortOrder = req.query?.order as string | undefined + const chatflowid = req.params?.id as string | undefined + const startDate = req.query?.startDate as string | undefined + const endDate = req.query?.endDate as string | undefined + const apiResponse = await upsertHistoryService.getAllUpsertHistory(sortOrder, chatflowid, startDate, endDate) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + +const patchDeleteUpsertHistory = async (req: Request, res: Response, next: NextFunction) => { + try { + const ids = req.body.ids ?? [] + const apiResponse = await upsertHistoryService.patchDeleteUpsertHistory(ids) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + +export default { + getAllUpsertHistory, + patchDeleteUpsertHistory +} diff --git a/packages/server/src/database/entities/Assistant.ts b/packages/server/src/database/entities/Assistant.ts index cf2f2633..e6be9edf 100644 --- a/packages/server/src/database/entities/Assistant.ts +++ b/packages/server/src/database/entities/Assistant.ts @@ -10,7 +10,7 @@ export class Assistant implements IAssistant { @Column({ type: 'text' }) details: string - @Column({ type: 'uuid'}) + @Column({ type: 'uuid' }) credential: string @Column({ nullable: true }) diff --git a/packages/server/src/database/entities/ChatFlow.ts b/packages/server/src/database/entities/ChatFlow.ts index 92f5a4b6..d654128d 100644 --- a/packages/server/src/database/entities/ChatFlow.ts +++ b/packages/server/src/database/entities/ChatFlow.ts @@ -34,11 +34,11 @@ export class ChatFlow implements IChatFlow { @Column({ nullable: true, type: 'text' }) speechToText?: string - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @CreateDateColumn() createdDate: Date - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @UpdateDateColumn() updatedDate: Date diff --git a/packages/server/src/database/entities/ChatMessage.ts b/packages/server/src/database/entities/ChatMessage.ts index 46cc0dc1..f0dcd0c5 100644 --- a/packages/server/src/database/entities/ChatMessage.ts +++ b/packages/server/src/database/entities/ChatMessage.ts @@ -41,7 +41,7 @@ export class ChatMessage implements IChatMessage { @Column({ type: 'varchar', nullable: true }) sessionId?: string - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @CreateDateColumn() createdDate: Date } diff --git a/packages/server/src/database/entities/ChatMessageFeedback.ts b/packages/server/src/database/entities/ChatMessageFeedback.ts index 7010a794..f7590b29 100644 --- a/packages/server/src/database/entities/ChatMessageFeedback.ts +++ b/packages/server/src/database/entities/ChatMessageFeedback.ts @@ -25,7 +25,7 @@ export class ChatMessageFeedback implements IChatMessageFeedback { @Column({ nullable: true, type: 'text' }) content?: string - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @CreateDateColumn() createdDate: Date } diff --git a/packages/server/src/database/entities/Credential.ts b/packages/server/src/database/entities/Credential.ts index 7b368ef3..daeb0595 100644 --- a/packages/server/src/database/entities/Credential.ts +++ b/packages/server/src/database/entities/Credential.ts @@ -16,11 +16,11 @@ export class Credential implements ICredential { @Column({ type: 'text' }) encryptedData: string - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @CreateDateColumn() createdDate: Date - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @UpdateDateColumn() updatedDate: Date } diff --git a/packages/server/src/database/entities/Tool.ts b/packages/server/src/database/entities/Tool.ts index 38f30480..49f7335e 100644 --- a/packages/server/src/database/entities/Tool.ts +++ b/packages/server/src/database/entities/Tool.ts @@ -25,11 +25,11 @@ export class Tool implements ITool { @Column({ nullable: true, type: 'text' }) func?: string - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @CreateDateColumn() createdDate: Date - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @UpdateDateColumn() updatedDate: Date } diff --git a/packages/server/src/database/entities/UpsertHistory.ts b/packages/server/src/database/entities/UpsertHistory.ts new file mode 100644 index 00000000..0d7674a6 --- /dev/null +++ b/packages/server/src/database/entities/UpsertHistory.ts @@ -0,0 +1,22 @@ +/* eslint-disable */ +import { Entity, Column, PrimaryGeneratedColumn, Index, CreateDateColumn } from 'typeorm' +import { IUpsertHistory } from '../../Interface' + +@Entity() +export class UpsertHistory implements IUpsertHistory { + @PrimaryGeneratedColumn('uuid') + id: string + + @Index() + @Column() + chatflowid: string + + @Column() + result: string + + @Column() + flowData: string + + @CreateDateColumn() + date: Date +} diff --git a/packages/server/src/database/entities/Variable.ts b/packages/server/src/database/entities/Variable.ts index 00df7267..2437e824 100644 --- a/packages/server/src/database/entities/Variable.ts +++ b/packages/server/src/database/entities/Variable.ts @@ -16,11 +16,11 @@ export class Variable implements IVariable { @Column({ default: 'string', type: 'text' }) type: string - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @CreateDateColumn() createdDate: Date - @Column({type:'timestamp'}) + @Column({ type: 'timestamp' }) @UpdateDateColumn() updatedDate: Date } diff --git a/packages/server/src/database/entities/index.ts b/packages/server/src/database/entities/index.ts index 6a0e2d22..d9b9536e 100644 --- a/packages/server/src/database/entities/index.ts +++ b/packages/server/src/database/entities/index.ts @@ -5,6 +5,7 @@ import { Credential } from './Credential' import { Tool } from './Tool' import { Assistant } from './Assistant' import { Variable } from './Variable' +import { UpsertHistory } from './UpsertHistory' export const entities = { ChatFlow, @@ -13,5 +14,6 @@ export const entities = { Credential, Tool, Assistant, - Variable + Variable, + UpsertHistory } diff --git a/packages/server/src/database/migrations/mysql/1709814301358-AddUpsertHistoryEntity.ts b/packages/server/src/database/migrations/mysql/1709814301358-AddUpsertHistoryEntity.ts new file mode 100644 index 00000000..5c114bb1 --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1709814301358-AddUpsertHistoryEntity.ts @@ -0,0 +1,21 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddUpsertHistoryEntity1709814301358 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE IF NOT EXISTS \`upsert_history\` ( + \`id\` varchar(36) NOT NULL, + \`chatflowid\` varchar(255) NOT NULL, + \`result\` text NOT NULL, + \`flowData\` text NOT NULL, + \`date\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (\`id\`) + KEY \`IDX_a0b59fd66f6e48d2b198123cb6\` (\`chatflowid\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;` + ) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE upsert_history`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index a4baf424..b0102422 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -13,6 +13,7 @@ import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-Ad import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage' import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity' import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText' +import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity' import { AddFeedback1707213626553 } from './1707213626553-AddFeedback' export const mysqlMigrations = [ @@ -31,5 +32,6 @@ export const mysqlMigrations = [ AddFileUploadsToChatMessage1701788586491, AddVariableEntity1699325775451, AddSpeechToText1706364937060, + AddUpsertHistoryEntity1709814301358, AddFeedback1707213626553 ] diff --git a/packages/server/src/database/migrations/postgres/1709814301358-AddUpsertHistoryEntity.ts b/packages/server/src/database/migrations/postgres/1709814301358-AddUpsertHistoryEntity.ts new file mode 100644 index 00000000..9b93156f --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1709814301358-AddUpsertHistoryEntity.ts @@ -0,0 +1,20 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddUpsertHistoryEntity1709814301358 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE IF NOT EXISTS upsert_history ( + id uuid NOT NULL DEFAULT uuid_generate_v4(), + "chatflowid" varchar NOT NULL, + "result" text NOT NULL, + "flowData" text NOT NULL, + "date" timestamp NOT NULL DEFAULT now(), + CONSTRAINT "PK_37327b22b6e246319bd5eeb0e88" PRIMARY KEY (id) + );` + ) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE upsert_history`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts index cec4ccbd..3f37c6e8 100644 --- a/packages/server/src/database/migrations/postgres/index.ts +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -13,6 +13,7 @@ import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-Ad import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage' import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity' import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText' +import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity' import { AddFeedback1707213601923 } from './1707213601923-AddFeedback' import { FieldTypes1710497452584 } from './1710497452584-FieldTypes' @@ -32,6 +33,7 @@ export const postgresMigrations = [ AddFileUploadsToChatMessage1701788586491, AddVariableEntity1699325775451, AddSpeechToText1706364937060, + AddUpsertHistoryEntity1709814301358, AddFeedback1707213601923, FieldTypes1710497452584 ] diff --git a/packages/server/src/database/migrations/sqlite/1709814301358-AddUpsertHistoryEntity.ts b/packages/server/src/database/migrations/sqlite/1709814301358-AddUpsertHistoryEntity.ts new file mode 100644 index 00000000..c5cfca7d --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1709814301358-AddUpsertHistoryEntity.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddUpsertHistoryEntity1709814301358 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE IF NOT EXISTS "upsert_history" ("id" varchar PRIMARY KEY NOT NULL, "chatflowid" varchar NOT NULL, "result" text NOT NULL, "flowData" text NOT NULL, "date" datetime NOT NULL DEFAULT (datetime('now')));` + ) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE upsert_history`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index 0b4f3e05..0cc692dd 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -13,6 +13,7 @@ import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-Ad import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage' import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity' import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText' +import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity' import { AddFeedback1707213619308 } from './1707213619308-AddFeedback' export const sqliteMigrations = [ @@ -31,5 +32,6 @@ export const sqliteMigrations = [ AddFileUploadsToChatMessage1701788586491, AddVariableEntity1699325775451, AddSpeechToText1706364937060, + AddUpsertHistoryEntity1709814301358, AddFeedback1707213619308 ] diff --git a/packages/server/src/routes/index.ts b/packages/server/src/routes/index.ts index c989880b..8d453879 100644 --- a/packages/server/src/routes/index.ts +++ b/packages/server/src/routes/index.ts @@ -35,6 +35,7 @@ import variablesRouter from './variables' import vectorRouter from './vectors' import verifyRouter from './verify' import versionRouter from './versions' +import upsertHistoryRouter from './upsert-history' const router = express.Router() @@ -74,5 +75,6 @@ router.use('/variables', variablesRouter) router.use('/vector', vectorRouter) router.use('/verify', verifyRouter) router.use('/version', versionRouter) +router.use('/upsert-history', upsertHistoryRouter) export default router diff --git a/packages/server/src/routes/upsert-history/index.ts b/packages/server/src/routes/upsert-history/index.ts new file mode 100644 index 00000000..1f4d9d02 --- /dev/null +++ b/packages/server/src/routes/upsert-history/index.ts @@ -0,0 +1,15 @@ +import express from 'express' +import upsertHistoryController from '../../controllers/upsert-history' +const router = express.Router() + +// CREATE + +// READ +router.get('/:id', upsertHistoryController.getAllUpsertHistory) + +// PATCH +router.patch('/', upsertHistoryController.patchDeleteUpsertHistory) + +// DELETE + +export default router diff --git a/packages/server/src/services/chatflows/index.ts b/packages/server/src/services/chatflows/index.ts index c6ee778a..78c5c695 100644 --- a/packages/server/src/services/chatflows/index.ts +++ b/packages/server/src/services/chatflows/index.ts @@ -14,6 +14,9 @@ import logger from '../../utils/logger' import { getStoragePath } from 'flowise-components' import { IReactFlowObject } from '../../Interface' import { utilGetUploadsConfig } from '../../utils/getUploadsConfig' +import { ChatMessage } from '../../database/entities/ChatMessage' +import { ChatMessageFeedback } from '../../database/entities/ChatMessageFeedback' +import { UpsertHistory } from '../../database/entities/UpsertHistory' // Check if chatflow valid for streaming const checkIfChatflowIsValidForStreaming = async (chatflowId: string): Promise => { @@ -105,9 +108,18 @@ const deleteChatflow = async (chatflowId: string): Promise => { const appServer = getRunningExpressApp() const dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).delete({ id: chatflowId }) try { - // Delete all uploads corresponding to this chatflow + // Delete all uploads corresponding to this chatflow const directory = path.join(getStoragePath(), chatflowId) deleteFolderRecursive(directory) + + // Delete all chat messages + await appServer.AppDataSource.getRepository(ChatMessage).delete({ chatflowid: chatflowId }) + + // Delete all chat feedback + await appServer.AppDataSource.getRepository(ChatMessageFeedback).delete({ chatflowid: chatflowId }) + + // Delete all upsert history + await appServer.AppDataSource.getRepository(UpsertHistory).delete({ chatflowid: chatflowId }) } catch (e) { logger.error(`[server]: Error deleting file storage for chatflow ${chatflowId}: ${e}`) } diff --git a/packages/server/src/services/upsert-history/index.ts b/packages/server/src/services/upsert-history/index.ts new file mode 100644 index 00000000..96d9d82d --- /dev/null +++ b/packages/server/src/services/upsert-history/index.ts @@ -0,0 +1,66 @@ +import { MoreThanOrEqual, LessThanOrEqual } from 'typeorm' +import { getRunningExpressApp } from '../../utils/getRunningExpressApp' +import { UpsertHistory } from '../../database/entities/UpsertHistory' + +const getAllUpsertHistory = async ( + sortOrder: string | undefined, + chatflowid: string | undefined, + startDate: string | undefined, + endDate: string | undefined +) => { + try { + const appServer = getRunningExpressApp() + + const setDateToStartOrEndOfDay = (dateTimeStr: string, setHours: 'start' | 'end') => { + const date = new Date(dateTimeStr) + if (isNaN(date.getTime())) { + return undefined + } + setHours === 'start' ? date.setHours(0, 0, 0, 0) : date.setHours(23, 59, 59, 999) + return date + } + + let fromDate + if (startDate) fromDate = setDateToStartOrEndOfDay(startDate, 'start') + + let toDate + if (endDate) toDate = setDateToStartOrEndOfDay(endDate, 'end') + + let upsertHistory = await appServer.AppDataSource.getRepository(UpsertHistory).find({ + where: { + chatflowid, + ...(fromDate && { date: MoreThanOrEqual(fromDate) }), + ...(toDate && { date: LessThanOrEqual(toDate) }) + }, + order: { + date: sortOrder === 'DESC' ? 'DESC' : 'ASC' + } + }) + upsertHistory = upsertHistory.map((hist) => { + return { + ...hist, + result: hist.result ? JSON.parse(hist.result) : {}, + flowData: hist.flowData ? JSON.parse(hist.flowData) : {} + } + }) + + return upsertHistory + } catch (error) { + throw new Error(`Error: upsertHistoryServices.getAllUpsertHistory - ${error}`) + } +} + +const patchDeleteUpsertHistory = async (ids: string[] = []): Promise => { + try { + const appServer = getRunningExpressApp() + const dbResponse = await appServer.AppDataSource.getRepository(UpsertHistory).delete(ids) + return dbResponse + } catch (error) { + throw new Error(`Error: upsertHistoryServices.patchUpsertHistory - ${error}`) + } +} + +export default { + getAllUpsertHistory, + patchDeleteUpsertHistory +} diff --git a/packages/server/src/services/variables/index.ts b/packages/server/src/services/variables/index.ts index 7153c940..37e1c073 100644 --- a/packages/server/src/services/variables/index.ts +++ b/packages/server/src/services/variables/index.ts @@ -18,7 +18,7 @@ const deleteVariable = async (variableId: string): Promise => { const dbResponse = await appServer.AppDataSource.getRepository(Variable).delete({ id: variableId }) return dbResponse } catch (error) { - throw new Error(`Error: variablesServices.createVariable - ${error}`) + throw new Error(`Error: variablesServices.deleteVariable - ${error}`) } } diff --git a/packages/server/src/utils/buildChatflow.ts b/packages/server/src/utils/buildChatflow.ts index 811782bc..b1043cc4 100644 --- a/packages/server/src/utils/buildChatflow.ts +++ b/packages/server/src/utils/buildChatflow.ts @@ -36,7 +36,6 @@ import { utilAddChatMessage } from './addChatMesage' * @param {Request} req * @param {Server} socketIO * @param {boolean} isInternal - * @param {boolean} isUpsert */ export const utilBuildChatflow = async (req: Request, socketIO?: Server, isInternal: boolean = false): Promise => { try { diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index e41e9f64..cfbb2535 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -237,6 +237,84 @@ export const getEndingNodes = (nodeDependencies: INodeDependencies, graph: INode return endingNodeIds } +/** + * Get file name from base64 string + * @param {string} fileBase64 + */ +export const getFileName = (fileBase64: string): string => { + let fileNames = [] + if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) { + const files = JSON.parse(fileBase64) + for (const file of files) { + const splitDataURI = file.split(',') + const filename = splitDataURI[splitDataURI.length - 1].split(':')[1] + fileNames.push(filename) + } + return fileNames.join(', ') + } else { + const splitDataURI = fileBase64.split(',') + const filename = splitDataURI[splitDataURI.length - 1].split(':')[1] + return filename + } +} + +/** + * Save upsert flowData + * @param {INodeData} nodeData + * @param {Record} upsertHistory + */ +export const saveUpsertFlowData = (nodeData: INodeData, upsertHistory: Record): ICommonObject[] => { + const existingUpsertFlowData = upsertHistory['flowData'] ?? [] + const paramValues: ICommonObject[] = [] + + for (const input in nodeData.inputs) { + const inputParam = nodeData.inputParams.find((inp) => inp.name === input) + if (!inputParam) continue + + let paramValue: ICommonObject = {} + + if (!nodeData.inputs[input]) { + continue + } + if ( + typeof nodeData.inputs[input] === 'string' && + nodeData.inputs[input].startsWith('{{') && + nodeData.inputs[input].endsWith('}}') + ) { + continue + } + // Get file name instead of the base64 string + if (nodeData.category === 'Document Loaders' && nodeData.inputParams.find((inp) => inp.name === input)?.type === 'file') { + paramValue = { + label: inputParam?.label, + name: inputParam?.name, + type: inputParam?.type, + value: getFileName(nodeData.inputs[input]) + } + paramValues.push(paramValue) + continue + } + + paramValue = { + label: inputParam?.label, + name: inputParam?.name, + type: inputParam?.type, + value: nodeData.inputs[input] + } + paramValues.push(paramValue) + } + + const newFlowData = { + label: nodeData.label, + name: nodeData.name, + category: nodeData.category, + id: nodeData.id, + paramValues + } + existingUpsertFlowData.push(newFlowData) + return existingUpsertFlowData +} + /** * Build langchain from start to end * @param {string[]} startingNodeIds @@ -272,6 +350,8 @@ export const buildFlow = async ( ) => { const flowNodes = cloneDeep(reactFlowNodes) + let upsertHistory: Record = {} + // Create a Queue and add our initial node in it const nodeQueue = [] as INodeQueue[] const exploredNode = {} as IExploredNode @@ -302,12 +382,15 @@ export const buildFlow = async ( let flowNodeData = cloneDeep(reactFlowNode.data) if (overrideConfig) flowNodeData = replaceInputsWithConfig(flowNodeData, overrideConfig) + + if (isUpsert) upsertHistory['flowData'] = saveUpsertFlowData(flowNodeData, upsertHistory) + const reactFlowNodeData: INodeData = resolveVariables(flowNodeData, flowNodes, question, chatHistory) // TODO: Avoid processing Text Splitter + Doc Loader once Upsert & Load Existing Vector Nodes are deprecated if (isUpsert && stopNodeId && nodeId === stopNodeId) { logger.debug(`[server]: Upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`) - await newNodeInstance.vectorStoreMethods!['upsert']!.call(newNodeInstance, reactFlowNodeData, { + const indexResult = await newNodeInstance.vectorStoreMethods!['upsert']!.call(newNodeInstance, reactFlowNodeData, { chatId, sessionId, chatflowid, @@ -319,6 +402,7 @@ export const buildFlow = async ( dynamicVariables, uploads }) + if (indexResult) upsertHistory['result'] = indexResult logger.debug(`[server]: Finished upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`) break } else { @@ -422,7 +506,7 @@ export const buildFlow = async ( flowNodes.push(flowNodes.splice(index, 1)[0]) } } - return flowNodes + return isUpsert ? (upsertHistory as any) : flowNodes } /** diff --git a/packages/server/src/utils/upsertVector.ts b/packages/server/src/utils/upsertVector.ts index 0fc90eb1..d8714502 100644 --- a/packages/server/src/utils/upsertVector.ts +++ b/packages/server/src/utils/upsertVector.ts @@ -1,5 +1,6 @@ import { Request, Response } from 'express' import * as fs from 'fs' +import { cloneDeep, omit } from 'lodash' import { ICommonObject } from 'flowise-components' import telemetryService from '../services/telemetry' import logger from '../utils/logger' @@ -18,7 +19,14 @@ import { utilValidateKey } from './validateKey' import { IncomingInput, INodeDirectedGraph, IReactFlowObject, chatType } from '../Interface' import { ChatFlow } from '../database/entities/ChatFlow' import { getRunningExpressApp } from '../utils/getRunningExpressApp' +import { UpsertHistory } from '../database/entities/UpsertHistory' +/** + * Upsert documents + * @param {Request} req + * @param {Response} res + * @param {boolean} isInternal + */ export const upsertVector = async (req: Request, res: Response, isInternal: boolean = false) => { try { const appServer = getRunningExpressApp() @@ -78,6 +86,8 @@ export const upsertVector = async (req: Request, res: Response, isInternal: bool (node) => node.data.category === 'Vector Stores' && !node.data.label.includes('Upsert') && !node.data.label.includes('Load Existing') ) + + // Check if multiple vector store nodes exist, and if stopNodeId is specified if (vsNodes.length > 1 && !stopNodeId) { return res.status(500).send('There are multiple vector nodes, please provide stopNodeId in body request') } else if (vsNodes.length === 1 && !stopNodeId) { @@ -99,7 +109,7 @@ export const upsertVector = async (req: Request, res: Response, isInternal: bool const { startingNodeIds, depthQueue } = getStartingNodes(filteredGraph, stopNodeId) - await buildFlow( + const upsertedResult = await buildFlow( startingNodeIds, nodes, edges, @@ -121,6 +131,19 @@ export const upsertVector = async (req: Request, res: Response, isInternal: bool const startingNodes = nodes.filter((nd) => startingNodeIds.includes(nd.data.id)) await appServer.chatflowPool.add(chatflowid, undefined, startingNodes, incomingInput?.overrideConfig) + + // Save to DB + if (upsertedResult['flowData'] && upsertedResult['result']) { + const result = cloneDeep(upsertedResult) + result['flowData'] = JSON.stringify(result['flowData']) + result['result'] = JSON.stringify(omit(result['result'], ['totalKeys', 'addedDocs'])) + result.chatflowid = chatflowid + const newUpsertHistory = new UpsertHistory() + Object.assign(newUpsertHistory, result) + const upsertHistory = appServer.AppDataSource.getRepository(UpsertHistory).create(newUpsertHistory) + await appServer.AppDataSource.getRepository(UpsertHistory).save(upsertHistory) + } + await telemetryService.createEvent({ name: `vector_upserted`, data: { @@ -131,7 +154,7 @@ export const upsertVector = async (req: Request, res: Response, isInternal: bool stopNodeId } }) - return res.status(201).send('Successfully Upserted') + return res.status(201).json(upsertedResult['result'] ?? { result: 'Successfully Upserted' }) } catch (e: any) { logger.error('[server]: Error:', e) return res.status(500).send(e.message) diff --git a/packages/ui/src/api/vectorstore.js b/packages/ui/src/api/vectorstore.js index 053f5112..64f2ce65 100644 --- a/packages/ui/src/api/vectorstore.js +++ b/packages/ui/src/api/vectorstore.js @@ -1,7 +1,11 @@ import client from './client' const upsertVectorStore = (id, input) => client.post(`/vector/internal-upsert/${id}`, input) +const getUpsertHistory = (id, params = {}) => client.get(`/upsert-history/${id}`, { params: { order: 'DESC', ...params } }) +const deleteUpsertHistory = (ids) => client.patch(`/upsert-history`, { ids }) export default { - upsertVectorStore + getUpsertHistory, + upsertVectorStore, + deleteUpsertHistory } diff --git a/packages/ui/src/assets/images/upsert_history_empty.svg b/packages/ui/src/assets/images/upsert_history_empty.svg new file mode 100644 index 00000000..9f341e2e --- /dev/null +++ b/packages/ui/src/assets/images/upsert_history_empty.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/ui/src/menu-items/settings.js b/packages/ui/src/menu-items/settings.js index 9b83aaa4..c45d7a20 100644 --- a/packages/ui/src/menu-items/settings.js +++ b/packages/ui/src/menu-items/settings.js @@ -1,5 +1,13 @@ // assets -import { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconMessage, IconAdjustmentsHorizontal } from '@tabler/icons' +import { + IconTrash, + IconFileUpload, + IconFileExport, + IconCopy, + IconMessage, + IconDatabaseExport, + IconAdjustmentsHorizontal +} from '@tabler/icons' // constant const icons = { @@ -8,6 +16,7 @@ const icons = { IconFileExport, IconCopy, IconMessage, + IconDatabaseExport, IconAdjustmentsHorizontal } @@ -25,6 +34,13 @@ const settings = { url: '', icon: icons.IconMessage }, + { + id: 'viewUpsertHistory', + title: 'Upsert History', + type: 'item', + url: '', + icon: icons.IconDatabaseExport + }, { id: 'chatflowConfiguration', title: 'Configuration', diff --git a/packages/ui/src/utils/genericHelper.js b/packages/ui/src/utils/genericHelper.js index 645435d2..ed445acd 100644 --- a/packages/ui/src/utils/genericHelper.js +++ b/packages/ui/src/utils/genericHelper.js @@ -355,6 +355,11 @@ export const getUpsertDetails = (nodes, edges) => { innerNodes.push(nodes.find((node) => node.data.id === embeddingsId)) } + if (vsNode.data.inputs.recordManager) { + const recordManagerId = vsNode.data.inputs.recordManager.replace(/{{|}}/g, '').split('.')[0] + innerNodes.push(nodes.find((node) => node.data.id === recordManagerId)) + } + for (const doc of connectedDocs) { const docId = doc.replace(/{{|}}/g, '').split('.')[0] const docNode = nodes.find((node) => node.data.id === docId) diff --git a/packages/ui/src/views/canvas/CanvasHeader.jsx b/packages/ui/src/views/canvas/CanvasHeader.jsx index 4728c6b0..2ec30a16 100644 --- a/packages/ui/src/views/canvas/CanvasHeader.jsx +++ b/packages/ui/src/views/canvas/CanvasHeader.jsx @@ -16,6 +16,7 @@ import SaveChatflowDialog from '@/ui-component/dialog/SaveChatflowDialog' import APICodeDialog from '@/views/chatflows/APICodeDialog' import ViewMessagesDialog from '@/ui-component/dialog/ViewMessagesDialog' import ChatflowConfigurationDialog from '@/ui-component/dialog/ChatflowConfigurationDialog' +import UpsertHistoryDialog from '@/views/vectorstore/UpsertHistoryDialog' // API import chatflowsApi from '@/api/chatflows' @@ -45,6 +46,8 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl const [apiDialogProps, setAPIDialogProps] = useState({}) const [viewMessagesDialogOpen, setViewMessagesDialogOpen] = useState(false) const [viewMessagesDialogProps, setViewMessagesDialogProps] = useState({}) + const [upsertHistoryDialogOpen, setUpsertHistoryDialogOpen] = useState(false) + const [upsertHistoryDialogProps, setUpsertHistoryDialogProps] = useState({}) const [chatflowConfigurationDialogOpen, setChatflowConfigurationDialogOpen] = useState(false) const [chatflowConfigurationDialogProps, setChatflowConfigurationDialogProps] = useState({}) @@ -62,6 +65,12 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl chatflow: chatflow }) setViewMessagesDialogOpen(true) + } else if (setting === 'viewUpsertHistory') { + setUpsertHistoryDialogProps({ + title: 'View Upsert History', + chatflow: chatflow + }) + setUpsertHistoryDialogOpen(true) } else if (setting === 'chatflowConfiguration') { setChatflowConfigurationDialogProps({ title: 'Chatflow Configuration', @@ -387,6 +396,11 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl dialogProps={viewMessagesDialogProps} onCancel={() => setViewMessagesDialogOpen(false)} /> + setUpsertHistoryDialogOpen(false)} + /> { let allMessages = [...cloneDeep(prevMessages)] diff --git a/packages/ui/src/views/vectorstore/UpsertHistoryDialog.jsx b/packages/ui/src/views/vectorstore/UpsertHistoryDialog.jsx new file mode 100644 index 00000000..5c976ece --- /dev/null +++ b/packages/ui/src/views/vectorstore/UpsertHistoryDialog.jsx @@ -0,0 +1,453 @@ +import PropTypes from 'prop-types' +import { createPortal } from 'react-dom' +import { useDispatch, useSelector } from 'react-redux' +import { useEffect, useState, forwardRef } from 'react' +import DatePicker from 'react-datepicker' +import moment from 'moment/moment' + +// MUI +import { + Stack, + Box, + Paper, + Table, + TableBody, + TableContainer, + TableHead, + TableRow, + TableCell, + IconButton, + Button, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + ListItemButton, + Collapse, + Accordion, + AccordionSummary, + Typography, + AccordionDetails, + Checkbox +} from '@mui/material' +import { useTheme } from '@mui/material/styles' +import ExpandMoreIcon from '@mui/icons-material/ExpandMore' +import { IconChevronsUp, IconChevronsDown, IconTrash, IconX } from '@tabler/icons' + +// Project imports +import { TableViewOnly } from '@/ui-component/table/Table' +import { TooltipWithParser } from '@/ui-component/tooltip/TooltipWithParser' +import HistoryEmptySVG from '@/assets/images/upsert_history_empty.svg' + +// Api +import vectorstoreApi from '@/api/vectorstore' +import useApi from '@/hooks/useApi' + +// Store +import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions' +import { baseURL } from '@/store/constant' +import useNotifier from '@/utils/useNotifier' +import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction } from '@/store/actions' + +const DatePickerCustomInput = forwardRef(function DatePickerCustomInput({ value, onClick }, ref) { + return ( + + {value} + + ) +}) + +DatePickerCustomInput.propTypes = { + value: PropTypes.string, + onClick: PropTypes.func +} + +function UpsertHistoryRow(props) { + const [open, setOpen] = useState(false) + const [nodeConfigExpanded, setNodeConfigExpanded] = useState({}) + + const handleAccordionChange = (nodeLabel) => (event, isExpanded) => { + const accordianNodes = { ...nodeConfigExpanded } + accordianNodes[nodeLabel] = isExpanded + setNodeConfigExpanded(accordianNodes) + } + + const isItemSelected = props.selected.indexOf(props.upsertHistory.id) !== -1 + + return ( + <> + + + props.handleSelect(event, props.upsertHistory.id)} + inputProps={{ + 'aria-labelledby': props.upsertHistory.id + }} + /> + + {moment(props.upsertHistory.date).format('MMMM Do YYYY, h:mm:ss a')} + {props.upsertHistory.result?.numAdded ?? '0'} + {props.upsertHistory.result?.numUpdated ?? '0'} + {props.upsertHistory.result?.numSkipped ?? '0'} + {props.upsertHistory.result?.numDeleted ?? '0'} + + setOpen(!open)}> + {open ? : } + + + + {open && ( + + + + + {(props.upsertHistory.flowData ?? []).map((node, index) => { + return ( + + } + aria-controls={`nodes-accordian-${node.name}`} + id={`nodes-accordian-header-${node.name}`} + > +
+
+ {node.name} +
+ {node.label} +
+ {node.id} +
+
+
+ + + +
+ ) + })} +
+
+
+
+ )} + + ) +} + +UpsertHistoryRow.propTypes = { + upsertHistory: PropTypes.object, + theme: PropTypes.any, + isDarkMode: PropTypes.bool, + selected: PropTypes.array, + handleSelect: PropTypes.func +} + +const UpsertHistoryDialog = ({ show, dialogProps, onCancel }) => { + const portalElement = document.getElementById('portal') + const dispatch = useDispatch() + const customization = useSelector((state) => state.customization) + const theme = useTheme() + const getUpsertHistoryApi = useApi(vectorstoreApi.getUpsertHistory) + + useNotifier() + const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) + const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args)) + + const [chatflowUpsertHistory, setChatflowUpsertHistory] = useState([]) + const [startDate, setStartDate] = useState(new Date().setMonth(new Date().getMonth() - 1)) + const [endDate, setEndDate] = useState(new Date()) + const [selected, setSelected] = useState([]) + + const onSelectAllClick = (event) => { + if (event.target.checked) { + const newSelected = chatflowUpsertHistory.map((n) => n.id) + setSelected(newSelected) + return + } + setSelected([]) + } + + const onStartDateSelected = (date) => { + setStartDate(date) + getUpsertHistoryApi.request(dialogProps.chatflow.id, { + startDate: date, + endDate: endDate + }) + } + + const onEndDateSelected = (date) => { + setEndDate(date) + getUpsertHistoryApi.request(dialogProps.chatflow.id, { + endDate: date, + startDate: startDate + }) + } + + const handleSelect = (event, id) => { + const selectedIndex = selected.indexOf(id) + let newSelected = [] + + if (selectedIndex === -1) { + newSelected = newSelected.concat(selected, id) + } else if (selectedIndex === 0) { + newSelected = newSelected.concat(selected.slice(1)) + } else if (selectedIndex === selected.length - 1) { + newSelected = newSelected.concat(selected.slice(0, -1)) + } else if (selectedIndex > 0) { + newSelected = newSelected.concat(selected.slice(0, selectedIndex), selected.slice(selectedIndex + 1)) + } + setSelected(newSelected) + } + + const handleRemoveHistory = async () => { + try { + await vectorstoreApi.deleteUpsertHistory(selected) + enqueueSnackbar({ + message: 'Succesfully deleted upsert history', + options: { + key: new Date().getTime() + Math.random(), + variant: 'success', + action: (key) => ( + + ) + } + }) + setChatflowUpsertHistory(chatflowUpsertHistory.filter((hist) => !selected.includes(hist.id))) + setSelected([]) + } catch (error) { + enqueueSnackbar({ + message: 'Error deleting upsert history', + options: { + key: new Date().getTime() + Math.random(), + variant: 'error', + persist: true, + action: (key) => ( + + ) + } + }) + } + } + + useEffect(() => { + if (getUpsertHistoryApi.data) { + setChatflowUpsertHistory(getUpsertHistoryApi.data) + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getUpsertHistoryApi.data]) + + useEffect(() => { + if (dialogProps.chatflow) { + getUpsertHistoryApi.request(dialogProps.chatflow.id) + } + + return () => { + setChatflowUpsertHistory([]) + setStartDate(new Date().setMonth(new Date().getMonth() - 1)) + setEndDate(new Date()) + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [dialogProps]) + + useEffect(() => { + if (show) dispatch({ type: SHOW_CANVAS_DIALOG }) + else dispatch({ type: HIDE_CANVAS_DIALOG }) + return () => dispatch({ type: HIDE_CANVAS_DIALOG }) + }, [show, dispatch]) + + const component = show ? ( + + + {dialogProps.title} + + + <> +
+
+ From Date + onStartDateSelected(date)} + selectsStart + startDate={startDate} + endDate={endDate} + customInput={} + /> +
+
+ To Date + onEndDateSelected(date)} + selectsEnd + startDate={startDate} + endDate={endDate} + minDate={startDate} + maxDate={new Date()} + customInput={} + /> +
+
+ {selected.length > 0 && ( + + )} + {chatflowUpsertHistory.length <= 0 && ( + + + HistoryEmptySVG + +
No Upsert History Yet
+
+ )} + {chatflowUpsertHistory.length > 0 && ( + + + + + + + + Date + + Added{' '} + + + + Updated{' '} + + + + Skipped{' '} + + + + Deleted{' '} + + + Details + + + + {chatflowUpsertHistory.map((upsertHistory, index) => ( + + ))} + +
+
+ )} + +
+ + + +
+ ) : null + + return createPortal(component, portalElement) +} + +UpsertHistoryDialog.propTypes = { + show: PropTypes.bool, + dialogProps: PropTypes.object, + onCancel: PropTypes.func +} + +export default UpsertHistoryDialog diff --git a/packages/ui/src/views/vectorstore/UpsertResultDialog.jsx b/packages/ui/src/views/vectorstore/UpsertResultDialog.jsx new file mode 100644 index 00000000..4276bd8a --- /dev/null +++ b/packages/ui/src/views/vectorstore/UpsertResultDialog.jsx @@ -0,0 +1,94 @@ +import PropTypes from 'prop-types' +import { createPortal } from 'react-dom' +import { useDispatch, useSelector } from 'react-redux' +import { useEffect } from 'react' +import ReactJson from 'flowise-react-json-view' +import { Typography, Card, CardContent, Button, Dialog, DialogActions, DialogContent, DialogTitle } from '@mui/material' +import StatsCard from '@/ui-component/cards/StatsCard' +import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions' + +const UpsertResultDialog = ({ show, dialogProps, onCancel }) => { + const portalElement = document.getElementById('portal') + const dispatch = useDispatch() + const customization = useSelector((state) => state.customization) + + useEffect(() => { + if (show) dispatch({ type: SHOW_CANVAS_DIALOG }) + else dispatch({ type: HIDE_CANVAS_DIALOG }) + return () => dispatch({ type: HIDE_CANVAS_DIALOG }) + }, [show, dispatch]) + + const component = show ? ( + + + Upsert Record + + + <> +
+ + + + +
+ {dialogProps.addedDocs && dialogProps.addedDocs.length > 0 && ( + {dialogProps.numAdded} Added Documents + )} + {dialogProps.addedDocs && + dialogProps.addedDocs.length > 0 && + dialogProps.addedDocs.map((docs, index) => { + return ( + + + + {docs.pageContent} + + + + + ) + })} + +
+ + + +
+ ) : null + + return createPortal(component, portalElement) +} + +UpsertResultDialog.propTypes = { + show: PropTypes.bool, + dialogProps: PropTypes.object, + onCancel: PropTypes.func, + onSave: PropTypes.func +} + +export default UpsertResultDialog diff --git a/packages/ui/src/views/vectorstore/VectorStoreDialog.jsx b/packages/ui/src/views/vectorstore/VectorStoreDialog.jsx index 6426a08f..059f0c6e 100644 --- a/packages/ui/src/views/vectorstore/VectorStoreDialog.jsx +++ b/packages/ui/src/views/vectorstore/VectorStoreDialog.jsx @@ -78,7 +78,7 @@ function a11yProps(index) { } } -const VectorStoreDialog = ({ show, dialogProps, onCancel }) => { +const VectorStoreDialog = ({ show, dialogProps, onCancel, onIndexResult }) => { const portalElement = document.getElementById('portal') const { reactFlowInstance } = useContext(flowContext) const dispatch = useDispatch() @@ -276,7 +276,7 @@ query(formData).then((response) => { const onUpsertClicked = async (vectorStoreNode) => { setLoading(true) try { - await vectorstoreApi.upsertVectorStore(dialogProps.chatflowid, { stopNodeId: vectorStoreNode.data.id }) + const res = await vectorstoreApi.upsertVectorStore(dialogProps.chatflowid, { stopNodeId: vectorStoreNode.data.id }) enqueueSnackbar({ message: 'Succesfully upserted vector store. You can start chatting now!', options: { @@ -290,6 +290,7 @@ query(formData).then((response) => { } }) setLoading(false) + if (res && res.data && typeof res.data === 'object') onIndexResult(res.data) } catch (error) { enqueueSnackbar({ message: error.response.data.message, @@ -549,7 +550,8 @@ query(formData).then((response) => { VectorStoreDialog.propTypes = { show: PropTypes.bool, dialogProps: PropTypes.object, - onCancel: PropTypes.func + onCancel: PropTypes.func, + onIndexResult: PropTypes.func } export default VectorStoreDialog diff --git a/packages/ui/src/views/vectorstore/VectorStorePopUp.jsx b/packages/ui/src/views/vectorstore/VectorStorePopUp.jsx index 13cd9043..657ff3b8 100644 --- a/packages/ui/src/views/vectorstore/VectorStorePopUp.jsx +++ b/packages/ui/src/views/vectorstore/VectorStorePopUp.jsx @@ -1,33 +1,19 @@ import { useState, useRef, useEffect } from 'react' -import { useDispatch } from 'react-redux' import PropTypes from 'prop-types' -import { Button } from '@mui/material' import { IconDatabaseImport, IconX } from '@tabler/icons' // project import import { StyledFab } from '@/ui-component/button/StyledFab' import VectorStoreDialog from './VectorStoreDialog' - -// api -import vectorstoreApi from '@/api/vectorstore' - -// Hooks -import useNotifier from '@/utils/useNotifier' - -// Const -import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction } from '@/store/actions' +import UpsertResultDialog from './UpsertResultDialog' export const VectorStorePopUp = ({ chatflowid }) => { - const dispatch = useDispatch() - - useNotifier() - const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) - const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args)) - const [open, setOpen] = useState(false) const [showExpandDialog, setShowExpandDialog] = useState(false) const [expandDialogProps, setExpandDialogProps] = useState({}) + const [showUpsertResultDialog, setShowUpsertResultDialog] = useState(false) + const [upsertResultDialogProps, setUpsertResultDialogProps] = useState({}) const anchorRef = useRef(null) const prevOpen = useRef(open) @@ -43,38 +29,6 @@ export const VectorStorePopUp = ({ chatflowid }) => { setShowExpandDialog(true) } - const onUpsert = async () => { - try { - await vectorstoreApi.upsertVectorStore(chatflowid, {}) - enqueueSnackbar({ - message: 'Succesfully upserted vector store', - options: { - key: new Date().getTime() + Math.random(), - variant: 'success', - action: (key) => ( - - ) - } - }) - } catch (error) { - enqueueSnackbar({ - message: error.response.data.message, - options: { - key: new Date().getTime() + Math.random(), - variant: 'error', - persist: true, - action: (key) => ( - - ) - } - }) - } - } - useEffect(() => { if (prevOpen.current === true && open === false) { anchorRef.current.focus() @@ -100,12 +54,24 @@ export const VectorStorePopUp = ({ chatflowid }) => { { setShowExpandDialog(false) setOpen((prevopen) => !prevopen) }} + onIndexResult={(indexRes) => { + setShowExpandDialog(false) + setShowUpsertResultDialog(true) + setUpsertResultDialogProps({ ...indexRes }) + }} > + { + setShowUpsertResultDialog(false) + setOpen(false) + }} + > ) } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2f9d6aa3..d0e4ccdb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,9 +1,5 @@ lockfileVersion: '6.0' -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - onlyBuiltDependencies: - faiss-node - sqlite3 @@ -34,7 +30,7 @@ importers: version: 8.10.0(eslint@8.57.0) eslint-config-react-app: specifier: ^7.0.1 - version: 7.0.1(@babel/plugin-syntax-flow@7.23.3)(@babel/plugin-transform-react-jsx@7.23.4)(eslint@8.57.0)(jest@27.5.1)(typescript@4.9.5) + version: 7.0.1(@babel/plugin-syntax-flow@7.23.3)(@babel/plugin-transform-react-jsx@7.23.4)(eslint@8.57.0)(typescript@4.9.5) eslint-plugin-jsx-a11y: specifier: ^6.6.1 version: 6.8.0(eslint@8.57.0) @@ -149,6 +145,9 @@ importers: '@langchain/pinecone': specifier: ^0.0.3 version: 0.0.3 + '@langchain/weaviate': + specifier: ^0.0.1 + version: 0.0.1(graphql@16.8.1) '@mistralai/mistralai': specifier: 0.1.3 version: 0.1.3 @@ -187,7 +186,7 @@ importers: version: 4.3.2 axios: specifier: 1.6.2 - version: 1.6.2(debug@4.3.4) + version: 1.6.2 cheerio: specifier: ^1.0.0-rc.12 version: 1.0.0-rc.12 @@ -278,7 +277,7 @@ importers: version: 3.9.2 node-fetch: specifier: ^2.6.11 - version: 2.7.0(encoding@0.1.13) + version: 2.7.0 node-html-markdown: specifier: ^1.3.0 version: 1.3.0 @@ -411,7 +410,7 @@ importers: version: 0.4.1 axios: specifier: 1.6.2 - version: 1.6.2(debug@4.3.4) + version: 1.6.2 content-disposition: specifier: 0.5.4 version: 0.5.4 @@ -586,7 +585,7 @@ importers: version: 4.21.24(@babel/runtime@7.24.0)(@codemirror/autocomplete@6.14.0)(@codemirror/language@6.10.1)(@codemirror/lint@6.5.0)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.25.1)(codemirror@6.0.1)(react-dom@18.2.0)(react@18.2.0) axios: specifier: 1.6.2 - version: 1.6.2(debug@4.3.4) + version: 1.6.2 clsx: specifier: ^1.1.1 version: 1.2.1 @@ -595,10 +594,10 @@ importers: version: 16.4.5 flowise-embed: specifier: latest - version: 1.2.0 + version: 1.2.1 flowise-embed-react: specifier: latest - version: 1.0.2(flowise-embed@1.2.0)(react-dom@18.2.0)(react@18.2.0)(sass@1.71.1)(typescript@4.9.5) + version: 1.0.2(flowise-embed@1.2.1)(react-dom@18.2.0)(react@18.2.0)(sass@1.71.1)(typescript@4.9.5) flowise-react-json-view: specifier: '*' version: 1.21.7(@types/react@18.2.65)(react-dom@18.2.0)(react@18.2.0) @@ -765,7 +764,7 @@ packages: digest-fetch: 1.3.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 web-streams-polyfill: 3.3.3 transitivePeerDependencies: - encoding @@ -781,7 +780,7 @@ packages: digest-fetch: 1.3.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 web-streams-polyfill: 3.3.3 transitivePeerDependencies: - encoding @@ -797,7 +796,7 @@ packages: digest-fetch: 1.3.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 web-streams-polyfill: 3.3.3 transitivePeerDependencies: - encoding @@ -1944,10 +1943,10 @@ packages: '@babel/helpers': 7.24.0 '@babel/parser': 7.24.0 '@babel/template': 7.24.0 - '@babel/traverse': 7.24.0(supports-color@5.5.0) + '@babel/traverse': 7.24.0 '@babel/types': 7.24.0 convert-source-map: 2.0.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -2035,7 +2034,7 @@ packages: '@babel/core': 7.24.0 '@babel/helper-compilation-targets': 7.23.6 '@babel/helper-plugin-utils': 7.24.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 lodash.debounce: 4.0.8 resolve: 1.22.8 transitivePeerDependencies: @@ -2049,7 +2048,7 @@ packages: '@babel/core': 7.24.0 '@babel/helper-compilation-targets': 7.23.6 '@babel/helper-plugin-utils': 7.24.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 lodash.debounce: 4.0.8 resolve: 1.22.8 transitivePeerDependencies: @@ -2172,7 +2171,7 @@ packages: engines: { node: '>=6.9.0' } dependencies: '@babel/template': 7.24.0 - '@babel/traverse': 7.24.0(supports-color@5.5.0) + '@babel/traverse': 7.24.0 '@babel/types': 7.24.0 transitivePeerDependencies: - supports-color @@ -3249,6 +3248,23 @@ packages: '@babel/parser': 7.24.0 '@babel/types': 7.24.0 + /@babel/traverse@7.24.0: + resolution: { integrity: sha512-HfuJlI8qq3dEDmNU5ChzzpZRWq+oxCZQyMzIMEqLho+AQnhMnKQUzH6ydo3RBl/YjPCuk68Y6s0Gx0AeyULiWw== } + engines: { node: '>=6.9.0' } + dependencies: + '@babel/code-frame': 7.23.5 + '@babel/generator': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/parser': 7.24.0 + '@babel/types': 7.24.0 + debug: 4.3.4 + globals: 11.12.0 + transitivePeerDependencies: + - supports-color + /@babel/traverse@7.24.0(supports-color@5.5.0): resolution: { integrity: sha512-HfuJlI8qq3dEDmNU5ChzzpZRWq+oxCZQyMzIMEqLho+AQnhMnKQUzH6ydo3RBl/YjPCuk68Y6s0Gx0AeyULiWw== } engines: { node: '>=6.9.0' } @@ -3265,6 +3281,7 @@ packages: globals: 11.12.0 transitivePeerDependencies: - supports-color + dev: false /@babel/types@7.24.0: resolution: { integrity: sha512-+j7a5c253RfKh8iABBhywc8NSfP5LURe7Uh4qpsh6jc+aLJguvmIUBdjSdEMQv2bENrCR5MfRdjGo7vzS/ob7w== } @@ -3595,7 +3612,7 @@ packages: engines: { node: '>=14.0.0' } hasBin: true dependencies: - axios: 1.6.2(debug@4.3.4) + axios: 1.6.2 bson: 6.4.0 winston: 3.12.0 transitivePeerDependencies: @@ -3620,7 +3637,7 @@ packages: resolution: { integrity: sha512-/SXVuVnuU5b4dq8OFY4izG+dmGla185PcoqgK6+AJMpmOeY1QYVNbWtCwvSvoAANN5D/wV+EBU8+x7Vf9EphbA== } engines: { node: '>=16' } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 hpagent: 1.2.0 ms: 2.1.3 secure-json-parse: 2.7.0 @@ -4146,7 +4163,7 @@ packages: engines: { node: ^12.22.0 || ^14.17.0 || >=16.0.0 } dependencies: ajv: 6.12.6 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -4365,7 +4382,7 @@ packages: engines: { node: '>=10.10.0' } dependencies: '@humanwhocodes/object-schema': 2.0.2 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -4758,7 +4775,7 @@ packages: '@babel/preset-typescript': 7.18.6(@babel/core@7.24.0) '@babel/runtime': 7.24.0 '@babel/template': 7.24.0 - '@babel/traverse': 7.24.0(supports-color@5.5.0) + '@babel/traverse': 7.24.0 '@babel/types': 7.24.0 '@ladle/react-context': 1.0.1(react-dom@18.2.0)(react@18.2.0) '@vitejs/plugin-react': 3.1.0(vite@4.5.2) @@ -4768,7 +4785,7 @@ packages: classnames: 2.5.1 commander: 9.5.0 cross-spawn: 7.0.3 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 default-browser: 3.1.0 express: 4.18.3 get-port: 6.1.2 @@ -5225,6 +5242,18 @@ packages: uuid: 9.0.1 dev: false + /@langchain/weaviate@0.0.1(graphql@16.8.1): + resolution: { integrity: sha512-Lf6zgTf6i/fsPNlkDxPRLA3LEz2Wwgk6LNe54dByt0oZM4W+N4n5n/gDwojsXAKNEF5alXUv2N6yAOcUuXSbxg== } + engines: { node: '>=18' } + dependencies: + '@langchain/core': 0.1.44 + uuid: 9.0.1 + weaviate-ts-client: 2.1.1(graphql@16.8.1) + transitivePeerDependencies: + - encoding + - graphql + dev: false + /@leichtgewicht/ip-codec@2.0.4: resolution: { integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A== } dev: true @@ -5272,7 +5301,7 @@ packages: '@types/qs': 6.9.12 form-data: 4.0.0 js-base64: 3.7.7 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 qs: 6.11.2 dev: false @@ -5296,7 +5325,7 @@ packages: detect-libc: 2.0.2 https-proxy-agent: 5.0.1 make-dir: 3.1.0 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 nopt: 5.0.0 npmlog: 5.0.1 rimraf: 3.0.2 @@ -5310,7 +5339,7 @@ packages: /@mistralai/mistralai@0.0.10: resolution: { integrity: sha512-fZOt7A32DcPSff58wTa44pKUBoJBH5toAuzNI9yoM7s5NjTupa1IYcSqqk2LigO8M5EtOEkFsD/XzdyWPnhaRA== } dependencies: - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 transitivePeerDependencies: - encoding dev: false @@ -5318,7 +5347,7 @@ packages: /@mistralai/mistralai@0.1.3: resolution: { integrity: sha512-WUHxC2xdeqX9PTXJEqdiNY54vT2ir72WSJrZTTBKRnkfhX6zIfCYA24faRlWjUB5WTpn+wfdGsTMl3ArijlXFA== } dependencies: - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 transitivePeerDependencies: - encoding dev: false @@ -5591,7 +5620,7 @@ packages: engines: { node: '>=12' } dependencies: '@types/node-fetch': 2.6.2 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 transitivePeerDependencies: - encoding dev: false @@ -5915,7 +5944,7 @@ packages: dependencies: '@oclif/core': 2.15.0(@types/node@20.11.26)(typescript@4.9.5) chalk: 4.1.2 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 http-call: 5.3.0 lodash.template: 4.5.0 semver: 7.6.0 @@ -6017,7 +6046,7 @@ packages: '@octokit/request-error': 2.1.0 '@octokit/types': 6.41.0 is-plain-object: 5.0.0 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 universal-user-agent: 6.0.1 transitivePeerDependencies: - encoding @@ -6045,7 +6074,7 @@ packages: engines: { node: '>=10' } dependencies: aws4: 1.12.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 hpagent: 0.1.2 ms: 2.1.3 secure-json-parse: 2.7.0 @@ -6165,7 +6194,7 @@ packages: typescript: optional: true dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 extract-zip: 2.0.1 progress: 2.0.3 proxy-agent: 6.3.0 @@ -8276,7 +8305,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/type-utils': 5.62.0(eslint@8.57.0)(typescript@4.9.5) '@typescript-eslint/utils': 5.62.0(eslint@8.57.0)(typescript@4.9.5) - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 eslint: 8.57.0 graphemer: 1.4.0 ignore: 5.3.1 @@ -8314,7 +8343,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/types': 5.62.0 '@typescript-eslint/typescript-estree': 5.62.0(typescript@4.9.5) - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 eslint: 8.57.0 typescript: 4.9.5 transitivePeerDependencies: @@ -8341,7 +8370,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 5.62.0(typescript@4.9.5) '@typescript-eslint/utils': 5.62.0(eslint@8.57.0)(typescript@4.9.5) - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 eslint: 8.57.0 tsutils: 3.21.0(typescript@4.9.5) typescript: 4.9.5 @@ -8365,7 +8394,7 @@ packages: dependencies: '@typescript-eslint/types': 5.62.0 '@typescript-eslint/visitor-keys': 5.62.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 globby: 11.1.0 is-glob: 4.0.3 semver: 7.6.0 @@ -8751,7 +8780,7 @@ packages: resolution: { integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== } engines: { node: '>= 6.0.0' } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 transitivePeerDependencies: - supports-color @@ -8759,7 +8788,7 @@ packages: resolution: { integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg== } engines: { node: '>= 14' } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 transitivePeerDependencies: - supports-color dev: false @@ -9779,6 +9808,16 @@ packages: engines: { node: '>=4' } dev: false + /axios@1.6.2: + resolution: { integrity: sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A== } + dependencies: + follow-redirects: 1.15.5 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + dev: false + /axios@1.6.2(debug@4.3.4): resolution: { integrity: sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A== } dependencies: @@ -9787,11 +9826,12 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug + dev: true /axios@1.6.7: resolution: { integrity: sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA== } dependencies: - follow-redirects: 1.15.5(debug@4.3.4) + follow-redirects: 1.15.5 form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -11610,7 +11650,7 @@ packages: dependencies: form-data: 4.0.0 js-base64: 3.7.2 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 qs: 6.11.2 url-join: 4.0.1 transitivePeerDependencies: @@ -12016,6 +12056,14 @@ packages: resolution: { integrity: sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g== } dev: false + /cross-fetch@3.1.8: + resolution: { integrity: sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg== } + dependencies: + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + dev: false + /cross-fetch@3.1.8(encoding@0.1.13): resolution: { integrity: sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg== } dependencies: @@ -12027,7 +12075,7 @@ packages: /cross-fetch@4.0.0: resolution: { integrity: sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g== } dependencies: - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 transitivePeerDependencies: - encoding dev: false @@ -12569,6 +12617,16 @@ packages: dependencies: ms: 2.0.0 + /debug@3.2.7: + resolution: { integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== } + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.3 + /debug@3.2.7(supports-color@5.5.0): resolution: { integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== } peerDependencies: @@ -12579,6 +12637,7 @@ packages: dependencies: ms: 2.1.3 supports-color: 5.5.0 + dev: true /debug@3.2.7(supports-color@8.1.1): resolution: { integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== } @@ -12592,6 +12651,17 @@ packages: supports-color: 8.1.1 dev: true + /debug@4.3.4: + resolution: { integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== } + engines: { node: '>=6.0' } + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + /debug@4.3.4(supports-color@5.5.0): resolution: { integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== } engines: { node: '>=6.0' } @@ -12603,6 +12673,7 @@ packages: dependencies: ms: 2.1.2 supports-color: 5.5.0 + dev: false /debug@4.3.4(supports-color@8.1.1): resolution: { integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== } @@ -13274,7 +13345,7 @@ packages: resolution: { integrity: sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q== } dependencies: '@socket.io/component-emitter': 3.1.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 engine.io-parser: 5.2.2 ws: 8.11.0 xmlhttprequest-ssl: 2.0.0 @@ -13300,7 +13371,7 @@ packages: base64id: 2.0.0 cookie: 0.4.2 cors: 2.8.5 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 engine.io-parser: 5.2.2 ws: 8.11.0 transitivePeerDependencies: @@ -13686,10 +13757,45 @@ packages: - supports-color dev: true + /eslint-config-react-app@7.0.1(@babel/plugin-syntax-flow@7.23.3)(@babel/plugin-transform-react-jsx@7.23.4)(eslint@8.57.0)(typescript@4.9.5): + resolution: { integrity: sha512-K6rNzvkIeHaTd8m/QEh1Zko0KI7BACWkkneSs6s9cKZC/J27X3eZR6Upt1jkmZ/4FK+XUOPPxMEN7+lbUXfSlA== } + engines: { node: '>=14.0.0' } + peerDependencies: + eslint: ^8.0.0 + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + '@babel/core': 7.24.0 + '@babel/eslint-parser': 7.23.10(@babel/core@7.24.0)(eslint@8.57.0) + '@rushstack/eslint-patch': 1.7.2 + '@typescript-eslint/eslint-plugin': 5.62.0(@typescript-eslint/parser@5.62.0)(eslint@8.57.0)(typescript@4.9.5) + '@typescript-eslint/parser': 5.62.0(eslint@8.57.0)(typescript@4.9.5) + babel-preset-react-app: 10.0.1 + confusing-browser-globals: 1.0.11 + eslint: 8.57.0 + eslint-plugin-flowtype: 8.0.3(@babel/plugin-syntax-flow@7.23.3)(@babel/plugin-transform-react-jsx@7.23.4)(eslint@8.57.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.62.0)(eslint@8.57.0) + eslint-plugin-jest: 25.7.0(@typescript-eslint/eslint-plugin@5.62.0)(eslint@8.57.0)(typescript@4.9.5) + eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0) + eslint-plugin-react: 7.34.0(eslint@8.57.0) + eslint-plugin-react-hooks: 4.6.0(eslint@8.57.0) + eslint-plugin-testing-library: 5.11.1(eslint@8.57.0)(typescript@4.9.5) + typescript: 4.9.5 + transitivePeerDependencies: + - '@babel/plugin-syntax-flow' + - '@babel/plugin-transform-react-jsx' + - eslint-import-resolver-typescript + - eslint-import-resolver-webpack + - jest + - supports-color + dev: true + /eslint-import-resolver-node@0.3.9: resolution: { integrity: sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g== } dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7 is-core-module: 2.13.1 resolve: 1.22.8 transitivePeerDependencies: @@ -13718,7 +13824,7 @@ packages: optional: true dependencies: '@typescript-eslint/parser': 5.62.0(eslint@8.57.0)(typescript@4.9.5) - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 transitivePeerDependencies: @@ -13755,7 +13861,7 @@ packages: array.prototype.findlastindex: 1.2.4 array.prototype.flat: 1.3.2 array.prototype.flatmap: 1.3.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7 doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 @@ -13797,6 +13903,27 @@ packages: - typescript dev: true + /eslint-plugin-jest@25.7.0(@typescript-eslint/eslint-plugin@5.62.0)(eslint@8.57.0)(typescript@4.9.5): + resolution: { integrity: sha512-PWLUEXeeF7C9QGKqvdSbzLOiLTx+bno7/HC9eefePfEb257QFHg7ye3dh80AZVkaa/RQsBB1Q/ORQvg2X7F0NQ== } + engines: { node: ^12.13.0 || ^14.15.0 || >=16.0.0 } + peerDependencies: + '@typescript-eslint/eslint-plugin': ^4.0.0 || ^5.0.0 + eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 + jest: '*' + peerDependenciesMeta: + '@typescript-eslint/eslint-plugin': + optional: true + jest: + optional: true + dependencies: + '@typescript-eslint/eslint-plugin': 5.62.0(@typescript-eslint/parser@5.62.0)(eslint@8.57.0)(typescript@4.9.5) + '@typescript-eslint/experimental-utils': 5.62.0(eslint@8.57.0)(typescript@4.9.5) + eslint: 8.57.0 + transitivePeerDependencies: + - supports-color + - typescript + dev: true + /eslint-plugin-jsx-a11y@6.8.0(eslint@8.57.0): resolution: { integrity: sha512-Hdh937BS3KdwwbBaKd5+PLCOmYY6U4f2h9Z2ktwtNKvIdIEu137rjYbcb9ApSbVJfWxANNuiKTD/9tOKjK9qOA== } engines: { node: '>=4.0' } @@ -13976,7 +14103,7 @@ packages: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -14449,7 +14576,7 @@ packages: engines: { node: '>= 10.17.0' } hasBin: true dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -14608,7 +14735,7 @@ packages: /fbjs@3.0.5: resolution: { integrity: sha512-ztsSx77JBtkuMrEypfhgc3cI0+0h+svqeie7xHbh1k/IKdcydnvadp/mUaGgjAOXQmQSxsqgaRhS3q9fy+1kxg== } dependencies: - cross-fetch: 3.1.8(encoding@0.1.13) + cross-fetch: 3.1.8 fbjs-css-vars: 1.0.2 loose-envify: 1.4.0 object-assign: 4.1.1 @@ -14920,14 +15047,14 @@ packages: resolution: { integrity: sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw== } dev: true - /flowise-embed-react@1.0.2(flowise-embed@1.2.0)(react-dom@18.2.0)(react@18.2.0)(sass@1.71.1)(typescript@4.9.5): + /flowise-embed-react@1.0.2(flowise-embed@1.2.1)(react-dom@18.2.0)(react@18.2.0)(sass@1.71.1)(typescript@4.9.5): resolution: { integrity: sha512-M6rDofJWTWI9rtZN7G3oTlqAcQaHoF/IUIoW1YitHsjKly24awq5sky+0Wfkcg4VfoXz3SiLHMZ/XOF4PDuvqA== } peerDependencies: flowise-embed: '*' react: 18.x dependencies: '@ladle/react': 2.5.1(react-dom@18.2.0)(react@18.2.0)(sass@1.71.1)(typescript@4.9.5) - flowise-embed: 1.2.0 + flowise-embed: 1.2.1 react: 18.2.0 transitivePeerDependencies: - '@types/node' @@ -14942,8 +15069,8 @@ packages: - typescript dev: false - /flowise-embed@1.2.0: - resolution: { integrity: sha512-WGVH/ix6fvb+bOvp8ugxgC9raQvF7HuhqTDasFmltkrDhDySXeyOiZOj5eUs5nOkk9WT4ZaDznjSeBGDXOiShQ== } + /flowise-embed@1.2.1: + resolution: { integrity: sha512-sn8An0dseo398RXx/yzQ3lC//+FpY5o/094yaWZs3NmYy6am+duCZJ18lhcYF5A4oCucrIoZCcCz5mxViYtznw== } dependencies: '@babel/core': 7.24.0 '@ts-stack/markdown': 1.5.0 @@ -14998,6 +15125,15 @@ packages: resolution: { integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== } dev: false + /follow-redirects@1.15.5: + resolution: { integrity: sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw== } + engines: { node: '>=4.0' } + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + /follow-redirects@1.15.5(debug@4.3.4): resolution: { integrity: sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw== } engines: { node: '>=4.0' } @@ -15007,7 +15143,8 @@ packages: debug: optional: true dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 + dev: true /for-each@0.3.3: resolution: { integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw== } @@ -15350,7 +15487,7 @@ packages: extend: 3.0.2 https-proxy-agent: 5.0.1 is-stream: 2.0.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 transitivePeerDependencies: - encoding - supports-color @@ -15363,7 +15500,7 @@ packages: extend: 3.0.2 https-proxy-agent: 7.0.4 is-stream: 2.0.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 transitivePeerDependencies: - encoding - supports-color @@ -15471,7 +15608,7 @@ packages: dependencies: basic-ftp: 5.0.5 data-uri-to-buffer: 6.0.2 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 fs-extra: 11.2.0 transitivePeerDependencies: - supports-color @@ -15839,7 +15976,7 @@ packages: fast-text-encoding: 1.0.6 google-auth-library: 8.9.0 is-stream-ended: 0.1.4 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 object-hash: 3.0.0 proto3-json-serializer: 1.1.1 protobufjs: 7.2.4 @@ -15897,7 +16034,7 @@ packages: graphql: 14 - 16 dependencies: '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.1) - cross-fetch: 3.1.8(encoding@0.1.13) + cross-fetch: 3.1.8 extract-files: 9.0.0 form-data: 3.0.1 graphql: 16.8.1 @@ -15930,7 +16067,7 @@ packages: digest-fetch: 1.3.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 web-streams-polyfill: 3.3.3 transitivePeerDependencies: - encoding @@ -16497,7 +16634,7 @@ packages: engines: { node: '>=8.0.0' } dependencies: content-type: 1.0.5 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 is-retry-allowed: 1.2.0 is-stream: 2.0.1 parse-json: 4.0.0 @@ -16540,7 +16677,7 @@ packages: dependencies: '@tootallnate/once': 1.1.2 agent-base: 6.0.2 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 transitivePeerDependencies: - supports-color @@ -16550,7 +16687,7 @@ packages: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 transitivePeerDependencies: - supports-color @@ -16559,7 +16696,7 @@ packages: engines: { node: '>= 14' } dependencies: agent-base: 7.1.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 transitivePeerDependencies: - supports-color dev: false @@ -16588,7 +16725,7 @@ packages: engines: { node: '>=8.0.0' } dependencies: eventemitter3: 4.0.7 - follow-redirects: 1.15.5(debug@4.3.4) + follow-redirects: 1.15.5 requires-port: 1.0.0 transitivePeerDependencies: - debug @@ -16620,7 +16757,7 @@ packages: engines: { node: '>= 6' } dependencies: agent-base: 6.0.2 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 transitivePeerDependencies: - supports-color @@ -16629,7 +16766,7 @@ packages: engines: { node: '>= 14' } dependencies: agent-base: 7.1.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 transitivePeerDependencies: - supports-color dev: false @@ -16873,7 +17010,7 @@ packages: dependencies: '@ioredis/commands': 1.2.0 cluster-key-slot: 1.1.2 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 denque: 2.1.0 lodash.defaults: 4.2.0 lodash.isarguments: 3.1.0 @@ -17560,7 +17697,7 @@ packages: /isomorphic-fetch@3.0.0: resolution: { integrity: sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA== } dependencies: - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 whatwg-fetch: 3.6.20 transitivePeerDependencies: - encoding @@ -17601,7 +17738,7 @@ packages: resolution: { integrity: sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw== } engines: { node: '>=10' } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: @@ -18080,7 +18217,7 @@ packages: '@babel/core': 7.24.0 '@babel/generator': 7.23.6 '@babel/plugin-syntax-typescript': 7.23.3(@babel/core@7.24.0) - '@babel/traverse': 7.24.0(supports-color@5.5.0) + '@babel/traverse': 7.24.0 '@babel/types': 7.24.0 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 @@ -18874,7 +19011,7 @@ packages: '@supabase/supabase-js': 2.39.8 apify-client: 2.9.3 assemblyai: 4.3.2 - axios: 1.6.2(debug@4.3.4) + axios: 1.6.2 binary-extensions: 2.2.0 cheerio: 1.0.0-rc.12 chromadb: 1.8.1(@google/generative-ai@0.1.3)(cohere-ai@6.2.2)(openai@4.28.4) @@ -19179,7 +19316,7 @@ packages: dependencies: chalk: 5.3.0 commander: 11.0.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 execa: 7.2.0 lilconfig: 2.1.0 listr2: 6.6.1 @@ -20299,6 +20436,27 @@ packages: resolution: { integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== } engines: { node: '>= 0.6' } + /mem-fs-editor@9.7.0: + resolution: { integrity: sha512-ReB3YD24GNykmu4WeUL/FDIQtkoyGB6zfJv60yfCo3QjKeimNcTqv2FT83bP0ccs6uu+sm5zyoBlspAzigmsdg== } + engines: { node: '>=12.10.0' } + peerDependencies: + mem-fs: ^2.1.0 + peerDependenciesMeta: + mem-fs: + optional: true + dependencies: + binaryextensions: 4.19.0 + commondir: 1.0.1 + deep-extend: 0.6.0 + ejs: 3.1.9 + globby: 11.1.0 + isbinaryfile: 5.0.2 + minimatch: 7.4.6 + multimatch: 5.0.0 + normalize-path: 3.0.0 + textextensions: 5.16.0 + dev: true + /mem-fs-editor@9.7.0(mem-fs@2.3.0): resolution: { integrity: sha512-ReB3YD24GNykmu4WeUL/FDIQtkoyGB6zfJv60yfCo3QjKeimNcTqv2FT83bP0ccs6uu+sm5zyoBlspAzigmsdg== } engines: { node: '>=12.10.0' } @@ -20656,7 +20814,7 @@ packages: /micromark@2.11.4: resolution: { integrity: sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA== } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 parse-entities: 2.0.0 transitivePeerDependencies: - supports-color @@ -20666,7 +20824,7 @@ packages: resolution: { integrity: sha512-uD66tJj54JLYq0De10AhWycZWGQNUvDI55xPgk2sQM5kn1JYlhbCMTtEeT27+vAhW2FBQxLlOmS3pmA7/2z4aA== } dependencies: '@types/debug': 4.1.12 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 decode-named-character-reference: 1.0.2 micromark-core-commonmark: 1.1.0 micromark-factory-space: 1.1.0 @@ -21248,6 +21406,17 @@ packages: resolution: { integrity: sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw== } dev: false + /node-fetch@2.7.0: + resolution: { integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== } + engines: { node: 4.x || >=6.0.0 } + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + dependencies: + whatwg-url: 5.0.0 + /node-fetch@2.7.0(encoding@0.1.13): resolution: { integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== } engines: { node: 4.x || >=6.0.0 } @@ -21259,6 +21428,7 @@ packages: dependencies: encoding: 0.1.13 whatwg-url: 5.0.0 + dev: false /node-forge@1.3.1: resolution: { integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== } @@ -21459,7 +21629,7 @@ packages: engines: { node: '>=12' } dependencies: markdown-table: 2.0.0 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 transitivePeerDependencies: - encoding dev: false @@ -21861,7 +22031,7 @@ packages: async-retry: 1.3.3 aws-sdk: 2.1575.0 concurrently: 7.6.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 find-yarn-workspace-root: 2.0.0 fs-extra: 8.1.0 github-slugger: 1.5.0 @@ -21997,7 +22167,7 @@ packages: digest-fetch: 1.3.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.7.0 web-streams-polyfill: 3.3.3 transitivePeerDependencies: - encoding @@ -22182,7 +22352,7 @@ packages: resolution: { integrity: sha512-UJKdSzgd3KOnXXAtqN5+/eeHcvTn1hBkesEmElVgvO/NAYcxAvmjzIGmnNd3Tb/gRAvMBdNRFD4qAWdHxY6QXg== } engines: { node: '>=12.10.0' } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 p-queue: 6.6.2 transitivePeerDependencies: - supports-color @@ -22199,7 +22369,7 @@ packages: dependencies: '@tootallnate/quickjs-emscripten': 0.23.0 agent-base: 7.1.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 get-uri: 6.0.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.4 @@ -22586,7 +22756,7 @@ packages: resolution: { integrity: sha512-v6ZJ/efsBpGrGGknjtq9J/oC8tZWq0KWL5vQrk2GlzLEQPUDB1ex+13Rmidl1neNN358Jn9EHZw5y07FFtaC7A== } engines: { node: '>=6.8.1' } dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7 node-ensure: 0.0.0 transitivePeerDependencies: - supports-color @@ -22596,6 +22766,8 @@ packages: resolution: { integrity: sha512-Un1yLbSlk/zfwrltgguskExIioXZlFSFwsyXU0cnBorLywbTbcdzmJJEebh+U2cFCtR7y8nDs5lPHAe7ldxjZg== } engines: { node: '>=18.12.1', npm: '>=8.19.2' } hasBin: true + dependencies: + '@xmldom/xmldom': 0.8.10 dev: false bundledDependencies: - '@xmldom/xmldom' @@ -23649,7 +23821,7 @@ packages: resolution: { integrity: sha512-JB+ei0LkwE+rKHyW5z79Nd1jUaGxU6TvkfjFqY9vQaHxU5aU8dRl0UUaEmZdZbHwjp3WmXCBQQRNyimwbNQfCw== } engines: { node: '>=15.0.0' } dependencies: - axios: 1.6.2(debug@4.3.4) + axios: 1.6.2 rusha: 0.8.14 transitivePeerDependencies: - debug @@ -24015,7 +24187,7 @@ packages: engines: { node: '>= 14' } dependencies: agent-base: 7.1.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.4 lru-cache: 7.18.3 @@ -24093,7 +24265,7 @@ packages: '@puppeteer/browsers': 1.4.6(typescript@4.9.5) chromium-bidi: 0.4.16(devtools-protocol@0.0.1147663) cross-fetch: 4.0.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 devtools-protocol: 0.0.1147663 typescript: 4.9.5 ws: 8.13.0 @@ -25540,7 +25712,7 @@ packages: resolution: { integrity: sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ== } engines: { node: '>=12' } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 extend: 3.0.2 transitivePeerDependencies: - supports-color @@ -26283,7 +26455,7 @@ packages: /socket.io-adapter@2.5.4: resolution: { integrity: sha512-wDNHGXGewWAjQPt3pyeYBtpWSq9cLE5UW1ZUPL/2eGK9jtse/FpXib7epSTsz0Q0m+6sg6Y4KtcFTlah1bdOVg== } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 ws: 8.11.0 transitivePeerDependencies: - bufferutil @@ -26296,7 +26468,7 @@ packages: engines: { node: '>=10.0.0' } dependencies: '@socket.io/component-emitter': 3.1.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 engine.io-client: 6.5.3 socket.io-parser: 4.2.4 transitivePeerDependencies: @@ -26310,7 +26482,7 @@ packages: engines: { node: '>=10.0.0' } dependencies: '@socket.io/component-emitter': 3.1.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 transitivePeerDependencies: - supports-color dev: false @@ -26322,7 +26494,7 @@ packages: accepts: 1.3.8 base64id: 2.0.0 cors: 2.8.5 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 engine.io: 6.5.4 socket.io-adapter: 2.5.4 socket.io-parser: 4.2.4 @@ -26345,7 +26517,7 @@ packages: engines: { node: '>= 10' } dependencies: agent-base: 6.0.2 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 socks: 2.8.1 transitivePeerDependencies: - supports-color @@ -26355,7 +26527,7 @@ packages: engines: { node: '>= 10' } dependencies: agent-base: 6.0.2 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 socks: 2.8.1 transitivePeerDependencies: - supports-color @@ -26366,7 +26538,7 @@ packages: engines: { node: '>= 14' } dependencies: agent-base: 7.1.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 socks: 2.8.1 transitivePeerDependencies: - supports-color @@ -26531,7 +26703,7 @@ packages: /spdy-transport@3.0.0: resolution: { integrity: sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw== } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 detect-node: 2.1.0 hpack.js: 2.1.6 obuf: 1.1.2 @@ -26545,7 +26717,7 @@ packages: resolution: { integrity: sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA== } engines: { node: '>=6.0.0' } dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 handle-thing: 2.0.1 http-deceiver: 1.2.7 select-hose: 2.0.0 @@ -26702,7 +26874,7 @@ packages: arg: 5.0.2 bluebird: 3.7.2 check-more-types: 2.24.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 execa: 5.1.1 lazy-ass: 1.6.0 ps-tree: 1.2.0 @@ -27831,7 +28003,7 @@ packages: engines: { node: ^14.17.0 || ^16.13.0 || >=18.0.0 } dependencies: '@tufjs/models': 1.0.4 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 make-fetch-happen: 11.1.1 transitivePeerDependencies: - supports-color @@ -28085,7 +28257,7 @@ packages: chalk: 4.1.2 cli-highlight: 2.1.11 dayjs: 1.11.10 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 dotenv: 16.4.5 glob: 10.3.10 ioredis: 5.3.2 @@ -28167,7 +28339,7 @@ packages: chalk: 4.1.2 cli-highlight: 2.1.11 dayjs: 1.11.10 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 dotenv: 16.4.5 glob: 10.3.10 mkdirp: 2.1.6 @@ -28958,7 +29130,7 @@ packages: workbox-build: ^7.0.0 workbox-window: ^7.0.0 dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 fast-glob: 3.3.2 pretty-bytes: 6.1.1 vite: 5.1.6(sass@1.71.1) @@ -28985,7 +29157,7 @@ packages: vite: optional: true dependencies: - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 globrex: 0.1.2 tsconfck: 3.0.3(typescript@4.9.5) vite: 4.5.2(sass@1.71.1) @@ -29168,6 +29340,17 @@ packages: - graphql dev: false + /weaviate-ts-client@2.1.1(graphql@16.8.1): + resolution: { integrity: sha512-d8yc2KnIEIV1beHAU8mhrElT3BoROoXGDsLlqFX8QGx3G+gOiPTRMc7SLy4F17+LvaUaTD0XkHvWX++4iehnsg== } + engines: { node: '>=16.0.0' } + dependencies: + graphql-request: 5.2.0(graphql@16.8.1) + uuid: 9.0.1 + transitivePeerDependencies: + - encoding + - graphql + dev: false + /web-namespaces@2.0.1: resolution: { integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ== } dev: false @@ -29735,7 +29918,6 @@ packages: /workbox-google-analytics@7.0.0: resolution: { integrity: sha512-MEYM1JTn/qiC3DbpvP2BVhyIH+dV/5BjHk756u9VbwuAhu0QHyKscTnisQuz21lfRpOwiS9z4XdqeVAKol0bzg== } - deprecated: It is not compatible with newer versions of GA starting with v4, as long as you are using GAv3 it should be ok, but the package is not longer being maintained dependencies: workbox-background-sync: 7.0.0 workbox-core: 7.0.0 @@ -30184,7 +30366,7 @@ packages: cli-table: 0.3.11 commander: 7.1.0 dateformat: 4.6.3 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 diff: 5.2.0 error: 10.4.0 escape-string-regexp: 4.0.0 @@ -30229,11 +30411,11 @@ packages: dependencies: chalk: 4.1.2 dargs: 7.0.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 execa: 5.1.1 github-username: 6.0.0 lodash: 4.17.21 - mem-fs-editor: 9.7.0(mem-fs@2.3.0) + mem-fs-editor: 9.7.0 minimist: 1.2.8 pacote: 15.2.0 read-pkg-up: 7.0.1 @@ -30307,3 +30489,7 @@ packages: /zwitch@2.0.4: resolution: { integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A== } dev: false + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false