Feature/Indexing (#1802)

* indexing

* fix for multiple files upsert

* fix default Postgres port

* fix SQLite node description

* add MySQLRecordManager node

* fix MySQL unique index

* add upsert history

* update jsx ui

* lint-fix

* update dialog details

* update llamaindex pinecone

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>

packages/server/src/database/migrations/mysql/1709814301358-AddUpsertHistoryEntity.ts

@@ -0,0 +1,21 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUpsertHistoryEntity1709814301358 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`upsert_history\` (
+                \`id\` varchar(36) NOT NULL,
+                \`chatflowid\` varchar(255) NOT NULL,
+                \`result\` text NOT NULL,
+                \`flowData\` text NOT NULL,
+                \`date\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`)
+                KEY \`IDX_a0b59fd66f6e48d2b198123cb6\` (\`chatflowid\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE upsert_history`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -13,6 +13,7 @@ import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-Ad
 import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
 import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
 import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
+import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213626553 } from './1707213626553-AddFeedback'
 
 export const mysqlMigrations = [
@@ -31,5 +32,6 @@ export const mysqlMigrations = [
     AddFileUploadsToChatMessage1701788586491,
     AddVariableEntity1699325775451,
     AddSpeechToText1706364937060,
+    AddUpsertHistoryEntity1709814301358,
     AddFeedback1707213626553
 ]

packages/server/src/database/migrations/postgres/1709814301358-AddUpsertHistoryEntity.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUpsertHistoryEntity1709814301358 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS upsert_history (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "chatflowid" varchar NOT NULL,
+                "result" text NOT NULL,
+                "flowData" text NOT NULL,
+                "date" timestamp NOT NULL DEFAULT now(),
+                CONSTRAINT "PK_37327b22b6e246319bd5eeb0e88" PRIMARY KEY (id)
+            );`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE upsert_history`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -13,6 +13,7 @@ import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-Ad
 import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
 import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
 import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
+import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213601923 } from './1707213601923-AddFeedback'
 import { FieldTypes1710497452584 } from './1710497452584-FieldTypes'
 
@@ -32,6 +33,7 @@ export const postgresMigrations = [
     AddFileUploadsToChatMessage1701788586491,
     AddVariableEntity1699325775451,
     AddSpeechToText1706364937060,
+    AddUpsertHistoryEntity1709814301358,
     AddFeedback1707213601923,
     FieldTypes1710497452584
 ]

packages/server/src/database/migrations/sqlite/1709814301358-AddUpsertHistoryEntity.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUpsertHistoryEntity1709814301358 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "upsert_history" ("id" varchar PRIMARY KEY NOT NULL, "chatflowid" varchar NOT NULL, "result" text NOT NULL, "flowData" text NOT NULL, "date" datetime NOT NULL DEFAULT (datetime('now')));`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE upsert_history`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -13,6 +13,7 @@ import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-Ad
 import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
 import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
 import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
+import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213619308 } from './1707213619308-AddFeedback'
 
 export const sqliteMigrations = [
@@ -31,5 +32,6 @@ export const sqliteMigrations = [
     AddFileUploadsToChatMessage1701788586491,
     AddVariableEntity1699325775451,
     AddSpeechToText1706364937060,
+    AddUpsertHistoryEntity1709814301358,
     AddFeedback1707213619308
 ]