Feature/DocumentStore (#2106)

* datasource: initial commit * datasource: datasource details and chunks * datasource: Document Store Node * more changes * Document Store - Base functionality * Document Store Loader Component * Document Store Loader Component * before merging the modularity PR * after merging the modularity PR * preview mode * initial draft PR * fixes * minor updates and fixes * preview with loader and splitter * preview with credential * show stored chunks * preview update... * edit config * save, preview and other changes * save, preview and other changes * save, process and other changes * save, process and other changes * alpha1 - for internal testing * rerouting urls * bug fix on new leader create * pagination support for chunks * delete document store * Update pnpm-lock.yaml * doc store card view * Update store files to use updated storage functions, Document Store Table View and other changes * ui changes * add expanded chunk dialog, improve ui * change throw Error to InternalError * Bug Fixes and removal of subFolder, adding of view chunks for store * lint fixes * merge changes * DocumentStoreStatus component * ui changes for doc store * add remove metadata key field, add custom document loader * add chatflows used doc store chips * add types/interfaces to DocumentStore Services * document loader list dialog title bar color change * update interfaces * Whereused Chatflow Name and Added chunkNo to retain order of created chunks. * use typeorm order chunkNo, ui changes --------- Co-authored-by: Henry <hzj94@hotmail.com> Co-authored-by: Henry Heng <henryheng@flowiseai.com>
2026-06-28 15:00:57 +03:00 · 2024-05-06 19:53:27 +05:30
parent af4e28aa91
commit 40e36d1b39
91 changed files with 38713 additions and 32791 deletions
@@ -0,0 +1,37 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddDocumentStore1711637331047 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`document_store\` (
+                \`id\` varchar(36) NOT NULL,
+                \`name\` varchar(255) NOT NULL,
+                \`description\` varchar(255),
+                \`loaders\` text,
+                \`whereUsed\` text,
+                \`status\` varchar(20) NOT NULL,
+                \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`document_store_file_chunk\` (
+                \`id\` varchar(36) NOT NULL,
+                \`docId\` varchar(36) NOT NULL,
+                \`storeId\` varchar(36) NOT NULL,
+                \`chunkNo\` INT NOT NULL,
+                \`pageContent\` text,
+                \`metadata\` text,
+                PRIMARY KEY (\`id\`),
+                KEY \`IDX_e76bae1780b77e56aab1h2asd4\` (\`docId\`),
+                KEY \`IDX_e213b811b01405a42309a6a410\` (\`storeId\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE document_store`)
+        await queryRunner.query(`DROP TABLE document_store_file_chunk`)
+    }
+}
@@ -15,6 +15,7 @@ import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntit
 import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
 import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213626553 } from './1707213626553-AddFeedback'
+import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
 import { AddLead1710832127079 } from './1710832127079-AddLead'
 import { AddLeadToChatMessage1711538023578 } from './1711538023578-AddLeadToChatMessage'

@@ -36,6 +37,7 @@ export const mysqlMigrations = [
    AddSpeechToText1706364937060,
    AddUpsertHistoryEntity1709814301358,
    AddFeedback1707213626553,
+    AddDocumentStore1711637331047,
    AddLead1710832127079,
    AddLeadToChatMessage1711538023578
 ]
@@ -0,0 +1,41 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddDocumentStore1711637331047 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS document_store (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "name" varchar NOT NULL,
+                "description" varchar,
+                "loaders" text,
+                "whereUsed" text,
+                "status" varchar NOT NULL,
+                "createdDate" timestamp NOT NULL DEFAULT now(),
+                "updatedDate" timestamp NOT NULL DEFAULT now(),
+                CONSTRAINT "PK_98495043dd774f54-9830ab78f9" PRIMARY KEY (id)
+            );`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS document_store_file_chunk (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "docId" uuid NOT NULL,
+                "chunkNo" integer NOT NULL,
+                "storeId" uuid NOT NULL,
+                "pageContent" text,
+                "metadata" text,
+                CONSTRAINT "PK_90005043dd774f54-9830ab78f9" PRIMARY KEY (id)
+            );`
+        )
+        await queryRunner.query(
+            `CREATE INDEX IF NOT EXISTS "IDX_e76bae1780b77e56aab1h2asd4" ON document_store_file_chunk USING btree (docId);`
+        )
+        await queryRunner.query(
+            `CREATE INDEX IF NOT EXISTS "IDX_e213b811b01405a42309a6a410" ON document_store_file_chunk USING btree (storeId);`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE document_store`)
+        await queryRunner.query(`DROP TABLE document_store_file_chunk`)
+    }
+}
@@ -16,6 +16,7 @@ import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
 import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213601923 } from './1707213601923-AddFeedback'
 import { FieldTypes1710497452584 } from './1710497452584-FieldTypes'
+import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
 import { AddLead1710832137905 } from './1710832137905-AddLead'
 import { AddLeadToChatMessage1711538016098 } from './1711538016098-AddLeadToChatMessage'

@@ -38,6 +39,7 @@ export const postgresMigrations = [
    AddUpsertHistoryEntity1709814301358,
    AddFeedback1707213601923,
    FieldTypes1710497452584,
+    AddDocumentStore1711637331047,
    AddLead1710832137905,
    AddLeadToChatMessage1711538016098
 ]
@@ -0,0 +1,34 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddDocumentStore1711637331047 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "document_store" (
+                "id" varchar PRIMARY KEY NOT NULL, 
+                "name" varchar NOT NULL, 
+                "description" varchar, 
+                "status" varchar NOT NULL, 
+                "loaders" text, 
+                "whereUsed" text, 
+                "updatedDate" datetime NOT NULL DEFAULT (datetime('now')),
+                "createdDate" datetime NOT NULL DEFAULT (datetime('now')));`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "document_store_file_chunk" (
+                "id" varchar PRIMARY KEY NOT NULL, 
+                "docId" varchar NOT NULL, 
+                "storeId" varchar NOT NULL, 
+                "chunkNo" INTEGER NOT NULL, 
+                "pageContent" text, 
+                "metadata" text 
+            );`
+        )
+        await queryRunner.query(`CREATE INDEX "IDX_e76bae1780b77e56aab1h2asd4" ON "document_store_file_chunk" ("docId") ;`)
+        await queryRunner.query(`CREATE INDEX "IDX_e213b811b01405a42309a6a410" ON "document_store_file_chunk" ("storeId") ;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS "document_store";`)
+        await queryRunner.query(`DROP TABLE IF EXISTS "document_store_file_chunk";`)
+    }
+}
@@ -15,6 +15,7 @@ import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntit
 import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
 import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213619308 } from './1707213619308-AddFeedback'
+import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
 import { AddLead1710832117612 } from './1710832117612-AddLead'
 import { AddLeadToChatMessage1711537986113 } from './1711537986113-AddLeadToChatMessage'

@@ -36,6 +37,7 @@ export const sqliteMigrations = [
    AddSpeechToText1706364937060,
    AddUpsertHistoryEntity1709814301358,
    AddFeedback1707213619308,
+    AddDocumentStore1711637331047,
    AddLead1710832117612,
    AddLeadToChatMessage1711537986113
 ]