Feature/DocumentStore (#2106)

* datasource: initial commit

* datasource: datasource details and chunks

* datasource: Document Store Node

* more changes

* Document Store - Base functionality

* Document Store Loader Component

* Document Store Loader Component

* before merging the modularity PR

* after merging the modularity PR

* preview mode

* initial draft PR

* fixes

* minor updates and  fixes

* preview with loader and splitter

* preview with credential

* show stored chunks

* preview update...

* edit config

* save, preview and other changes

* save, preview and other changes

* save, process and other changes

* save, process and other changes

* alpha1 - for internal testing

* rerouting urls

* bug fix on new leader create

* pagination support for chunks

* delete document store

* Update pnpm-lock.yaml

* doc store card view

* Update store files to use updated storage functions, Document Store Table View and other changes

* ui changes

* add expanded chunk dialog, improve ui

* change throw Error to InternalError

* Bug Fixes and removal of subFolder, adding of view chunks for store

* lint fixes

* merge changes

* DocumentStoreStatus component

* ui changes for doc store

* add remove metadata key field, add custom document loader

* add chatflows used doc store chips

* add types/interfaces to DocumentStore Services

* document loader list dialog title bar color change

* update interfaces

* Whereused Chatflow Name and Added chunkNo to retain order of created chunks.

* use typeorm order chunkNo, ui changes

---------

Co-authored-by: Henry <hzj94@hotmail.com>
Co-authored-by: Henry Heng <henryheng@flowiseai.com>

packages/server/src/database/migrations/sqlite/1711637331047-AddDocumentStore.ts

@@ -0,0 +1,34 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddDocumentStore1711637331047 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "document_store" (
+                "id" varchar PRIMARY KEY NOT NULL, 
+                "name" varchar NOT NULL, 
+                "description" varchar, 
+                "status" varchar NOT NULL, 
+                "loaders" text, 
+                "whereUsed" text, 
+                "updatedDate" datetime NOT NULL DEFAULT (datetime('now')),
+                "createdDate" datetime NOT NULL DEFAULT (datetime('now')));`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "document_store_file_chunk" (
+                "id" varchar PRIMARY KEY NOT NULL, 
+                "docId" varchar NOT NULL, 
+                "storeId" varchar NOT NULL, 
+                "chunkNo" INTEGER NOT NULL, 
+                "pageContent" text, 
+                "metadata" text 
+            );`
+        )
+        await queryRunner.query(`CREATE INDEX "IDX_e76bae1780b77e56aab1h2asd4" ON "document_store_file_chunk" ("docId") ;`)
+        await queryRunner.query(`CREATE INDEX "IDX_e213b811b01405a42309a6a410" ON "document_store_file_chunk" ("storeId") ;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS "document_store";`)
+        await queryRunner.query(`DROP TABLE IF EXISTS "document_store_file_chunk";`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -15,6 +15,7 @@ import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntit
 import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
 import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddFeedback1707213619308 } from './1707213619308-AddFeedback'
+import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
 import { AddLead1710832117612 } from './1710832117612-AddLead'
 import { AddLeadToChatMessage1711537986113 } from './1711537986113-AddLeadToChatMessage'
 
@@ -36,6 +37,7 @@ export const sqliteMigrations = [
     AddSpeechToText1706364937060,
     AddUpsertHistoryEntity1709814301358,
     AddFeedback1707213619308,
+    AddDocumentStore1711637331047,
     AddLead1710832117612,
     AddLeadToChatMessage1711537986113
 ]