Feature/DocumentStore (#2106)

* datasource: initial commit * datasource: datasource details and chunks * datasource: Document Store Node * more changes * Document Store - Base functionality * Document Store Loader Component * Document Store Loader Component * before merging the modularity PR * after merging the modularity PR * preview mode * initial draft PR * fixes * minor updates and fixes * preview with loader and splitter * preview with credential * show stored chunks * preview update... * edit config * save, preview and other changes * save, preview and other changes * save, process and other changes * save, process and other changes * alpha1 - for internal testing * rerouting urls * bug fix on new leader create * pagination support for chunks * delete document store * Update pnpm-lock.yaml * doc store card view * Update store files to use updated storage functions, Document Store Table View and other changes * ui changes * add expanded chunk dialog, improve ui * change throw Error to InternalError * Bug Fixes and removal of subFolder, adding of view chunks for store * lint fixes * merge changes * DocumentStoreStatus component * ui changes for doc store * add remove metadata key field, add custom document loader * add chatflows used doc store chips * add types/interfaces to DocumentStore Services * document loader list dialog title bar color change * update interfaces * Whereused Chatflow Name and Added chunkNo to retain order of created chunks. * use typeorm order chunkNo, ui changes --------- Co-authored-by: Henry <hzj94@hotmail.com> Co-authored-by: Henry Heng <henryheng@flowiseai.com>
2026-06-28 17:01:00 +03:00 · 2024-05-06 19:53:27 +05:30
parent af4e28aa91
commit 40e36d1b39
91 changed files with 38713 additions and 32791 deletions
@@ -0,0 +1,165 @@
+import { DocumentStore } from './database/entities/DocumentStore'
+
+export enum DocumentStoreStatus {
+    EMPTY_SYNC = 'EMPTY',
+    SYNC = 'SYNC',
+    SYNCING = 'SYNCING',
+    STALE = 'STALE',
+    NEW = 'NEW'
+}
+
+export interface IDocumentStore {
+    id: string
+    name: string
+    description: string
+    loaders: string // JSON string
+    whereUsed: string // JSON string
+    updatedDate: Date
+    createdDate: Date
+    status: DocumentStoreStatus
+}
+
+export interface IDocumentStoreFileChunk {
+    id: string
+    chunkNo: number
+    docId: string
+    storeId: string
+    pageContent: string
+    metadata: string
+}
+
+export interface IDocumentStoreFileChunkPagedResponse {
+    chunks: IDocumentStoreFileChunk[]
+    count: number
+    file?: IDocumentStoreLoader
+    currentPage: number
+    storeName: string
+    description: string
+}
+
+export interface IDocumentStoreLoader {
+    id: string
+    loaderId: string
+    loaderName: string
+    loaderConfig: any // JSON string
+    splitterId: string
+    splitterName: string
+    splitterConfig: any // JSON string
+    totalChunks: number
+    totalChars: number
+    status: DocumentStoreStatus
+    storeId?: string
+    files?: IDocumentStoreLoaderFile[]
+    source?: string
+    credential?: string
+}
+
+export interface IDocumentStoreLoaderForPreview extends IDocumentStoreLoader {
+    rehydrated: boolean
+    preview: boolean
+    previewChunkCount: number
+}
+
+export interface IDocumentStoreLoaderFile {
+    id: string
+    name: string
+    mimePrefix: string
+    size: number
+    status: DocumentStoreStatus
+    uploaded: Date
+}
+
+export interface IDocumentStoreWhereUsed {
+    id: string
+    name: string
+}
+
+export class DocumentStoreDTO {
+    id: string
+    name: string
+    description: string
+    files: IDocumentStoreLoaderFile[]
+    whereUsed: IDocumentStoreWhereUsed[]
+    createdDate: Date
+    updatedDate: Date
+    status: DocumentStoreStatus
+    chunkOverlap: number
+    splitter: string
+    totalChunks: number
+    totalChars: number
+    chunkSize: number
+    loaders: IDocumentStoreLoader[]
+
+    constructor() {}
+
+    static fromEntity(entity: DocumentStore): DocumentStoreDTO {
+        let documentStoreDTO = new DocumentStoreDTO()
+
+        Object.assign(documentStoreDTO, entity)
+        documentStoreDTO.id = entity.id
+        documentStoreDTO.name = entity.name
+        documentStoreDTO.description = entity.description
+        documentStoreDTO.status = entity.status
+        documentStoreDTO.totalChars = 0
+        documentStoreDTO.totalChunks = 0
+
+        if (entity.whereUsed) {
+            documentStoreDTO.whereUsed = JSON.parse(entity.whereUsed)
+        } else {
+            documentStoreDTO.whereUsed = []
+        }
+
+        if (entity.loaders) {
+            documentStoreDTO.loaders = JSON.parse(entity.loaders)
+            documentStoreDTO.loaders.map((loader) => {
+                documentStoreDTO.totalChars += loader.totalChars
+                documentStoreDTO.totalChunks += loader.totalChunks
+                switch (loader.loaderId) {
+                    case 'pdfFile':
+                        loader.source = loader.loaderConfig.pdfFile.replace('FILE-STORAGE::', '')
+                        break
+                    case 'apiLoader':
+                        loader.source = loader.loaderConfig.url + ' (' + loader.loaderConfig.method + ')'
+                        break
+                    case 'cheerioWebScraper':
+                        loader.source = loader.loaderConfig.url
+                        break
+                    case 'jsonFile':
+                        loader.source = loader.loaderConfig.jsonFile.replace('FILE-STORAGE::', '')
+                        break
+                    case 'docxFile':
+                        loader.source = loader.loaderConfig.docxFile.replace('FILE-STORAGE::', '')
+                        break
+                    case 'textFile':
+                        loader.source = loader.loaderConfig.txtFile.replace('FILE-STORAGE::', '')
+                        break
+                    case 'unstructuredFileLoader':
+                        loader.source = loader.loaderConfig.filePath
+                        break
+                    default:
+                        loader.source = 'None'
+                        break
+                }
+                if (loader.status !== 'SYNC') {
+                    documentStoreDTO.status = DocumentStoreStatus.STALE
+                }
+            })
+        }
+
+        return documentStoreDTO
+    }
+
+    static fromEntities(entities: DocumentStore[]): DocumentStoreDTO[] {
+        return entities.map((entity) => this.fromEntity(entity))
+    }
+
+    static toEntity(body: any): DocumentStore {
+        const docStore = new DocumentStore()
+        Object.assign(docStore, body)
+        docStore.loaders = '[]'
+        docStore.whereUsed = '[]'
+        // when a new document store is created, it is empty and in sync
+        docStore.status = DocumentStoreStatus.EMPTY_SYNC
+        return docStore
+    }
+}