Flowise/packages/server/src/Interface.DocumentStore.ts

import { DocumentStore } from './database/entities/DocumentStore'

export enum DocumentStoreStatus {
    EMPTY_SYNC = 'EMPTY',
    SYNC = 'SYNC',
    SYNCING = 'SYNCING',
    STALE = 'STALE',
    NEW = 'NEW',
    UPSERTING = 'UPSERTING',
    UPSERTED = 'UPSERTED'
}

export interface IDocumentStore {
    id: string
    name: string
    description: string
    loaders: string // JSON string
    whereUsed: string // JSON string
    updatedDate: Date
    createdDate: Date
    status: DocumentStoreStatus
    vectorStoreConfig: string | null // JSON string
    embeddingConfig: string | null // JSON string
    recordManagerConfig: string | null // JSON string
}

export interface IDocumentStoreFileChunk {
    id: string
    chunkNo: number
    docId: string
    storeId: string
    pageContent: string
    metadata: string
}

export interface IDocumentStoreFileChunkPagedResponse {
    chunks: IDocumentStoreFileChunk[]
    count: number
    file?: IDocumentStoreLoader
    currentPage: number
    storeName: string
    description: string
}

export interface IDocumentStoreLoader {
    id: string
    loaderId: string
    loaderName: string
    loaderConfig: any // JSON string
    splitterId: string
    splitterName: string
    splitterConfig: any // JSON string
    totalChunks: number
    totalChars: number
    status: DocumentStoreStatus
    storeId?: string
    files?: IDocumentStoreLoaderFile[]
    source?: string
    credential?: string
}

export interface IDocumentStoreLoaderForPreview extends IDocumentStoreLoader {
    rehydrated: boolean
    preview: boolean
    previewChunkCount: number
}

export interface IDocumentStoreLoaderFile {
    id: string
    name: string
    mimePrefix: string
    size: number
    status: DocumentStoreStatus
    uploaded: Date
}

export interface IDocumentStoreWhereUsed {
    id: string
    name: string
}

export class DocumentStoreDTO {
    id: string
    name: string
    description: string
    files: IDocumentStoreLoaderFile[]
    whereUsed: IDocumentStoreWhereUsed[]
    createdDate: Date
    updatedDate: Date
    status: DocumentStoreStatus
    chunkOverlap: number
    splitter: string
    totalChunks: number
    totalChars: number
    chunkSize: number
    loaders: IDocumentStoreLoader[]
    vectorStoreConfig: any
    embeddingConfig: any
    recordManagerConfig: any

    constructor() {}

    static fromEntity(entity: DocumentStore): DocumentStoreDTO {
        let documentStoreDTO = new DocumentStoreDTO()

        Object.assign(documentStoreDTO, entity)
        documentStoreDTO.id = entity.id
        documentStoreDTO.name = entity.name
        documentStoreDTO.description = entity.description
        documentStoreDTO.status = entity.status
        documentStoreDTO.totalChars = 0
        documentStoreDTO.totalChunks = 0

        if (entity.whereUsed) {
            documentStoreDTO.whereUsed = JSON.parse(entity.whereUsed)
        } else {
            documentStoreDTO.whereUsed = []
        }

        if (entity.vectorStoreConfig) {
            documentStoreDTO.vectorStoreConfig = JSON.parse(entity.vectorStoreConfig)
        }
        if (entity.embeddingConfig) {
            documentStoreDTO.embeddingConfig = JSON.parse(entity.embeddingConfig)
        }
        if (entity.recordManagerConfig) {
            documentStoreDTO.recordManagerConfig = JSON.parse(entity.recordManagerConfig)
        }

        if (entity.loaders) {
            documentStoreDTO.loaders = JSON.parse(entity.loaders)
            documentStoreDTO.loaders.map((loader) => {
                documentStoreDTO.totalChars += loader.totalChars
                documentStoreDTO.totalChunks += loader.totalChunks
                switch (loader.loaderId) {
                    case 'pdfFile':
                        loader.source = loader.loaderConfig.pdfFile.replace('FILE-STORAGE::', '')
                        break
                    case 'apiLoader':
                        loader.source = loader.loaderConfig.url + ' (' + loader.loaderConfig.method + ')'
                        break
                    case 'cheerioWebScraper':
                        loader.source = loader.loaderConfig.url
                        break
                    case 'playwrightWebScraper':
                        loader.source = loader.loaderConfig.url
                        break
                    case 'puppeteerWebScraper':
                        loader.source = loader.loaderConfig.url
                        break
                    case 'jsonFile':
                        loader.source = loader.loaderConfig.jsonFile.replace('FILE-STORAGE::', '')
                        break
                    case 'docxFile':
                        loader.source = loader.loaderConfig.docxFile.replace('FILE-STORAGE::', '')
                        break
                    case 'textFile':
                        loader.source = loader.loaderConfig.txtFile.replace('FILE-STORAGE::', '')
                        break
                    case 'unstructuredFileLoader':
                        loader.source = loader.loaderConfig.filePath
                        break
                    default:
                        loader.source = 'None'
                        break
                }
                if (loader.status !== 'SYNC') {
                    documentStoreDTO.status = DocumentStoreStatus.STALE
                }
            })
        }

        return documentStoreDTO
    }

    static fromEntities(entities: DocumentStore[]): DocumentStoreDTO[] {
        return entities.map((entity) => this.fromEntity(entity))
    }

    static toEntity(body: any): DocumentStore {
        const docStore = new DocumentStore()
        Object.assign(docStore, body)
        docStore.loaders = '[]'
        docStore.whereUsed = '[]'
        // when a new document store is created, it is empty and in sync
        docStore.status = DocumentStoreStatus.EMPTY_SYNC
        return docStore
    }
}