Feature/DocumentStore (#2106)

* datasource: initial commit * datasource: datasource details and chunks * datasource: Document Store Node * more changes * Document Store - Base functionality * Document Store Loader Component * Document Store Loader Component * before merging the modularity PR * after merging the modularity PR * preview mode * initial draft PR * fixes * minor updates and fixes * preview with loader and splitter * preview with credential * show stored chunks * preview update... * edit config * save, preview and other changes * save, preview and other changes * save, process and other changes * save, process and other changes * alpha1 - for internal testing * rerouting urls * bug fix on new leader create * pagination support for chunks * delete document store * Update pnpm-lock.yaml * doc store card view * Update store files to use updated storage functions, Document Store Table View and other changes * ui changes * add expanded chunk dialog, improve ui * change throw Error to InternalError * Bug Fixes and removal of subFolder, adding of view chunks for store * lint fixes * merge changes * DocumentStoreStatus component * ui changes for doc store * add remove metadata key field, add custom document loader * add chatflows used doc store chips * add types/interfaces to DocumentStore Services * document loader list dialog title bar color change * update interfaces * Whereused Chatflow Name and Added chunkNo to retain order of created chunks. * use typeorm order chunkNo, ui changes --------- Co-authored-by: Henry <hzj94@hotmail.com> Co-authored-by: Henry Heng <henryheng@flowiseai.com>
2026-06-28 17:01:00 +03:00 · 2024-05-06 19:53:27 +05:30
parent af4e28aa91
commit 40e36d1b39
91 changed files with 38713 additions and 32791 deletions
@@ -0,0 +1,263 @@
+import { NextFunction, Request, Response } from 'express'
+import { StatusCodes } from 'http-status-codes'
+import documentStoreService from '../../services/documentstore'
+import { DocumentStore } from '../../database/entities/DocumentStore'
+import { InternalFlowiseError } from '../../errors/internalFlowiseError'
+import { DocumentStoreDTO } from '../../Interface'
+
+const createDocumentStore = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.body === 'undefined') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.createDocumentStore - body not provided!`
+            )
+        }
+        const body = req.body
+        const docStore = DocumentStoreDTO.toEntity(body)
+        const apiResponse = await documentStoreService.createDocumentStore(docStore)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const getAllDocumentStores = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const apiResponse = await documentStoreService.getAllDocumentStores()
+        return res.json(DocumentStoreDTO.fromEntities(apiResponse))
+    } catch (error) {
+        next(error)
+    }
+}
+
+const deleteLoaderFromDocumentStore = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const storeId = req.params.id
+        const loaderId = req.params.loaderId
+
+        if (!storeId || !loaderId) {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.deleteLoaderFromDocumentStore - missing storeId or loaderId.`
+            )
+        }
+        const apiResponse = await documentStoreService.deleteLoaderFromDocumentStore(storeId, loaderId)
+        return res.json(DocumentStoreDTO.fromEntity(apiResponse))
+    } catch (error) {
+        next(error)
+    }
+}
+
+const getDocumentStoreById = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.id === 'undefined' || req.params.id === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.getDocumentStoreById - id not provided!`
+            )
+        }
+        const apiResponse = await documentStoreService.getDocumentStoreById(req.params.id)
+        if (apiResponse && apiResponse.whereUsed) {
+            apiResponse.whereUsed = JSON.stringify(await documentStoreService.getUsedChatflowNames(apiResponse))
+        }
+        return res.json(DocumentStoreDTO.fromEntity(apiResponse))
+    } catch (error) {
+        next(error)
+    }
+}
+
+const getDocumentStoreFileChunks = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.storeId === 'undefined' || req.params.storeId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.getDocumentStoreFileChunks - storeId not provided!`
+            )
+        }
+        if (typeof req.params.fileId === 'undefined' || req.params.fileId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.getDocumentStoreFileChunks - fileId not provided!`
+            )
+        }
+        const page = req.params.pageNo ? parseInt(req.params.pageNo) : 1
+        const apiResponse = await documentStoreService.getDocumentStoreFileChunks(req.params.storeId, req.params.fileId, page)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const deleteDocumentStoreFileChunk = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.storeId === 'undefined' || req.params.storeId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.deleteDocumentStoreFileChunk - storeId not provided!`
+            )
+        }
+        if (typeof req.params.loaderId === 'undefined' || req.params.loaderId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.deleteDocumentStoreFileChunk - loaderId not provided!`
+            )
+        }
+        if (typeof req.params.chunkId === 'undefined' || req.params.chunkId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.deleteDocumentStoreFileChunk - chunkId not provided!`
+            )
+        }
+        const apiResponse = await documentStoreService.deleteDocumentStoreFileChunk(
+            req.params.storeId,
+            req.params.loaderId,
+            req.params.chunkId
+        )
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const editDocumentStoreFileChunk = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.storeId === 'undefined' || req.params.storeId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.editDocumentStoreFileChunk - storeId not provided!`
+            )
+        }
+        if (typeof req.params.loaderId === 'undefined' || req.params.loaderId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.editDocumentStoreFileChunk - loaderId not provided!`
+            )
+        }
+        if (typeof req.params.chunkId === 'undefined' || req.params.chunkId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.editDocumentStoreFileChunk - chunkId not provided!`
+            )
+        }
+        const body = req.body
+        if (typeof body === 'undefined' || body.pageContent === 'undefined' || body.pageContent === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.editDocumentStoreFileChunk - body not provided!`
+            )
+        }
+        const apiResponse = await documentStoreService.editDocumentStoreFileChunk(
+            req.params.storeId,
+            req.params.loaderId,
+            req.params.chunkId,
+            body.pageContent
+        )
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const processFileChunks = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.body === 'undefined') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.processFileChunks - body not provided!`
+            )
+        }
+        const body = req.body
+        const apiResponse = await documentStoreService.processAndSaveChunks(body)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const updateDocumentStore = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.id === 'undefined' || req.params.id === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.updateDocumentStore - storeId not provided!`
+            )
+        }
+        if (typeof req.body === 'undefined') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.updateDocumentStore - body not provided!`
+            )
+        }
+        const store = await documentStoreService.getDocumentStoreById(req.params.id)
+        if (!store) {
+            throw new InternalFlowiseError(
+                StatusCodes.NOT_FOUND,
+                `Error: documentStoreController.updateDocumentStore - DocumentStore ${req.params.id} not found in the database`
+            )
+        }
+        const body = req.body
+        const updateDocStore = new DocumentStore()
+        Object.assign(updateDocStore, body)
+        const apiResponse = await documentStoreService.updateDocumentStore(store, updateDocStore)
+        return res.json(DocumentStoreDTO.fromEntity(apiResponse))
+    } catch (error) {
+        next(error)
+    }
+}
+
+const deleteDocumentStore = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.id === 'undefined' || req.params.id === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.deleteDocumentStore - storeId not provided!`
+            )
+        }
+        const apiResponse = await documentStoreService.deleteDocumentStore(req.params.id)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const previewFileChunks = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.body === 'undefined') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.previewFileChunks - body not provided!`
+            )
+        }
+        const body = req.body
+        body.preview = true
+        const apiResponse = await documentStoreService.previewChunks(body)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const getDocumentLoaders = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const apiResponse = await documentStoreService.getDocumentLoaders()
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+export default {
+    deleteDocumentStore,
+    createDocumentStore,
+    getAllDocumentStores,
+    deleteLoaderFromDocumentStore,
+    getDocumentStoreById,
+    getDocumentStoreFileChunks,
+    updateDocumentStore,
+    processFileChunks,
+    previewFileChunks,
+    getDocumentLoaders,
+    deleteDocumentStoreFileChunk,
+    editDocumentStoreFileChunk
+}
@@ -1,4 +1,5 @@
 import { Request, Response, NextFunction } from 'express'
+import _ from 'lodash'
 import nodesService from '../../services/nodes'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
@@ -24,6 +25,22 @@ const getNodeByName = async (req: Request, res: Response, next: NextFunction) =>
    }
 }

+const getNodesByCategory = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.name === 'undefined' || req.params.name === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: nodesController.getNodesByCategory - name not provided!`
+            )
+        }
+        const name = _.unescape(req.params.name)
+        const apiResponse = await nodesService.getAllNodesForCategory(name)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
 const getSingleNodeIcon = async (req: Request, res: Response, next: NextFunction) => {
    try {
        if (typeof req.params === 'undefined' || !req.params.name) {
@@ -77,5 +94,6 @@ export default {
    getNodeByName,
    getSingleNodeIcon,
    getSingleNodeAsyncOptions,
-    executeCustomFunction
+    executeCustomFunction,
+    getNodesByCategory
 }