Feature/export import stage 2 (#3063)

* add export all function

* modify exportAll to reuse existing code from other services

* modify routes of export-import

* add exportAll function into UI

* add errorhandler

* add importAll Function into UI for ChatFlow

* modify importAll Function to import tools

* remove appServer variable

* modify exportAll to exportData for new requirement in backend

* chore modify type camelCase to PascalCase in exportImportService

* add import export for variables, assistants, and checkboxes for UI

---------

Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/src/Interface.ts

@@ -1,5 +1,4 @@
-import { IAction } from 'flowise-components'
-import { ICommonObject, IFileUpload, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components'
+import { IAction, ICommonObject, IFileUpload, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components'
 
 export type MessageType = 'apiMessage' | 'userMessage'
 

packages/server/src/controllers/export-import/index.ts

@@ -0,0 +1,26 @@
+import { NextFunction, Request, Response } from 'express'
+import exportImportService from '../../services/export-import'
+
+const exportData = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const apiResponse = await exportImportService.exportData(exportImportService.convertExportInput(req.body))
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const importData = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const importData = req.body
+        await exportImportService.importData(importData)
+        return res.json({ message: 'success' })
+    } catch (error) {
+        next(error)
+    }
+}
+
+export default {
+    exportData,
+    importData
+}

packages/server/src/routes/export-import/index.ts

@@ -0,0 +1,9 @@
+import express from 'express'
+import exportImportController from '../../controllers/export-import'
+const router = express.Router()
+
+router.post('/export', exportImportController.exportData)
+
+router.post('/import', exportImportController.importData)
+
+export default router

packages/server/src/routes/index.ts

@@ -9,6 +9,7 @@ import componentsCredentialsRouter from './components-credentials'
 import componentsCredentialsIconRouter from './components-credentials-icon'
 import credentialsRouter from './credentials'
 import documentStoreRouter from './documentstore'
+import exportImportRouter from './export-import'
 import feedbackRouter from './feedback'
 import fetchLinksRouter from './fetch-links'
 import flowConfigRouter from './flow-config'
@@ -53,6 +54,7 @@ router.use('/components-credentials-icon', componentsCredentialsIconRouter)
 router.use('/chatflows-uploads', chatflowsUploadsRouter)
 router.use('/credentials', credentialsRouter)
 router.use('/document-store', documentStoreRouter)
+router.use('/export-import', exportImportRouter)
 router.use('/feedback', feedbackRouter)
 router.use('/fetch-links', fetchLinksRouter)
 router.use('/flow-config', flowConfigRouter)

packages/server/src/services/assistants/index.ts

@@ -289,10 +289,60 @@ const updateAssistant = async (assistantId: string, requestBody: any): Promise<A
     }
 }
 
+const importAssistants = async (newAssistants: Partial<Assistant>[]): Promise<any> => {
+    try {
+        const appServer = getRunningExpressApp()
+
+        // step 1 - check whether array is zero
+        if (newAssistants.length == 0) return
+
+        // step 2 - check whether ids are duplicate in database
+        let ids = '('
+        let count: number = 0
+        const lastCount = newAssistants.length - 1
+        newAssistants.forEach((newAssistant) => {
+            ids += `'${newAssistant.id}'`
+            if (lastCount != count) ids += ','
+            if (lastCount == count) ids += ')'
+            count += 1
+        })
+
+        const selectResponse = await appServer.AppDataSource.getRepository(Assistant)
+            .createQueryBuilder('assistant')
+            .select('assistant.id')
+            .where(`assistant.id IN ${ids}`)
+            .getMany()
+        const foundIds = selectResponse.map((response) => {
+            return response.id
+        })
+
+        // step 3 - remove ids that are only duplicate
+        const prepVariables: Partial<Assistant>[] = newAssistants.map((newAssistant) => {
+            let id: string = ''
+            if (newAssistant.id) id = newAssistant.id
+            if (foundIds.includes(id)) {
+                newAssistant.id = undefined
+            }
+            return newAssistant
+        })
+
+        // step 4 - transactional insert array of entities
+        const insertResponse = await appServer.AppDataSource.getRepository(Assistant).insert(prepVariables)
+
+        return insertResponse
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: variableService.importVariables - ${getErrorMessage(error)}`
+        )
+    }
+}
+
 export default {
     createAssistant,
     deleteAssistant,
     getAllAssistants,
     getAssistantById,
-    updateAssistant
+    updateAssistant,
+    importAssistants
 }

packages/server/src/services/chatflows/index.ts

@@ -1,6 +1,6 @@
 import { removeFolderFromStorage } from 'flowise-components'
 import { StatusCodes } from 'http-status-codes'
-import { ChatflowType, IChatFlow, IReactFlowObject } from '../../Interface'
+import { ChatflowType, IReactFlowObject } from '../../Interface'
 import { ChatFlow } from '../../database/entities/ChatFlow'
 import { ChatMessage } from '../../database/entities/ChatMessage'
 import { ChatMessageFeedback } from '../../database/entities/ChatMessageFeedback'
@@ -103,14 +103,17 @@ const deleteChatflow = async (chatflowId: string): Promise<any> => {
     }
 }
 
-const getAllChatflows = async (type?: ChatflowType): Promise<IChatFlow[]> => {
+const getAllChatflows = async (type?: ChatflowType): Promise<ChatFlow[]> => {
     try {
         const appServer = getRunningExpressApp()
         const dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).find()
         if (type === 'MULTIAGENT') {
-            return dbResponse.filter((chatflow) => chatflow.type === type)
+            return dbResponse.filter((chatflow) => chatflow.type === 'MULTIAGENT')
+        } else if (type === 'CHATFLOW') {
+            // fetch all chatflows that are not agentflow
+            return dbResponse.filter((chatflow) => chatflow.type === 'CHATFLOW' || !chatflow.type)
         }
-        return dbResponse.filter((chatflow) => chatflow.type === 'CHATFLOW' || !chatflow.type)
+        return dbResponse
     } catch (error) {
         throw new InternalFlowiseError(
             StatusCodes.INTERNAL_SERVER_ERROR,
@@ -202,7 +205,7 @@ const importChatflows = async (newChatflows: Partial<ChatFlow>[]): Promise<any>
         const appServer = getRunningExpressApp()
 
         // step 1 - check whether file chatflows array is zero
-        if (newChatflows.length == 0) throw new Error('No chatflows in this file.')
+        if (newChatflows.length == 0) return
 
         // step 2 - check whether ids are duplicate in database
         let ids = '('
@@ -232,9 +235,8 @@ const importChatflows = async (newChatflows: Partial<ChatFlow>[]): Promise<any>
             if (newChatflow.flowData) flowData = newChatflow.flowData
             if (foundIds.includes(id)) {
                 newChatflow.id = undefined
-                newChatflow.name += ' with new id'
+                newChatflow.name += ' (1)'
             }
-            newChatflow.type = 'CHATFLOW'
             newChatflow.flowData = JSON.stringify(JSON.parse(flowData))
             return newChatflow
         })

packages/server/src/services/documentstore/index.ts

@@ -62,6 +62,19 @@ const getAllDocumentStores = async () => {
     }
 }
 
+const getAllDocumentFileChunks = async () => {
+    try {
+        const appServer = getRunningExpressApp()
+        const entities = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).find()
+        return entities
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.getAllDocumentFileChunks - ${getErrorMessage(error)}`
+        )
+    }
+}
+
 const deleteLoaderFromDocumentStore = async (storeId: string, loaderId: string) => {
     try {
         const appServer = getRunningExpressApp()
@@ -1225,6 +1238,7 @@ export default {
     createDocumentStore,
     deleteLoaderFromDocumentStore,
     getAllDocumentStores,
+    getAllDocumentFileChunks,
     getDocumentStoreById,
     getUsedChatflowNames,
     getDocumentStoreFileChunks,

packages/server/src/services/export-import/index.ts

@@ -0,0 +1,119 @@
+import { StatusCodes } from 'http-status-codes'
+import { ChatFlow } from '../../database/entities/ChatFlow'
+import { Tool } from '../../database/entities/Tool'
+import { Variable } from '../../database/entities/Variable'
+import { Assistant } from '../../database/entities/Assistant'
+import { InternalFlowiseError } from '../../errors/internalFlowiseError'
+import { getErrorMessage } from '../../errors/utils'
+import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
+import chatflowService from '../chatflows'
+import toolsService from '../tools'
+import variableService from '../variables'
+import assistantService from '../assistants'
+
+type ExportInput = {
+    tool: boolean
+    chatflow: boolean
+    agentflow: boolean
+    variable: boolean
+    assistant: boolean
+}
+
+type ExportData = {
+    Tool: Tool[]
+    ChatFlow: ChatFlow[]
+    AgentFlow: ChatFlow[]
+    Variable: Variable[]
+    Assistant: Assistant[]
+}
+
+const convertExportInput = (body: any): ExportInput => {
+    try {
+        if (!body || typeof body !== 'object') throw new Error('Invalid ExportInput object in request body')
+        if (body.tool && typeof body.tool !== 'boolean') throw new Error('Invalid tool property in ExportInput object')
+        if (body.chatflow && typeof body.chatflow !== 'boolean') throw new Error('Invalid chatflow property in ExportInput object')
+        if (body.agentflow && typeof body.agentflow !== 'boolean') throw new Error('Invalid agentflow property in ExportInput object')
+        if (body.variable && typeof body.variable !== 'boolean') throw new Error('Invalid variable property in ExportInput object')
+        if (body.assistant && typeof body.assistant !== 'boolean') throw new Error('Invalid assistant property in ExportInput object')
+        return body as ExportInput
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: exportImportService.convertExportInput - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const FileDefaultName = 'ExportData.json'
+const exportData = async (exportInput: ExportInput): Promise<{ FileDefaultName: string } & ExportData> => {
+    try {
+        // step 1 - get all Tool
+        let allTool: Tool[] = []
+        if (exportInput.tool === true) allTool = await toolsService.getAllTools()
+
+        // step 2 - get all ChatFlow
+        let allChatflow: ChatFlow[] = []
+        if (exportInput.chatflow === true) allChatflow = await chatflowService.getAllChatflows('CHATFLOW')
+
+        // step 3 - get all MultiAgent
+        let allMultiAgent: ChatFlow[] = []
+        if (exportInput.agentflow === true) allMultiAgent = await chatflowService.getAllChatflows('MULTIAGENT')
+
+        let allVars: Variable[] = []
+        if (exportInput.variable === true) allVars = await variableService.getAllVariables()
+
+        let allAssistants: Assistant[] = []
+        if (exportInput.assistant === true) allAssistants = await assistantService.getAllAssistants()
+
+        return {
+            FileDefaultName,
+            Tool: allTool,
+            ChatFlow: allChatflow,
+            AgentFlow: allMultiAgent,
+            Variable: allVars,
+            Assistant: allAssistants
+        }
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: exportImportService.exportData - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const importData = async (importData: ExportData) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const queryRunner = appServer.AppDataSource.createQueryRunner()
+
+        try {
+            queryRunner.startTransaction()
+
+            // step 1 - importTools
+            if (importData.Tool.length > 0) await toolsService.importTools(importData.Tool)
+            // step 2 - importChatflows
+            if (importData.ChatFlow.length > 0) await chatflowService.importChatflows(importData.ChatFlow)
+            // step 3 - importAgentlows
+            if (importData.AgentFlow.length > 0) await chatflowService.importChatflows(importData.AgentFlow)
+            if (importData.Variable.length > 0) await variableService.importVariables(importData.Variable)
+            if (importData.Assistant.length > 0) await assistantService.importAssistants(importData.Assistant)
+            queryRunner.commitTransaction()
+        } catch (error) {
+            queryRunner.rollbackTransaction()
+            throw error
+        } finally {
+            queryRunner.release()
+        }
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: exportImportService.importAll - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+export default {
+    convertExportInput,
+    exportData,
+    importData
+}

packages/server/src/services/tools/index.ts

@@ -1,9 +1,9 @@
 import { StatusCodes } from 'http-status-codes'
-import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { Tool } from '../../database/entities/Tool'
-import { getAppVersion } from '../../utils'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { getErrorMessage } from '../../errors/utils'
+import { getAppVersion } from '../../utils'
+import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 
 const createTool = async (requestBody: any): Promise<any> => {
     try {
@@ -35,7 +35,7 @@ const deleteTool = async (toolId: string): Promise<any> => {
     }
 }
 
-const getAllTools = async (): Promise<any> => {
+const getAllTools = async (): Promise<Tool[]> => {
     try {
         const appServer = getRunningExpressApp()
         const dbResponse = await appServer.AppDataSource.getRepository(Tool).find()
@@ -79,10 +79,58 @@ const updateTool = async (toolId: string, toolBody: any): Promise<any> => {
     }
 }
 
+const importTools = async (newTools: Partial<Tool>[]) => {
+    try {
+        const appServer = getRunningExpressApp()
+
+        // step 1 - check whether file tools array is zero
+        if (newTools.length == 0) return
+
+        // step 2 - check whether ids are duplicate in database
+        let ids = '('
+        let count: number = 0
+        const lastCount = newTools.length - 1
+        newTools.forEach((newTools) => {
+            ids += `'${newTools.id}'`
+            if (lastCount != count) ids += ','
+            if (lastCount == count) ids += ')'
+            count += 1
+        })
+
+        const selectResponse = await appServer.AppDataSource.getRepository(Tool)
+            .createQueryBuilder('t')
+            .select('t.id')
+            .where(`t.id IN ${ids}`)
+            .getMany()
+        const foundIds = selectResponse.map((response) => {
+            return response.id
+        })
+
+        // step 3 - remove ids that are only duplicate
+        const prepTools: Partial<Tool>[] = newTools.map((newTool) => {
+            let id: string = ''
+            if (newTool.id) id = newTool.id
+            if (foundIds.includes(id)) {
+                newTool.id = undefined
+                newTool.name += ' (1)'
+            }
+            return newTool
+        })
+
+        // step 4 - transactional insert array of entities
+        const insertResponse = await appServer.AppDataSource.getRepository(Tool).insert(prepTools)
+
+        return insertResponse
+    } catch (error) {
+        throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Error: toolsService.importTools - ${getErrorMessage(error)}`)
+    }
+}
+
 export default {
     createTool,
     deleteTool,
     getAllTools,
     getToolById,
-    updateTool
+    updateTool,
+    importTools
 }

packages/server/src/services/variables/index.ts

@@ -73,10 +73,61 @@ const updateVariable = async (variable: Variable, updatedVariable: Variable) =>
     }
 }
 
+const importVariables = async (newVariables: Partial<Variable>[]): Promise<any> => {
+    try {
+        const appServer = getRunningExpressApp()
+
+        // step 1 - check whether array is zero
+        if (newVariables.length == 0) return
+
+        // step 2 - check whether ids are duplicate in database
+        let ids = '('
+        let count: number = 0
+        const lastCount = newVariables.length - 1
+        newVariables.forEach((newVariable) => {
+            ids += `'${newVariable.id}'`
+            if (lastCount != count) ids += ','
+            if (lastCount == count) ids += ')'
+            count += 1
+        })
+
+        const selectResponse = await appServer.AppDataSource.getRepository(Variable)
+            .createQueryBuilder('v')
+            .select('v.id')
+            .where(`v.id IN ${ids}`)
+            .getMany()
+        const foundIds = selectResponse.map((response) => {
+            return response.id
+        })
+
+        // step 3 - remove ids that are only duplicate
+        const prepVariables: Partial<Variable>[] = newVariables.map((newVariable) => {
+            let id: string = ''
+            if (newVariable.id) id = newVariable.id
+            if (foundIds.includes(id)) {
+                newVariable.id = undefined
+                newVariable.name += ' (1)'
+            }
+            return newVariable
+        })
+
+        // step 4 - transactional insert array of entities
+        const insertResponse = await appServer.AppDataSource.getRepository(Variable).insert(prepVariables)
+
+        return insertResponse
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: variableService.importVariables - ${getErrorMessage(error)}`
+        )
+    }
+}
+
 export default {
     createVariable,
     deleteVariable,
     getAllVariables,
     getVariableById,
-    updateVariable
+    updateVariable,
+    importVariables
 }