mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 13:00:56 +03:00
Feature/export import stage 2 (#3063)
* add export all function * modify exportAll to reuse existing code from other services * modify routes of export-import * add exportAll function into UI * add errorhandler * add importAll Function into UI for ChatFlow * modify importAll Function to import tools * remove appServer variable * modify exportAll to exportData for new requirement in backend * chore modify type camelCase to PascalCase in exportImportService * add import export for variables, assistants, and checkboxes for UI --------- Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Ong Chung Yau
@@ -289,10 +289,60 @@ const updateAssistant = async (assistantId: string, requestBody: any): Promise<A
|
||||
}
|
||||
}
|
||||
|
||||
const importAssistants = async (newAssistants: Partial<Assistant>[]): Promise<any> => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
|
||||
// step 1 - check whether array is zero
|
||||
if (newAssistants.length == 0) return
|
||||
|
||||
// step 2 - check whether ids are duplicate in database
|
||||
let ids = '('
|
||||
let count: number = 0
|
||||
const lastCount = newAssistants.length - 1
|
||||
newAssistants.forEach((newAssistant) => {
|
||||
ids += `'${newAssistant.id}'`
|
||||
if (lastCount != count) ids += ','
|
||||
if (lastCount == count) ids += ')'
|
||||
count += 1
|
||||
})
|
||||
|
||||
const selectResponse = await appServer.AppDataSource.getRepository(Assistant)
|
||||
.createQueryBuilder('assistant')
|
||||
.select('assistant.id')
|
||||
.where(`assistant.id IN ${ids}`)
|
||||
.getMany()
|
||||
const foundIds = selectResponse.map((response) => {
|
||||
return response.id
|
||||
})
|
||||
|
||||
// step 3 - remove ids that are only duplicate
|
||||
const prepVariables: Partial<Assistant>[] = newAssistants.map((newAssistant) => {
|
||||
let id: string = ''
|
||||
if (newAssistant.id) id = newAssistant.id
|
||||
if (foundIds.includes(id)) {
|
||||
newAssistant.id = undefined
|
||||
}
|
||||
return newAssistant
|
||||
})
|
||||
|
||||
// step 4 - transactional insert array of entities
|
||||
const insertResponse = await appServer.AppDataSource.getRepository(Assistant).insert(prepVariables)
|
||||
|
||||
return insertResponse
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: variableService.importVariables - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
createAssistant,
|
||||
deleteAssistant,
|
||||
getAllAssistants,
|
||||
getAssistantById,
|
||||
updateAssistant
|
||||
updateAssistant,
|
||||
importAssistants
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { removeFolderFromStorage } from 'flowise-components'
|
||||
import { StatusCodes } from 'http-status-codes'
|
||||
import { ChatflowType, IChatFlow, IReactFlowObject } from '../../Interface'
|
||||
import { ChatflowType, IReactFlowObject } from '../../Interface'
|
||||
import { ChatFlow } from '../../database/entities/ChatFlow'
|
||||
import { ChatMessage } from '../../database/entities/ChatMessage'
|
||||
import { ChatMessageFeedback } from '../../database/entities/ChatMessageFeedback'
|
||||
@@ -103,14 +103,17 @@ const deleteChatflow = async (chatflowId: string): Promise<any> => {
|
||||
}
|
||||
}
|
||||
|
||||
const getAllChatflows = async (type?: ChatflowType): Promise<IChatFlow[]> => {
|
||||
const getAllChatflows = async (type?: ChatflowType): Promise<ChatFlow[]> => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).find()
|
||||
if (type === 'MULTIAGENT') {
|
||||
return dbResponse.filter((chatflow) => chatflow.type === type)
|
||||
return dbResponse.filter((chatflow) => chatflow.type === 'MULTIAGENT')
|
||||
} else if (type === 'CHATFLOW') {
|
||||
// fetch all chatflows that are not agentflow
|
||||
return dbResponse.filter((chatflow) => chatflow.type === 'CHATFLOW' || !chatflow.type)
|
||||
}
|
||||
return dbResponse.filter((chatflow) => chatflow.type === 'CHATFLOW' || !chatflow.type)
|
||||
return dbResponse
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
@@ -202,7 +205,7 @@ const importChatflows = async (newChatflows: Partial<ChatFlow>[]): Promise<any>
|
||||
const appServer = getRunningExpressApp()
|
||||
|
||||
// step 1 - check whether file chatflows array is zero
|
||||
if (newChatflows.length == 0) throw new Error('No chatflows in this file.')
|
||||
if (newChatflows.length == 0) return
|
||||
|
||||
// step 2 - check whether ids are duplicate in database
|
||||
let ids = '('
|
||||
@@ -232,9 +235,8 @@ const importChatflows = async (newChatflows: Partial<ChatFlow>[]): Promise<any>
|
||||
if (newChatflow.flowData) flowData = newChatflow.flowData
|
||||
if (foundIds.includes(id)) {
|
||||
newChatflow.id = undefined
|
||||
newChatflow.name += ' with new id'
|
||||
newChatflow.name += ' (1)'
|
||||
}
|
||||
newChatflow.type = 'CHATFLOW'
|
||||
newChatflow.flowData = JSON.stringify(JSON.parse(flowData))
|
||||
return newChatflow
|
||||
})
|
||||
|
||||
@@ -62,6 +62,19 @@ const getAllDocumentStores = async () => {
|
||||
}
|
||||
}
|
||||
|
||||
const getAllDocumentFileChunks = async () => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entities = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).find()
|
||||
return entities
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.getAllDocumentFileChunks - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const deleteLoaderFromDocumentStore = async (storeId: string, loaderId: string) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
@@ -1225,6 +1238,7 @@ export default {
|
||||
createDocumentStore,
|
||||
deleteLoaderFromDocumentStore,
|
||||
getAllDocumentStores,
|
||||
getAllDocumentFileChunks,
|
||||
getDocumentStoreById,
|
||||
getUsedChatflowNames,
|
||||
getDocumentStoreFileChunks,
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { StatusCodes } from 'http-status-codes'
|
||||
import { ChatFlow } from '../../database/entities/ChatFlow'
|
||||
import { Tool } from '../../database/entities/Tool'
|
||||
import { Variable } from '../../database/entities/Variable'
|
||||
import { Assistant } from '../../database/entities/Assistant'
|
||||
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
|
||||
import { getErrorMessage } from '../../errors/utils'
|
||||
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
|
||||
import chatflowService from '../chatflows'
|
||||
import toolsService from '../tools'
|
||||
import variableService from '../variables'
|
||||
import assistantService from '../assistants'
|
||||
|
||||
type ExportInput = {
|
||||
tool: boolean
|
||||
chatflow: boolean
|
||||
agentflow: boolean
|
||||
variable: boolean
|
||||
assistant: boolean
|
||||
}
|
||||
|
||||
type ExportData = {
|
||||
Tool: Tool[]
|
||||
ChatFlow: ChatFlow[]
|
||||
AgentFlow: ChatFlow[]
|
||||
Variable: Variable[]
|
||||
Assistant: Assistant[]
|
||||
}
|
||||
|
||||
const convertExportInput = (body: any): ExportInput => {
|
||||
try {
|
||||
if (!body || typeof body !== 'object') throw new Error('Invalid ExportInput object in request body')
|
||||
if (body.tool && typeof body.tool !== 'boolean') throw new Error('Invalid tool property in ExportInput object')
|
||||
if (body.chatflow && typeof body.chatflow !== 'boolean') throw new Error('Invalid chatflow property in ExportInput object')
|
||||
if (body.agentflow && typeof body.agentflow !== 'boolean') throw new Error('Invalid agentflow property in ExportInput object')
|
||||
if (body.variable && typeof body.variable !== 'boolean') throw new Error('Invalid variable property in ExportInput object')
|
||||
if (body.assistant && typeof body.assistant !== 'boolean') throw new Error('Invalid assistant property in ExportInput object')
|
||||
return body as ExportInput
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: exportImportService.convertExportInput - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const FileDefaultName = 'ExportData.json'
|
||||
const exportData = async (exportInput: ExportInput): Promise<{ FileDefaultName: string } & ExportData> => {
|
||||
try {
|
||||
// step 1 - get all Tool
|
||||
let allTool: Tool[] = []
|
||||
if (exportInput.tool === true) allTool = await toolsService.getAllTools()
|
||||
|
||||
// step 2 - get all ChatFlow
|
||||
let allChatflow: ChatFlow[] = []
|
||||
if (exportInput.chatflow === true) allChatflow = await chatflowService.getAllChatflows('CHATFLOW')
|
||||
|
||||
// step 3 - get all MultiAgent
|
||||
let allMultiAgent: ChatFlow[] = []
|
||||
if (exportInput.agentflow === true) allMultiAgent = await chatflowService.getAllChatflows('MULTIAGENT')
|
||||
|
||||
let allVars: Variable[] = []
|
||||
if (exportInput.variable === true) allVars = await variableService.getAllVariables()
|
||||
|
||||
let allAssistants: Assistant[] = []
|
||||
if (exportInput.assistant === true) allAssistants = await assistantService.getAllAssistants()
|
||||
|
||||
return {
|
||||
FileDefaultName,
|
||||
Tool: allTool,
|
||||
ChatFlow: allChatflow,
|
||||
AgentFlow: allMultiAgent,
|
||||
Variable: allVars,
|
||||
Assistant: allAssistants
|
||||
}
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: exportImportService.exportData - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const importData = async (importData: ExportData) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const queryRunner = appServer.AppDataSource.createQueryRunner()
|
||||
|
||||
try {
|
||||
queryRunner.startTransaction()
|
||||
|
||||
// step 1 - importTools
|
||||
if (importData.Tool.length > 0) await toolsService.importTools(importData.Tool)
|
||||
// step 2 - importChatflows
|
||||
if (importData.ChatFlow.length > 0) await chatflowService.importChatflows(importData.ChatFlow)
|
||||
// step 3 - importAgentlows
|
||||
if (importData.AgentFlow.length > 0) await chatflowService.importChatflows(importData.AgentFlow)
|
||||
if (importData.Variable.length > 0) await variableService.importVariables(importData.Variable)
|
||||
if (importData.Assistant.length > 0) await assistantService.importAssistants(importData.Assistant)
|
||||
queryRunner.commitTransaction()
|
||||
} catch (error) {
|
||||
queryRunner.rollbackTransaction()
|
||||
throw error
|
||||
} finally {
|
||||
queryRunner.release()
|
||||
}
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: exportImportService.importAll - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
convertExportInput,
|
||||
exportData,
|
||||
importData
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
import { StatusCodes } from 'http-status-codes'
|
||||
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
|
||||
import { Tool } from '../../database/entities/Tool'
|
||||
import { getAppVersion } from '../../utils'
|
||||
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
|
||||
import { getErrorMessage } from '../../errors/utils'
|
||||
import { getAppVersion } from '../../utils'
|
||||
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
|
||||
|
||||
const createTool = async (requestBody: any): Promise<any> => {
|
||||
try {
|
||||
@@ -35,7 +35,7 @@ const deleteTool = async (toolId: string): Promise<any> => {
|
||||
}
|
||||
}
|
||||
|
||||
const getAllTools = async (): Promise<any> => {
|
||||
const getAllTools = async (): Promise<Tool[]> => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const dbResponse = await appServer.AppDataSource.getRepository(Tool).find()
|
||||
@@ -79,10 +79,58 @@ const updateTool = async (toolId: string, toolBody: any): Promise<any> => {
|
||||
}
|
||||
}
|
||||
|
||||
const importTools = async (newTools: Partial<Tool>[]) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
|
||||
// step 1 - check whether file tools array is zero
|
||||
if (newTools.length == 0) return
|
||||
|
||||
// step 2 - check whether ids are duplicate in database
|
||||
let ids = '('
|
||||
let count: number = 0
|
||||
const lastCount = newTools.length - 1
|
||||
newTools.forEach((newTools) => {
|
||||
ids += `'${newTools.id}'`
|
||||
if (lastCount != count) ids += ','
|
||||
if (lastCount == count) ids += ')'
|
||||
count += 1
|
||||
})
|
||||
|
||||
const selectResponse = await appServer.AppDataSource.getRepository(Tool)
|
||||
.createQueryBuilder('t')
|
||||
.select('t.id')
|
||||
.where(`t.id IN ${ids}`)
|
||||
.getMany()
|
||||
const foundIds = selectResponse.map((response) => {
|
||||
return response.id
|
||||
})
|
||||
|
||||
// step 3 - remove ids that are only duplicate
|
||||
const prepTools: Partial<Tool>[] = newTools.map((newTool) => {
|
||||
let id: string = ''
|
||||
if (newTool.id) id = newTool.id
|
||||
if (foundIds.includes(id)) {
|
||||
newTool.id = undefined
|
||||
newTool.name += ' (1)'
|
||||
}
|
||||
return newTool
|
||||
})
|
||||
|
||||
// step 4 - transactional insert array of entities
|
||||
const insertResponse = await appServer.AppDataSource.getRepository(Tool).insert(prepTools)
|
||||
|
||||
return insertResponse
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Error: toolsService.importTools - ${getErrorMessage(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
createTool,
|
||||
deleteTool,
|
||||
getAllTools,
|
||||
getToolById,
|
||||
updateTool
|
||||
updateTool,
|
||||
importTools
|
||||
}
|
||||
|
||||
@@ -73,10 +73,61 @@ const updateVariable = async (variable: Variable, updatedVariable: Variable) =>
|
||||
}
|
||||
}
|
||||
|
||||
const importVariables = async (newVariables: Partial<Variable>[]): Promise<any> => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
|
||||
// step 1 - check whether array is zero
|
||||
if (newVariables.length == 0) return
|
||||
|
||||
// step 2 - check whether ids are duplicate in database
|
||||
let ids = '('
|
||||
let count: number = 0
|
||||
const lastCount = newVariables.length - 1
|
||||
newVariables.forEach((newVariable) => {
|
||||
ids += `'${newVariable.id}'`
|
||||
if (lastCount != count) ids += ','
|
||||
if (lastCount == count) ids += ')'
|
||||
count += 1
|
||||
})
|
||||
|
||||
const selectResponse = await appServer.AppDataSource.getRepository(Variable)
|
||||
.createQueryBuilder('v')
|
||||
.select('v.id')
|
||||
.where(`v.id IN ${ids}`)
|
||||
.getMany()
|
||||
const foundIds = selectResponse.map((response) => {
|
||||
return response.id
|
||||
})
|
||||
|
||||
// step 3 - remove ids that are only duplicate
|
||||
const prepVariables: Partial<Variable>[] = newVariables.map((newVariable) => {
|
||||
let id: string = ''
|
||||
if (newVariable.id) id = newVariable.id
|
||||
if (foundIds.includes(id)) {
|
||||
newVariable.id = undefined
|
||||
newVariable.name += ' (1)'
|
||||
}
|
||||
return newVariable
|
||||
})
|
||||
|
||||
// step 4 - transactional insert array of entities
|
||||
const insertResponse = await appServer.AppDataSource.getRepository(Variable).insert(prepVariables)
|
||||
|
||||
return insertResponse
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: variableService.importVariables - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
createVariable,
|
||||
deleteVariable,
|
||||
getAllVariables,
|
||||
getVariableById,
|
||||
updateVariable
|
||||
updateVariable,
|
||||
importVariables
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user