Feature/s3 storage (#2226)

* centralizing file writing....

* allowing s3 as storage option

* allowing s3 as storage option

* update s3 storage

---------

Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/src/utils/buildChatflow.ts

@@ -1,8 +1,7 @@
 import { Request } from 'express'
-import { IFileUpload, getStoragePath, convertSpeechToText, ICommonObject } from 'flowise-components'
+import { IFileUpload, convertSpeechToText, ICommonObject, addFileToStorage } from 'flowise-components'
 import { StatusCodes } from 'http-status-codes'
 import { IncomingInput, IMessage, INodeData, IReactFlowObject, IReactFlowNode, IDepthQueue, chatType, IChatMessage } from '../Interface'
-import path from 'path'
 import { InternalFlowiseError } from '../errors/internalFlowiseError'
 import { ChatFlow } from '../database/entities/ChatFlow'
 import { Server } from 'socket.io'
@@ -69,17 +68,12 @@ export const utilBuildChatflow = async (req: Request, socketIO?: Server, isInter
 
                 if ((upload.type === 'file' || upload.type === 'audio') && upload.data) {
                     const filename = upload.name
-                    const dir = path.join(getStoragePath(), chatflowid, chatId)
-                    if (!fs.existsSync(dir)) {
-                        fs.mkdirSync(dir, { recursive: true })
-                    }
-                    const filePath = path.join(dir, filename)
                     const splitDataURI = upload.data.split(',')
                     const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-                    fs.writeFileSync(filePath, bf)
-
-                    // Omit upload.data since we don't store the content in database
+                    const mime = splitDataURI[0].split(':')[1].split(';')[0]
+                    await addFileToStorage(mime, bf, filename, chatflowid, chatId)
                     upload.type = 'stored-file'
+                    // Omit upload.data since we don't store the content in database
                     fileUploads[i] = omit(upload, ['data'])
                 }
 

packages/server/src/utils/fileRepository.ts

@@ -1,8 +1,6 @@
 import { ChatFlow } from '../database/entities/ChatFlow'
-import path from 'path'
-import { getStoragePath } from 'flowise-components'
-import fs from 'fs'
 import { IReactFlowObject } from '../Interface'
+import { addBase64FilesToStorage } from 'flowise-components'
 
 export const containsBase64File = (chatflow: ChatFlow) => {
     const parsedFlowData: IReactFlowObject = JSON.parse(chatflow.flowData)
@@ -48,23 +46,7 @@ export const containsBase64File = (chatflow: ChatFlow) => {
     return found
 }
 
-function addFileToStorage(file: string, chatflowid: string, fileNames: string[]) {
-    const dir = path.join(getStoragePath(), chatflowid)
-    if (!fs.existsSync(dir)) {
-        fs.mkdirSync(dir, { recursive: true })
-    }
-
-    const splitDataURI = file.split(',')
-    const filename = splitDataURI.pop()?.split(':')[1] ?? ''
-    const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-
-    const filePath = path.join(dir, filename)
-    fs.writeFileSync(filePath, bf)
-    fileNames.push(filename)
-    return 'FILE-STORAGE::' + JSON.stringify(fileNames)
-}
-
-export const updateFlowDataWithFilePaths = (chatflowid: string, flowData: string) => {
+export const updateFlowDataWithFilePaths = async (chatflowid: string, flowData: string) => {
     try {
         const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
         const re = new RegExp('^data.*;base64', 'i')
@@ -93,14 +75,14 @@ export const updateFlowDataWithFilePaths = (chatflowid: string, flowData: string
                             for (let j = 0; j < files.length; j++) {
                                 const file = files[j]
                                 if (re.test(file)) {
-                                    node.data.inputs[key] = addFileToStorage(file, chatflowid, fileNames)
+                                    node.data.inputs[key] = await addBase64FilesToStorage(file, chatflowid, fileNames)
                                 }
                             }
                         } catch (e) {
                             continue
                         }
                     } else if (re.test(input)) {
-                        node.data.inputs[key] = addFileToStorage(input, chatflowid, fileNames)
+                        node.data.inputs[key] = await addBase64FilesToStorage(input, chatflowid, fileNames)
                     }
                 }
             }

packages/server/src/utils/index.ts

@@ -1324,34 +1324,6 @@ export const getAllValuesFromJson = (obj: any): any[] => {
     return values
 }
 
-/**
- * Delete file & folder recursively
- * @param {string} directory
- */
-export const deleteFolderRecursive = (directory: string) => {
-    if (fs.existsSync(directory)) {
-        fs.readdir(directory, (error, files) => {
-            if (error) throw new Error('Could not read directory')
-
-            files.forEach((file) => {
-                const file_path = path.join(directory, file)
-
-                fs.stat(file_path, (error, stat) => {
-                    if (error) throw new Error('File do not exist')
-
-                    if (!stat.isDirectory()) {
-                        fs.unlink(file_path, (error) => {
-                            if (error) throw new Error('Could not delete file')
-                        })
-                    } else {
-                        deleteFolderRecursive(file_path)
-                    }
-                })
-            })
-        })
-    }
-}
-
 /**
  * Get only essential flow data items for telemetry
  * @param {IReactFlowNode[]} nodes