Feat/add multer s3 (#3854)

* add multer s3

* add types multer s3

* update multer s3 implementation

* Revert "update multer s3 implementation"

This reverts commit 9a25bf57a93f77074bdb084921f0591dbe0b44e1.

* update storage utils

* update multer storage type on routes

* revert getMulterStorage

* revert getMulterStorage

* update getmulterstorage

* update getmulterstorage

* update getmulterstorage

packages/server/src/utils/buildChatflow.ts

@@ -9,7 +9,9 @@ import {
     mapMimeTypeToInputField,
     mapExtToInputField,
     generateFollowUpPrompts,
-    IServerSideEventStreamer
+    IServerSideEventStreamer,
+    getFileFromUpload,
+    removeSpecificFileFromUpload
 } from 'flowise-components'
 import { StatusCodes } from 'http-status-codes'
 import {
@@ -49,7 +51,6 @@ import { validateChatflowAPIKey } from './validateKey'
 import { databaseEntities } from '.'
 import { v4 as uuidv4 } from 'uuid'
 import { omit } from 'lodash'
-import * as fs from 'fs'
 import logger from './logger'
 import { utilAddChatMessage } from './addChatMesage'
 import { buildAgentGraph } from './buildAgentGraph'
@@ -162,7 +163,7 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
             const overrideConfig: ICommonObject = { ...req.body }
             const fileNames: string[] = []
             for (const file of files) {
-                const fileBuffer = fs.readFileSync(file.path)
+                const fileBuffer = await getFileFromUpload(file.path ?? file.key)
                 // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
                 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
                 const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid)
@@ -195,7 +196,7 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
                     overrideConfig[fileInputField] = storagePath
                 }
 
-                fs.unlinkSync(file.path)
+                await removeSpecificFileFromUpload(file.path ?? file.key)
             }
             if (overrideConfig.vars && typeof overrideConfig.vars === 'string') {
                 overrideConfig.vars = JSON.parse(overrideConfig.vars)

packages/server/src/utils/createAttachment.ts

@@ -1,7 +1,13 @@
 import { Request } from 'express'
 import * as path from 'path'
-import * as fs from 'fs'
-import { addArrayFilesToStorage, IDocument, mapExtToInputField, mapMimeTypeToInputField } from 'flowise-components'
+import {
+    addArrayFilesToStorage,
+    getFileFromUpload,
+    IDocument,
+    mapExtToInputField,
+    mapMimeTypeToInputField,
+    removeSpecificFileFromUpload
+} from 'flowise-components'
 import { getRunningExpressApp } from './getRunningExpressApp'
 import { getErrorMessage } from '../errors/utils'
 
@@ -41,7 +47,7 @@ export const createFileAttachment = async (req: Request) => {
     if (files.length) {
         const isBase64 = req.body.base64
         for (const file of files) {
-            const fileBuffer = fs.readFileSync(file.path)
+            const fileBuffer = await getFileFromUpload(file.path ?? file.key)
             const fileNames: string[] = []
 
             // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
@@ -63,7 +69,7 @@ export const createFileAttachment = async (req: Request) => {
                 fileInputField = fileInputFieldFromExt
             }
 
-            fs.unlinkSync(file.path)
+            await removeSpecificFileFromUpload(file.path ?? file.key)
 
             try {
                 const nodeData = {

packages/server/src/utils/index.ts

@@ -36,11 +36,13 @@ import {
     IDatabaseEntity,
     IMessage,
     FlowiseMemory,
-    IFileUpload
+    IFileUpload,
+    getS3Config
 } from 'flowise-components'
 import { randomBytes } from 'crypto'
 import { AES, enc } from 'crypto-js'
-
+import multer from 'multer'
+import multerS3 from 'multer-s3'
 import { ChatFlow } from '../database/entities/ChatFlow'
 import { ChatMessage } from '../database/entities/ChatMessage'
 import { Credential } from '../database/entities/Credential'
@@ -1779,3 +1781,38 @@ export const getUploadPath = (): string => {
         ? path.join(process.env.BLOB_STORAGE_PATH, 'uploads')
         : path.join(getUserHome(), '.flowise', 'uploads')
 }
+
+const getOrgId = () => {
+    const settingsContent = fs.readFileSync(getUserSettingsFilePath(), 'utf8')
+    try {
+        const settings = JSON.parse(settingsContent)
+        return settings.instanceId
+    } catch (error) {
+        return ''
+    }
+}
+
+export const getMulterStorage = () => {
+    const storageType = process.env.STORAGE_TYPE ? process.env.STORAGE_TYPE : 'local'
+
+    if (storageType === 's3') {
+        const s3Client = getS3Config().s3Client
+        const Bucket = getS3Config().Bucket
+
+        const upload = multer({
+            storage: multerS3({
+                s3: s3Client,
+                bucket: Bucket,
+                metadata: function (req, file, cb) {
+                    cb(null, { fieldName: file.fieldname, originalName: file.originalname, orgId: getOrgId() })
+                },
+                key: function (req, file, cb) {
+                    cb(null, `${getOrgId()}/${Date.now().toString()}`)
+                }
+            })
+        })
+        return upload
+    } else {
+        return multer({ dest: getUploadPath() })
+    }
+}

packages/server/src/utils/upsertVector.ts

@@ -1,8 +1,15 @@
 import { Request } from 'express'
-import * as fs from 'fs'
 import * as path from 'path'
 import { cloneDeep, omit } from 'lodash'
-import { ICommonObject, IMessage, addArrayFilesToStorage, mapMimeTypeToInputField, mapExtToInputField } from 'flowise-components'
+import {
+    ICommonObject,
+    IMessage,
+    addArrayFilesToStorage,
+    mapMimeTypeToInputField,
+    mapExtToInputField,
+    getFileFromUpload,
+    removeSpecificFileFromUpload
+} from 'flowise-components'
 import logger from '../utils/logger'
 import {
     buildFlow,
@@ -57,7 +64,7 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
             const overrideConfig: ICommonObject = { ...req.body }
             for (const file of files) {
                 const fileNames: string[] = []
-                const fileBuffer = fs.readFileSync(file.path)
+                const fileBuffer = await getFileFromUpload(file.path ?? file.key)
                 // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
                 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
                 const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid)
@@ -90,7 +97,7 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
                     overrideConfig[fileInputField] = storagePath
                 }
 
-                fs.unlinkSync(file.path)
+                await removeSpecificFileFromUpload(file.path ?? file.key)
             }
             if (overrideConfig.vars && typeof overrideConfig.vars === 'string') {
                 overrideConfig.vars = JSON.parse(overrideConfig.vars)