farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feat: Support Google Cloud Storage (#4061)

Browse Source 
* support google cloud storage

* update example and docs for supporting google cloud storage

* recover the indent of pnpm-lock-yaml

* populate the logs to google logging

* normalize gcs storage paths

---------

Co-authored-by: Ilango <rajagopalilango@gmail.com>
Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
allen
2025-04-14 07:52:54 -07:00
committed by GitHub
parent d53b1b657f
commit c318fc57e9
11 changed files with 529 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/src/storageUtils.ts
+110
View File
@@ -8,6 +8,7 @@ import {
S3Client,
S3ClientConfig
} from '@aws-sdk/client-s3'
import { Storage } from '@google-cloud/storage'
import { Readable } from 'node:stream'
import { getUserHome } from './utils'
import sanitize from 'sanitize-filename'
@@ -34,6 +35,25 @@ export const addBase64FilesToStorage = async (fileBase64: string, chatflowid: st
})
await s3Client.send(putObjCmd)
fileNames.push(sanitizedFilename)
return 'FILE-STORAGE::' + JSON.stringify(fileNames)
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
const splitDataURI = fileBase64.split(',')
const filename = splitDataURI.pop()?.split(':')[1] ?? ''
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
const mime = splitDataURI[0].split(':')[1].split(';')[0]
const sanitizedFilename = _sanitizeFilename(filename)
const normalizedChatflowid = chatflowid.replace(/\\/g, '/')
const normalizedFilename = sanitizedFilename.replace(/\\/g, '/')
const filePath = `${normalizedChatflowid}/${normalizedFilename}`
const file = bucket.file(filePath)
await new Promise<void>((resolve, reject) => {
file.createWriteStream({ contentType: mime, metadata: { contentEncoding: 'base64' } })
.on('error', (err) => reject(err))
.on('finish', () => resolve())
.end(bf)
})
fileNames.push(sanitizedFilename)
return 'FILE-STORAGE::' + JSON.stringify(fileNames)
} else {
@@ -76,6 +96,20 @@ export const addArrayFilesToStorage = async (mime: string, bf: Buffer, fileName:
await s3Client.send(putObjCmd)
fileNames.push(sanitizedFilename)
return 'FILE-STORAGE::' + JSON.stringify(fileNames)
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
const normalizedPaths = paths.map((p) => p.replace(/\\/g, '/'))
const normalizedFilename = sanitizedFilename.replace(/\\/g, '/')
const filePath = [...normalizedPaths, normalizedFilename].join('/')
const file = bucket.file(filePath)
await new Promise<void>((resolve, reject) => {
file.createWriteStream()
.on('error', (err) => reject(err))
.on('finish', () => resolve())
.end(bf)
})
fileNames.push(sanitizedFilename)
return 'FILE-STORAGE::' + JSON.stringify(fileNames)
} else {
const dir = path.join(getStoragePath(), ...paths.map(_sanitizeFilename))
if (!fs.existsSync(dir)) {
@@ -109,6 +143,19 @@ export const addSingleFileToStorage = async (mime: string, bf: Buffer, fileName:
})
await s3Client.send(putObjCmd)
return 'FILE-STORAGE::' + sanitizedFilename
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
const normalizedPaths = paths.map((p) => p.replace(/\\/g, '/'))
const normalizedFilename = sanitizedFilename.replace(/\\/g, '/')
const filePath = [...normalizedPaths, normalizedFilename].join('/')
const file = bucket.file(filePath)
await new Promise<void>((resolve, reject) => {
file.createWriteStream({ contentType: mime, metadata: { contentEncoding: 'base64' } })
.on('error', (err) => reject(err))
.on('finish', () => resolve())
.end(bf)
})
return 'FILE-STORAGE::' + sanitizedFilename
} else {
const dir = path.join(getStoragePath(), ...paths.map(_sanitizeFilename))
if (!fs.existsSync(dir)) {
@@ -146,6 +193,11 @@ export const getFileFromUpload = async (filePath: string): Promise<Buffer> => {
// @ts-ignore
const buffer = Buffer.concat(response.Body.toArray())
return buffer
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
const file = bucket.file(filePath)
const [buffer] = await file.download()
return buffer
} else {
return fs.readFileSync(filePath)
}
@@ -179,6 +231,14 @@ export const getFileFromStorage = async (file: string, ...paths: string[]): Prom
// @ts-ignore
const buffer = Buffer.concat(response.Body.toArray())
return buffer
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
const normalizedPaths = paths.map((p) => p.replace(/\\/g, '/'))
const normalizedFilename = sanitizedFilename.replace(/\\/g, '/')
const filePath = [...normalizedPaths, normalizedFilename].join('/')
const file = bucket.file(filePath)
const [buffer] = await file.download()
return buffer
} else {
const fileInStorage = path.join(getStoragePath(), ...paths.map(_sanitizeFilename), sanitizedFilename)
return fs.readFileSync(fileInStorage)
@@ -208,6 +268,10 @@ export const removeFilesFromStorage = async (...paths: string[]) => {
Key = Key.substring(1)
}
await _deleteS3Folder(Key)
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
const normalizedPath = paths.map((p) => p.replace(/\\/g, '/')).join('/')
await bucket.deleteFiles({ prefix: `${normalizedPath}/` })
} else {
const directory = path.join(getStoragePath(), ...paths.map(_sanitizeFilename))
_deleteLocalFolderRecursive(directory)
@@ -223,6 +287,9 @@ export const removeSpecificFileFromUpload = async (filePath: string) => {
Key = Key.substring(1)
}
await _deleteS3Folder(Key)
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
await bucket.file(filePath).delete()
} else {
fs.unlinkSync(filePath)
}
@@ -237,6 +304,15 @@ export const removeSpecificFileFromStorage = async (...paths: string[]) => {
Key = Key.substring(1)
}
await _deleteS3Folder(Key)
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
const fileName = paths.pop()
if (fileName) {
const sanitizedFilename = _sanitizeFilename(fileName)
paths.push(sanitizedFilename)
}
const normalizedPath = paths.map((p) => p.replace(/\\/g, '/')).join('/')
await bucket.file(normalizedPath).delete()
} else {
const fileName = paths.pop()
if (fileName) {
@@ -257,6 +333,10 @@ export const removeFolderFromStorage = async (...paths: string[]) => {
Key = Key.substring(1)
}
await _deleteS3Folder(Key)
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
const normalizedPath = paths.map((p) => p.replace(/\\/g, '/')).join('/')
await bucket.deleteFiles({ prefix: `${normalizedPath}/` })
} else {
const directory = path.join(getStoragePath(), ...paths.map(_sanitizeFilename))
_deleteLocalFolderRecursive(directory, true)
@@ -355,6 +435,14 @@ export const streamStorageFile = async (
const blob = await body.transformToByteArray()
return Buffer.from(blob)
}
} else if (storageType === 'gcs') {
const { bucket } = getGcsClient()
const normalizedChatflowId = chatflowId.replace(/\\/g, '/')
const normalizedChatId = chatId.replace(/\\/g, '/')
const normalizedFilename = sanitizedFilename.replace(/\\/g, '/')
const filePath = `${normalizedChatflowId}/${normalizedChatId}/${normalizedFilename}`
const [buffer] = await bucket.file(filePath).download()
return buffer
} else {
const filePath = path.join(getStoragePath(), chatflowId, chatId, sanitizedFilename)
//raise error if file path is not absolute
@@ -372,6 +460,28 @@ export const streamStorageFile = async (
}
}
export const getGcsClient = () => {
const pathToGcsCredential = process.env.GOOGLE_CLOUD_STORAGE_CREDENTIAL
const projectId = process.env.GOOGLE_CLOUD_STORAGE_PROJ_ID
const bucketName = process.env.GOOGLE_CLOUD_STORAGE_BUCKET_NAME
if (!pathToGcsCredential) {
throw new Error('GOOGLE_CLOUD_STORAGE_CREDENTIAL env variable is required')
}
if (!bucketName) {
throw new Error('GOOGLE_CLOUD_STORAGE_BUCKET_NAME env variable is required')
}
const storageConfig = {
keyFilename: pathToGcsCredential,
...(projectId ? { projectId } : {})
}
const storage = new Storage(storageConfig)
const bucket = storage.bucket(bucketName)
return { storage, bucket }
}
export const getS3Config = () => {
const accessKeyId = process.env.S3_STORAGE_ACCESS_KEY_ID
const secretAccessKey = process.env.S3_STORAGE_SECRET_ACCESS_KEY