Feat: Support Google Cloud Storage (#4061)

* support google cloud storage

* update example and docs for supporting google cloud storage

* recover the indent of pnpm-lock-yaml

* populate the logs to google logging

* normalize gcs storage paths

---------

Co-authored-by: Ilango <rajagopalilango@gmail.com>
Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/.env.example

@@ -54,6 +54,10 @@ PORT=3000
 # S3_STORAGE_REGION=us-west-2
 # S3_ENDPOINT_URL=<custom-s3-endpoint-url>
 # S3_FORCE_PATH_STYLE=false
+# GOOGLE_CLOUD_STORAGE_CREDENTIAL=/the/keyfilename/path
+# GOOGLE_CLOUD_STORAGE_PROJ_ID=<your-gcp-project-id>
+# GOOGLE_CLOUD_STORAGE_BUCKET_NAME=<the-bucket-name>
+# GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS=true
 
 # SHOW_COMMUNITY_NODES=true
 # DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable)

packages/server/package.json

@@ -57,6 +57,7 @@
     "license": "SEE LICENSE IN LICENSE.md",
     "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.699.0",
+        "@google-cloud/logging-winston": "^6.0.0",
         "@oclif/core": "4.0.7",
         "@opentelemetry/api": "^1.3.0",
         "@opentelemetry/auto-instrumentations-node": "^0.52.0",
@@ -95,6 +96,7 @@
         "moment": "^2.29.3",
         "moment-timezone": "^0.5.34",
         "multer": "^1.4.5-lts.1",
+        "multer-cloud-storage": "^4.0.0",
         "multer-s3": "^3.0.1",
         "mysql2": "^3.11.3",
         "flowise-nim-container-manager": "^1.0.11",

packages/server/src/commands/base.ts

@@ -49,6 +49,10 @@ export abstract class BaseCommand extends Command {
         S3_STORAGE_REGION: Flags.string(),
         S3_ENDPOINT_URL: Flags.string(),
         S3_FORCE_PATH_STYLE: Flags.string(),
+        GOOGLE_CLOUD_STORAGE_CREDENTIAL: Flags.string(),
+        GOOGLE_CLOUD_STORAGE_PROJ_ID: Flags.string(),
+        GOOGLE_CLOUD_STORAGE_BUCKET_NAME: Flags.string(),
+        GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS: Flags.string(),
         SHOW_COMMUNITY_NODES: Flags.string(),
         SECRETKEY_STORAGE_TYPE: Flags.string(),
         SECRETKEY_PATH: Flags.string(),
@@ -184,6 +188,11 @@ export abstract class BaseCommand extends Command {
         if (flags.S3_STORAGE_REGION) process.env.S3_STORAGE_REGION = flags.S3_STORAGE_REGION
         if (flags.S3_ENDPOINT_URL) process.env.S3_ENDPOINT_URL = flags.S3_ENDPOINT_URL
         if (flags.S3_FORCE_PATH_STYLE) process.env.S3_FORCE_PATH_STYLE = flags.S3_FORCE_PATH_STYLE
+        if (flags.GOOGLE_CLOUD_STORAGE_CREDENTIAL) process.env.GOOGLE_CLOUD_STORAGE_CREDENTIAL = flags.GOOGLE_CLOUD_STORAGE_CREDENTIAL
+        if (flags.GOOGLE_CLOUD_STORAGE_PROJ_ID) process.env.GOOGLE_CLOUD_STORAGE_PROJ_ID = flags.GOOGLE_CLOUD_STORAGE_PROJ_ID
+        if (flags.GOOGLE_CLOUD_STORAGE_BUCKET_NAME) process.env.GOOGLE_CLOUD_STORAGE_BUCKET_NAME = flags.GOOGLE_CLOUD_STORAGE_BUCKET_NAME
+        if (flags.GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS)
+            process.env.GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS = flags.GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS
 
         // Queue
         if (flags.MODE) process.env.MODE = flags.MODE

packages/server/src/utils/index.ts

@@ -43,6 +43,7 @@ import { randomBytes } from 'crypto'
 import { AES, enc } from 'crypto-js'
 import multer from 'multer'
 import multerS3 from 'multer-s3'
+import MulterGoogleCloudStorage from 'multer-cloud-storage'
 import { ChatFlow } from '../database/entities/ChatFlow'
 import { ChatMessage } from '../database/entities/ChatMessage'
 import { Credential } from '../database/entities/Credential'
@@ -1799,6 +1800,16 @@ export const getMulterStorage = () => {
             })
         })
         return upload
+    } else if (storageType === 'gcs') {
+        return multer({
+            storage: new MulterGoogleCloudStorage({
+                projectId: process.env.GOOGLE_CLOUD_STORAGE_PROJ_ID,
+                bucket: process.env.GOOGLE_CLOUD_STORAGE_BUCKET_NAME,
+                keyFilename: process.env.GOOGLE_CLOUD_STORAGE_CREDENTIAL,
+                uniformBucketLevelAccess: Boolean(process.env.GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS) ?? true,
+                destination: `uploads/${getOrgId()}`
+            })
+        })
     } else {
         return multer({ dest: getUploadPath() })
     }

packages/server/src/utils/logger.ts

@@ -5,6 +5,7 @@ import config from './config' // should be replaced by node-config or similar
 import { createLogger, transports, format } from 'winston'
 import { NextFunction, Request, Response } from 'express'
 import { S3ClientConfig } from '@aws-sdk/client-s3'
+import { LoggingWinston } from '@google-cloud/logging-winston'
 
 const { S3StreamLogger } = require('s3-streamlogger')
 
@@ -13,6 +14,11 @@ const { combine, timestamp, printf, errors } = format
 let s3ServerStream: any
 let s3ErrorStream: any
 let s3ServerReqStream: any
+
+let gcsServerStream: any
+let gcsErrorStream: any
+let gcsServerReqStream: any
+
 if (process.env.STORAGE_TYPE === 's3') {
     const accessKeyId = process.env.S3_STORAGE_ACCESS_KEY_ID
     const secretAccessKey = process.env.S3_STORAGE_SECRET_ACCESS_KEY
@@ -60,6 +66,29 @@ if (process.env.STORAGE_TYPE === 's3') {
     })
 }
 
+if (process.env.STORAGE_TYPE === 'gcs') {
+    const config = {
+        projectId: process.env.GOOGLE_CLOUD_STORAGE_PROJ_ID,
+        keyFilename: process.env.GOOGLE_CLOUD_STORAGE_CREDENTIAL,
+        defaultCallback: (err: any) => {
+            if (err) {
+                console.error('Error logging to GCS: ' + err)
+            }
+        }
+    }
+    gcsServerStream = new LoggingWinston({
+        ...config,
+        logName: 'server'
+    })
+    gcsErrorStream = new LoggingWinston({
+        ...config,
+        logName: 'error'
+    })
+    gcsServerReqStream = new LoggingWinston({
+        ...config,
+        logName: 'requests'
+    })
+}
 // expect the log dir be relative to the projects root
 const logDir = config.logging.dir
 
@@ -101,7 +130,8 @@ const logger = createLogger({
                       stream: s3ServerStream
                   })
               ]
-            : [])
+            : []),
+        ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsServerStream] : [])
     ],
     exceptionHandlers: [
         ...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
@@ -117,7 +147,8 @@ const logger = createLogger({
                       stream: s3ErrorStream
                   })
               ]
-            : [])
+            : []),
+        ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsErrorStream] : [])
     ],
     rejectionHandlers: [
         ...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
@@ -133,7 +164,8 @@ const logger = createLogger({
                       stream: s3ErrorStream
                   })
               ]
-            : [])
+            : []),
+        ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsErrorStream] : [])
     ]
 })
 
@@ -168,7 +200,8 @@ export function expressRequestLogger(req: Request, res: Response, next: NextFunc
                               stream: s3ServerReqStream
                           })
                       ]
-                    : [])
+                    : []),
+                ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsServerReqStream] : [])
             ]
         })