Feat: Support Google Cloud Storage (#4061)

* support google cloud storage

* update example and docs for supporting google cloud storage

* recover the indent of pnpm-lock-yaml

* populate the logs to google logging

* normalize gcs storage paths

---------

Co-authored-by: Ilango <rajagopalilango@gmail.com>
Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/src/utils/index.ts

@@ -43,6 +43,7 @@ import { randomBytes } from 'crypto'
 import { AES, enc } from 'crypto-js'
 import multer from 'multer'
 import multerS3 from 'multer-s3'
+import MulterGoogleCloudStorage from 'multer-cloud-storage'
 import { ChatFlow } from '../database/entities/ChatFlow'
 import { ChatMessage } from '../database/entities/ChatMessage'
 import { Credential } from '../database/entities/Credential'
@@ -1799,6 +1800,16 @@ export const getMulterStorage = () => {
             })
         })
         return upload
+    } else if (storageType === 'gcs') {
+        return multer({
+            storage: new MulterGoogleCloudStorage({
+                projectId: process.env.GOOGLE_CLOUD_STORAGE_PROJ_ID,
+                bucket: process.env.GOOGLE_CLOUD_STORAGE_BUCKET_NAME,
+                keyFilename: process.env.GOOGLE_CLOUD_STORAGE_CREDENTIAL,
+                uniformBucketLevelAccess: Boolean(process.env.GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS) ?? true,
+                destination: `uploads/${getOrgId()}`
+            })
+        })
     } else {
         return multer({ dest: getUploadPath() })
     }

packages/server/src/utils/logger.ts

@@ -5,6 +5,7 @@ import config from './config' // should be replaced by node-config or similar
 import { createLogger, transports, format } from 'winston'
 import { NextFunction, Request, Response } from 'express'
 import { S3ClientConfig } from '@aws-sdk/client-s3'
+import { LoggingWinston } from '@google-cloud/logging-winston'
 
 const { S3StreamLogger } = require('s3-streamlogger')
 
@@ -13,6 +14,11 @@ const { combine, timestamp, printf, errors } = format
 let s3ServerStream: any
 let s3ErrorStream: any
 let s3ServerReqStream: any
+
+let gcsServerStream: any
+let gcsErrorStream: any
+let gcsServerReqStream: any
+
 if (process.env.STORAGE_TYPE === 's3') {
     const accessKeyId = process.env.S3_STORAGE_ACCESS_KEY_ID
     const secretAccessKey = process.env.S3_STORAGE_SECRET_ACCESS_KEY
@@ -60,6 +66,29 @@ if (process.env.STORAGE_TYPE === 's3') {
     })
 }
 
+if (process.env.STORAGE_TYPE === 'gcs') {
+    const config = {
+        projectId: process.env.GOOGLE_CLOUD_STORAGE_PROJ_ID,
+        keyFilename: process.env.GOOGLE_CLOUD_STORAGE_CREDENTIAL,
+        defaultCallback: (err: any) => {
+            if (err) {
+                console.error('Error logging to GCS: ' + err)
+            }
+        }
+    }
+    gcsServerStream = new LoggingWinston({
+        ...config,
+        logName: 'server'
+    })
+    gcsErrorStream = new LoggingWinston({
+        ...config,
+        logName: 'error'
+    })
+    gcsServerReqStream = new LoggingWinston({
+        ...config,
+        logName: 'requests'
+    })
+}
 // expect the log dir be relative to the projects root
 const logDir = config.logging.dir
 
@@ -101,7 +130,8 @@ const logger = createLogger({
                       stream: s3ServerStream
                   })
               ]
-            : [])
+            : []),
+        ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsServerStream] : [])
     ],
     exceptionHandlers: [
         ...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
@@ -117,7 +147,8 @@ const logger = createLogger({
                       stream: s3ErrorStream
                   })
               ]
-            : [])
+            : []),
+        ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsErrorStream] : [])
     ],
     rejectionHandlers: [
         ...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
@@ -133,7 +164,8 @@ const logger = createLogger({
                       stream: s3ErrorStream
                   })
               ]
-            : [])
+            : []),
+        ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsErrorStream] : [])
     ]
 })
 
@@ -168,7 +200,8 @@ export function expressRequestLogger(req: Request, res: Response, next: NextFunc
                               stream: s3ServerReqStream
                           })
                       ]
-                    : [])
+                    : []),
+                ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsServerReqStream] : [])
             ]
         })