[Feat] Allow AWS SECRETS MANAGER instead of storing AES Encrypted in db (#3616)

* AWS Secrets

* AWS Secrets support

* add examples

* remove test compose

* fix lint

* update aws secret manager implementation

* update secret manager client

* update comments

---------

Co-authored-by: Henry <hzj94@hotmail.com>

docker/.env.example

@@ -5,6 +5,8 @@ SECRETKEY_PATH=/root/.flowise
 LOG_PATH=/root/.flowise/logs
 BLOB_STORAGE_PATH=/root/.flowise/storage
 
+# APIKEY_STORAGE_TYPE=json (json | db)
+
 # NUMBER_OF_PROXIES= 1
 # CORS_ORIGINS=*
 # IFRAME_ORIGINS=*
@@ -18,6 +20,13 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
 # DATABASE_SSL=true
 # DATABASE_SSL_KEY_BASE64=<Self signed certificate in BASE64>
 
+# SECRETKEY_STORAGE_TYPE=local #(local | aws)
+# SECRETKEY_PATH=/your_api_key_path/.flowise
+# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
+# SECRETKEY_AWS_ACCESS_KEY=<your-access-key>
+# SECRETKEY_AWS_SECRET_KEY=<your-secret-key>
+# SECRETKEY_AWS_REGION=us-west-2
+
 # FLOWISE_USERNAME=user
 # FLOWISE_PASSWORD=1234
 # FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
@@ -50,7 +59,6 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
 # S3_ENDPOINT_URL=<custom-s3-endpoint-url>
 # S3_FORCE_PATH_STYLE=false
 
-# APIKEY_STORAGE_TYPE=json (json | db)
 # SHOW_COMMUNITY_NODES=true
 # DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable)
 

packages/components/package.json

@@ -24,6 +24,7 @@
         "@aws-sdk/client-bedrock-runtime": "3.422.0",
         "@aws-sdk/client-dynamodb": "^3.360.0",
         "@aws-sdk/client-s3": "^3.427.0",
+        "@aws-sdk/client-secrets-manager": "^3.699.0",
         "@datastax/astra-db-ts": "1.5.0",
         "@dqbd/tiktoken": "^1.0.7",
         "@e2b/code-interpreter": "^0.0.5",

packages/components/src/utils.ts

@@ -9,12 +9,30 @@ import { ICommonObject, IDatabaseEntity, IDocument, IMessage, INodeData, IVariab
 import { AES, enc } from 'crypto-js'
 import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages'
 import { getFileFromStorage } from './storageUtils'
+import { GetSecretValueCommand, SecretsManagerClient, SecretsManagerClientConfig } from '@aws-sdk/client-secrets-manager'
 import { customGet } from '../nodes/sequentialagents/commonUtils'
 
 export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
 export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank
 export const FLOWISE_CHATID = 'flowise_chatId'
 
+let secretsManagerClient: SecretsManagerClient | null = null
+const USE_AWS_SECRETS_MANAGER = process.env.SECRETKEY_STORAGE_TYPE === 'aws'
+if (USE_AWS_SECRETS_MANAGER) {
+    const region = process.env.SECRETKEY_AWS_REGION || 'us-east-1' // Default region if not provided
+    const accessKeyId = process.env.SECRETKEY_AWS_ACCESS_KEY
+    const secretAccessKey = process.env.SECRETKEY_AWS_SECRET_KEY
+
+    let credentials: SecretsManagerClientConfig['credentials'] | undefined
+    if (accessKeyId && secretAccessKey) {
+        credentials = {
+            accessKeyId,
+            secretAccessKey
+        }
+    }
+    secretsManagerClient = new SecretsManagerClient({ credentials, region })
+}
+
 /*
  * List of dependencies allowed to be import in @flowiseai/nodevm
  */
@@ -503,10 +521,33 @@ const getEncryptionKey = async (): Promise<string> => {
  * @returns {Promise<ICommonObject>}
  */
 const decryptCredentialData = async (encryptedData: string): Promise<ICommonObject> => {
-    const encryptKey = await getEncryptionKey()
+    let decryptedDataStr: string
-    const decryptedData = AES.decrypt(encryptedData, encryptKey)
+
+    if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
+        try {
+            const command = new GetSecretValueCommand({ SecretId: encryptedData })
+            const response = await secretsManagerClient.send(command)
+
+            if (response.SecretString) {
+                const secretObj = JSON.parse(response.SecretString)
+                decryptedDataStr = JSON.stringify(secretObj)
+            } else {
+                throw new Error('Failed to retrieve secret value.')
+            }
+        } catch (error) {
+            console.error(error)
+            throw new Error('Credentials could not be decrypted.')
+        }
+    } else {
+        // Fallback to existing code
+        const encryptKey = await getEncryptionKey()
+        const decryptedData = AES.decrypt(encryptedData, encryptKey)
+        decryptedDataStr = decryptedData.toString(enc.Utf8)
+    }
+
+    if (!decryptedDataStr) return {}
     try {
-        return JSON.parse(decryptedData.toString(enc.Utf8))
+        return JSON.parse(decryptedDataStr)
     } catch (e) {
         console.error(e)
         throw new Error('Credentials could not be decrypted.')

packages/server/.env.example

@@ -1,7 +1,14 @@
 PORT=3000
 
+# APIKEY_STORAGE_TYPE=json (json | db)
 # APIKEY_PATH=/your_api_key_path/.flowise
+
+# SECRETKEY_STORAGE_TYPE=local #(local | aws)
 # SECRETKEY_PATH=/your_api_key_path/.flowise
+# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
+# SECRETKEY_AWS_ACCESS_KEY=<your-access-key>
+# SECRETKEY_AWS_SECRET_KEY=<your-secret-key>
+# SECRETKEY_AWS_REGION=us-west-2
 
 # NUMBER_OF_PROXIES= 1
 # CORS_ORIGINS=*
@@ -19,7 +26,6 @@ PORT=3000
 
 # FLOWISE_USERNAME=user
 # FLOWISE_PASSWORD=1234
-# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
 # FLOWISE_FILE_SIZE_LIMIT=50mb
 
 # DISABLE_CHATFLOW_REUSE=true
@@ -50,7 +56,6 @@ PORT=3000
 # S3_ENDPOINT_URL=<custom-s3-endpoint-url>
 # S3_FORCE_PATH_STYLE=false
 
-# APIKEY_STORAGE_TYPE=json (json | db)
 # SHOW_COMMUNITY_NODES=true
 # DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable)
 

packages/server/package.json

@@ -54,8 +54,10 @@
     },
     "license": "SEE LICENSE IN LICENSE.md",
     "dependencies": {
+        "@aws-sdk/client-secrets-manager": "^3.699.0",
         "@oclif/core": "^1.13.10",
         "@opentelemetry/api": "^1.3.0",
+        "@opentelemetry/auto-instrumentations-node": "^0.52.0",
         "@opentelemetry/core": "1.27.0",
         "@opentelemetry/exporter-metrics-otlp-grpc": "0.54.0",
         "@opentelemetry/exporter-metrics-otlp-http": "0.54.0",
@@ -65,10 +67,9 @@
         "@opentelemetry/exporter-trace-otlp-proto": "0.54.0",
         "@opentelemetry/resources": "1.27.0",
         "@opentelemetry/sdk-metrics": "1.27.0",
+        "@opentelemetry/sdk-node": "^0.54.0",
         "@opentelemetry/sdk-trace-base": "1.27.0",
         "@opentelemetry/semantic-conventions": "1.27.0",
-        "@opentelemetry/auto-instrumentations-node": "^0.52.0",
-        "@opentelemetry/sdk-node": "^0.54.0",
         "@types/lodash": "^4.14.202",
         "@types/uuid": "^9.0.7",
         "async-mutex": "^0.4.0",
@@ -82,6 +83,7 @@
         "express-rate-limit": "^6.9.0",
         "flowise-components": "workspace:^",
         "flowise-ui": "workspace:^",
+        "global-agent": "^3.0.0",
         "http-errors": "^2.0.0",
         "http-status-codes": "^2.3.0",
         "langchainhub": "^0.0.11",
@@ -100,8 +102,7 @@
         "sqlite3": "^5.1.6",
         "typeorm": "^0.3.6",
         "uuid": "^9.0.1",
-        "winston": "^3.9.0",
+        "winston": "^3.9.0"
-        "global-agent": "^3.0.0"
     },
     "devDependencies": {
         "@types/content-disposition": "0.5.8",

packages/server/src/commands/start.ts

@@ -26,8 +26,6 @@ export default class Start extends Command {
         BLOB_STORAGE_PATH: Flags.string(),
         APIKEY_STORAGE_TYPE: Flags.string(),
         APIKEY_PATH: Flags.string(),
-        SECRETKEY_PATH: Flags.string(),
-        FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
         LOG_PATH: Flags.string(),
         LOG_LEVEL: Flags.string(),
         TOOL_FUNCTION_BUILTIN_DEP: Flags.string(),
@@ -57,6 +55,12 @@ export default class Start extends Command {
         S3_ENDPOINT_URL: Flags.string(),
         S3_FORCE_PATH_STYLE: Flags.string(),
         SHOW_COMMUNITY_NODES: Flags.string(),
+        SECRETKEY_STORAGE_TYPE: Flags.string(),
+        SECRETKEY_PATH: Flags.string(),
+        FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
+        SECRETKEY_AWS_ACCESS_KEY: Flags.string(),
+        SECRETKEY_AWS_SECRET_KEY: Flags.string(),
+        SECRETKEY_AWS_REGION: Flags.string(),
         DISABLED_NODES: Flags.string()
     }
 
@@ -113,8 +117,12 @@ export default class Start extends Command {
         if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT
 
         // Credentials
+        if (flags.SECRETKEY_STORAGE_TYPE) process.env.SECRETKEY_STORAGE_TYPE = flags.SECRETKEY_STORAGE_TYPE
         if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
         if (flags.FLOWISE_SECRETKEY_OVERWRITE) process.env.FLOWISE_SECRETKEY_OVERWRITE = flags.FLOWISE_SECRETKEY_OVERWRITE
+        if (flags.SECRETKEY_AWS_ACCESS_KEY) process.env.SECRETKEY_AWS_ACCESS_KEY = flags.SECRETKEY_AWS_ACCESS_KEY
+        if (flags.SECRETKEY_AWS_SECRET_KEY) process.env.SECRETKEY_AWS_SECRET_KEY = flags.SECRETKEY_AWS_SECRET_KEY
+        if (flags.SECRETKEY_AWS_REGION) process.env.SECRETKEY_AWS_REGION = flags.SECRETKEY_AWS_REGION
 
         // Logs
         if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH

packages/server/src/utils/index.ts

@@ -54,12 +54,36 @@ import { DocumentStore } from '../database/entities/DocumentStore'
 import { DocumentStoreFileChunk } from '../database/entities/DocumentStoreFileChunk'
 import { InternalFlowiseError } from '../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
+import {
+    CreateSecretCommand,
+    GetSecretValueCommand,
+    PutSecretValueCommand,
+    SecretsManagerClient,
+    SecretsManagerClientConfig
+} from '@aws-sdk/client-secrets-manager'
 
 const QUESTION_VAR_PREFIX = 'question'
 const FILE_ATTACHMENT_PREFIX = 'file_attachment'
 const CHAT_HISTORY_VAR_PREFIX = 'chat_history'
 const REDACTED_CREDENTIAL_VALUE = '_FLOWISE_BLANK_07167752-1a71-43b1-bf8f-4f32252165db'
 
+let secretsManagerClient: SecretsManagerClient | null = null
+const USE_AWS_SECRETS_MANAGER = process.env.SECRETKEY_STORAGE_TYPE === 'aws'
+if (USE_AWS_SECRETS_MANAGER) {
+    const region = process.env.SECRETKEY_AWS_REGION || 'us-east-1' // Default region if not provided
+    const accessKeyId = process.env.SECRETKEY_AWS_ACCESS_KEY
+    const secretAccessKey = process.env.SECRETKEY_AWS_SECRET_KEY
+
+    let credentials: SecretsManagerClientConfig['credentials'] | undefined
+    if (accessKeyId && secretAccessKey) {
+        credentials = {
+            accessKeyId,
+            secretAccessKey
+        }
+    }
+    secretsManagerClient = new SecretsManagerClient({ credentials, region })
+}
+
 export const databaseEntities: IDatabaseEntity = {
     ChatFlow: ChatFlow,
     ChatMessage: ChatMessage,
@@ -1360,14 +1384,6 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
     return isChatOrLLMsExist && isValidChainOrAgent && !isOutputParserExist
 }
 
-/**
- * Generate an encryption key
- * @returns {string}
- */
-export const generateEncryptKey = (): string => {
-    return randomBytes(24).toString('base64')
-}
-
 /**
  * Returns the encryption key
  * @returns {Promise<string>}
@@ -1394,7 +1410,39 @@ export const getEncryptionKey = async (): Promise<string> => {
  * @returns {Promise<string>}
  */
 export const encryptCredentialData = async (plainDataObj: ICredentialDataDecrypted): Promise<string> => {
+    if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
+        const secretName = `FlowiseCredential_${randomBytes(12).toString('hex')}`
+
+        logger.info(`[server]: Upserting AWS Secret: ${secretName}`)
+
+        const secretString = JSON.stringify({ ...plainDataObj })
+
+        try {
+            // Try to update the secret if it exists
+            const putCommand = new PutSecretValueCommand({
+                SecretId: secretName,
+                SecretString: secretString
+            })
+            await secretsManagerClient.send(putCommand)
+        } catch (error: any) {
+            if (error.name === 'ResourceNotFoundException') {
+                // Secret doesn't exist, so create it
+                const createCommand = new CreateSecretCommand({
+                    Name: secretName,
+                    SecretString: secretString
+                })
+                await secretsManagerClient.send(createCommand)
+            } else {
+                // Rethrow any other errors
+                throw error
+            }
+        }
+        return secretName
+    }
+
     const encryptKey = await getEncryptionKey()
+
+    // Fallback to existing code
     return AES.encrypt(JSON.stringify(plainDataObj), encryptKey).toString()
 }
 
@@ -1410,22 +1458,52 @@ export const decryptCredentialData = async (
     componentCredentialName?: string,
     componentCredentials?: IComponentCredentials
 ): Promise<ICredentialDataDecrypted> => {
-    const encryptKey = await getEncryptionKey()
+    let decryptedDataStr: string
-    const decryptedData = AES.decrypt(encryptedData, encryptKey)
+
-    const decryptedDataStr = decryptedData.toString(enc.Utf8)
+    if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
+        try {
+            logger.info(`[server]: Reading AWS Secret: ${encryptedData}`)
+            const command = new GetSecretValueCommand({ SecretId: encryptedData })
+            const response = await secretsManagerClient.send(command)
+
+            if (response.SecretString) {
+                const secretObj = JSON.parse(response.SecretString)
+                decryptedDataStr = JSON.stringify(secretObj)
+            } else {
+                throw new Error('Failed to retrieve secret value.')
+            }
+        } catch (error) {
+            console.error(error)
+            throw new Error('Failed to decrypt credential data.')
+        }
+    } else {
+        // Fallback to existing code
+        const encryptKey = await getEncryptionKey()
+        const decryptedData = AES.decrypt(encryptedData, encryptKey)
+        decryptedDataStr = decryptedData.toString(enc.Utf8)
+    }
+
     if (!decryptedDataStr) return {}
     try {
         if (componentCredentialName && componentCredentials) {
-            const plainDataObj = JSON.parse(decryptedData.toString(enc.Utf8))
+            const plainDataObj = JSON.parse(decryptedDataStr)
             return redactCredentialWithPasswordType(componentCredentialName, plainDataObj, componentCredentials)
         }
-        return JSON.parse(decryptedData.toString(enc.Utf8))
+        return JSON.parse(decryptedDataStr)
     } catch (e) {
         console.error(e)
         return {}
     }
 }
 
+/**
+ * Generate an encryption key
+ * @returns {string}
+ */
+export const generateEncryptKey = (): string => {
+    return randomBytes(24).toString('base64')
+}
+
 /**
  * Transform ICredentialBody from req to Credential entity
  * @param {ICredentialReqBody} body