[Feat] Allow AWS SECRETS MANAGER instead of storing AES Encrypted in db (#3616)

* AWS Secrets * AWS Secrets support * add examples * remove test compose * fix lint * update aws secret manager implementation * update secret manager client * update comments --------- Co-authored-by: Henry <hzj94@hotmail.com>
2026-06-28 19:00:59 +03:00 · 2025-01-08 14:24:57 -06:00
parent c2c1ca9162
commit 1ae78c2739
10 changed files with 1393 additions and 271 deletions
@@ -26,8 +26,6 @@ export default class Start extends Command {
        BLOB_STORAGE_PATH: Flags.string(),
        APIKEY_STORAGE_TYPE: Flags.string(),
        APIKEY_PATH: Flags.string(),
-        SECRETKEY_PATH: Flags.string(),
-        FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
        LOG_PATH: Flags.string(),
        LOG_LEVEL: Flags.string(),
        TOOL_FUNCTION_BUILTIN_DEP: Flags.string(),
@@ -57,6 +55,12 @@ export default class Start extends Command {
        S3_ENDPOINT_URL: Flags.string(),
        S3_FORCE_PATH_STYLE: Flags.string(),
        SHOW_COMMUNITY_NODES: Flags.string(),
+        SECRETKEY_STORAGE_TYPE: Flags.string(),
+        SECRETKEY_PATH: Flags.string(),
+        FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
+        SECRETKEY_AWS_ACCESS_KEY: Flags.string(),
+        SECRETKEY_AWS_SECRET_KEY: Flags.string(),
+        SECRETKEY_AWS_REGION: Flags.string(),
        DISABLED_NODES: Flags.string()
    }

@@ -113,8 +117,12 @@ export default class Start extends Command {
        if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT

        // Credentials
+        if (flags.SECRETKEY_STORAGE_TYPE) process.env.SECRETKEY_STORAGE_TYPE = flags.SECRETKEY_STORAGE_TYPE
        if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
        if (flags.FLOWISE_SECRETKEY_OVERWRITE) process.env.FLOWISE_SECRETKEY_OVERWRITE = flags.FLOWISE_SECRETKEY_OVERWRITE
+        if (flags.SECRETKEY_AWS_ACCESS_KEY) process.env.SECRETKEY_AWS_ACCESS_KEY = flags.SECRETKEY_AWS_ACCESS_KEY
+        if (flags.SECRETKEY_AWS_SECRET_KEY) process.env.SECRETKEY_AWS_SECRET_KEY = flags.SECRETKEY_AWS_SECRET_KEY
+        if (flags.SECRETKEY_AWS_REGION) process.env.SECRETKEY_AWS_REGION = flags.SECRETKEY_AWS_REGION

        // Logs
        if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
@@ -54,12 +54,36 @@ import { DocumentStore } from '../database/entities/DocumentStore'
 import { DocumentStoreFileChunk } from '../database/entities/DocumentStoreFileChunk'
 import { InternalFlowiseError } from '../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
+import {
+    CreateSecretCommand,
+    GetSecretValueCommand,
+    PutSecretValueCommand,
+    SecretsManagerClient,
+    SecretsManagerClientConfig
+} from '@aws-sdk/client-secrets-manager'

 const QUESTION_VAR_PREFIX = 'question'
 const FILE_ATTACHMENT_PREFIX = 'file_attachment'
 const CHAT_HISTORY_VAR_PREFIX = 'chat_history'
 const REDACTED_CREDENTIAL_VALUE = '_FLOWISE_BLANK_07167752-1a71-43b1-bf8f-4f32252165db'

+let secretsManagerClient: SecretsManagerClient | null = null
+const USE_AWS_SECRETS_MANAGER = process.env.SECRETKEY_STORAGE_TYPE === 'aws'
+if (USE_AWS_SECRETS_MANAGER) {
+    const region = process.env.SECRETKEY_AWS_REGION || 'us-east-1' // Default region if not provided
+    const accessKeyId = process.env.SECRETKEY_AWS_ACCESS_KEY
+    const secretAccessKey = process.env.SECRETKEY_AWS_SECRET_KEY
+
+    let credentials: SecretsManagerClientConfig['credentials'] | undefined
+    if (accessKeyId && secretAccessKey) {
+        credentials = {
+            accessKeyId,
+            secretAccessKey
+        }
+    }
+    secretsManagerClient = new SecretsManagerClient({ credentials, region })
+}
+
 export const databaseEntities: IDatabaseEntity = {
    ChatFlow: ChatFlow,
    ChatMessage: ChatMessage,
@@ -1360,14 +1384,6 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
    return isChatOrLLMsExist && isValidChainOrAgent && !isOutputParserExist
 }

-/**
- * Generate an encryption key
- * @returns {string}
- */
-export const generateEncryptKey = (): string => {
-    return randomBytes(24).toString('base64')
-}
-
 /**
 * Returns the encryption key
 * @returns {Promise<string>}
@@ -1394,7 +1410,39 @@ export const getEncryptionKey = async (): Promise<string> => {
 * @returns {Promise<string>}
 */
 export const encryptCredentialData = async (plainDataObj: ICredentialDataDecrypted): Promise<string> => {
+    if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
+        const secretName = `FlowiseCredential_${randomBytes(12).toString('hex')}`
+
+        logger.info(`[server]: Upserting AWS Secret: ${secretName}`)
+
+        const secretString = JSON.stringify({ ...plainDataObj })
+
+        try {
+            // Try to update the secret if it exists
+            const putCommand = new PutSecretValueCommand({
+                SecretId: secretName,
+                SecretString: secretString
+            })
+            await secretsManagerClient.send(putCommand)
+        } catch (error: any) {
+            if (error.name === 'ResourceNotFoundException') {
+                // Secret doesn't exist, so create it
+                const createCommand = new CreateSecretCommand({
+                    Name: secretName,
+                    SecretString: secretString
+                })
+                await secretsManagerClient.send(createCommand)
+            } else {
+                // Rethrow any other errors
+                throw error
+            }
+        }
+        return secretName
+    }
+
    const encryptKey = await getEncryptionKey()
+
+    // Fallback to existing code
    return AES.encrypt(JSON.stringify(plainDataObj), encryptKey).toString()
 }

@@ -1410,22 +1458,52 @@ export const decryptCredentialData = async (
    componentCredentialName?: string,
    componentCredentials?: IComponentCredentials
 ): Promise<ICredentialDataDecrypted> => {
-    const encryptKey = await getEncryptionKey()
-    const decryptedData = AES.decrypt(encryptedData, encryptKey)
-    const decryptedDataStr = decryptedData.toString(enc.Utf8)
+    let decryptedDataStr: string
+
+    if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
+        try {
+            logger.info(`[server]: Reading AWS Secret: ${encryptedData}`)
+            const command = new GetSecretValueCommand({ SecretId: encryptedData })
+            const response = await secretsManagerClient.send(command)
+
+            if (response.SecretString) {
+                const secretObj = JSON.parse(response.SecretString)
+                decryptedDataStr = JSON.stringify(secretObj)
+            } else {
+                throw new Error('Failed to retrieve secret value.')
+            }
+        } catch (error) {
+            console.error(error)
+            throw new Error('Failed to decrypt credential data.')
+        }
+    } else {
+        // Fallback to existing code
+        const encryptKey = await getEncryptionKey()
+        const decryptedData = AES.decrypt(encryptedData, encryptKey)
+        decryptedDataStr = decryptedData.toString(enc.Utf8)
+    }
+
    if (!decryptedDataStr) return {}
    try {
        if (componentCredentialName && componentCredentials) {
-            const plainDataObj = JSON.parse(decryptedData.toString(enc.Utf8))
+            const plainDataObj = JSON.parse(decryptedDataStr)
            return redactCredentialWithPasswordType(componentCredentialName, plainDataObj, componentCredentials)
        }
-        return JSON.parse(decryptedData.toString(enc.Utf8))
+        return JSON.parse(decryptedDataStr)
    } catch (e) {
        console.error(e)
        return {}
    }
 }

+/**
+ * Generate an encryption key
+ * @returns {string}
+ */
+export const generateEncryptKey = (): string => {
+    return randomBytes(24).toString('base64')
+}
+
 /**
 * Transform ICredentialBody from req to Credential entity
 * @param {ICredentialReqBody} body