farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 23:01:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

[Feat] Allow AWS SECRETS MANAGER instead of storing AES Encrypted in db (#3616)

Browse Source 
* AWS Secrets

* AWS Secrets support

* add examples

* remove test compose

* fix lint

* update aws secret manager implementation

* update secret manager client

* update comments

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Lucas Mohallem Ferraz
2025-01-08 14:24:57 -06:00
committed by GitHub
parent c2c1ca9162
commit 1ae78c2739
10 changed files with 1393 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/.env.example
+7 -2
View File
@@ -1,7 +1,14 @@
PORT=3000
# APIKEY_STORAGE_TYPE=json (json | db)
# APIKEY_PATH=/your_api_key_path/.flowise
# SECRETKEY_STORAGE_TYPE=local #(local | aws)
# SECRETKEY_PATH=/your_api_key_path/.flowise
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
# SECRETKEY_AWS_ACCESS_KEY=<your-access-key>
# SECRETKEY_AWS_SECRET_KEY=<your-secret-key>
# SECRETKEY_AWS_REGION=us-west-2
# NUMBER_OF_PROXIES= 1
# CORS_ORIGINS=*
@@ -19,7 +26,6 @@ PORT=3000
# FLOWISE_USERNAME=user
# FLOWISE_PASSWORD=1234
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
# FLOWISE_FILE_SIZE_LIMIT=50mb
# DISABLE_CHATFLOW_REUSE=true
@@ -50,7 +56,6 @@ PORT=3000
# S3_ENDPOINT_URL=<custom-s3-endpoint-url>
# S3_FORCE_PATH_STYLE=false
# APIKEY_STORAGE_TYPE=json (json | db)
# SHOW_COMMUNITY_NODES=true
# DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable)
packages/server/package.json
+5 -4
View File
@@ -54,8 +54,10 @@
},
"license": "SEE LICENSE IN LICENSE.md",
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.699.0",
"@oclif/core": "^1.13.10",
"@opentelemetry/api": "^1.3.0",
"@opentelemetry/auto-instrumentations-node": "^0.52.0",
"@opentelemetry/core": "1.27.0",
"@opentelemetry/exporter-metrics-otlp-grpc": "0.54.0",
"@opentelemetry/exporter-metrics-otlp-http": "0.54.0",
@@ -65,10 +67,9 @@
"@opentelemetry/exporter-trace-otlp-proto": "0.54.0",
"@opentelemetry/resources": "1.27.0",
"@opentelemetry/sdk-metrics": "1.27.0",
"@opentelemetry/sdk-node": "^0.54.0",
"@opentelemetry/sdk-trace-base": "1.27.0",
"@opentelemetry/semantic-conventions": "1.27.0",
"@opentelemetry/auto-instrumentations-node": "^0.52.0",
"@opentelemetry/sdk-node": "^0.54.0",
"@types/lodash": "^4.14.202",
"@types/uuid": "^9.0.7",
"async-mutex": "^0.4.0",
@@ -82,6 +83,7 @@
"express-rate-limit": "^6.9.0",
"flowise-components": "workspace:^",
"flowise-ui": "workspace:^",
"global-agent": "^3.0.0",
"http-errors": "^2.0.0",
"http-status-codes": "^2.3.0",
"langchainhub": "^0.0.11",
@@ -100,8 +102,7 @@
"sqlite3": "^5.1.6",
"typeorm": "^0.3.6",
"uuid": "^9.0.1",
"winston": "^3.9.0",
"global-agent": "^3.0.0"
"winston": "^3.9.0"
},
"devDependencies": {
"@types/content-disposition": "0.5.8",
packages/server/src/commands/start.ts
+10 -2
View File
@@ -26,8 +26,6 @@ export default class Start extends Command {
BLOB_STORAGE_PATH: Flags.string(),
APIKEY_STORAGE_TYPE: Flags.string(),
APIKEY_PATH: Flags.string(),
SECRETKEY_PATH: Flags.string(),
FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
LOG_PATH: Flags.string(),
LOG_LEVEL: Flags.string(),
TOOL_FUNCTION_BUILTIN_DEP: Flags.string(),
@@ -57,6 +55,12 @@ export default class Start extends Command {
S3_ENDPOINT_URL: Flags.string(),
S3_FORCE_PATH_STYLE: Flags.string(),
SHOW_COMMUNITY_NODES: Flags.string(),
SECRETKEY_STORAGE_TYPE: Flags.string(),
SECRETKEY_PATH: Flags.string(),
FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
SECRETKEY_AWS_ACCESS_KEY: Flags.string(),
SECRETKEY_AWS_SECRET_KEY: Flags.string(),
SECRETKEY_AWS_REGION: Flags.string(),
DISABLED_NODES: Flags.string()
}
@@ -113,8 +117,12 @@ export default class Start extends Command {
if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT
// Credentials
if (flags.SECRETKEY_STORAGE_TYPE) process.env.SECRETKEY_STORAGE_TYPE = flags.SECRETKEY_STORAGE_TYPE
if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
if (flags.FLOWISE_SECRETKEY_OVERWRITE) process.env.FLOWISE_SECRETKEY_OVERWRITE = flags.FLOWISE_SECRETKEY_OVERWRITE
if (flags.SECRETKEY_AWS_ACCESS_KEY) process.env.SECRETKEY_AWS_ACCESS_KEY = flags.SECRETKEY_AWS_ACCESS_KEY
if (flags.SECRETKEY_AWS_SECRET_KEY) process.env.SECRETKEY_AWS_SECRET_KEY = flags.SECRETKEY_AWS_SECRET_KEY
if (flags.SECRETKEY_AWS_REGION) process.env.SECRETKEY_AWS_REGION = flags.SECRETKEY_AWS_REGION
// Logs
if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
packages/server/src/utils/index.ts
+91 -13
View File
@@ -54,12 +54,36 @@ import { DocumentStore } from '../database/entities/DocumentStore'
import { DocumentStoreFileChunk } from '../database/entities/DocumentStoreFileChunk'
import { InternalFlowiseError } from '../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import {
CreateSecretCommand,
GetSecretValueCommand,
PutSecretValueCommand,
SecretsManagerClient,
SecretsManagerClientConfig
} from '@aws-sdk/client-secrets-manager'
const QUESTION_VAR_PREFIX = 'question'
const FILE_ATTACHMENT_PREFIX = 'file_attachment'
const CHAT_HISTORY_VAR_PREFIX = 'chat_history'
const REDACTED_CREDENTIAL_VALUE = '_FLOWISE_BLANK_07167752-1a71-43b1-bf8f-4f32252165db'
let secretsManagerClient: SecretsManagerClient | null = null
const USE_AWS_SECRETS_MANAGER = process.env.SECRETKEY_STORAGE_TYPE === 'aws'
if (USE_AWS_SECRETS_MANAGER) {
const region = process.env.SECRETKEY_AWS_REGION || 'us-east-1' // Default region if not provided
const accessKeyId = process.env.SECRETKEY_AWS_ACCESS_KEY
const secretAccessKey = process.env.SECRETKEY_AWS_SECRET_KEY
let credentials: SecretsManagerClientConfig['credentials'] | undefined
if (accessKeyId && secretAccessKey) {
credentials = {
accessKeyId,
secretAccessKey
}
}
secretsManagerClient = new SecretsManagerClient({ credentials, region })
}
export const databaseEntities: IDatabaseEntity = {
ChatFlow: ChatFlow,
ChatMessage: ChatMessage,
@@ -1360,14 +1384,6 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
return isChatOrLLMsExist && isValidChainOrAgent && !isOutputParserExist
}
/**
* Generate an encryption key
* @returns {string}
*/
export const generateEncryptKey = (): string => {
return randomBytes(24).toString('base64')
}
/**
* Returns the encryption key
* @returns {Promise<string>}
@@ -1394,7 +1410,39 @@ export const getEncryptionKey = async (): Promise<string> => {
* @returns {Promise<string>}
*/
export const encryptCredentialData = async (plainDataObj: ICredentialDataDecrypted): Promise<string> => {
if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
const secretName = `FlowiseCredential_${randomBytes(12).toString('hex')}`
logger.info(`[server]: Upserting AWS Secret: ${secretName}`)
const secretString = JSON.stringify({ ...plainDataObj })
try {
// Try to update the secret if it exists
const putCommand = new PutSecretValueCommand({
SecretId: secretName,
SecretString: secretString
})
await secretsManagerClient.send(putCommand)
} catch (error: any) {
if (error.name === 'ResourceNotFoundException') {
// Secret doesn't exist, so create it
const createCommand = new CreateSecretCommand({
Name: secretName,
SecretString: secretString
})
await secretsManagerClient.send(createCommand)
} else {
// Rethrow any other errors
throw error
}
}
return secretName
}
const encryptKey = await getEncryptionKey()
// Fallback to existing code
return AES.encrypt(JSON.stringify(plainDataObj), encryptKey).toString()
}
@@ -1410,22 +1458,52 @@ export const decryptCredentialData = async (
componentCredentialName?: string,
componentCredentials?: IComponentCredentials
): Promise<ICredentialDataDecrypted> => {
const encryptKey = await getEncryptionKey()
const decryptedData = AES.decrypt(encryptedData, encryptKey)
const decryptedDataStr = decryptedData.toString(enc.Utf8)
let decryptedDataStr: string
if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
try {
logger.info(`[server]: Reading AWS Secret: ${encryptedData}`)
const command = new GetSecretValueCommand({ SecretId: encryptedData })
const response = await secretsManagerClient.send(command)
if (response.SecretString) {
const secretObj = JSON.parse(response.SecretString)
decryptedDataStr = JSON.stringify(secretObj)
} else {
throw new Error('Failed to retrieve secret value.')
}
} catch (error) {
console.error(error)
throw new Error('Failed to decrypt credential data.')
}
} else {
// Fallback to existing code
const encryptKey = await getEncryptionKey()
const decryptedData = AES.decrypt(encryptedData, encryptKey)
decryptedDataStr = decryptedData.toString(enc.Utf8)
}
if (!decryptedDataStr) return {}
try {
if (componentCredentialName && componentCredentials) {
const plainDataObj = JSON.parse(decryptedData.toString(enc.Utf8))
const plainDataObj = JSON.parse(decryptedDataStr)
return redactCredentialWithPasswordType(componentCredentialName, plainDataObj, componentCredentials)
}
return JSON.parse(decryptedData.toString(enc.Utf8))
return JSON.parse(decryptedDataStr)
} catch (e) {
console.error(e)
return {}
}
}
/**
* Generate an encryption key
* @returns {string}
*/
export const generateEncryptKey = (): string => {
return randomBytes(24).toString('base64')
}
/**
* Transform ICredentialBody from req to Credential entity
* @param {ICredentialReqBody} body