farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 21:00:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

[Feat] Allow AWS SECRETS MANAGER instead of storing AES Encrypted in db (#3616)

Browse Source 
* AWS Secrets

* AWS Secrets support

* add examples

* remove test compose

* fix lint

* update aws secret manager implementation

* update secret manager client

* update comments

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Lucas Mohallem Ferraz
2025-01-08 14:24:57 -06:00
committed by GitHub
parent c2c1ca9162
commit 1ae78c2739
10 changed files with 1393 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/.env.example
+9 -1
View File
@@ -5,6 +5,8 @@ SECRETKEY_PATH=/root/.flowise
LOG_PATH=/root/.flowise/logs LOG_PATH=/root/.flowise/logs
BLOB_STORAGE_PATH=/root/.flowise/storage BLOB_STORAGE_PATH=/root/.flowise/storage
# APIKEY_STORAGE_TYPE=json (json | db)
# NUMBER_OF_PROXIES= 1 # NUMBER_OF_PROXIES= 1
# CORS_ORIGINS=* # CORS_ORIGINS=*
# IFRAME_ORIGINS=* # IFRAME_ORIGINS=*
@@ -18,6 +20,13 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
# DATABASE_SSL=true # DATABASE_SSL=true
# DATABASE_SSL_KEY_BASE64=<Self signed certificate in BASE64> # DATABASE_SSL_KEY_BASE64=<Self signed certificate in BASE64>
# SECRETKEY_STORAGE_TYPE=local #(local | aws)
# SECRETKEY_PATH=/your_api_key_path/.flowise
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
# SECRETKEY_AWS_ACCESS_KEY=<your-access-key>
# SECRETKEY_AWS_SECRET_KEY=<your-secret-key>
# SECRETKEY_AWS_REGION=us-west-2
# FLOWISE_USERNAME=user # FLOWISE_USERNAME=user
# FLOWISE_PASSWORD=1234 # FLOWISE_PASSWORD=1234
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey # FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
@@ -50,7 +59,6 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
# S3_ENDPOINT_URL=<custom-s3-endpoint-url> # S3_ENDPOINT_URL=<custom-s3-endpoint-url>
# S3_FORCE_PATH_STYLE=false # S3_FORCE_PATH_STYLE=false
# APIKEY_STORAGE_TYPE=json (json | db)
# SHOW_COMMUNITY_NODES=true # SHOW_COMMUNITY_NODES=true
# DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable) # DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable)
packages/components/nodes/chatmodels/Deepseek/Deepseek.ts
+197 -197
View File
@@ -1,197 +1,197 @@
import { BaseCache } from '@langchain/core/caches' import { BaseCache } from '@langchain/core/caches'
import { BaseChatModelParams } from '@langchain/core/language_models/chat_models' import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
import { ChatOpenAI, LegacyOpenAIInput, OpenAIChatInput } from '@langchain/openai' import { ChatOpenAI, LegacyOpenAIInput, OpenAIChatInput } from '@langchain/openai'
import type { ClientOptions } from 'openai' import type { ClientOptions } from 'openai'
import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface' import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
import { getModels, MODEL_TYPE } from '../../../src/modelLoader' import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
class Deepseek_ChatModels implements INode { class Deepseek_ChatModels implements INode {
readonly baseURL: string = 'https://api.deepseek.com' readonly baseURL: string = 'https://api.deepseek.com'
label: string label: string
name: string name: string
version: number version: number
type: string type: string
icon: string icon: string
category: string category: string
description: string description: string
baseClasses: string[] baseClasses: string[]
credential: INodeParams credential: INodeParams
inputs: INodeParams[] inputs: INodeParams[]
constructor() { constructor() {
this.label = 'ChatDeepseek' this.label = 'ChatDeepseek'
this.name = 'chatDeepseek' this.name = 'chatDeepseek'
this.version = 1.0 this.version = 1.0
this.type = 'chatDeepseek' this.type = 'chatDeepseek'
this.icon = 'deepseek.svg' this.icon = 'deepseek.svg'
this.category = 'Chat Models' this.category = 'Chat Models'
this.description = 'Wrapper around Deepseek large language models that use the Chat endpoint' this.description = 'Wrapper around Deepseek large language models that use the Chat endpoint'
this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)] this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)]
this.credential = { this.credential = {
label: 'Connect Credential', label: 'Connect Credential',
name: 'credential', name: 'credential',
type: 'credential', type: 'credential',
credentialNames: ['deepseekApi'] credentialNames: ['deepseekApi']
} }
this.inputs = [ this.inputs = [
{ {
label: 'Cache', label: 'Cache',
name: 'cache', name: 'cache',
type: 'BaseCache', type: 'BaseCache',
optional: true optional: true
}, },
{ {
label: 'Model Name', label: 'Model Name',
name: 'modelName', name: 'modelName',
type: 'asyncOptions', type: 'asyncOptions',
loadMethod: 'listModels', loadMethod: 'listModels',
default: 'deepseek-chat' default: 'deepseek-chat'
}, },
{ {
label: 'Temperature', label: 'Temperature',
name: 'temperature', name: 'temperature',
type: 'number', type: 'number',
step: 0.1, step: 0.1,
default: 0.7, default: 0.7,
optional: true optional: true
}, },
{ {
label: 'Streaming', label: 'Streaming',
name: 'streaming', name: 'streaming',
type: 'boolean', type: 'boolean',
default: true, default: true,
optional: true, optional: true,
additionalParams: true additionalParams: true
}, },
{ {
label: 'Max Tokens', label: 'Max Tokens',
name: 'maxTokens', name: 'maxTokens',
type: 'number', type: 'number',
step: 1, step: 1,
optional: true, optional: true,
additionalParams: true additionalParams: true
}, },
{ {
label: 'Top Probability', label: 'Top Probability',
name: 'topP', name: 'topP',
type: 'number', type: 'number',
step: 0.1, step: 0.1,
optional: true, optional: true,
additionalParams: true additionalParams: true
}, },
{ {
label: 'Frequency Penalty', label: 'Frequency Penalty',
name: 'frequencyPenalty', name: 'frequencyPenalty',
type: 'number', type: 'number',
step: 0.1, step: 0.1,
optional: true, optional: true,
additionalParams: true additionalParams: true
}, },
{ {
label: 'Presence Penalty', label: 'Presence Penalty',
name: 'presencePenalty', name: 'presencePenalty',
type: 'number', type: 'number',
step: 0.1, step: 0.1,
optional: true, optional: true,
additionalParams: true additionalParams: true
}, },
{ {
label: 'Timeout', label: 'Timeout',
name: 'timeout', name: 'timeout',
type: 'number', type: 'number',
step: 1, step: 1,
optional: true, optional: true,
additionalParams: true additionalParams: true
}, },
{ {
label: 'Stop Sequence', label: 'Stop Sequence',
name: 'stopSequence', name: 'stopSequence',
type: 'string', type: 'string',
rows: 4, rows: 4,
optional: true, optional: true,
description: 'List of stop words to use when generating. Use comma to separate multiple stop words.', description: 'List of stop words to use when generating. Use comma to separate multiple stop words.',
additionalParams: true additionalParams: true
}, },
{ {
label: 'Base Options', label: 'Base Options',
name: 'baseOptions', name: 'baseOptions',
type: 'json', type: 'json',
optional: true, optional: true,
additionalParams: true, additionalParams: true,
description: 'Additional options to pass to the Deepseek client. This should be a JSON object.' description: 'Additional options to pass to the Deepseek client. This should be a JSON object.'
} }
] ]
} }
//@ts-ignore //@ts-ignore
loadMethods = { loadMethods = {
async listModels(): Promise<INodeOptionsValue[]> { async listModels(): Promise<INodeOptionsValue[]> {
return await getModels(MODEL_TYPE.CHAT, 'deepseek') return await getModels(MODEL_TYPE.CHAT, 'deepseek')
} }
} }
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> { async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const temperature = nodeData.inputs?.temperature as string const temperature = nodeData.inputs?.temperature as string
const modelName = nodeData.inputs?.modelName as string const modelName = nodeData.inputs?.modelName as string
const maxTokens = nodeData.inputs?.maxTokens as string const maxTokens = nodeData.inputs?.maxTokens as string
const topP = nodeData.inputs?.topP as string const topP = nodeData.inputs?.topP as string
const frequencyPenalty = nodeData.inputs?.frequencyPenalty as string const frequencyPenalty = nodeData.inputs?.frequencyPenalty as string
const presencePenalty = nodeData.inputs?.presencePenalty as string const presencePenalty = nodeData.inputs?.presencePenalty as string
const timeout = nodeData.inputs?.timeout as string const timeout = nodeData.inputs?.timeout as string
const stopSequence = nodeData.inputs?.stopSequence as string const stopSequence = nodeData.inputs?.stopSequence as string
const streaming = nodeData.inputs?.streaming as boolean const streaming = nodeData.inputs?.streaming as boolean
const baseOptions = nodeData.inputs?.baseOptions const baseOptions = nodeData.inputs?.baseOptions
if (nodeData.inputs?.credentialId) { if (nodeData.inputs?.credentialId) {
nodeData.credential = nodeData.inputs?.credentialId nodeData.credential = nodeData.inputs?.credentialId
} }
const credentialData = await getCredentialData(nodeData.credential ?? '', options) const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const openAIApiKey = getCredentialParam('deepseekApiKey', credentialData, nodeData) const openAIApiKey = getCredentialParam('deepseekApiKey', credentialData, nodeData)
const cache = nodeData.inputs?.cache as BaseCache const cache = nodeData.inputs?.cache as BaseCache
const obj: Partial<OpenAIChatInput> & BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput } = { const obj: Partial<OpenAIChatInput> & BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput } = {
temperature: parseFloat(temperature), temperature: parseFloat(temperature),
modelName, modelName,
openAIApiKey, openAIApiKey,
streaming: streaming ?? true streaming: streaming ?? true
} }
if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10) if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10)
if (topP) obj.topP = parseFloat(topP) if (topP) obj.topP = parseFloat(topP)
if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty) if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty)
if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty)
if (timeout) obj.timeout = parseInt(timeout, 10) if (timeout) obj.timeout = parseInt(timeout, 10)
if (cache) obj.cache = cache if (cache) obj.cache = cache
if (stopSequence) { if (stopSequence) {
const stopSequenceArray = stopSequence.split(',').map((item) => item.trim()) const stopSequenceArray = stopSequence.split(',').map((item) => item.trim())
obj.stop = stopSequenceArray obj.stop = stopSequenceArray
} }
let parsedBaseOptions: any | undefined = undefined let parsedBaseOptions: any | undefined = undefined
if (baseOptions) { if (baseOptions) {
try { try {
parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions) parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions)
if (parsedBaseOptions.baseURL) { if (parsedBaseOptions.baseURL) {
console.warn("The 'baseURL' parameter is not allowed when using the ChatDeepseek node.") console.warn("The 'baseURL' parameter is not allowed when using the ChatDeepseek node.")
parsedBaseOptions.baseURL = undefined parsedBaseOptions.baseURL = undefined
} }
} catch (exception) { } catch (exception) {
throw new Error('Invalid JSON in the BaseOptions: ' + exception) throw new Error('Invalid JSON in the BaseOptions: ' + exception)
} }
} }
const model = new ChatOpenAI({ const model = new ChatOpenAI({
...obj, ...obj,
configuration: { configuration: {
baseURL: this.baseURL, baseURL: this.baseURL,
...parsedBaseOptions ...parsedBaseOptions
} }
}) })
return model return model
} }
} }
module.exports = { nodeClass: Deepseek_ChatModels } module.exports = { nodeClass: Deepseek_ChatModels }
packages/components/nodes/chatmodels/Deepseek/deepseek.svg
+30 -30
View File
@@ -1,30 +1,30 @@
<?xml version="1.0" standalone="no"?> <?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN" <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd"> "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<svg version="1.0" xmlns="http://www.w3.org/2000/svg" <svg version="1.0" xmlns="http://www.w3.org/2000/svg"
width="225.000000pt" height="225.000000pt" viewBox="0 0 225.000000 225.000000" width="225.000000pt" height="225.000000pt" viewBox="0 0 225.000000 225.000000"
preserveAspectRatio="xMidYMid meet"> preserveAspectRatio="xMidYMid meet">
<g transform="translate(0.000000,225.000000) scale(0.100000,-0.100000)" <g transform="translate(0.000000,225.000000) scale(0.100000,-0.100000)"
fill="#000000" stroke="none"> fill="#000000" stroke="none">
<path d="M1527 1893 c-18 -39 -22 -66 -22 -143 0 -125 30 -194 118 -276 l61 <path d="M1527 1893 c-18 -39 -22 -66 -22 -143 0 -125 30 -194 118 -276 l61
-56 -13 -47 c-7 -25 -17 -50 -22 -56 -21 -22 -123 49 -264 185 -78 74 -164 -56 -13 -47 c-7 -25 -17 -50 -22 -56 -21 -22 -123 49 -264 185 -78 74 -164
151 -192 170 -75 50 -91 96 -53 150 11 16 29 32 40 35 24 8 26 30 4 39 -43 16 151 -192 170 -75 50 -91 96 -53 150 11 16 29 32 40 35 24 8 26 30 4 39 -43 16
-140 5 -239 -28 -101 -34 -104 -35 -231 -29 -147 7 -261 -11 -351 -53 -233 -140 5 -239 -28 -101 -34 -104 -35 -231 -29 -147 7 -261 -11 -351 -53 -233
-110 -377 -369 -360 -644 23 -365 263 -673 617 -792 92 -31 100 -32 265 -32 -110 -377 -369 -360 -644 23 -365 263 -673 617 -792 92 -31 100 -32 265 -32
133 0 184 3 235 18 83 23 174 66 232 110 l45 35 49 -16 c69 -21 247 -22 297 0 133 0 184 3 235 18 83 23 174 66 232 110 l45 35 49 -16 c69 -21 247 -22 297 0
31 13 37 21 37 46 0 26 -6 32 -54 54 -30 14 -71 31 -93 38 -68 23 -68 26 -5 31 13 37 21 37 46 0 26 -6 32 -54 54 -30 14 -71 31 -93 38 -68 23 -68 26 -5
98 32 36 73 87 91 113 86 127 155 330 168 489 l6 77 56 11 c107 21 209 99 256 98 32 36 73 87 91 113 86 127 155 330 168 489 l6 77 56 11 c107 21 209 99 256
196 33 68 54 188 36 209 -19 23 -34 20 -67 -15 -41 -44 -99 -68 -164 -69 -57 196 33 68 54 188 36 209 -19 23 -34 20 -67 -15 -41 -44 -99 -68 -164 -69 -57
0 -125 -25 -149 -53 -19 -23 -28 -21 -35 7 -10 39 -55 85 -110 113 -70 35 -93 0 -125 -25 -149 -53 -19 -23 -28 -21 -35 7 -10 39 -55 85 -110 113 -70 35 -93
58 -111 114 -22 66 -48 67 -78 2z m-1139 -549 c180 -37 351 -133 478 -266 38 58 -111 114 -22 66 -48 67 -78 2z m-1139 -549 c180 -37 351 -133 478 -266 38
-40 114 -138 169 -217 103 -148 211 -273 284 -326 22 -16 41 -34 41 -39 0 -13 -40 114 -138 169 -217 103 -148 211 -273 284 -326 22 -16 41 -34 41 -39 0 -13
-148 1 -199 19 -24 8 -83 44 -130 79 -139 104 -217 149 -259 150 -33 1 -37 -2 -148 1 -199 19 -24 8 -83 44 -130 79 -139 104 -217 149 -259 150 -33 1 -37 -2
-40 -26 -2 -14 8 -49 22 -77 30 -58 26 -77 -19 -90 -88 -24 -196 30 -336 169 -40 -26 -2 -14 8 -49 22 -77 30 -58 26 -77 -19 -90 -88 -24 -196 30 -336 169
-84 84 -101 107 -147 200 -59 121 -89 222 -98 330 -5 58 -3 78 7 87 31 25 124 -84 84 -101 107 -147 200 -59 121 -89 222 -98 330 -5 58 -3 78 7 87 31 25 124
28 227 7z m920 -73 c65 -33 178 -175 184 -231 6 -48 -88 -67 -148 -29 -45 27 28 227 7z m920 -73 c65 -33 178 -175 184 -231 6 -48 -88 -67 -148 -29 -45 27
-54 47 -54 113 0 63 -22 96 -64 96 -36 0 -56 10 -56 29 0 44 73 55 138 22z -54 47 -54 113 0 63 -22 96 -64 96 -36 0 -56 10 -56 29 0 44 73 55 138 22z
m-78 -106 c10 -12 10 -18 0 -30 -25 -30 -61 -7 -46 30 3 8 12 15 19 15 8 0 20 m-78 -106 c10 -12 10 -18 0 -30 -25 -30 -61 -7 -46 30 3 8 12 15 19 15 8 0 20
-7 27 -15z"/> -7 27 -15z"/>
</g> </g>
</svg> </svg>
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.8 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

packages/components/package.json
+1
View File
@@ -24,6 +24,7 @@
"@aws-sdk/client-bedrock-runtime": "3.422.0", "@aws-sdk/client-bedrock-runtime": "3.422.0",
"@aws-sdk/client-dynamodb": "^3.360.0", "@aws-sdk/client-dynamodb": "^3.360.0",
"@aws-sdk/client-s3": "^3.427.0", "@aws-sdk/client-s3": "^3.427.0",
"@aws-sdk/client-secrets-manager": "^3.699.0",
"@datastax/astra-db-ts": "1.5.0", "@datastax/astra-db-ts": "1.5.0",
"@dqbd/tiktoken": "^1.0.7", "@dqbd/tiktoken": "^1.0.7",
"@e2b/code-interpreter": "^0.0.5", "@e2b/code-interpreter": "^0.0.5",
packages/components/src/utils.ts
+44 -3
View File
@@ -9,12 +9,30 @@ import { ICommonObject, IDatabaseEntity, IDocument, IMessage, INodeData, IVariab
import { AES, enc } from 'crypto-js' import { AES, enc } from 'crypto-js'
import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages' import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages'
import { getFileFromStorage } from './storageUtils' import { getFileFromStorage } from './storageUtils'
import { GetSecretValueCommand, SecretsManagerClient, SecretsManagerClientConfig } from '@aws-sdk/client-secrets-manager'
import { customGet } from '../nodes/sequentialagents/commonUtils' import { customGet } from '../nodes/sequentialagents/commonUtils'
export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}} export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank
export const FLOWISE_CHATID = 'flowise_chatId' export const FLOWISE_CHATID = 'flowise_chatId'
let secretsManagerClient: SecretsManagerClient | null = null
const USE_AWS_SECRETS_MANAGER = process.env.SECRETKEY_STORAGE_TYPE === 'aws'
if (USE_AWS_SECRETS_MANAGER) {
const region = process.env.SECRETKEY_AWS_REGION || 'us-east-1' // Default region if not provided
const accessKeyId = process.env.SECRETKEY_AWS_ACCESS_KEY
const secretAccessKey = process.env.SECRETKEY_AWS_SECRET_KEY
let credentials: SecretsManagerClientConfig['credentials'] | undefined
if (accessKeyId && secretAccessKey) {
credentials = {
accessKeyId,
secretAccessKey
}
}
secretsManagerClient = new SecretsManagerClient({ credentials, region })
}
/* /*
* List of dependencies allowed to be import in @flowiseai/nodevm * List of dependencies allowed to be import in @flowiseai/nodevm
*/ */
@@ -503,10 +521,33 @@ const getEncryptionKey = async (): Promise<string> => {
* @returns {Promise<ICommonObject>} * @returns {Promise<ICommonObject>}
*/ */
const decryptCredentialData = async (encryptedData: string): Promise<ICommonObject> => { const decryptCredentialData = async (encryptedData: string): Promise<ICommonObject> => {
const encryptKey = await getEncryptionKey() let decryptedDataStr: string
const decryptedData = AES.decrypt(encryptedData, encryptKey)
if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
try {
const command = new GetSecretValueCommand({ SecretId: encryptedData })
const response = await secretsManagerClient.send(command)
if (response.SecretString) {
const secretObj = JSON.parse(response.SecretString)
decryptedDataStr = JSON.stringify(secretObj)
} else {
throw new Error('Failed to retrieve secret value.')
}
} catch (error) {
console.error(error)
throw new Error('Credentials could not be decrypted.')
}
} else {
// Fallback to existing code
const encryptKey = await getEncryptionKey()
const decryptedData = AES.decrypt(encryptedData, encryptKey)
decryptedDataStr = decryptedData.toString(enc.Utf8)
}
if (!decryptedDataStr) return {}
try { try {
return JSON.parse(decryptedData.toString(enc.Utf8)) return JSON.parse(decryptedDataStr)
} catch (e) { } catch (e) {
console.error(e) console.error(e)
throw new Error('Credentials could not be decrypted.') throw new Error('Credentials could not be decrypted.')
packages/server/.env.example
+7 -2
View File
@@ -1,7 +1,14 @@
PORT=3000 PORT=3000
# APIKEY_STORAGE_TYPE=json (json | db)
# APIKEY_PATH=/your_api_key_path/.flowise # APIKEY_PATH=/your_api_key_path/.flowise
# SECRETKEY_STORAGE_TYPE=local #(local | aws)
# SECRETKEY_PATH=/your_api_key_path/.flowise # SECRETKEY_PATH=/your_api_key_path/.flowise
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
# SECRETKEY_AWS_ACCESS_KEY=<your-access-key>
# SECRETKEY_AWS_SECRET_KEY=<your-secret-key>
# SECRETKEY_AWS_REGION=us-west-2
# NUMBER_OF_PROXIES= 1 # NUMBER_OF_PROXIES= 1
# CORS_ORIGINS=* # CORS_ORIGINS=*
@@ -19,7 +26,6 @@ PORT=3000
# FLOWISE_USERNAME=user # FLOWISE_USERNAME=user
# FLOWISE_PASSWORD=1234 # FLOWISE_PASSWORD=1234
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
# FLOWISE_FILE_SIZE_LIMIT=50mb # FLOWISE_FILE_SIZE_LIMIT=50mb
# DISABLE_CHATFLOW_REUSE=true # DISABLE_CHATFLOW_REUSE=true
@@ -50,7 +56,6 @@ PORT=3000
# S3_ENDPOINT_URL=<custom-s3-endpoint-url> # S3_ENDPOINT_URL=<custom-s3-endpoint-url>
# S3_FORCE_PATH_STYLE=false # S3_FORCE_PATH_STYLE=false
# APIKEY_STORAGE_TYPE=json (json | db)
# SHOW_COMMUNITY_NODES=true # SHOW_COMMUNITY_NODES=true
# DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable) # DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable)
packages/server/package.json
+5 -4
View File
@@ -54,8 +54,10 @@
}, },
"license": "SEE LICENSE IN LICENSE.md", "license": "SEE LICENSE IN LICENSE.md",
"dependencies": { "dependencies": {
"@aws-sdk/client-secrets-manager": "^3.699.0",
"@oclif/core": "^1.13.10", "@oclif/core": "^1.13.10",
"@opentelemetry/api": "^1.3.0", "@opentelemetry/api": "^1.3.0",
"@opentelemetry/auto-instrumentations-node": "^0.52.0",
"@opentelemetry/core": "1.27.0", "@opentelemetry/core": "1.27.0",
"@opentelemetry/exporter-metrics-otlp-grpc": "0.54.0", "@opentelemetry/exporter-metrics-otlp-grpc": "0.54.0",
"@opentelemetry/exporter-metrics-otlp-http": "0.54.0", "@opentelemetry/exporter-metrics-otlp-http": "0.54.0",
@@ -65,10 +67,9 @@
"@opentelemetry/exporter-trace-otlp-proto": "0.54.0", "@opentelemetry/exporter-trace-otlp-proto": "0.54.0",
"@opentelemetry/resources": "1.27.0", "@opentelemetry/resources": "1.27.0",
"@opentelemetry/sdk-metrics": "1.27.0", "@opentelemetry/sdk-metrics": "1.27.0",
"@opentelemetry/sdk-node": "^0.54.0",
"@opentelemetry/sdk-trace-base": "1.27.0", "@opentelemetry/sdk-trace-base": "1.27.0",
"@opentelemetry/semantic-conventions": "1.27.0", "@opentelemetry/semantic-conventions": "1.27.0",
"@opentelemetry/auto-instrumentations-node": "^0.52.0",
"@opentelemetry/sdk-node": "^0.54.0",
"@types/lodash": "^4.14.202", "@types/lodash": "^4.14.202",
"@types/uuid": "^9.0.7", "@types/uuid": "^9.0.7",
"async-mutex": "^0.4.0", "async-mutex": "^0.4.0",
@@ -82,6 +83,7 @@
"express-rate-limit": "^6.9.0", "express-rate-limit": "^6.9.0",
"flowise-components": "workspace:^", "flowise-components": "workspace:^",
"flowise-ui": "workspace:^", "flowise-ui": "workspace:^",
"global-agent": "^3.0.0",
"http-errors": "^2.0.0", "http-errors": "^2.0.0",
"http-status-codes": "^2.3.0", "http-status-codes": "^2.3.0",
"langchainhub": "^0.0.11", "langchainhub": "^0.0.11",
@@ -100,8 +102,7 @@
"sqlite3": "^5.1.6", "sqlite3": "^5.1.6",
"typeorm": "^0.3.6", "typeorm": "^0.3.6",
"uuid": "^9.0.1", "uuid": "^9.0.1",
"winston": "^3.9.0", "winston": "^3.9.0"
"global-agent": "^3.0.0"
}, },
"devDependencies": { "devDependencies": {
"@types/content-disposition": "0.5.8", "@types/content-disposition": "0.5.8",
packages/server/src/commands/start.ts
+10 -2
View File
@@ -26,8 +26,6 @@ export default class Start extends Command {
BLOB_STORAGE_PATH: Flags.string(), BLOB_STORAGE_PATH: Flags.string(),
APIKEY_STORAGE_TYPE: Flags.string(), APIKEY_STORAGE_TYPE: Flags.string(),
APIKEY_PATH: Flags.string(), APIKEY_PATH: Flags.string(),
SECRETKEY_PATH: Flags.string(),
FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
LOG_PATH: Flags.string(), LOG_PATH: Flags.string(),
LOG_LEVEL: Flags.string(), LOG_LEVEL: Flags.string(),
TOOL_FUNCTION_BUILTIN_DEP: Flags.string(), TOOL_FUNCTION_BUILTIN_DEP: Flags.string(),
@@ -57,6 +55,12 @@ export default class Start extends Command {
S3_ENDPOINT_URL: Flags.string(), S3_ENDPOINT_URL: Flags.string(),
S3_FORCE_PATH_STYLE: Flags.string(), S3_FORCE_PATH_STYLE: Flags.string(),
SHOW_COMMUNITY_NODES: Flags.string(), SHOW_COMMUNITY_NODES: Flags.string(),
SECRETKEY_STORAGE_TYPE: Flags.string(),
SECRETKEY_PATH: Flags.string(),
FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
SECRETKEY_AWS_ACCESS_KEY: Flags.string(),
SECRETKEY_AWS_SECRET_KEY: Flags.string(),
SECRETKEY_AWS_REGION: Flags.string(),
DISABLED_NODES: Flags.string() DISABLED_NODES: Flags.string()
} }
@@ -113,8 +117,12 @@ export default class Start extends Command {
if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT
// Credentials // Credentials
if (flags.SECRETKEY_STORAGE_TYPE) process.env.SECRETKEY_STORAGE_TYPE = flags.SECRETKEY_STORAGE_TYPE
if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
if (flags.FLOWISE_SECRETKEY_OVERWRITE) process.env.FLOWISE_SECRETKEY_OVERWRITE = flags.FLOWISE_SECRETKEY_OVERWRITE if (flags.FLOWISE_SECRETKEY_OVERWRITE) process.env.FLOWISE_SECRETKEY_OVERWRITE = flags.FLOWISE_SECRETKEY_OVERWRITE
if (flags.SECRETKEY_AWS_ACCESS_KEY) process.env.SECRETKEY_AWS_ACCESS_KEY = flags.SECRETKEY_AWS_ACCESS_KEY
if (flags.SECRETKEY_AWS_SECRET_KEY) process.env.SECRETKEY_AWS_SECRET_KEY = flags.SECRETKEY_AWS_SECRET_KEY
if (flags.SECRETKEY_AWS_REGION) process.env.SECRETKEY_AWS_REGION = flags.SECRETKEY_AWS_REGION
// Logs // Logs
if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
packages/server/src/utils/index.ts
+91 -13
View File
@@ -54,12 +54,36 @@ import { DocumentStore } from '../database/entities/DocumentStore'
import { DocumentStoreFileChunk } from '../database/entities/DocumentStoreFileChunk' import { DocumentStoreFileChunk } from '../database/entities/DocumentStoreFileChunk'
import { InternalFlowiseError } from '../errors/internalFlowiseError' import { InternalFlowiseError } from '../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes' import { StatusCodes } from 'http-status-codes'
import {
CreateSecretCommand,
GetSecretValueCommand,
PutSecretValueCommand,
SecretsManagerClient,
SecretsManagerClientConfig
} from '@aws-sdk/client-secrets-manager'
const QUESTION_VAR_PREFIX = 'question' const QUESTION_VAR_PREFIX = 'question'
const FILE_ATTACHMENT_PREFIX = 'file_attachment' const FILE_ATTACHMENT_PREFIX = 'file_attachment'
const CHAT_HISTORY_VAR_PREFIX = 'chat_history' const CHAT_HISTORY_VAR_PREFIX = 'chat_history'
const REDACTED_CREDENTIAL_VALUE = '_FLOWISE_BLANK_07167752-1a71-43b1-bf8f-4f32252165db' const REDACTED_CREDENTIAL_VALUE = '_FLOWISE_BLANK_07167752-1a71-43b1-bf8f-4f32252165db'
let secretsManagerClient: SecretsManagerClient | null = null
const USE_AWS_SECRETS_MANAGER = process.env.SECRETKEY_STORAGE_TYPE === 'aws'
if (USE_AWS_SECRETS_MANAGER) {
const region = process.env.SECRETKEY_AWS_REGION || 'us-east-1' // Default region if not provided
const accessKeyId = process.env.SECRETKEY_AWS_ACCESS_KEY
const secretAccessKey = process.env.SECRETKEY_AWS_SECRET_KEY
let credentials: SecretsManagerClientConfig['credentials'] | undefined
if (accessKeyId && secretAccessKey) {
credentials = {
accessKeyId,
secretAccessKey
}
}
secretsManagerClient = new SecretsManagerClient({ credentials, region })
}
export const databaseEntities: IDatabaseEntity = { export const databaseEntities: IDatabaseEntity = {
ChatFlow: ChatFlow, ChatFlow: ChatFlow,
ChatMessage: ChatMessage, ChatMessage: ChatMessage,
@@ -1360,14 +1384,6 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
return isChatOrLLMsExist && isValidChainOrAgent && !isOutputParserExist return isChatOrLLMsExist && isValidChainOrAgent && !isOutputParserExist
} }
/**
* Generate an encryption key
* @returns {string}
*/
export const generateEncryptKey = (): string => {
return randomBytes(24).toString('base64')
}
/** /**
* Returns the encryption key * Returns the encryption key
* @returns {Promise<string>} * @returns {Promise<string>}
@@ -1394,7 +1410,39 @@ export const getEncryptionKey = async (): Promise<string> => {
* @returns {Promise<string>} * @returns {Promise<string>}
*/ */
export const encryptCredentialData = async (plainDataObj: ICredentialDataDecrypted): Promise<string> => { export const encryptCredentialData = async (plainDataObj: ICredentialDataDecrypted): Promise<string> => {
if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
const secretName = `FlowiseCredential_${randomBytes(12).toString('hex')}`
logger.info(`[server]: Upserting AWS Secret: ${secretName}`)
const secretString = JSON.stringify({ ...plainDataObj })
try {
// Try to update the secret if it exists
const putCommand = new PutSecretValueCommand({
SecretId: secretName,
SecretString: secretString
})
await secretsManagerClient.send(putCommand)
} catch (error: any) {
if (error.name === 'ResourceNotFoundException') {
// Secret doesn't exist, so create it
const createCommand = new CreateSecretCommand({
Name: secretName,
SecretString: secretString
})
await secretsManagerClient.send(createCommand)
} else {
// Rethrow any other errors
throw error
}
}
return secretName
}
const encryptKey = await getEncryptionKey() const encryptKey = await getEncryptionKey()
// Fallback to existing code
return AES.encrypt(JSON.stringify(plainDataObj), encryptKey).toString() return AES.encrypt(JSON.stringify(plainDataObj), encryptKey).toString()
} }
@@ -1410,22 +1458,52 @@ export const decryptCredentialData = async (
componentCredentialName?: string, componentCredentialName?: string,
componentCredentials?: IComponentCredentials componentCredentials?: IComponentCredentials
): Promise<ICredentialDataDecrypted> => { ): Promise<ICredentialDataDecrypted> => {
const encryptKey = await getEncryptionKey() let decryptedDataStr: string
const decryptedData = AES.decrypt(encryptedData, encryptKey)
const decryptedDataStr = decryptedData.toString(enc.Utf8) if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
try {
logger.info(`[server]: Reading AWS Secret: ${encryptedData}`)
const command = new GetSecretValueCommand({ SecretId: encryptedData })
const response = await secretsManagerClient.send(command)
if (response.SecretString) {
const secretObj = JSON.parse(response.SecretString)
decryptedDataStr = JSON.stringify(secretObj)
} else {
throw new Error('Failed to retrieve secret value.')
}
} catch (error) {
console.error(error)
throw new Error('Failed to decrypt credential data.')
}
} else {
// Fallback to existing code
const encryptKey = await getEncryptionKey()
const decryptedData = AES.decrypt(encryptedData, encryptKey)
decryptedDataStr = decryptedData.toString(enc.Utf8)
}
if (!decryptedDataStr) return {} if (!decryptedDataStr) return {}
try { try {
if (componentCredentialName && componentCredentials) { if (componentCredentialName && componentCredentials) {
const plainDataObj = JSON.parse(decryptedData.toString(enc.Utf8)) const plainDataObj = JSON.parse(decryptedDataStr)
return redactCredentialWithPasswordType(componentCredentialName, plainDataObj, componentCredentials) return redactCredentialWithPasswordType(componentCredentialName, plainDataObj, componentCredentials)
} }
return JSON.parse(decryptedData.toString(enc.Utf8)) return JSON.parse(decryptedDataStr)
} catch (e) { } catch (e) {
console.error(e) console.error(e)
return {} return {}
} }
} }
/**
* Generate an encryption key
* @returns {string}
*/
export const generateEncryptKey = (): string => {
return randomBytes(24).toString('base64')
}
/** /**
* Transform ICredentialBody from req to Credential entity * Transform ICredentialBody from req to Credential entity
* @param {ICredentialReqBody} body * @param {ICredentialReqBody} body
pnpm-lock.yaml
Generated
+999 -19
View File
File diff suppressed because it is too large Load Diff