farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-22 07:01:07 +03:00
Code Issues Packages Projects Releases Wiki Activity

[Feat] Allow AWS SECRETS MANAGER instead of storing AES Encrypted in db (#3616)

Browse Source 
* AWS Secrets

* AWS Secrets support

* add examples

* remove test compose

* fix lint

* update aws secret manager implementation

* update secret manager client

* update comments

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Lucas Mohallem Ferraz
2025-01-08 14:24:57 -06:00
committed by GitHub
parent c2c1ca9162
commit 1ae78c2739
10 changed files with 1393 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/.env.example
+9 -1
View File
@@ -5,6 +5,8 @@ SECRETKEY_PATH=/root/.flowise
LOG_PATH=/root/.flowise/logs
BLOB_STORAGE_PATH=/root/.flowise/storage
# APIKEY_STORAGE_TYPE=json (json | db)
# NUMBER_OF_PROXIES= 1
# CORS_ORIGINS=*
# IFRAME_ORIGINS=*
@@ -18,6 +20,13 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
# DATABASE_SSL=true
# DATABASE_SSL_KEY_BASE64=<Self signed certificate in BASE64>
# SECRETKEY_STORAGE_TYPE=local #(local | aws)
# SECRETKEY_PATH=/your_api_key_path/.flowise
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
# SECRETKEY_AWS_ACCESS_KEY=<your-access-key>
# SECRETKEY_AWS_SECRET_KEY=<your-secret-key>
# SECRETKEY_AWS_REGION=us-west-2
# FLOWISE_USERNAME=user
# FLOWISE_PASSWORD=1234
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
@@ -50,7 +59,6 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
# S3_ENDPOINT_URL=<custom-s3-endpoint-url>
# S3_FORCE_PATH_STYLE=false
# APIKEY_STORAGE_TYPE=json (json | db)
# SHOW_COMMUNITY_NODES=true
# DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable)
packages/components/nodes/chatmodels/Deepseek/Deepseek.ts
+197 -197
View File
@@ -1,197 +1,197 @@
import { BaseCache } from '@langchain/core/caches'
import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
import { ChatOpenAI, LegacyOpenAIInput, OpenAIChatInput } from '@langchain/openai'
import type { ClientOptions } from 'openai'
import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
class Deepseek_ChatModels implements INode {
readonly baseURL: string = 'https://api.deepseek.com'
label: string
name: string
version: number
type: string
icon: string
category: string
description: string
baseClasses: string[]
credential: INodeParams
inputs: INodeParams[]
constructor() {
this.label = 'ChatDeepseek'
this.name = 'chatDeepseek'
this.version = 1.0
this.type = 'chatDeepseek'
this.icon = 'deepseek.svg'
this.category = 'Chat Models'
this.description = 'Wrapper around Deepseek large language models that use the Chat endpoint'
this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)]
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['deepseekApi']
}
this.inputs = [
{
label: 'Cache',
name: 'cache',
type: 'BaseCache',
optional: true
},
{
label: 'Model Name',
name: 'modelName',
type: 'asyncOptions',
loadMethod: 'listModels',
default: 'deepseek-chat'
},
{
label: 'Temperature',
name: 'temperature',
type: 'number',
step: 0.1,
default: 0.7,
optional: true
},
{
label: 'Streaming',
name: 'streaming',
type: 'boolean',
default: true,
optional: true,
additionalParams: true
},
{
label: 'Max Tokens',
name: 'maxTokens',
type: 'number',
step: 1,
optional: true,
additionalParams: true
},
{
label: 'Top Probability',
name: 'topP',
type: 'number',
step: 0.1,
optional: true,
additionalParams: true
},
{
label: 'Frequency Penalty',
name: 'frequencyPenalty',
type: 'number',
step: 0.1,
optional: true,
additionalParams: true
},
{
label: 'Presence Penalty',
name: 'presencePenalty',
type: 'number',
step: 0.1,
optional: true,
additionalParams: true
},
{
label: 'Timeout',
name: 'timeout',
type: 'number',
step: 1,
optional: true,
additionalParams: true
},
{
label: 'Stop Sequence',
name: 'stopSequence',
type: 'string',
rows: 4,
optional: true,
description: 'List of stop words to use when generating. Use comma to separate multiple stop words.',
additionalParams: true
},
{
label: 'Base Options',
name: 'baseOptions',
type: 'json',
optional: true,
additionalParams: true,
description: 'Additional options to pass to the Deepseek client. This should be a JSON object.'
}
]
}
//@ts-ignore
loadMethods = {
async listModels(): Promise<INodeOptionsValue[]> {
return await getModels(MODEL_TYPE.CHAT, 'deepseek')
}
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const temperature = nodeData.inputs?.temperature as string
const modelName = nodeData.inputs?.modelName as string
const maxTokens = nodeData.inputs?.maxTokens as string
const topP = nodeData.inputs?.topP as string
const frequencyPenalty = nodeData.inputs?.frequencyPenalty as string
const presencePenalty = nodeData.inputs?.presencePenalty as string
const timeout = nodeData.inputs?.timeout as string
const stopSequence = nodeData.inputs?.stopSequence as string
const streaming = nodeData.inputs?.streaming as boolean
const baseOptions = nodeData.inputs?.baseOptions
if (nodeData.inputs?.credentialId) {
nodeData.credential = nodeData.inputs?.credentialId
}
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const openAIApiKey = getCredentialParam('deepseekApiKey', credentialData, nodeData)
const cache = nodeData.inputs?.cache as BaseCache
const obj: Partial<OpenAIChatInput> & BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput } = {
temperature: parseFloat(temperature),
modelName,
openAIApiKey,
streaming: streaming ?? true
}
if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10)
if (topP) obj.topP = parseFloat(topP)
if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty)
if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty)
if (timeout) obj.timeout = parseInt(timeout, 10)
if (cache) obj.cache = cache
if (stopSequence) {
const stopSequenceArray = stopSequence.split(',').map((item) => item.trim())
obj.stop = stopSequenceArray
}
let parsedBaseOptions: any | undefined = undefined
if (baseOptions) {
try {
parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions)
if (parsedBaseOptions.baseURL) {
console.warn("The 'baseURL' parameter is not allowed when using the ChatDeepseek node.")
parsedBaseOptions.baseURL = undefined
}
} catch (exception) {
throw new Error('Invalid JSON in the BaseOptions: ' + exception)
}
}
const model = new ChatOpenAI({
...obj,
configuration: {
baseURL: this.baseURL,
...parsedBaseOptions
}
})
return model
}
}
module.exports = { nodeClass: Deepseek_ChatModels }
import { BaseCache } from '@langchain/core/caches'
import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
import { ChatOpenAI, LegacyOpenAIInput, OpenAIChatInput } from '@langchain/openai'
import type { ClientOptions } from 'openai'
import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
class Deepseek_ChatModels implements INode {
readonly baseURL: string = 'https://api.deepseek.com'
label: string
name: string
version: number
type: string
icon: string
category: string
description: string
baseClasses: string[]
credential: INodeParams
inputs: INodeParams[]
constructor() {
this.label = 'ChatDeepseek'
this.name = 'chatDeepseek'
this.version = 1.0
this.type = 'chatDeepseek'
this.icon = 'deepseek.svg'
this.category = 'Chat Models'
this.description = 'Wrapper around Deepseek large language models that use the Chat endpoint'
this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)]
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['deepseekApi']
}
this.inputs = [
{
label: 'Cache',
name: 'cache',
type: 'BaseCache',
optional: true
},
{
label: 'Model Name',
name: 'modelName',
type: 'asyncOptions',
loadMethod: 'listModels',
default: 'deepseek-chat'
},
{
label: 'Temperature',
name: 'temperature',
type: 'number',
step: 0.1,
default: 0.7,
optional: true
},
{
label: 'Streaming',
name: 'streaming',
type: 'boolean',
default: true,
optional: true,
additionalParams: true
},
{
label: 'Max Tokens',
name: 'maxTokens',
type: 'number',
step: 1,
optional: true,
additionalParams: true
},
{
label: 'Top Probability',
name: 'topP',
type: 'number',
step: 0.1,
optional: true,
additionalParams: true
},
{
label: 'Frequency Penalty',
name: 'frequencyPenalty',
type: 'number',
step: 0.1,
optional: true,
additionalParams: true
},
{
label: 'Presence Penalty',
name: 'presencePenalty',
type: 'number',
step: 0.1,
optional: true,
additionalParams: true
},
{
label: 'Timeout',
name: 'timeout',
type: 'number',
step: 1,
optional: true,
additionalParams: true
},
{
label: 'Stop Sequence',
name: 'stopSequence',
type: 'string',
rows: 4,
optional: true,
description: 'List of stop words to use when generating. Use comma to separate multiple stop words.',
additionalParams: true
},
{
label: 'Base Options',
name: 'baseOptions',
type: 'json',
optional: true,
additionalParams: true,
description: 'Additional options to pass to the Deepseek client. This should be a JSON object.'
}
]
}
//@ts-ignore
loadMethods = {
async listModels(): Promise<INodeOptionsValue[]> {
return await getModels(MODEL_TYPE.CHAT, 'deepseek')
}
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const temperature = nodeData.inputs?.temperature as string
const modelName = nodeData.inputs?.modelName as string
const maxTokens = nodeData.inputs?.maxTokens as string
const topP = nodeData.inputs?.topP as string
const frequencyPenalty = nodeData.inputs?.frequencyPenalty as string
const presencePenalty = nodeData.inputs?.presencePenalty as string
const timeout = nodeData.inputs?.timeout as string
const stopSequence = nodeData.inputs?.stopSequence as string
const streaming = nodeData.inputs?.streaming as boolean
const baseOptions = nodeData.inputs?.baseOptions
if (nodeData.inputs?.credentialId) {
nodeData.credential = nodeData.inputs?.credentialId
}
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const openAIApiKey = getCredentialParam('deepseekApiKey', credentialData, nodeData)
const cache = nodeData.inputs?.cache as BaseCache
const obj: Partial<OpenAIChatInput> & BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput } = {
temperature: parseFloat(temperature),
modelName,
openAIApiKey,
streaming: streaming ?? true
}
if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10)
if (topP) obj.topP = parseFloat(topP)
if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty)
if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty)
if (timeout) obj.timeout = parseInt(timeout, 10)
if (cache) obj.cache = cache
if (stopSequence) {
const stopSequenceArray = stopSequence.split(',').map((item) => item.trim())
obj.stop = stopSequenceArray
}
let parsedBaseOptions: any | undefined = undefined
if (baseOptions) {
try {
parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions)
if (parsedBaseOptions.baseURL) {
console.warn("The 'baseURL' parameter is not allowed when using the ChatDeepseek node.")
parsedBaseOptions.baseURL = undefined
}
} catch (exception) {
throw new Error('Invalid JSON in the BaseOptions: ' + exception)
}
}
const model = new ChatOpenAI({
...obj,
configuration: {
baseURL: this.baseURL,
...parsedBaseOptions
}
})
return model
}
}
module.exports = { nodeClass: Deepseek_ChatModels }
packages/components/nodes/chatmodels/Deepseek/deepseek.svg
+30 -30
View File
@@ -1,30 +1,30 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
width="225.000000pt" height="225.000000pt" viewBox="0 0 225.000000 225.000000"
preserveAspectRatio="xMidYMid meet">
<g transform="translate(0.000000,225.000000) scale(0.100000,-0.100000)"
fill="#000000" stroke="none">
<path d="M1527 1893 c-18 -39 -22 -66 -22 -143 0 -125 30 -194 118 -276 l61
-56 -13 -47 c-7 -25 -17 -50 -22 -56 -21 -22 -123 49 -264 185 -78 74 -164
151 -192 170 -75 50 -91 96 -53 150 11 16 29 32 40 35 24 8 26 30 4 39 -43 16
-140 5 -239 -28 -101 -34 -104 -35 -231 -29 -147 7 -261 -11 -351 -53 -233
-110 -377 -369 -360 -644 23 -365 263 -673 617 -792 92 -31 100 -32 265 -32
133 0 184 3 235 18 83 23 174 66 232 110 l45 35 49 -16 c69 -21 247 -22 297 0
31 13 37 21 37 46 0 26 -6 32 -54 54 -30 14 -71 31 -93 38 -68 23 -68 26 -5
98 32 36 73 87 91 113 86 127 155 330 168 489 l6 77 56 11 c107 21 209 99 256
196 33 68 54 188 36 209 -19 23 -34 20 -67 -15 -41 -44 -99 -68 -164 -69 -57
0 -125 -25 -149 -53 -19 -23 -28 -21 -35 7 -10 39 -55 85 -110 113 -70 35 -93
58 -111 114 -22 66 -48 67 -78 2z m-1139 -549 c180 -37 351 -133 478 -266 38
-40 114 -138 169 -217 103 -148 211 -273 284 -326 22 -16 41 -34 41 -39 0 -13
-148 1 -199 19 -24 8 -83 44 -130 79 -139 104 -217 149 -259 150 -33 1 -37 -2
-40 -26 -2 -14 8 -49 22 -77 30 -58 26 -77 -19 -90 -88 -24 -196 30 -336 169
-84 84 -101 107 -147 200 -59 121 -89 222 -98 330 -5 58 -3 78 7 87 31 25 124
28 227 7z m920 -73 c65 -33 178 -175 184 -231 6 -48 -88 -67 -148 -29 -45 27
-54 47 -54 113 0 63 -22 96 -64 96 -36 0 -56 10 -56 29 0 44 73 55 138 22z
m-78 -106 c10 -12 10 -18 0 -30 -25 -30 -61 -7 -46 30 3 8 12 15 19 15 8 0 20
-7 27 -15z"/>
</g>
</svg>
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
width="225.000000pt" height="225.000000pt" viewBox="0 0 225.000000 225.000000"
preserveAspectRatio="xMidYMid meet">
<g transform="translate(0.000000,225.000000) scale(0.100000,-0.100000)"
fill="#000000" stroke="none">
<path d="M1527 1893 c-18 -39 -22 -66 -22 -143 0 -125 30 -194 118 -276 l61
-56 -13 -47 c-7 -25 -17 -50 -22 -56 -21 -22 -123 49 -264 185 -78 74 -164
151 -192 170 -75 50 -91 96 -53 150 11 16 29 32 40 35 24 8 26 30 4 39 -43 16
-140 5 -239 -28 -101 -34 -104 -35 -231 -29 -147 7 -261 -11 -351 -53 -233
-110 -377 -369 -360 -644 23 -365 263 -673 617 -792 92 -31 100 -32 265 -32
133 0 184 3 235 18 83 23 174 66 232 110 l45 35 49 -16 c69 -21 247 -22 297 0
31 13 37 21 37 46 0 26 -6 32 -54 54 -30 14 -71 31 -93 38 -68 23 -68 26 -5
98 32 36 73 87 91 113 86 127 155 330 168 489 l6 77 56 11 c107 21 209 99 256
196 33 68 54 188 36 209 -19 23 -34 20 -67 -15 -41 -44 -99 -68 -164 -69 -57
0 -125 -25 -149 -53 -19 -23 -28 -21 -35 7 -10 39 -55 85 -110 113 -70 35 -93
58 -111 114 -22 66 -48 67 -78 2z m-1139 -549 c180 -37 351 -133 478 -266 38
-40 114 -138 169 -217 103 -148 211 -273 284 -326 22 -16 41 -34 41 -39 0 -13
-148 1 -199 19 -24 8 -83 44 -130 79 -139 104 -217 149 -259 150 -33 1 -37 -2
-40 -26 -2 -14 8 -49 22 -77 30 -58 26 -77 -19 -90 -88 -24 -196 30 -336 169
-84 84 -101 107 -147 200 -59 121 -89 222 -98 330 -5 58 -3 78 7 87 31 25 124
28 227 7z m920 -73 c65 -33 178 -175 184 -231 6 -48 -88 -67 -148 -29 -45 27
-54 47 -54 113 0 63 -22 96 -64 96 -36 0 -56 10 -56 29 0 44 73 55 138 22z
m-78 -106 c10 -12 10 -18 0 -30 -25 -30 -61 -7 -46 30 3 8 12 15 19 15 8 0 20
-7 27 -15z"/>
</g>
</svg>
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.8 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

packages/components/package.json
+1
View File
@@ -24,6 +24,7 @@
"@aws-sdk/client-bedrock-runtime": "3.422.0",
"@aws-sdk/client-dynamodb": "^3.360.0",
"@aws-sdk/client-s3": "^3.427.0",
"@aws-sdk/client-secrets-manager": "^3.699.0",
"@datastax/astra-db-ts": "1.5.0",
"@dqbd/tiktoken": "^1.0.7",
"@e2b/code-interpreter": "^0.0.5",
packages/components/src/utils.ts
+44 -3
View File
@@ -9,12 +9,30 @@ import { ICommonObject, IDatabaseEntity, IDocument, IMessage, INodeData, IVariab
import { AES, enc } from 'crypto-js'
import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages'
import { getFileFromStorage } from './storageUtils'
import { GetSecretValueCommand, SecretsManagerClient, SecretsManagerClientConfig } from '@aws-sdk/client-secrets-manager'
import { customGet } from '../nodes/sequentialagents/commonUtils'
export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank
export const FLOWISE_CHATID = 'flowise_chatId'
let secretsManagerClient: SecretsManagerClient | null = null
const USE_AWS_SECRETS_MANAGER = process.env.SECRETKEY_STORAGE_TYPE === 'aws'
if (USE_AWS_SECRETS_MANAGER) {
const region = process.env.SECRETKEY_AWS_REGION || 'us-east-1' // Default region if not provided
const accessKeyId = process.env.SECRETKEY_AWS_ACCESS_KEY
const secretAccessKey = process.env.SECRETKEY_AWS_SECRET_KEY
let credentials: SecretsManagerClientConfig['credentials'] | undefined
if (accessKeyId && secretAccessKey) {
credentials = {
accessKeyId,
secretAccessKey
}
}
secretsManagerClient = new SecretsManagerClient({ credentials, region })
}
/*
* List of dependencies allowed to be import in @flowiseai/nodevm
*/
@@ -503,10 +521,33 @@ const getEncryptionKey = async (): Promise<string> => {
* @returns {Promise<ICommonObject>}
*/
const decryptCredentialData = async (encryptedData: string): Promise<ICommonObject> => {
const encryptKey = await getEncryptionKey()
const decryptedData = AES.decrypt(encryptedData, encryptKey)
let decryptedDataStr: string
if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
try {
const command = new GetSecretValueCommand({ SecretId: encryptedData })
const response = await secretsManagerClient.send(command)
if (response.SecretString) {
const secretObj = JSON.parse(response.SecretString)
decryptedDataStr = JSON.stringify(secretObj)
} else {
throw new Error('Failed to retrieve secret value.')
}
} catch (error) {
console.error(error)
throw new Error('Credentials could not be decrypted.')
}
} else {
// Fallback to existing code
const encryptKey = await getEncryptionKey()
const decryptedData = AES.decrypt(encryptedData, encryptKey)
decryptedDataStr = decryptedData.toString(enc.Utf8)
}
if (!decryptedDataStr) return {}
try {
return JSON.parse(decryptedData.toString(enc.Utf8))
return JSON.parse(decryptedDataStr)
} catch (e) {
console.error(e)
throw new Error('Credentials could not be decrypted.')
packages/server/.env.example
+7 -2
View File
@@ -1,7 +1,14 @@
PORT=3000
# APIKEY_STORAGE_TYPE=json (json | db)
# APIKEY_PATH=/your_api_key_path/.flowise
# SECRETKEY_STORAGE_TYPE=local #(local | aws)
# SECRETKEY_PATH=/your_api_key_path/.flowise
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
# SECRETKEY_AWS_ACCESS_KEY=<your-access-key>
# SECRETKEY_AWS_SECRET_KEY=<your-secret-key>
# SECRETKEY_AWS_REGION=us-west-2
# NUMBER_OF_PROXIES= 1
# CORS_ORIGINS=*
@@ -19,7 +26,6 @@ PORT=3000
# FLOWISE_USERNAME=user
# FLOWISE_PASSWORD=1234
# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
# FLOWISE_FILE_SIZE_LIMIT=50mb
# DISABLE_CHATFLOW_REUSE=true
@@ -50,7 +56,6 @@ PORT=3000
# S3_ENDPOINT_URL=<custom-s3-endpoint-url>
# S3_FORCE_PATH_STYLE=false
# APIKEY_STORAGE_TYPE=json (json | db)
# SHOW_COMMUNITY_NODES=true
# DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable)
packages/server/package.json
+5 -4
View File
@@ -54,8 +54,10 @@
},
"license": "SEE LICENSE IN LICENSE.md",
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.699.0",
"@oclif/core": "^1.13.10",
"@opentelemetry/api": "^1.3.0",
"@opentelemetry/auto-instrumentations-node": "^0.52.0",
"@opentelemetry/core": "1.27.0",
"@opentelemetry/exporter-metrics-otlp-grpc": "0.54.0",
"@opentelemetry/exporter-metrics-otlp-http": "0.54.0",
@@ -65,10 +67,9 @@
"@opentelemetry/exporter-trace-otlp-proto": "0.54.0",
"@opentelemetry/resources": "1.27.0",
"@opentelemetry/sdk-metrics": "1.27.0",
"@opentelemetry/sdk-node": "^0.54.0",
"@opentelemetry/sdk-trace-base": "1.27.0",
"@opentelemetry/semantic-conventions": "1.27.0",
"@opentelemetry/auto-instrumentations-node": "^0.52.0",
"@opentelemetry/sdk-node": "^0.54.0",
"@types/lodash": "^4.14.202",
"@types/uuid": "^9.0.7",
"async-mutex": "^0.4.0",
@@ -82,6 +83,7 @@
"express-rate-limit": "^6.9.0",
"flowise-components": "workspace:^",
"flowise-ui": "workspace:^",
"global-agent": "^3.0.0",
"http-errors": "^2.0.0",
"http-status-codes": "^2.3.0",
"langchainhub": "^0.0.11",
@@ -100,8 +102,7 @@
"sqlite3": "^5.1.6",
"typeorm": "^0.3.6",
"uuid": "^9.0.1",
"winston": "^3.9.0",
"global-agent": "^3.0.0"
"winston": "^3.9.0"
},
"devDependencies": {
"@types/content-disposition": "0.5.8",
packages/server/src/commands/start.ts
+10 -2
View File
@@ -26,8 +26,6 @@ export default class Start extends Command {
BLOB_STORAGE_PATH: Flags.string(),
APIKEY_STORAGE_TYPE: Flags.string(),
APIKEY_PATH: Flags.string(),
SECRETKEY_PATH: Flags.string(),
FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
LOG_PATH: Flags.string(),
LOG_LEVEL: Flags.string(),
TOOL_FUNCTION_BUILTIN_DEP: Flags.string(),
@@ -57,6 +55,12 @@ export default class Start extends Command {
S3_ENDPOINT_URL: Flags.string(),
S3_FORCE_PATH_STYLE: Flags.string(),
SHOW_COMMUNITY_NODES: Flags.string(),
SECRETKEY_STORAGE_TYPE: Flags.string(),
SECRETKEY_PATH: Flags.string(),
FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
SECRETKEY_AWS_ACCESS_KEY: Flags.string(),
SECRETKEY_AWS_SECRET_KEY: Flags.string(),
SECRETKEY_AWS_REGION: Flags.string(),
DISABLED_NODES: Flags.string()
}
@@ -113,8 +117,12 @@ export default class Start extends Command {
if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT
// Credentials
if (flags.SECRETKEY_STORAGE_TYPE) process.env.SECRETKEY_STORAGE_TYPE = flags.SECRETKEY_STORAGE_TYPE
if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
if (flags.FLOWISE_SECRETKEY_OVERWRITE) process.env.FLOWISE_SECRETKEY_OVERWRITE = flags.FLOWISE_SECRETKEY_OVERWRITE
if (flags.SECRETKEY_AWS_ACCESS_KEY) process.env.SECRETKEY_AWS_ACCESS_KEY = flags.SECRETKEY_AWS_ACCESS_KEY
if (flags.SECRETKEY_AWS_SECRET_KEY) process.env.SECRETKEY_AWS_SECRET_KEY = flags.SECRETKEY_AWS_SECRET_KEY
if (flags.SECRETKEY_AWS_REGION) process.env.SECRETKEY_AWS_REGION = flags.SECRETKEY_AWS_REGION
// Logs
if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
packages/server/src/utils/index.ts
+91 -13
View File
@@ -54,12 +54,36 @@ import { DocumentStore } from '../database/entities/DocumentStore'
import { DocumentStoreFileChunk } from '../database/entities/DocumentStoreFileChunk'
import { InternalFlowiseError } from '../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import {
CreateSecretCommand,
GetSecretValueCommand,
PutSecretValueCommand,
SecretsManagerClient,
SecretsManagerClientConfig
} from '@aws-sdk/client-secrets-manager'
const QUESTION_VAR_PREFIX = 'question'
const FILE_ATTACHMENT_PREFIX = 'file_attachment'
const CHAT_HISTORY_VAR_PREFIX = 'chat_history'
const REDACTED_CREDENTIAL_VALUE = '_FLOWISE_BLANK_07167752-1a71-43b1-bf8f-4f32252165db'
let secretsManagerClient: SecretsManagerClient | null = null
const USE_AWS_SECRETS_MANAGER = process.env.SECRETKEY_STORAGE_TYPE === 'aws'
if (USE_AWS_SECRETS_MANAGER) {
const region = process.env.SECRETKEY_AWS_REGION || 'us-east-1' // Default region if not provided
const accessKeyId = process.env.SECRETKEY_AWS_ACCESS_KEY
const secretAccessKey = process.env.SECRETKEY_AWS_SECRET_KEY
let credentials: SecretsManagerClientConfig['credentials'] | undefined
if (accessKeyId && secretAccessKey) {
credentials = {
accessKeyId,
secretAccessKey
}
}
secretsManagerClient = new SecretsManagerClient({ credentials, region })
}
export const databaseEntities: IDatabaseEntity = {
ChatFlow: ChatFlow,
ChatMessage: ChatMessage,
@@ -1360,14 +1384,6 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
return isChatOrLLMsExist && isValidChainOrAgent && !isOutputParserExist
}
/**
* Generate an encryption key
* @returns {string}
*/
export const generateEncryptKey = (): string => {
return randomBytes(24).toString('base64')
}
/**
* Returns the encryption key
* @returns {Promise<string>}
@@ -1394,7 +1410,39 @@ export const getEncryptionKey = async (): Promise<string> => {
* @returns {Promise<string>}
*/
export const encryptCredentialData = async (plainDataObj: ICredentialDataDecrypted): Promise<string> => {
if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
const secretName = `FlowiseCredential_${randomBytes(12).toString('hex')}`
logger.info(`[server]: Upserting AWS Secret: ${secretName}`)
const secretString = JSON.stringify({ ...plainDataObj })
try {
// Try to update the secret if it exists
const putCommand = new PutSecretValueCommand({
SecretId: secretName,
SecretString: secretString
})
await secretsManagerClient.send(putCommand)
} catch (error: any) {
if (error.name === 'ResourceNotFoundException') {
// Secret doesn't exist, so create it
const createCommand = new CreateSecretCommand({
Name: secretName,
SecretString: secretString
})
await secretsManagerClient.send(createCommand)
} else {
// Rethrow any other errors
throw error
}
}
return secretName
}
const encryptKey = await getEncryptionKey()
// Fallback to existing code
return AES.encrypt(JSON.stringify(plainDataObj), encryptKey).toString()
}
@@ -1410,22 +1458,52 @@ export const decryptCredentialData = async (
componentCredentialName?: string,
componentCredentials?: IComponentCredentials
): Promise<ICredentialDataDecrypted> => {
const encryptKey = await getEncryptionKey()
const decryptedData = AES.decrypt(encryptedData, encryptKey)
const decryptedDataStr = decryptedData.toString(enc.Utf8)
let decryptedDataStr: string
if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
try {
logger.info(`[server]: Reading AWS Secret: ${encryptedData}`)
const command = new GetSecretValueCommand({ SecretId: encryptedData })
const response = await secretsManagerClient.send(command)
if (response.SecretString) {
const secretObj = JSON.parse(response.SecretString)
decryptedDataStr = JSON.stringify(secretObj)
} else {
throw new Error('Failed to retrieve secret value.')
}
} catch (error) {
console.error(error)
throw new Error('Failed to decrypt credential data.')
}
} else {
// Fallback to existing code
const encryptKey = await getEncryptionKey()
const decryptedData = AES.decrypt(encryptedData, encryptKey)
decryptedDataStr = decryptedData.toString(enc.Utf8)
}
if (!decryptedDataStr) return {}
try {
if (componentCredentialName && componentCredentials) {
const plainDataObj = JSON.parse(decryptedData.toString(enc.Utf8))
const plainDataObj = JSON.parse(decryptedDataStr)
return redactCredentialWithPasswordType(componentCredentialName, plainDataObj, componentCredentials)
}
return JSON.parse(decryptedData.toString(enc.Utf8))
return JSON.parse(decryptedDataStr)
} catch (e) {
console.error(e)
return {}
}
}
/**
* Generate an encryption key
* @returns {string}
*/
export const generateEncryptKey = (): string => {
return randomBytes(24).toString('base64')
}
/**
* Transform ICredentialBody from req to Credential entity
* @param {ICredentialReqBody} body
pnpm-lock.yaml
Generated
+999 -19
View File
File diff suppressed because it is too large Load Diff