add credentials

packages/server/package.json

@@ -48,6 +48,7 @@
         "@oclif/core": "^1.13.10",
         "axios": "^0.27.2",
         "cors": "^2.8.5",
+        "crypto-js": "^4.1.1",
         "dotenv": "^16.0.0",
         "express": "^4.17.3",
         "express-basic-auth": "^1.2.1",
@@ -63,6 +64,7 @@
     },
     "devDependencies": {
         "@types/cors": "^2.8.12",
+        "@types/crypto-js": "^4.1.1",
         "@types/multer": "^1.4.7",
         "concurrently": "^7.1.0",
         "nodemon": "^2.0.15",

packages/server/src/ChildProcess.ts

@@ -5,6 +5,7 @@ import { DataSource } from 'typeorm'
 import { ChatFlow } from './entity/ChatFlow'
 import { ChatMessage } from './entity/ChatMessage'
 import { Tool } from './entity/Tool'
+import { Credential } from './entity/Credential'
 
 export class ChildProcess {
     /**
@@ -133,7 +134,7 @@ async function initDB() {
         type: 'sqlite',
         database: path.resolve(homePath, 'database.sqlite'),
         synchronize: true,
-        entities: [ChatFlow, ChatMessage, Tool],
+        entities: [ChatFlow, ChatMessage, Tool, Credential],
         migrations: []
     })
     return await childAppDataSource.initialize()

packages/server/src/DataSource.ts

@@ -3,6 +3,7 @@ import path from 'path'
 import { DataSource } from 'typeorm'
 import { ChatFlow } from './entity/ChatFlow'
 import { ChatMessage } from './entity/ChatMessage'
+import { Credential } from './entity/Credential'
 import { Tool } from './entity/Tool'
 import { getUserHome } from './utils'
 
@@ -15,7 +16,7 @@ export const init = async (): Promise<void> => {
         type: 'sqlite',
         database: path.resolve(homePath, 'database.sqlite'),
         synchronize: true,
-        entities: [ChatFlow, ChatMessage, Tool],
+        entities: [ChatFlow, ChatMessage, Tool, Credential],
         migrations: []
     })
 }

packages/server/src/Interface.ts

@@ -38,10 +38,23 @@ export interface ITool {
     createdDate: Date
 }
 
+export interface ICredential {
+    id: string
+    name: string
+    credentialName: string
+    encryptedData: string
+    updatedDate: Date
+    createdDate: Date
+}
+
 export interface IComponentNodes {
     [key: string]: INode
 }
 
+export interface IComponentCredentials {
+    [key: string]: INode
+}
+
 export interface IVariableDict {
     [key: string]: string
 }
@@ -167,3 +180,17 @@ export interface IChildProcessMessage {
     key: string
     value?: any
 }
+
+export type ICredentialDataDecrypted = ICommonObject
+
+// Plain credential object sent to server
+export interface ICredentialReqBody {
+    name: string
+    credentialName: string
+    plainDataObj: ICredentialDataDecrypted
+}
+
+// Decrypted credential object sent back to client
+export interface ICredentialReturnResponse extends ICredential {
+    plainDataObj: ICredentialDataDecrypted
+}

packages/server/src/NodesPool.ts

@@ -1,23 +1,33 @@
-import { IComponentNodes } from './Interface'
-
+import { IComponentNodes, IComponentCredentials } from './Interface'
 import path from 'path'
 import { Dirent } from 'fs'
 import { getNodeModulesPackagePath } from './utils'
 import { promises } from 'fs'
+import { ICommonObject } from 'flowise-components'
 
 export class NodesPool {
     componentNodes: IComponentNodes = {}
+    componentCredentials: IComponentCredentials = {}
+    private credentialIconPath: ICommonObject = {}
 
     /**
-     * Initialize to get all nodes
+     * Initialize to get all nodes & credentials
      */
     async initialize() {
+        await this.initializeNodes()
+        await this.initializeCrdentials()
+    }
+
+    /**
+     * Initialize nodes
+     */
+    private async initializeNodes() {
         const packagePath = getNodeModulesPackagePath('flowise-components')
         const nodesPath = path.join(packagePath, 'dist', 'nodes')
         const nodeFiles = await this.getFiles(nodesPath)
         return Promise.all(
             nodeFiles.map(async (file) => {
-                if (file.endsWith('.js')) {
+                if (file.endsWith('.js') && !file.endsWith('.credential.js')) {
                     const nodeModule = await require(file)
 
                     if (nodeModule.nodeClass) {
@@ -37,6 +47,13 @@ export class NodesPool {
                             filePath.pop()
                             const nodeIconAbsolutePath = `${filePath.join('/')}/${newNodeInstance.icon}`
                             this.componentNodes[newNodeInstance.name].icon = nodeIconAbsolutePath
+
+                            // Store icon path for componentCredentials
+                            if (newNodeInstance.credential) {
+                                for (const credName of newNodeInstance.credential.credentialNames) {
+                                    this.credentialIconPath[credName] = nodeIconAbsolutePath
+                                }
+                            }
                         }
                     }
                 }
@@ -44,12 +61,33 @@ export class NodesPool {
         )
     }
 
+    /**
+     * Initialize credentials
+     */
+    private async initializeCrdentials() {
+        const packagePath = getNodeModulesPackagePath('flowise-components')
+        const nodesPath = path.join(packagePath, 'dist', 'nodes')
+        const nodeFiles = await this.getFiles(nodesPath)
+        return Promise.all(
+            nodeFiles.map(async (file) => {
+                if (file.endsWith('.credential.js')) {
+                    const credentialModule = await require(file)
+                    if (credentialModule.credClass) {
+                        const newCredInstance = new credentialModule.credClass()
+                        newCredInstance.icon = this.credentialIconPath[newCredInstance.name] ?? ''
+                        this.componentCredentials[newCredInstance.name] = newCredInstance
+                    }
+                }
+            })
+        )
+    }
+
     /**
      * Recursive function to get node files
      * @param {string} dir
      * @returns {string[]}
      */
-    async getFiles(dir: string): Promise<string[]> {
+    private async getFiles(dir: string): Promise<string[]> {
         const dirents = await promises.readdir(dir, { withFileTypes: true })
         const files = await Promise.all(
             dirents.map((dirent: Dirent) => {

packages/server/src/entity/Credential.ts

@@ -0,0 +1,24 @@
+/* eslint-disable */
+import { Entity, Column, PrimaryGeneratedColumn, Index, CreateDateColumn, UpdateDateColumn } from 'typeorm'
+import { ICredential } from '../Interface'
+
+@Entity()
+export class Credential implements ICredential {
+    @PrimaryGeneratedColumn('uuid')
+    id: string
+
+    @Column()
+    name: string
+
+    @Column()
+    credentialName: string
+
+    @Column()
+    encryptedData: string
+
+    @CreateDateColumn()
+    createdDate: Date
+
+    @UpdateDateColumn()
+    updatedDate: Date
+}

packages/server/src/index.ts

@@ -17,7 +17,8 @@ import {
     INodeData,
     IDatabaseExport,
     IRunChatflowMessageValue,
-    IChildProcessMessage
+    IChildProcessMessage,
+    ICredentialReturnResponse
 } from './Interface'
 import {
     getNodeModulesPackagePath,
@@ -39,17 +40,20 @@ import {
     isFlowValidForStream,
     isVectorStoreFaiss,
     databaseEntities,
-    getApiKey
+    getApiKey,
+    transformToCredentialEntity,
+    decryptCredentialData
 } from './utils'
-import { cloneDeep } from 'lodash'
+import { cloneDeep, omit } from 'lodash'
 import { getDataSource } from './DataSource'
 import { NodesPool } from './NodesPool'
 import { ChatFlow } from './entity/ChatFlow'
 import { ChatMessage } from './entity/ChatMessage'
+import { Credential } from './entity/Credential'
+import { Tool } from './entity/Tool'
 import { ChatflowPool } from './ChatflowPool'
 import { ICommonObject, INodeOptionsValue } from 'flowise-components'
 import { fork } from 'child_process'
-import { Tool } from './entity/Tool'
 
 export class App {
     app: express.Application
@@ -70,10 +74,11 @@ export class App {
             .then(async () => {
                 logger.info('📦 [server]: Data Source has been initialized!')
 
-                // Initialize pools
+                // Initialize nodes pool
                 this.nodesPool = new NodesPool()
                 await this.nodesPool.initialize()
 
+                // Initialize chatflow pool
                 this.chatflowPool = new ChatflowPool()
 
                 // Initialize API keys
@@ -104,6 +109,7 @@ export class App {
                 '/api/v1/public-chatflows',
                 '/api/v1/prediction/',
                 '/api/v1/node-icon/',
+                '/api/v1/components-credentials-icon/',
                 '/api/v1/chatflows-streaming'
             ]
             this.app.use((req, res, next) => {
@@ -116,7 +122,7 @@ export class App {
         const upload = multer({ dest: `${path.join(__dirname, '..', 'uploads')}/` })
 
         // ----------------------------------------
-        // Nodes
+        // Components
         // ----------------------------------------
 
         // Get all component nodes
@@ -129,6 +135,16 @@ export class App {
             return res.json(returnData)
         })
 
+        // Get all component credentials
+        this.app.get('/api/v1/components-credentials', async (req: Request, res: Response) => {
+            const returnData = []
+            for (const credName in this.nodesPool.componentCredentials) {
+                const clonedCred = cloneDeep(this.nodesPool.componentCredentials[credName])
+                returnData.push(clonedCred)
+            }
+            return res.json(returnData)
+        })
+
         // Get specific component node via name
         this.app.get('/api/v1/nodes/:name', (req: Request, res: Response) => {
             if (Object.prototype.hasOwnProperty.call(this.nodesPool.componentNodes, req.params.name)) {
@@ -138,6 +154,27 @@ export class App {
             }
         })
 
+        // Get component credential via name
+        this.app.get('/api/v1/components-credentials/:name', (req: Request, res: Response) => {
+            if (!req.params.name.includes('&')) {
+                if (Object.prototype.hasOwnProperty.call(this.nodesPool.componentCredentials, req.params.name)) {
+                    return res.json(this.nodesPool.componentCredentials[req.params.name])
+                } else {
+                    throw new Error(`Credential ${req.params.name} not found`)
+                }
+            } else {
+                const returnResponse = []
+                for (const name of req.params.name.split('&')) {
+                    if (Object.prototype.hasOwnProperty.call(this.nodesPool.componentCredentials, name)) {
+                        returnResponse.push(this.nodesPool.componentCredentials[name])
+                    } else {
+                        throw new Error(`Credential ${name} not found`)
+                    }
+                }
+                return res.json(returnResponse)
+            }
+        })
+
         // Returns specific component node icon via name
         this.app.get('/api/v1/node-icon/:name', (req: Request, res: Response) => {
             if (Object.prototype.hasOwnProperty.call(this.nodesPool.componentNodes, req.params.name)) {
@@ -157,6 +194,25 @@ export class App {
             }
         })
 
+        // Returns specific component credential icon via name
+        this.app.get('/api/v1/components-credentials-icon/:name', (req: Request, res: Response) => {
+            if (Object.prototype.hasOwnProperty.call(this.nodesPool.componentCredentials, req.params.name)) {
+                const credInstance = this.nodesPool.componentCredentials[req.params.name]
+                if (credInstance.icon === undefined) {
+                    throw new Error(`Credential ${req.params.name} icon not found`)
+                }
+
+                if (credInstance.icon.endsWith('.svg') || credInstance.icon.endsWith('.png') || credInstance.icon.endsWith('.jpg')) {
+                    const filepath = credInstance.icon
+                    res.sendFile(filepath)
+                } else {
+                    throw new Error(`Credential ${req.params.name} icon is missing icon`)
+                }
+            } else {
+                throw new Error(`Credential ${req.params.name} not found`)
+            }
+        })
+
         // load async options
         this.app.post('/api/v1/node-load-method/:name', async (req: Request, res: Response) => {
             const nodeData: INodeData = req.body
@@ -324,6 +380,91 @@ export class App {
             return res.json(results)
         })
 
+        // ----------------------------------------
+        // Credentials
+        // ----------------------------------------
+
+        // Create new credential
+        this.app.post('/api/v1/credentials', async (req: Request, res: Response) => {
+            const body = req.body
+            const newCredential = await transformToCredentialEntity(body)
+            const credential = this.AppDataSource.getRepository(Credential).create(newCredential)
+            const results = await this.AppDataSource.getRepository(Credential).save(credential)
+            return res.json(results)
+        })
+
+        // Get all credentials
+        this.app.get('/api/v1/credentials', async (req: Request, res: Response) => {
+            if (req.query.credentialName) {
+                let returnCredentials = []
+                if (Array.isArray(req.query.credentialName)) {
+                    for (let i = 0; i < req.query.credentialName.length; i += 1) {
+                        const name = req.query.credentialName[i] as string
+                        const credentials = await this.AppDataSource.getRepository(Credential).findBy({
+                            credentialName: name
+                        })
+                        returnCredentials.push(...credentials)
+                    }
+                } else {
+                    const credentials = await this.AppDataSource.getRepository(Credential).findBy({
+                        credentialName: req.query.credentialName as string
+                    })
+                    returnCredentials = [...credentials]
+                }
+                return res.json(returnCredentials)
+            } else {
+                const credentials = await this.AppDataSource.getRepository(Credential).find()
+                const returnCredentials = []
+                for (const credential of credentials) {
+                    returnCredentials.push(omit(credential, ['encryptedData']))
+                }
+                return res.json(returnCredentials)
+            }
+        })
+
+        // Get specific credential
+        this.app.get('/api/v1/credentials/:id', async (req: Request, res: Response) => {
+            const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
+                id: req.params.id
+            })
+
+            if (!credential) return res.status(404).send(`Credential ${req.params.id} not found`)
+
+            // Decrpyt credentialData
+            const decryptedCredentialData = await decryptCredentialData(
+                credential.encryptedData,
+                credential.credentialName,
+                this.nodesPool.componentCredentials
+            )
+            const returnCredential: ICredentialReturnResponse = {
+                ...credential,
+                plainDataObj: decryptedCredentialData
+            }
+            return res.json(omit(returnCredential, ['encryptedData']))
+        })
+
+        // Update credential
+        this.app.put('/api/v1/credentials/:id', async (req: Request, res: Response) => {
+            const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
+                id: req.params.id
+            })
+
+            if (!credential) return res.status(404).send(`Credential ${req.params.id} not found`)
+
+            const body = req.body
+            const updateCredential = await transformToCredentialEntity(body)
+            this.AppDataSource.getRepository(Credential).merge(credential, updateCredential)
+            const result = await this.AppDataSource.getRepository(Credential).save(credential)
+
+            return res.json(result)
+        })
+
+        // Delete all chatmessages from chatflowid
+        this.app.delete('/api/v1/credentials/:id', async (req: Request, res: Response) => {
+            const results = await this.AppDataSource.getRepository(Credential).delete({ id: req.params.id })
+            return res.json(results)
+        })
+
         // ----------------------------------------
         // Tools
         // ----------------------------------------
@@ -393,7 +534,7 @@ export class App {
             const flowData = chatflow.flowData
             const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
             const nodes = parsedFlowData.nodes
-            const availableConfigs = findAvailableConfigs(nodes)
+            const availableConfigs = findAvailableConfigs(nodes, this.nodesPool.componentCredentials)
             return res.json(availableConfigs)
         })
 

packages/server/src/utils/index.ts

@@ -13,18 +13,26 @@ import {
     IReactFlowNode,
     IVariableDict,
     INodeData,
-    IOverrideConfig
+    IOverrideConfig,
+    ICredentialDataDecrypted,
+    IComponentCredentials,
+    ICredentialReqBody
 } from '../Interface'
 import { cloneDeep, get, omit, merge } from 'lodash'
 import { ICommonObject, getInputVariables, IDatabaseEntity } from 'flowise-components'
 import { scryptSync, randomBytes, timingSafeEqual } from 'crypto'
+import { lib, PBKDF2, AES, enc } from 'crypto-js'
+
 import { ChatFlow } from '../entity/ChatFlow'
 import { ChatMessage } from '../entity/ChatMessage'
+import { Credential } from '../entity/Credential'
 import { Tool } from '../entity/Tool'
 import { DataSource } from 'typeorm'
 
 const QUESTION_VAR_PREFIX = 'question'
-export const databaseEntities: IDatabaseEntity = { ChatFlow: ChatFlow, ChatMessage: ChatMessage, Tool: Tool }
+const REDACTED_CREDENTIAL_VALUE = '_FLOWISE_BLANK_07167752-1a71-43b1-bf8f-4f32252165db'
+
+export const databaseEntities: IDatabaseEntity = { ChatFlow: ChatFlow, ChatMessage: ChatMessage, Tool: Tool, Credential: Credential }
 
 /**
  * Returns the home folder path of the user if
@@ -399,9 +407,8 @@ export const replaceInputsWithConfig = (flowNodeData: INodeData, overrideConfig:
     const types = 'inputs'
 
     const getParamValues = (paramsObj: ICommonObject) => {
-        for (const key in paramsObj) {
-            const paramValue: string = paramsObj[key]
-            paramsObj[key] = overrideConfig[key] ?? paramValue
+        for (const config in overrideConfig) {
+            paramsObj[config] = overrideConfig[config]
         }
     }
 
@@ -623,11 +630,12 @@ export const mapMimeTypeToInputField = (mimeType: string) => {
 }
 
 /**
- * Find all available inpur params config
+ * Find all available input params config
  * @param {IReactFlowNode[]} reactFlowNodes
- * @returns {Promise<IOverrideConfig[]>}
+ * @param {IComponentCredentials} componentCredentials
+ * @returns {IOverrideConfig[]}
  */
-export const findAvailableConfigs = (reactFlowNodes: IReactFlowNode[]) => {
+export const findAvailableConfigs = (reactFlowNodes: IReactFlowNode[], componentCredentials: IComponentCredentials) => {
     const configs: IOverrideConfig[] = []
 
     for (const flowNode of reactFlowNodes) {
@@ -653,6 +661,23 @@ export const findAvailableConfigs = (reactFlowNodes: IReactFlowNode[]) => {
                               .join(', ')
                         : 'string'
                 }
+            } else if (inputParam.type === 'credential') {
+                // get component credential inputs
+                for (const name of inputParam.credentialNames ?? []) {
+                    if (Object.prototype.hasOwnProperty.call(componentCredentials, name)) {
+                        const inputs = componentCredentials[name]?.inputs ?? []
+                        for (const input of inputs) {
+                            obj = {
+                                node: flowNode.data.label,
+                                label: input.label,
+                                name: input.name,
+                                type: input.type === 'password' ? 'string' : input.type
+                            }
+                            configs.push(obj)
+                        }
+                    }
+                }
+                continue
             } else {
                 obj = {
                     node: flowNode.data.label,
@@ -705,3 +730,118 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
 
     return isChatOrLLMsExist && isValidChainOrAgent && !isVectorStoreFaiss(endingNodeData) && process.env.EXECUTION_MODE !== 'child'
 }
+
+/**
+ * Returns the path of encryption key
+ * @returns {string}
+ */
+export const getEncryptionKeyPath = (): string => {
+    return process.env.SECRETKEY_PATH
+        ? path.join(process.env.SECRETKEY_PATH, 'encryption.key')
+        : path.join(__dirname, '..', '..', 'encryption.key')
+}
+
+/**
+ * Generate an encryption key
+ * @returns {string}
+ */
+export const generateEncryptKey = (): string => {
+    const salt = lib.WordArray.random(128 / 8)
+    const key256Bits = PBKDF2(process.env.PASSPHRASE || 'MYPASSPHRASE', salt, {
+        keySize: 256 / 32,
+        iterations: 1000
+    })
+    return key256Bits.toString()
+}
+
+/**
+ * Returns the encryption key
+ * @returns {Promise<string>}
+ */
+export const getEncryptionKey = async (): Promise<string> => {
+    try {
+        return await fs.promises.readFile(getEncryptionKeyPath(), 'utf8')
+    } catch (error) {
+        const encryptKey = generateEncryptKey()
+        await fs.promises.writeFile(getEncryptionKeyPath(), encryptKey)
+        return encryptKey
+    }
+}
+
+/**
+ * Encrypt credential data
+ * @param {ICredentialDataDecrypted} plainDataObj
+ * @returns {Promise<string>}
+ */
+export const encryptCredentialData = async (plainDataObj: ICredentialDataDecrypted): Promise<string> => {
+    const encryptKey = await getEncryptionKey()
+    return AES.encrypt(JSON.stringify(plainDataObj), encryptKey).toString()
+}
+
+/**
+ * Decrypt credential data
+ * @param {string} encryptedData
+ * @param {string} componentCredentialName
+ * @param {IComponentCredentials} componentCredentials
+ * @returns {Promise<ICredentialDataDecrypted>}
+ */
+export const decryptCredentialData = async (
+    encryptedData: string,
+    componentCredentialName?: string,
+    componentCredentials?: IComponentCredentials
+): Promise<ICredentialDataDecrypted> => {
+    const encryptKey = await getEncryptionKey()
+    const decryptedData = AES.decrypt(encryptedData, encryptKey)
+    try {
+        if (componentCredentialName && componentCredentials) {
+            const plainDataObj = JSON.parse(decryptedData.toString(enc.Utf8))
+            return redactCredentialWithPasswordType(componentCredentialName, plainDataObj, componentCredentials)
+        }
+        return JSON.parse(decryptedData.toString(enc.Utf8))
+    } catch (e) {
+        console.error(e)
+        throw new Error('Credentials could not be decrypted.')
+    }
+}
+
+/**
+ * Transform ICredentialBody from req to Credential entity
+ * @param {ICredentialReqBody} body
+ * @returns {Credential}
+ */
+export const transformToCredentialEntity = async (body: ICredentialReqBody): Promise<Credential> => {
+    const encryptedData = await encryptCredentialData(body.plainDataObj)
+
+    const credentialBody = {
+        name: body.name,
+        credentialName: body.credentialName,
+        encryptedData
+    }
+
+    const newCredential = new Credential()
+    Object.assign(newCredential, credentialBody)
+
+    return newCredential
+}
+
+/**
+ * Redact values that are of password type to avoid sending back to client
+ * @param {string} componentCredentialName
+ * @param {ICredentialDataDecrypted} decryptedCredentialObj
+ * @param {IComponentCredentials} componentCredentials
+ * @returns {ICredentialDataDecrypted}
+ */
+export const redactCredentialWithPasswordType = (
+    componentCredentialName: string,
+    decryptedCredentialObj: ICredentialDataDecrypted,
+    componentCredentials: IComponentCredentials
+): ICredentialDataDecrypted => {
+    const plainDataObj = cloneDeep(decryptedCredentialObj)
+    for (const cred in plainDataObj) {
+        const inputParam = componentCredentials[componentCredentialName].inputs?.find((inp) => inp.type === 'password' && inp.name === cred)
+        if (inputParam) {
+            plainDataObj[cred] = REDACTED_CREDENTIAL_VALUE
+        }
+    }
+    return plainDataObj
+}

packages/server/src/utils/logger.ts

@@ -81,7 +81,8 @@ export function expressRequestLogger(req: Request, res: Response, next: NextFunc
             GET: '⬇️',
             POST: '⬆️',
             PUT: '🖊',
-            DELETE: '❌'
+            DELETE: '❌',
+            OPTION: '🔗'
         }
 
         return requetsEmojis[method] || '?'