farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 13:00:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feature/Credential

Browse Source 
# Conflicts:
#	README.md
#	docker/.env.example
#	packages/components/nodes/documentloaders/Notion/NotionDB.ts
#	packages/components/nodes/memory/DynamoDb/DynamoDb.ts
#	packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts
#	packages/components/nodes/memory/ZepMemory/ZepMemory.ts
#	packages/components/package.json
#	packages/components/src/utils.ts
#	packages/server/.env.example
#	packages/server/README.md
#	packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json
#	packages/server/src/ChildProcess.ts
#	packages/server/src/DataSource.ts
#	packages/server/src/commands/start.ts
#	packages/server/src/index.ts
#	packages/server/src/utils/index.ts
#	packages/server/src/utils/logger.ts
This commit is contained in:
Henry
2023-07-27 11:26:34 +01:00
parent 0665dcef55 e2d3f27842
commit 61dabbb7da
107 changed files with 4347 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/ChildProcess.ts
+64 -9
View File
@@ -1,6 +1,14 @@
import path from 'path'
import { IChildProcessMessage, IReactFlowNode, IReactFlowObject, IRunChatflowMessageValue, INodeData } from './Interface'
import { buildLangchain, constructGraphs, getEndingNode, getStartingNodes, getUserHome, resolveVariables } from './utils'
import {
buildLangchain,
constructGraphs,
getEndingNode,
getStartingNodes,
getUserHome,
replaceInputsWithConfig,
resolveVariables
} from './utils'
import { DataSource } from 'typeorm'
import { ChatFlow } from './entity/ChatFlow'
import { ChatMessage } from './entity/ChatMessage'
@@ -110,6 +118,8 @@ export class ChildProcess {
return
}
if (incomingInput.overrideConfig)
nodeToExecute.data = replaceInputsWithConfig(nodeToExecute.data, incomingInput.overrideConfig)
const reactFlowNodeData: INodeData = resolveVariables(nodeToExecute.data, reactFlowNodes, incomingInput.question)
nodeToExecuteData = reactFlowNodeData
@@ -143,14 +153,59 @@ export class ChildProcess {
* @returns {DataSource}
*/
async function initDB() {
const homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise')
const childAppDataSource = new DataSource({
type: 'sqlite',
database: path.resolve(homePath, 'database.sqlite'),
synchronize: true,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
let childAppDataSource
let homePath
const synchronize = process.env.OVERRIDE_DATABASE === 'false' ? false : true
switch (process.env.DATABASE_TYPE) {
case 'sqlite':
homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise')
childAppDataSource = new DataSource({
type: 'sqlite',
database: path.resolve(homePath, 'database.sqlite'),
synchronize,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
break
case 'mysql':
childAppDataSource = new DataSource({
type: 'mysql',
host: process.env.DATABASE_HOST,
port: parseInt(process.env.DATABASE_PORT || '3306'),
username: process.env.DATABASE_USER,
password: process.env.DATABASE_PASSWORD,
database: process.env.DATABASE_NAME,
charset: 'utf8mb4',
synchronize,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
break
case 'postgres':
childAppDataSource = new DataSource({
type: 'postgres',
host: process.env.DATABASE_HOST,
port: parseInt(process.env.DATABASE_PORT || '5432'),
username: process.env.DATABASE_USER,
password: process.env.DATABASE_PASSWORD,
database: process.env.DATABASE_NAME,
synchronize,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
break
default:
homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise')
childAppDataSource = new DataSource({
type: 'sqlite',
database: path.resolve(homePath, 'database.sqlite'),
synchronize,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
break
}
return await childAppDataSource.initialize()
}
packages/server/src/DataSource.ts
+51 -9
View File
@@ -10,15 +10,57 @@ import { getUserHome } from './utils'
let appDataSource: DataSource
export const init = async (): Promise<void> => {
const homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise')
appDataSource = new DataSource({
type: 'sqlite',
database: path.resolve(homePath, 'database.sqlite'),
synchronize: true,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
let homePath
const synchronize = process.env.OVERRIDE_DATABASE === 'false' ? false : true
switch (process.env.DATABASE_TYPE) {
case 'sqlite':
homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise')
appDataSource = new DataSource({
type: 'sqlite',
database: path.resolve(homePath, 'database.sqlite'),
synchronize,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
break
case 'mysql':
appDataSource = new DataSource({
type: 'mysql',
host: process.env.DATABASE_HOST,
port: parseInt(process.env.DATABASE_PORT || '3306'),
username: process.env.DATABASE_USER,
password: process.env.DATABASE_PASSWORD,
database: process.env.DATABASE_NAME,
charset: 'utf8mb4',
synchronize,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
break
case 'postgres':
appDataSource = new DataSource({
type: 'postgres',
host: process.env.DATABASE_HOST,
port: parseInt(process.env.DATABASE_PORT || '5432'),
username: process.env.DATABASE_USER,
password: process.env.DATABASE_PASSWORD,
database: process.env.DATABASE_NAME,
synchronize,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
break
default:
homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise')
appDataSource = new DataSource({
type: 'sqlite',
database: path.resolve(homePath, 'database.sqlite'),
synchronize,
entities: [ChatFlow, ChatMessage, Tool, Credential],
migrations: []
})
break
}
}
export function getDataSource(): DataSource {
packages/server/src/commands/start.ts
+39 -10
View File
@@ -21,12 +21,21 @@ export default class Start extends Command {
PORT: Flags.string(),
PASSPHRASE: Flags.string(),
DEBUG: Flags.string(),
DATABASE_PATH: Flags.string(),
APIKEY_PATH: Flags.string(),
SECRETKEY_PATH: Flags.string(),
LOG_PATH: Flags.string(),
LOG_LEVEL: Flags.string(),
EXECUTION_MODE: Flags.string()
EXECUTION_MODE: Flags.string(),
TOOL_FUNCTION_BUILTIN_DEP: Flags.string(),
TOOL_FUNCTION_EXTERNAL_DEP: Flags.string(),
OVERRIDE_DATABASE: Flags.string(),
DATABASE_TYPE: Flags.string(),
DATABASE_PATH: Flags.string(),
DATABASE_PORT: Flags.string(),
DATABASE_HOST: Flags.string(),
DATABASE_NAME: Flags.string(),
DATABASE_USER: Flags.string(),
DATABASE_PASSWORD: Flags.string()
}
async stopProcess() {
@@ -58,18 +67,38 @@ export default class Start extends Command {
})
const { flags } = await this.parse(Start)
if (flags.FLOWISE_USERNAME) process.env.FLOWISE_USERNAME = flags.FLOWISE_USERNAME
if (flags.FLOWISE_PASSWORD) process.env.FLOWISE_PASSWORD = flags.FLOWISE_PASSWORD
if (flags.PORT) process.env.PORT = flags.PORT
if (flags.PASSPHRASE) process.env.PASSPHRASE = flags.PASSPHRASE
if (flags.DATABASE_PATH) process.env.DATABASE_PATH = flags.DATABASE_PATH
if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH
if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
if (flags.LOG_LEVEL) process.env.LOG_LEVEL = flags.LOG_LEVEL
if (flags.EXECUTION_MODE) process.env.EXECUTION_MODE = flags.EXECUTION_MODE
if (flags.DEBUG) process.env.DEBUG = flags.DEBUG
// Authorization
if (flags.FLOWISE_USERNAME) process.env.FLOWISE_USERNAME = flags.FLOWISE_USERNAME
if (flags.FLOWISE_PASSWORD) process.env.FLOWISE_PASSWORD = flags.FLOWISE_PASSWORD
if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH
// Credentials
if (flags.PASSPHRASE) process.env.PASSPHRASE = flags.PASSPHRASE
if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
// Logs
if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
if (flags.LOG_LEVEL) process.env.LOG_LEVEL = flags.LOG_LEVEL
// Tool functions
if (flags.TOOL_FUNCTION_BUILTIN_DEP) process.env.TOOL_FUNCTION_BUILTIN_DEP = flags.TOOL_FUNCTION_BUILTIN_DEP
if (flags.TOOL_FUNCTION_EXTERNAL_DEP) process.env.TOOL_FUNCTION_EXTERNAL_DEP = flags.TOOL_FUNCTION_EXTERNAL_DEP
// Database config
if (flags.OVERRIDE_DATABASE) process.env.OVERRIDE_DATABASE = flags.OVERRIDE_DATABASE
if (flags.DATABASE_TYPE) process.env.DATABASE_TYPE = flags.DATABASE_TYPE
if (flags.DATABASE_PATH) process.env.DATABASE_PATH = flags.DATABASE_PATH
if (flags.DATABASE_PORT) process.env.DATABASE_PORT = flags.DATABASE_PORT
if (flags.DATABASE_HOST) process.env.DATABASE_HOST = flags.DATABASE_HOST
if (flags.DATABASE_NAME) process.env.DATABASE_NAME = flags.DATABASE_NAME
if (flags.DATABASE_USER) process.env.DATABASE_USER = flags.DATABASE_USER
if (flags.DATABASE_PASSWORD) process.env.DATABASE_PASSWORD = flags.DATABASE_PASSWORD
await (async () => {
try {
logger.info('Starting Flowise...')
packages/server/src/entity/ChatFlow.ts
+1 -1
View File
@@ -10,7 +10,7 @@ export class ChatFlow implements IChatFlow {
@Column()
name: string
@Column()
@Column({ type: 'text' })
flowData: string
@Column({ nullable: true })
packages/server/src/entity/ChatMessage.ts
+1 -1
View File
@@ -14,7 +14,7 @@ export class ChatMessage implements IChatMessage {
@Column()
chatflowid: string
@Column()
@Column({ type: 'text' })
content: string
@Column({ nullable: true })
packages/server/src/entity/Tool.ts
+1 -1
View File
@@ -10,7 +10,7 @@ export class Tool implements ITool {
@Column()
name: string
@Column()
@Column({ type: 'text' })
description: string
@Column()
packages/server/src/index.ts
+31 -7
View File
@@ -42,7 +42,9 @@ import {
databaseEntities,
getApiKey,
transformToCredentialEntity,
decryptCredentialData
decryptCredentialData,
clearSessionMemory,
replaceInputsWithConfig
} from './utils'
import { cloneDeep, omit } from 'lodash'
import { getDataSource } from './DataSource'
@@ -63,9 +65,6 @@ export class App {
constructor() {
this.app = express()
// Add the expressRequestLogger middleware to log all requests
this.app.use(expressRequestLogger)
}
async initDatabase() {
@@ -97,6 +96,9 @@ export class App {
// Allow access from *
this.app.use(cors())
// Add the expressRequestLogger middleware to log all requests
this.app.use(expressRequestLogger)
if (process.env.FLOWISE_USERNAME && process.env.FLOWISE_PASSWORD) {
const username = process.env.FLOWISE_USERNAME
const password = process.env.FLOWISE_PASSWORD
@@ -362,8 +364,13 @@ export class App {
// Get all chatmessages from chatflowid
this.app.get('/api/v1/chatmessage/:id', async (req: Request, res: Response) => {
const chatmessages = await this.AppDataSource.getRepository(ChatMessage).findBy({
chatflowid: req.params.id
const chatmessages = await this.AppDataSource.getRepository(ChatMessage).find({
where: {
chatflowid: req.params.id
},
order: {
createdDate: 'ASC'
}
})
return res.json(chatmessages)
})
@@ -382,6 +389,19 @@ export class App {
// Delete all chatmessages from chatflowid
this.app.delete('/api/v1/chatmessage/:id', async (req: Request, res: Response) => {
const chatflow = await this.AppDataSource.getRepository(ChatFlow).findOneBy({
id: req.params.id
})
if (!chatflow) {
res.status(404).send(`Chatflow ${req.params.id} not found`)
return
}
const flowData = chatflow.flowData
const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
const nodes = parsedFlowData.nodes
let chatId = await getChatId(chatflow.id)
if (!chatId) chatId = chatflow.id
clearSessionMemory(nodes, this.nodesPool.componentNodes, chatId, req.query.sessionId as string)
const results = await this.AppDataSource.getRepository(ChatMessage).delete({ chatflowid: req.params.id })
return res.json(results)
})
@@ -630,6 +650,8 @@ export class App {
}
templates.push(template)
})
const FlowiseDocsQnA = templates.find((tmp) => tmp.name === 'Flowise Docs QnA')
if (FlowiseDocsQnA) templates.unshift(FlowiseDocsQnA)
return res.json(templates)
})
@@ -809,7 +831,7 @@ export class App {
if (!chatflow) return res.status(404).send(`Chatflow ${chatflowid} not found`)
let chatId = await getChatId(chatflow.id)
if (!chatId) chatId = Date.now().toString()
if (!chatId) chatId = chatflowid
if (!isInternal) {
await this.validateKey(req, res, chatflow)
@@ -941,6 +963,8 @@ export class App {
const nodeToExecute = reactFlowNodes.find((node: IReactFlowNode) => node.id === endingNodeId)
if (!nodeToExecute) return res.status(404).send(`Node ${endingNodeId} not found`)
if (incomingInput.overrideConfig)
nodeToExecute.data = replaceInputsWithConfig(nodeToExecute.data, incomingInput.overrideConfig)
const reactFlowNodeData: INodeData = resolveVariables(nodeToExecute.data, reactFlowNodes, incomingInput.question)
nodeToExecuteData = reactFlowNodeData
packages/server/src/utils/config.ts
+1 -1
View File
@@ -7,7 +7,7 @@ dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true }
// default config
const loggingConfig = {
dir: process.env.LOG_PATH ?? path.join(__dirname, '..', '..', '..', '..', 'logs'),
dir: process.env.LOG_PATH ?? path.join(__dirname, '..', '..', 'logs'),
server: {
level: process.env.LOG_LEVEL ?? 'info',
filename: 'server.log',
packages/server/src/utils/index.ts
+36 -4
View File
@@ -19,7 +19,7 @@ import {
ICredentialReqBody
} from '../Interface'
import { cloneDeep, get, omit, merge } from 'lodash'
import { ICommonObject, getInputVariables, IDatabaseEntity } from 'flowise-components'
import { ICommonObject, getInputVariables, IDatabaseEntity, handleEscapeCharacters } from 'flowise-components'
import { scryptSync, randomBytes, timingSafeEqual } from 'crypto'
import { lib, PBKDF2, AES, enc } from 'crypto-js'
@@ -281,6 +281,29 @@ export const buildLangchain = async (
return flowNodes
}
/**
* Clear memory
* @param {IReactFlowNode[]} reactFlowNodes
* @param {IComponentNodes} componentNodes
* @param {string} chatId
* @param {string} sessionId
*/
export const clearSessionMemory = async (
reactFlowNodes: IReactFlowNode[],
componentNodes: IComponentNodes,
chatId: string,
sessionId?: string
) => {
for (const node of reactFlowNodes) {
if (node.data.category !== 'Memory') continue
const nodeInstanceFilePath = componentNodes[node.data.name].filePath as string
const nodeModule = await import(nodeInstanceFilePath)
const newNodeInstance = new nodeModule.nodeClass()
if (sessionId && node.data.inputs) node.data.inputs.sessionId = sessionId
if (newNodeInstance.clearSessionMemory) await newNodeInstance?.clearSessionMemory(node.data, { chatId })
}
}
/**
* Get variable value from outputResponses.output
* @param {string} paramValue
@@ -310,8 +333,13 @@ export const getVariableValue = (paramValue: string, reactFlowNodes: IReactFlowN
const variableEndIdx = startIdx
const variableFullPath = returnVal.substring(variableStartIdx, variableEndIdx)
/**
* Apply string transformation to convert special chars:
* FROM: hello i am ben\n\n\thow are you?
* TO: hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?
*/
if (isAcceptVariable && variableFullPath === QUESTION_VAR_PREFIX) {
variableDict[`{{${variableFullPath}}}`] = question
variableDict[`{{${variableFullPath}}}`] = handleEscapeCharacters(question, false)
}
// Split by first occurrence of '.' to get just nodeId
@@ -414,7 +442,11 @@ export const replaceInputsWithConfig = (flowNodeData: INodeData, overrideConfig:
const getParamValues = (paramsObj: ICommonObject) => {
for (const config in overrideConfig) {
paramsObj[config] = overrideConfig[config]
let paramValue = overrideConfig[config] ?? paramsObj[config]
// Check if boolean
if (paramValue === 'true') paramValue = true
else if (paramValue === 'false') paramValue = false
paramsObj[config] = paramValue
}
}
@@ -730,7 +762,7 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
isValidChainOrAgent = !blacklistChains.includes(endingNodeData.name)
} else if (endingNodeData.category === 'Agents') {
// Agent that are available to stream
const whitelistAgents = ['openAIFunctionAgent']
const whitelistAgents = ['openAIFunctionAgent', 'csvAgent', 'airtableAgent']
isValidChainOrAgent = whitelistAgents.includes(endingNodeData.name)
}
packages/server/src/utils/logger.ts
+38 -35
View File
@@ -57,44 +57,47 @@ const logger = createLogger({
* this.app.use(expressRequestLogger)
*/
export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void {
const fileLogger = createLogger({
format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json(), errors({ stack: true })),
defaultMeta: {
package: 'server',
request: {
method: req.method,
url: req.url,
body: req.body,
query: req.query,
params: req.params,
headers: req.headers
}
},
transports: [
new transports.File({
filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
level: config.logging.express.level ?? 'debug'
})
]
})
const unwantedLogURLs = ['/api/v1/node-icon/']
if (req.url.includes('/api/v1/') && !unwantedLogURLs.some((url) => req.url.includes(url))) {
const fileLogger = createLogger({
format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json(), errors({ stack: true })),
defaultMeta: {
package: 'server',
request: {
method: req.method,
url: req.url,
body: req.body,
query: req.query,
params: req.params,
headers: req.headers
}
},
transports: [
new transports.File({
filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
level: config.logging.express.level ?? 'debug'
})
]
})
const getRequestEmoji = (method: string) => {
const requetsEmojis: Record<string, string> = {
GET: '⬇️',
POST: '⬆️',
PUT: '🖊',
DELETE: '❌',
OPTION: '🔗'
const getRequestEmoji = (method: string) => {
const requetsEmojis: Record<string, string> = {
GET: '⬇️',
POST: '⬆️',
PUT: '🖊',
DELETE: '❌',
OPTION: '🔗'
}
return requetsEmojis[method] || '?'
}
return requetsEmojis[method] || '?'
}
if (req.method !== 'GET') {
fileLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
logger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
} else {
fileLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
if (req.method !== 'GET') {
fileLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
logger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
} else {
fileLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
}
}
next()