From 6827a13e37a0f7acbf5a67a7a6192b66944a9a15 Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 11 Jul 2023 16:29:30 +0100 Subject: [PATCH 01/22] clear session memory --- .../nodes/memory/DynamoDb/DynamoDb.ts | 66 +++++++++++-------- .../memory/MotorheadMemory/MotorheadMemory.ts | 63 ++++++++++-------- .../RedisBackedChatMemory.ts | 55 +++++++++------- .../nodes/memory/ZepMemory/ZepMemory.ts | 53 +++++++++------ packages/components/src/Interface.ts | 1 + packages/server/src/index.ts | 18 ++++- packages/server/src/utils/index.ts | 23 +++++++ 7 files changed, 178 insertions(+), 101 deletions(-) diff --git a/packages/components/nodes/memory/DynamoDb/DynamoDb.ts b/packages/components/nodes/memory/DynamoDb/DynamoDb.ts index 49d15cb6..2ea7ba04 100644 --- a/packages/components/nodes/memory/DynamoDb/DynamoDb.ts +++ b/packages/components/nodes/memory/DynamoDb/DynamoDb.ts @@ -65,37 +65,47 @@ class DynamoDb_Memory implements INode { } ] } + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { - const tableName = nodeData.inputs?.tableName as string - const partitionKey = nodeData.inputs?.partitionKey as string - const sessionId = nodeData.inputs?.sessionId as string - const region = nodeData.inputs?.region as string - const accessKey = nodeData.inputs?.accessKey as string - const secretAccessKey = nodeData.inputs?.secretAccessKey as string - const memoryKey = nodeData.inputs?.memoryKey as string + return initalizeDynamoDB(nodeData, options) + } - const chatId = options.chatId - - const dynamoDb = new DynamoDBChatMessageHistory({ - tableName, - partitionKey, - sessionId: sessionId ? sessionId : chatId, - config: { - region, - credentials: { - accessKeyId: accessKey, - secretAccessKey - } - } - }) - - const memory = new BufferMemory({ - memoryKey, - chatHistory: dynamoDb, - returnMessages: true - }) - return memory + async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise { + const dynamodbMemory = initalizeDynamoDB(nodeData, options) + dynamodbMemory.clear() } } +const initalizeDynamoDB = (nodeData: INodeData, options: ICommonObject): BufferMemory => { + const tableName = nodeData.inputs?.tableName as string + const partitionKey = nodeData.inputs?.partitionKey as string + const sessionId = nodeData.inputs?.sessionId as string + const region = nodeData.inputs?.region as string + const accessKey = nodeData.inputs?.accessKey as string + const secretAccessKey = nodeData.inputs?.secretAccessKey as string + const memoryKey = nodeData.inputs?.memoryKey as string + + const chatId = options.chatId + + const dynamoDb = new DynamoDBChatMessageHistory({ + tableName, + partitionKey, + sessionId: sessionId ? sessionId : chatId, + config: { + region, + credentials: { + accessKeyId: accessKey, + secretAccessKey + } + } + }) + + const memory = new BufferMemory({ + memoryKey, + chatHistory: dynamoDb, + returnMessages: true + }) + return memory +} + module.exports = { nodeClass: DynamoDb_Memory } diff --git a/packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts b/packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts index 01d57614..b710b30a 100644 --- a/packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts +++ b/packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts @@ -64,35 +64,44 @@ class MotorMemory_Memory implements INode { } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { - const memoryKey = nodeData.inputs?.memoryKey as string - const baseURL = nodeData.inputs?.baseURL as string - const sessionId = nodeData.inputs?.sessionId as string - const apiKey = nodeData.inputs?.apiKey as string - const clientId = nodeData.inputs?.clientId as string + return initalizeMotorhead(nodeData, options) + } - const chatId = options?.chatId as string - - let obj: MotorheadMemoryInput = { - returnMessages: true, - sessionId: sessionId ? sessionId : chatId, - memoryKey - } - - if (baseURL) { - obj = { - ...obj, - url: baseURL - } - } else { - obj = { - ...obj, - apiKey, - clientId - } - } - - return new MotorheadMemory(obj) + async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise { + const motorhead = initalizeMotorhead(nodeData, options) + motorhead.clear() } } +const initalizeMotorhead = (nodeData: INodeData, options: ICommonObject): MotorheadMemory => { + const memoryKey = nodeData.inputs?.memoryKey as string + const baseURL = nodeData.inputs?.baseURL as string + const sessionId = nodeData.inputs?.sessionId as string + const apiKey = nodeData.inputs?.apiKey as string + const clientId = nodeData.inputs?.clientId as string + + const chatId = options?.chatId as string + + let obj: MotorheadMemoryInput = { + returnMessages: true, + sessionId: sessionId ? sessionId : chatId, + memoryKey + } + + if (baseURL) { + obj = { + ...obj, + url: baseURL + } + } else { + obj = { + ...obj, + apiKey, + clientId + } + } + + return new MotorheadMemory(obj) +} + module.exports = { nodeClass: MotorMemory_Memory } diff --git a/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts b/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts index 2b4e51c2..cd406f59 100644 --- a/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts +++ b/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts @@ -56,31 +56,40 @@ class RedisBackedChatMemory_Memory implements INode { } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { - const baseURL = nodeData.inputs?.baseURL as string - const sessionId = nodeData.inputs?.sessionId as string - const sessionTTL = nodeData.inputs?.sessionTTL as number - const memoryKey = nodeData.inputs?.memoryKey as string + return initalizeRedis(nodeData, options) + } - const chatId = options?.chatId as string - - const redisClient = createClient({ url: baseURL }) - let obj: RedisChatMessageHistoryInput = { - sessionId: sessionId ? sessionId : chatId, - client: redisClient - } - - if (sessionTTL) { - obj = { - ...obj, - sessionTTL - } - } - - let redisChatMessageHistory = new RedisChatMessageHistory(obj) - let redis = new BufferMemory({ memoryKey, chatHistory: redisChatMessageHistory, returnMessages: true }) - - return redis + async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise { + const redis = initalizeRedis(nodeData, options) + redis.clear() } } +const initalizeRedis = (nodeData: INodeData, options: ICommonObject): BufferMemory => { + const baseURL = nodeData.inputs?.baseURL as string + const sessionId = nodeData.inputs?.sessionId as string + const sessionTTL = nodeData.inputs?.sessionTTL as number + const memoryKey = nodeData.inputs?.memoryKey as string + + const chatId = options?.chatId as string + + const redisClient = createClient({ url: baseURL }) + let obj: RedisChatMessageHistoryInput = { + sessionId: sessionId ? sessionId : chatId, + client: redisClient + } + + if (sessionTTL) { + obj = { + ...obj, + sessionTTL + } + } + + let redisChatMessageHistory = new RedisChatMessageHistory(obj) + let redis = new BufferMemory({ memoryKey, chatHistory: redisChatMessageHistory, returnMessages: true }) + + return redis +} + module.exports = { nodeClass: RedisBackedChatMemory_Memory } diff --git a/packages/components/nodes/memory/ZepMemory/ZepMemory.ts b/packages/components/nodes/memory/ZepMemory/ZepMemory.ts index 6e1a14bd..d9bac948 100644 --- a/packages/components/nodes/memory/ZepMemory/ZepMemory.ts +++ b/packages/components/nodes/memory/ZepMemory/ZepMemory.ts @@ -104,31 +104,11 @@ class ZepMemory_Memory implements INode { } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { - const baseURL = nodeData.inputs?.baseURL as string - const aiPrefix = nodeData.inputs?.aiPrefix as string - const humanPrefix = nodeData.inputs?.humanPrefix as string - const memoryKey = nodeData.inputs?.memoryKey as string - const inputKey = nodeData.inputs?.inputKey as string const autoSummaryTemplate = nodeData.inputs?.autoSummaryTemplate as string const autoSummary = nodeData.inputs?.autoSummary as boolean - const sessionId = nodeData.inputs?.sessionId as string - const apiKey = nodeData.inputs?.apiKey as string const k = nodeData.inputs?.k as string - const chatId = options?.chatId as string - - const obj: ZepMemoryInput = { - baseURL, - sessionId: sessionId ? sessionId : chatId, - aiPrefix, - humanPrefix, - returnMessages: true, - memoryKey, - inputKey - } - if (apiKey) obj.apiKey = apiKey - - let zep = new ZepMemory(obj) + let zep = initalizeZep(nodeData, options) // hack to support summary let tmpFunc = zep.loadMemoryVariables @@ -153,6 +133,37 @@ class ZepMemory_Memory implements INode { } return zep } + + async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise { + const zep = initalizeZep(nodeData, options) + zep.clear() + } +} + +const initalizeZep = (nodeData: INodeData, options: ICommonObject) => { + const baseURL = nodeData.inputs?.baseURL as string + const aiPrefix = nodeData.inputs?.aiPrefix as string + const humanPrefix = nodeData.inputs?.humanPrefix as string + const memoryKey = nodeData.inputs?.memoryKey as string + const inputKey = nodeData.inputs?.inputKey as string + + const sessionId = nodeData.inputs?.sessionId as string + const apiKey = nodeData.inputs?.apiKey as string + + const chatId = options?.chatId as string + + const obj: ZepMemoryInput = { + baseURL, + sessionId: sessionId ? sessionId : chatId, + aiPrefix, + humanPrefix, + returnMessages: true, + memoryKey, + inputKey + } + if (apiKey) obj.apiKey = apiKey + + return new ZepMemory(obj) } module.exports = { nodeClass: ZepMemory_Memory } diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index d9233e49..862b81ac 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -96,6 +96,7 @@ export interface INode extends INodeProperties { } init?(nodeData: INodeData, input: string, options?: ICommonObject): Promise run?(nodeData: INodeData, input: string, options?: ICommonObject): Promise + clearSessionMemory?(nodeData: INodeData, options?: ICommonObject): Promise } export interface INodeData extends INodeProperties { diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 0f87aeba..e11fa283 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -39,7 +39,8 @@ import { isFlowValidForStream, isVectorStoreFaiss, databaseEntities, - getApiKey + getApiKey, + clearSessionMemory } from './utils' import { cloneDeep } from 'lodash' import { getDataSource } from './DataSource' @@ -320,6 +321,19 @@ export class App { // Delete all chatmessages from chatflowid this.app.delete('/api/v1/chatmessage/:id', async (req: Request, res: Response) => { + const chatflow = await this.AppDataSource.getRepository(ChatFlow).findOneBy({ + id: req.params.id + }) + if (!chatflow) { + res.status(404).send(`Chatflow ${req.params.id} not found`) + return + } + const flowData = chatflow.flowData + const parsedFlowData: IReactFlowObject = JSON.parse(flowData) + const nodes = parsedFlowData.nodes + let chatId = await getChatId(chatflow.id) + if (!chatId) chatId = chatflow.id + clearSessionMemory(nodes, this.nodesPool.componentNodes, chatId, req.query.sessionId as string) const results = await this.AppDataSource.getRepository(ChatMessage).delete({ chatflowid: req.params.id }) return res.json(results) }) @@ -662,7 +676,7 @@ export class App { if (!chatflow) return res.status(404).send(`Chatflow ${chatflowid} not found`) let chatId = await getChatId(chatflow.id) - if (!chatId) chatId = Date.now().toString() + if (!chatId) chatId = chatflowid if (!isInternal) { await this.validateKey(req, res, chatflow) diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 3ee7a25b..e13bca97 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -267,6 +267,29 @@ export const buildLangchain = async ( return flowNodes } +/** + * Clear memory + * @param {IReactFlowNode[]} reactFlowNodes + * @param {IComponentNodes} componentNodes + * @param {string} chatId + * @param {string} sessionId + */ +export const clearSessionMemory = async ( + reactFlowNodes: IReactFlowNode[], + componentNodes: IComponentNodes, + chatId: string, + sessionId?: string +) => { + for (const node of reactFlowNodes) { + if (node.data.category !== 'Memory') continue + const nodeInstanceFilePath = componentNodes[node.data.name].filePath as string + const nodeModule = await import(nodeInstanceFilePath) + const newNodeInstance = new nodeModule.nodeClass() + if (sessionId && node.data.inputs) node.data.inputs.sessionId = sessionId + if (newNodeInstance.clearSessionMemory) await newNodeInstance?.clearSessionMemory(node.data, { chatId }) + } +} + /** * Get variable value from outputResponses.output * @param {string} paramValue From f893edcc02f9b830a5d51a26e2d77166f863ad48 Mon Sep 17 00:00:00 2001 From: Atish Amte Date: Sat, 15 Jul 2023 00:49:31 +0530 Subject: [PATCH 02/22] ChatMessage Order Fixed --- packages/server/src/index.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 74c4d07e..ff7d375d 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -306,8 +306,13 @@ export class App { // Get all chatmessages from chatflowid this.app.get('/api/v1/chatmessage/:id', async (req: Request, res: Response) => { - const chatmessages = await this.AppDataSource.getRepository(ChatMessage).findBy({ - chatflowid: req.params.id + const chatmessages = await this.AppDataSource.getRepository(ChatMessage).find({ + where: { + chatflowid: req.params.id + }, + order: { + createdDate: 'ASC' + }, }) return res.json(chatmessages) }) From 647bb1f8649d11f27686b827ae67d40e9644fae4 Mon Sep 17 00:00:00 2001 From: denchi Date: Sat, 15 Jul 2023 13:34:08 +0100 Subject: [PATCH 03/22] Added BraveSearch Api Tool --- .../tools/BraveSearchAPI/BraveSearchAPI.ts | 38 +++++++++++++++ .../nodes/tools/BraveSearchAPI/brave-logo.svg | 46 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts create mode 100644 packages/components/nodes/tools/BraveSearchAPI/brave-logo.svg diff --git a/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts b/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts new file mode 100644 index 00000000..75d0d5c5 --- /dev/null +++ b/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts @@ -0,0 +1,38 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses } from '../../../src/utils' +import { BraveSearch } from 'langchain/tools' + +class BraveSearchAPI_Tools implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'BraveSearch API' + this.name = 'braveSearchAPI' + this.type = 'BraveSearchAPI' + this.icon = 'brave-logo.svg' + this.category = 'Tools' + this.description = 'Wrapper around BraveSearch API - a real-time API to access Brave search results' + this.inputs = [ + { + label: 'BraveSearch API Key', + name: 'apiKey', + type: 'password' + } + ] + this.baseClasses = [this.type, ...getBaseClasses(BraveSearch)] + } + + async init(nodeData: INodeData): Promise { + const apiKey = nodeData.inputs?.apiKey as string + return new BraveSearch({apiKey}) + } +} + +module.exports = { nodeClass: BraveSearchAPI_Tools } diff --git a/packages/components/nodes/tools/BraveSearchAPI/brave-logo.svg b/packages/components/nodes/tools/BraveSearchAPI/brave-logo.svg new file mode 100644 index 00000000..686b5c41 --- /dev/null +++ b/packages/components/nodes/tools/BraveSearchAPI/brave-logo.svg @@ -0,0 +1,46 @@ + + + + Logotypes/bat/logo-dark@1x + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From c4e4e92fb213d5a208474b60ca057306b84200d7 Mon Sep 17 00:00:00 2001 From: denchi Date: Sat, 15 Jul 2023 15:56:52 +0100 Subject: [PATCH 04/22] Fix linting error --- .../components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts b/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts index 75d0d5c5..76629392 100644 --- a/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts +++ b/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts @@ -31,7 +31,7 @@ class BraveSearchAPI_Tools implements INode { async init(nodeData: INodeData): Promise { const apiKey = nodeData.inputs?.apiKey as string - return new BraveSearch({apiKey}) + return new BraveSearch({ apiKey }) } } From 78ee45565f2395f25e68aefe2769e2d0ee9eb5b2 Mon Sep 17 00:00:00 2001 From: Atish Amte Date: Sun, 16 Jul 2023 01:06:38 +0530 Subject: [PATCH 05/22] Update index.ts --- packages/server/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index ff7d375d..1c2d4ebc 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -312,7 +312,7 @@ export class App { }, order: { createdDate: 'ASC' - }, + } }) return res.json(chatmessages) }) From 97eb80b9b54ed186c4994938edc10bd79d5654c2 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Sun, 16 Jul 2023 09:49:11 +0800 Subject: [PATCH 06/22] add content type text/xml --- packages/components/src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index 027ec8db..78486643 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -320,7 +320,7 @@ export async function xmlScrape(currentURL: string, limit: number): Promise Date: Sat, 15 Jul 2023 18:53:57 -0300 Subject: [PATCH 07/22] HtmlToMarkdownTextSplitter --- .../HtmlToMarkdownTextSplitter.ts | 75 +++++++++++++++++++ .../htmlToMarkdownTextSplitter.svg | 6 ++ packages/components/package.json | 1 + 3 files changed, 82 insertions(+) create mode 100644 packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/HtmlToMarkdownTextSplitter.ts create mode 100644 packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/htmlToMarkdownTextSplitter.svg diff --git a/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/HtmlToMarkdownTextSplitter.ts b/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/HtmlToMarkdownTextSplitter.ts new file mode 100644 index 00000000..d05298fe --- /dev/null +++ b/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/HtmlToMarkdownTextSplitter.ts @@ -0,0 +1,75 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses } from '../../../src/utils' +import { MarkdownTextSplitter, MarkdownTextSplitterParams } from 'langchain/text_splitter' +import { NodeHtmlMarkdown } from 'node-html-markdown' + +class HtmlToMarkdownTextSplitter_TextSplitters implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'HtmlToMarkdown Text Splitter' + this.name = 'htmlToMarkdownTextSplitter' + this.type = 'HtmlToMarkdownTextSplitter' + this.icon = 'htmlToMarkdownTextSplitter.svg' + this.category = 'Text Splitters' + this.description = `Converts Html to Markdown and then split your content into documents based on the Markdown headers` + this.baseClasses = [this.type, ...getBaseClasses(HtmlToMarkdownTextSplitter)] + this.inputs = [ + { + label: 'Chunk Size', + name: 'chunkSize', + type: 'number', + default: 1000, + optional: true + }, + { + label: 'Chunk Overlap', + name: 'chunkOverlap', + type: 'number', + optional: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const chunkSize = nodeData.inputs?.chunkSize as string + const chunkOverlap = nodeData.inputs?.chunkOverlap as string + + const obj = {} as MarkdownTextSplitterParams + + if (chunkSize) obj.chunkSize = parseInt(chunkSize, 10) + if (chunkOverlap) obj.chunkOverlap = parseInt(chunkOverlap, 10) + + const splitter = new HtmlToMarkdownTextSplitter(obj) + + return splitter + } +} +class HtmlToMarkdownTextSplitter extends MarkdownTextSplitter implements MarkdownTextSplitterParams { + constructor(fields?: Partial) { + { + super(fields) + } + } + splitText(text: string): Promise { + return new Promise((resolve, reject) => { + const markdown = NodeHtmlMarkdown.translate( + /* html */ text, + /* options (optional) */ {}, + /* customTranslators (optional) */ undefined, + /* customCodeBlockTranslators (optional) */ undefined + ) + super.splitText(markdown).then((result) => { + resolve(result) + }) + }) + } +} +module.exports = { nodeClass: HtmlToMarkdownTextSplitter_TextSplitters } diff --git a/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/htmlToMarkdownTextSplitter.svg b/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/htmlToMarkdownTextSplitter.svg new file mode 100644 index 00000000..f7d45d60 --- /dev/null +++ b/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/htmlToMarkdownTextSplitter.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/packages/components/package.json b/packages/components/package.json index 3459a372..d3ac06c3 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -42,6 +42,7 @@ "mammoth": "^1.5.1", "moment": "^2.29.3", "node-fetch": "^2.6.11", + "node-html-markdown": "^1.3.0", "pdf-parse": "^1.1.1", "pdfjs-dist": "^3.7.107", "playwright": "^1.35.0", From b4b7ccdbd78cd8ff11b9f5894ea0893360beb3f4 Mon Sep 17 00:00:00 2001 From: Rafael Reis Date: Sun, 16 Jul 2023 01:34:18 -0300 Subject: [PATCH 08/22] clean code --- .../HtmlToMarkdownTextSplitter.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/HtmlToMarkdownTextSplitter.ts b/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/HtmlToMarkdownTextSplitter.ts index d05298fe..161cb89e 100644 --- a/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/HtmlToMarkdownTextSplitter.ts +++ b/packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/HtmlToMarkdownTextSplitter.ts @@ -59,13 +59,8 @@ class HtmlToMarkdownTextSplitter extends MarkdownTextSplitter implements Markdow } } splitText(text: string): Promise { - return new Promise((resolve, reject) => { - const markdown = NodeHtmlMarkdown.translate( - /* html */ text, - /* options (optional) */ {}, - /* customTranslators (optional) */ undefined, - /* customCodeBlockTranslators (optional) */ undefined - ) + return new Promise((resolve) => { + const markdown = NodeHtmlMarkdown.translate(text) super.splitText(markdown).then((result) => { resolve(result) }) From b5a41be7db9224c85d632488835eef4612246226 Mon Sep 17 00:00:00 2001 From: Yongtae Date: Sun, 16 Jul 2023 17:59:25 +0900 Subject: [PATCH 09/22] add -H "Content-Type: application/json" in curl command --- packages/ui/src/views/chatflows/APICodeDialog.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/ui/src/views/chatflows/APICodeDialog.js b/packages/ui/src/views/chatflows/APICodeDialog.js index 5e32c1d4..ed9d8986 100644 --- a/packages/ui/src/views/chatflows/APICodeDialog.js +++ b/packages/ui/src/views/chatflows/APICodeDialog.js @@ -204,7 +204,8 @@ query({"question": "Hey, how are you?"}).then((response) => { } else if (codeLang === 'cURL') { return `curl ${baseURL}/api/v1/prediction/${dialogProps.chatflowid} \\ -X POST \\ - -d '{"question": "Hey, how are you?"}'` + -d '{"question": "Hey, how are you?"}' \\ + -H "Content-Type: application/json"` } return '' } @@ -246,6 +247,7 @@ query({"question": "Hey, how are you?"}).then((response) => { return `curl ${baseURL}/api/v1/prediction/${dialogProps.chatflowid} \\ -X POST \\ -d '{"question": "Hey, how are you?"}' \\ + -H "Content-Type: application/json" \\ -H "Authorization: Bearer ${selectedApiKey?.apiKey}"` } return '' @@ -316,7 +318,8 @@ query(formData).then((response) => { ` } else if (codeLang === 'cURL') { return `curl ${baseURL}/api/v1/prediction/${dialogProps.chatflowid} \\ - -X POST \\${getConfigExamplesForCurl(configData, 'formData')}` + -X POST \\${getConfigExamplesForCurl(configData, 'formData')} \\ + -H "Content-Type: application/json"` } return '' } @@ -363,6 +366,7 @@ query(formData).then((response) => { } else if (codeLang === 'cURL') { return `curl ${baseURL}/api/v1/prediction/${dialogProps.chatflowid} \\ -X POST \\${getConfigExamplesForCurl(configData, 'formData')} \\ + -H "Content-Type: application/json" \\ -H "Authorization: Bearer ${selectedApiKey?.apiKey}"` } return '' @@ -410,7 +414,8 @@ query({ } else if (codeLang === 'cURL') { return `curl ${baseURL}/api/v1/prediction/${dialogProps.chatflowid} \\ -X POST \\ - -d '{"question": "Hey, how are you?", "overrideConfig": {${getConfigExamplesForCurl(configData, 'json')}}'` + -d '{"question": "Hey, how are you?", "overrideConfig": {${getConfigExamplesForCurl(configData, 'json')}}' \\ + -H "Content-Type: application/json"` } return '' } @@ -460,6 +465,7 @@ query({ return `curl ${baseURL}/api/v1/prediction/${dialogProps.chatflowid} \\ -X POST \\ -d '{"question": "Hey, how are you?", "overrideConfig": {${getConfigExamplesForCurl(configData, 'json')}}' \\ + -H "Content-Type: application/json" \\ -H "Authorization: Bearer ${selectedApiKey?.apiKey}"` } return '' From 832064634bdae63ed4fadd9019c6d5a8a3c225a1 Mon Sep 17 00:00:00 2001 From: Yongtae Date: Sun, 16 Jul 2023 18:22:31 +0900 Subject: [PATCH 10/22] Refactor API request to include Content-Type header --- .../ui/src/views/chatflows/APICodeDialog.js | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/packages/ui/src/views/chatflows/APICodeDialog.js b/packages/ui/src/views/chatflows/APICodeDialog.js index 5e32c1d4..9a036948 100644 --- a/packages/ui/src/views/chatflows/APICodeDialog.js +++ b/packages/ui/src/views/chatflows/APICodeDialog.js @@ -190,7 +190,10 @@ output = query({ "${baseURL}/api/v1/prediction/${dialogProps.chatflowid}", { method: "POST", - body: data + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify(data) } ); const result = await response.json(); @@ -229,9 +232,12 @@ output = query({ const response = await fetch( "${baseURL}/api/v1/prediction/${dialogProps.chatflowid}", { - headers: { Authorization: "Bearer ${selectedApiKey?.apiKey}" }, + headers: { + Authorization: "Bearer ${selectedApiKey?.apiKey}", + "Content-Type": "application/json" + }, method: "POST", - body: data + body: JSON.stringify(data) } ); const result = await response.json(); @@ -303,7 +309,10 @@ async function query(formData) { "${baseURL}/api/v1/prediction/${dialogProps.chatflowid}", { method: "POST", - body: formData + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify(formData) } ); const result = await response.json(); @@ -347,9 +356,12 @@ async function query(formData) { const response = await fetch( "${baseURL}/api/v1/prediction/${dialogProps.chatflowid}", { - headers: { Authorization: "Bearer ${selectedApiKey?.apiKey}" }, + headers: { + Authorization: "Bearer ${selectedApiKey?.apiKey}", + "Content-Type": "application/json" + }, method: "POST", - body: formData + body: JSON.stringify(formData) } ); const result = await response.json(); @@ -392,7 +404,10 @@ output = query({ "${baseURL}/api/v1/prediction/${dialogProps.chatflowid}", { method: "POST", - body: data + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify(data) } ); const result = await response.json(); @@ -439,9 +454,12 @@ output = query({ const response = await fetch( "${baseURL}/api/v1/prediction/${dialogProps.chatflowid}", { - headers: { Authorization: "Bearer ${selectedApiKey?.apiKey}" }, + headers: { + Authorization: "Bearer ${selectedApiKey?.apiKey}", + "Content-Type": "application/json" + }, method: "POST", - body: data + body: JSON.stringify(data) } ); const result = await response.json(); From 7ec8f8ea2841db6ad7d9738ddc81f7c133dedf62 Mon Sep 17 00:00:00 2001 From: Henry Date: Sun, 16 Jul 2023 14:32:47 +0100 Subject: [PATCH 11/22] address vm2 security --- README.md | 24 +++++++------ docker/.env.example | 2 ++ docker/docker-compose.yml | 2 ++ .../components/nodes/tools/CustomTool/core.ts | 36 ++++++++++++------- packages/server/.env.example | 2 ++ packages/server/README.md | 24 +++++++------ packages/server/src/commands/start.ts | 6 +++- 7 files changed, 61 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 5023a1b7..b4d4b860 100644 --- a/README.md +++ b/README.md @@ -130,17 +130,19 @@ FLOWISE_PASSWORD=1234 Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. -| Variable | Description | Type | Default | -| ---------------- | ---------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | -| PORT | The HTTP port Flowise runs on | Number | 3000 | -| FLOWISE_USERNAME | Username to login | String | -| FLOWISE_PASSWORD | Password to login | String | -| DEBUG | Print logs from components | Boolean | -| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/logs` | -| LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | -| DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | -| APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | -| EXECUTION_MODE | Whether predictions run in their own process or the main process | Enum String: `child`, `main` | `main` | +| Variable | Description | Type | Default | +| -------------------------- | ---------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | +| PORT | The HTTP port Flowise runs on | Number | 3000 | +| FLOWISE_USERNAME | Username to login | String | +| FLOWISE_PASSWORD | Password to login | String | +| DEBUG | Print logs from components | Boolean | +| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/logs` | +| LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | +| DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | +| APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | +| EXECUTION_MODE | Whether predictions run in their own process or the main process | Enum String: `child`, `main` | `main` | +| TOOL_FUNCTION_BUILTIN_DEP | NodeJS built-in modules to be used for Tool Function | String | | +| TOOL_FUNCTION_EXTERNAL_DEP | External modules to be used for Tool Function | String | | You can also specify the env variables when using `npx`. For example: diff --git a/docker/.env.example b/docker/.env.example index 262e08a6..a0b1fb26 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -6,3 +6,5 @@ PORT=3000 # APIKEY_PATH=/your_api_key_path/.flowise # LOG_PATH=/your_log_path/logs # EXECUTION_MODE=child or main +# TOOL_FUNCTION_BUILTIN_DEP=crypto,fs +# TOOL_FUNCTION_EXTERNAL_DEP=moment,lodash \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3077c43d..4a8c4dbb 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -13,6 +13,8 @@ services: - LOG_PATH=${LOG_PATH} - EXECUTION_MODE=${EXECUTION_MODE} - DEBUG=${DEBUG} + - TOOL_FUNCTION_BUILTIN_DEP=${TOOL_FUNCTION_BUILTIN_DEP} + - TOOL_FUNCTION_EXTERNAL_DEP=${TOOL_FUNCTION_EXTERNAL_DEP} ports: - '${PORT}:${PORT}' volumes: diff --git a/packages/components/nodes/tools/CustomTool/core.ts b/packages/components/nodes/tools/CustomTool/core.ts index 0d3d7bcd..2aa06b54 100644 --- a/packages/components/nodes/tools/CustomTool/core.ts +++ b/packages/components/nodes/tools/CustomTool/core.ts @@ -51,25 +51,37 @@ export class DynamicStructuredTool< } } + const defaultAllowBuiltInDep = [ + 'assert', + 'buffer', + 'crypto', + 'events', + 'http', + 'https', + 'net', + 'path', + 'querystring', + 'timers', + 'tls', + 'url', + 'zlib' + ] + + const builtinDeps = process.env.TOOL_FUNCTION_BUILTIN_DEP + ? defaultAllowBuiltInDep.concat(process.env.TOOL_FUNCTION_BUILTIN_DEP.split(',')) + : defaultAllowBuiltInDep + const externalDeps = process.env.TOOL_FUNCTION_EXTERNAL_DEP ? process.env.TOOL_FUNCTION_EXTERNAL_DEP.split(',') : [] + const deps = availableDependencies.concat(externalDeps) + const options = { console: 'inherit', sandbox, require: { - external: false as boolean | { modules: string[] }, - builtin: ['*'] + external: { modules: deps }, + builtin: builtinDeps } } as any - const external = JSON.stringify(availableDependencies) - if (external) { - const deps = JSON.parse(external) - if (deps && deps.length) { - options.require.external = { - modules: deps - } - } - } - const vm = new NodeVM(options) const response = await vm.run(`module.exports = async function() {${this.code}}()`, __dirname) diff --git a/packages/server/.env.example b/packages/server/.env.example index d9b2da76..81bac2dc 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -7,3 +7,5 @@ PORT=3000 # LOG_PATH=/your_log_path/logs # LOG_LEVEL=debug (error | warn | info | verbose | debug) # EXECUTION_MODE=main (child | main) +# TOOL_FUNCTION_BUILTIN_DEP=crypto,fs +# TOOL_FUNCTION_EXTERNAL_DEP=moment,lodash diff --git a/packages/server/README.md b/packages/server/README.md index fb3a0c12..e7aacf32 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -33,17 +33,19 @@ FLOWISE_PASSWORD=1234 Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. -| Variable | Description | Type | Default | -| ---------------- | ---------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | -| PORT | The HTTP port Flowise runs on | Number | 3000 | -| FLOWISE_USERNAME | Username to login | String | -| FLOWISE_PASSWORD | Password to login | String | -| DEBUG | Print logs from components | Boolean | -| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/logs` | -| LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | -| DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | -| APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | -| EXECUTION_MODE | Whether predictions run in their own process or the main process | Enum String: `child`, `main` | `main` | +| Variable | Description | Type | Default | +| -------------------------- | ---------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | +| PORT | The HTTP port Flowise runs on | Number | 3000 | +| FLOWISE_USERNAME | Username to login | String | +| FLOWISE_PASSWORD | Password to login | String | +| DEBUG | Print logs from components | Boolean | +| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/logs` | +| LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | +| DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | +| APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | +| EXECUTION_MODE | Whether predictions run in their own process or the main process | Enum String: `child`, `main` | `main` | +| TOOL_FUNCTION_BUILTIN_DEP | NodeJS built-in modules to be used for Tool Function | String | | +| TOOL_FUNCTION_EXTERNAL_DEP | External modules to be used for Tool Function | String | | You can also specify the env variables when using `npx`. For example: diff --git a/packages/server/src/commands/start.ts b/packages/server/src/commands/start.ts index 276a3036..ffb6e449 100644 --- a/packages/server/src/commands/start.ts +++ b/packages/server/src/commands/start.ts @@ -24,7 +24,9 @@ export default class Start extends Command { APIKEY_PATH: Flags.string(), LOG_PATH: Flags.string(), LOG_LEVEL: Flags.string(), - EXECUTION_MODE: Flags.string() + EXECUTION_MODE: Flags.string(), + TOOL_FUNCTION_BUILTIN_DEP: Flags.string(), + TOOL_FUNCTION_EXTERNAL_DEP: Flags.string() } async stopProcess() { @@ -65,6 +67,8 @@ export default class Start extends Command { if (flags.LOG_LEVEL) process.env.LOG_LEVEL = flags.LOG_LEVEL if (flags.EXECUTION_MODE) process.env.EXECUTION_MODE = flags.EXECUTION_MODE if (flags.DEBUG) process.env.DEBUG = flags.DEBUG + if (flags.TOOL_FUNCTION_BUILTIN_DEP) process.env.TOOL_FUNCTION_BUILTIN_DEP = flags.TOOL_FUNCTION_BUILTIN_DEP + if (flags.TOOL_FUNCTION_EXTERNAL_DEP) process.env.TOOL_FUNCTION_EXTERNAL_DEP = flags.TOOL_FUNCTION_EXTERNAL_DEP await (async () => { try { From be49c11f6b712ef781a8f1c7672c7d9357e2c3e1 Mon Sep 17 00:00:00 2001 From: Yongtae Date: Mon, 17 Jul 2023 13:49:16 +0900 Subject: [PATCH 12/22] Update content-type header to application/x-www-form-urlencoded in APICodeDialog.js --- packages/ui/src/views/chatflows/APICodeDialog.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ui/src/views/chatflows/APICodeDialog.js b/packages/ui/src/views/chatflows/APICodeDialog.js index 9a036948..b28cee73 100644 --- a/packages/ui/src/views/chatflows/APICodeDialog.js +++ b/packages/ui/src/views/chatflows/APICodeDialog.js @@ -310,9 +310,9 @@ async function query(formData) { { method: "POST", headers: { - "Content-Type": "application/json" + "Content-Type": "application/x-www-form-urlencoded" }, - body: JSON.stringify(formData) + body: new URLSearchParams(formData) } ); const result = await response.json(); @@ -358,10 +358,10 @@ async function query(formData) { { headers: { Authorization: "Bearer ${selectedApiKey?.apiKey}", - "Content-Type": "application/json" + "Content-Type": "application/x-www-form-urlencoded" }, method: "POST", - body: JSON.stringify(formData) + body: new URLSearchParams(formData) } ); const result = await response.json(); From 6804216c6a7e2e6bd72178a88a902f353a41d2e0 Mon Sep 17 00:00:00 2001 From: Yongtae Date: Mon, 17 Jul 2023 14:11:06 +0900 Subject: [PATCH 13/22] Update cURL command with new Content-Type header --- packages/ui/src/views/chatflows/APICodeDialog.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ui/src/views/chatflows/APICodeDialog.js b/packages/ui/src/views/chatflows/APICodeDialog.js index ed9d8986..c9d5565a 100644 --- a/packages/ui/src/views/chatflows/APICodeDialog.js +++ b/packages/ui/src/views/chatflows/APICodeDialog.js @@ -319,7 +319,7 @@ query(formData).then((response) => { } else if (codeLang === 'cURL') { return `curl ${baseURL}/api/v1/prediction/${dialogProps.chatflowid} \\ -X POST \\${getConfigExamplesForCurl(configData, 'formData')} \\ - -H "Content-Type: application/json"` + -H "Content-Type: multipart/form-data"` } return '' } @@ -366,7 +366,7 @@ query(formData).then((response) => { } else if (codeLang === 'cURL') { return `curl ${baseURL}/api/v1/prediction/${dialogProps.chatflowid} \\ -X POST \\${getConfigExamplesForCurl(configData, 'formData')} \\ - -H "Content-Type: application/json" \\ + -H "Content-Type: multipart/form-data" \\ -H "Authorization: Bearer ${selectedApiKey?.apiKey}"` } return '' From d03d77d870b85d7fea47ebedad27e4d7f5f66596 Mon Sep 17 00:00:00 2001 From: yejiezhao Date: Mon, 17 Jul 2023 15:40:35 +0800 Subject: [PATCH 14/22] fix: bug inputType --- packages/ui/src/ui-component/input/Input.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ui/src/ui-component/input/Input.js b/packages/ui/src/ui-component/input/Input.js index e7744764..5a7f45b7 100644 --- a/packages/ui/src/ui-component/input/Input.js +++ b/packages/ui/src/ui-component/input/Input.js @@ -61,7 +61,7 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo Input.propTypes = { inputParam: PropTypes.object, - value: PropTypes.string, + value: PropTypes.oneOfType([PropTypes.string, PropTypes.number]), onChange: PropTypes.func, disabled: PropTypes.bool, showDialog: PropTypes.bool, From 551b0d515a8706f6ed452b6cef7d819543777c42 Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Jul 2023 15:53:53 +0100 Subject: [PATCH 15/22] update log path --- README.md | 2 +- packages/server/README.md | 2 +- packages/server/src/utils/config.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5023a1b7..2c7002d5 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Flowise support different environment variables to configure your instance. You | FLOWISE_USERNAME | Username to login | String | | FLOWISE_PASSWORD | Password to login | String | | DEBUG | Print logs from components | Boolean | -| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/logs` | +| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/packages/server` | | LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | | DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | | APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | diff --git a/packages/server/README.md b/packages/server/README.md index fb3a0c12..087483e1 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -39,7 +39,7 @@ Flowise support different environment variables to configure your instance. You | FLOWISE_USERNAME | Username to login | String | | FLOWISE_PASSWORD | Password to login | String | | DEBUG | Print logs from components | Boolean | -| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/logs` | +| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/packages/server` | | LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | | DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | | APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | diff --git a/packages/server/src/utils/config.ts b/packages/server/src/utils/config.ts index 8540b3b1..b5f5884e 100644 --- a/packages/server/src/utils/config.ts +++ b/packages/server/src/utils/config.ts @@ -7,7 +7,7 @@ dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true } // default config const loggingConfig = { - dir: process.env.LOG_PATH ?? path.join(__dirname, '..', '..', '..', '..', 'logs'), + dir: process.env.LOG_PATH ?? path.join(__dirname, '..', '..', 'logs'), server: { level: process.env.LOG_LEVEL ?? 'info', filename: 'server.log', From 3fc496ecb0182de8a853733b3564a0aa15142aac Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Jul 2023 18:11:56 +0100 Subject: [PATCH 16/22] added unwanted log urls to prevent logging node-icon api calls --- packages/server/src/index.ts | 6 +-- packages/server/src/utils/logger.ts | 71 +++++++++++++++-------------- 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 1c2d4ebc..35545bc3 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -59,9 +59,6 @@ export class App { constructor() { this.app = express() - - // Add the expressRequestLogger middleware to log all requests - this.app.use(expressRequestLogger) } async initDatabase() { @@ -92,6 +89,9 @@ export class App { // Allow access from * this.app.use(cors()) + // Add the expressRequestLogger middleware to log all requests + this.app.use(expressRequestLogger) + if (process.env.FLOWISE_USERNAME && process.env.FLOWISE_PASSWORD) { const username = process.env.FLOWISE_USERNAME const password = process.env.FLOWISE_PASSWORD diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts index 5d7ffedf..57d5f64a 100644 --- a/packages/server/src/utils/logger.ts +++ b/packages/server/src/utils/logger.ts @@ -57,43 +57,46 @@ const logger = createLogger({ * this.app.use(expressRequestLogger) */ export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void { - const fileLogger = createLogger({ - format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json(), errors({ stack: true })), - defaultMeta: { - package: 'server', - request: { - method: req.method, - url: req.url, - body: req.body, - query: req.query, - params: req.params, - headers: req.headers - } - }, - transports: [ - new transports.File({ - filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'), - level: config.logging.express.level ?? 'debug' - }) - ] - }) + const unwantedLogURLs = ['/api/v1/node-icon/'] + if (req.url.includes('/api/v1/') && !unwantedLogURLs.some((url) => req.url.includes(url))) { + const fileLogger = createLogger({ + format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json(), errors({ stack: true })), + defaultMeta: { + package: 'server', + request: { + method: req.method, + url: req.url, + body: req.body, + query: req.query, + params: req.params, + headers: req.headers + } + }, + transports: [ + new transports.File({ + filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'), + level: config.logging.express.level ?? 'debug' + }) + ] + }) - const getRequestEmoji = (method: string) => { - const requetsEmojis: Record = { - GET: '⬇️', - POST: '⬆️', - PUT: '🖊', - DELETE: '❌' + const getRequestEmoji = (method: string) => { + const requetsEmojis: Record = { + GET: '⬇️', + POST: '⬆️', + PUT: '🖊', + DELETE: '❌' + } + + return requetsEmojis[method] || '?' } - return requetsEmojis[method] || '?' - } - - if (req.method !== 'GET') { - fileLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) - logger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) - } else { - fileLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + if (req.method !== 'GET') { + fileLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + logger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + } else { + fileLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + } } next() From 25b160261a92fb4f075228334f7c03113ef39271 Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Jul 2023 18:25:53 +0100 Subject: [PATCH 17/22] update README --- README.md | 4 ++-- packages/server/README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2c7002d5..de7d218b 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,9 @@ Flowise support different environment variables to configure your instance. You | PORT | The HTTP port Flowise runs on | Number | 3000 | | FLOWISE_USERNAME | Username to login | String | | FLOWISE_PASSWORD | Password to login | String | -| DEBUG | Print logs from components | Boolean | +| DEBUG | Print logs onto terminal/console | Boolean | | LOG_PATH | Location where log files are stored | String | `your-path/Flowise/packages/server` | -| LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | +| LOG_LEVEL | Different log levels for loggers to be saved | Enum String: `error`, `info`, `verbose`, `debug` | `info` | | DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | | APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | | EXECUTION_MODE | Whether predictions run in their own process or the main process | Enum String: `child`, `main` | `main` | diff --git a/packages/server/README.md b/packages/server/README.md index 087483e1..3d039c26 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -38,9 +38,9 @@ Flowise support different environment variables to configure your instance. You | PORT | The HTTP port Flowise runs on | Number | 3000 | | FLOWISE_USERNAME | Username to login | String | | FLOWISE_PASSWORD | Password to login | String | -| DEBUG | Print logs from components | Boolean | +| DEBUG | Print logs onto terminal/console | Boolean | | LOG_PATH | Location where log files are stored | String | `your-path/Flowise/packages/server` | -| LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | +| LOG_LEVEL | Different log levels for loggers to be saved | Enum String: `error`, `info`, `verbose`, `debug` | `info` | | DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | | APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | | EXECUTION_MODE | Whether predictions run in their own process or the main process | Enum String: `child`, `main` | `main` | From 745a42224fed43102e93de8caf489776a56b1a4d Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Jul 2023 20:02:04 +0100 Subject: [PATCH 18/22] update docs on Docker env path --- README.md | 2 +- docker/.env.example | 7 ++++--- docker/README.md | 24 +++++++++++++++++++++++- docker/docker-compose.yml | 3 ++- packages/server/.env.example | 2 +- packages/server/README.md | 2 +- 6 files changed, 32 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index de7d218b..15e781c9 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ FLOWISE_PASSWORD=1234 ## 🌱 Env Variables -Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. +Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. Read [more](https://docs.flowiseai.com/environment-variables) | Variable | Description | Type | Default | | ---------------- | ---------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | diff --git a/docker/.env.example b/docker/.env.example index 262e08a6..cbee9c89 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1,8 +1,9 @@ PORT=3000 +DATABASE_PATH=/root/.flowise +APIKEY_PATH=/root/.flowise +LOG_PATH=/root/.flowise/logs # FLOWISE_USERNAME=user # FLOWISE_PASSWORD=1234 # DEBUG=true -# DATABASE_PATH=/your_database_path/.flowise -# APIKEY_PATH=/your_api_key_path/.flowise -# LOG_PATH=/your_log_path/logs +# LOG_LEVEL=debug (error | warn | info | verbose | debug) # EXECUTION_MODE=child or main diff --git a/docker/README.md b/docker/README.md index 7f991a04..995db6ff 100644 --- a/docker/README.md +++ b/docker/README.md @@ -9,7 +9,7 @@ Starts Flowise from [DockerHub Image](https://hub.docker.com/repository/docker/f 3. Open [http://localhost:3000](http://localhost:3000) 4. You can bring the containers down by `docker-compose stop` -## With Authrorization +## 🔒 Authrorization 1. Create `.env` file and specify the `PORT`, `FLOWISE_USERNAME`, and `FLOWISE_PASSWORD` (refer to `.env.example`) 2. Pass `FLOWISE_USERNAME` and `FLOWISE_PASSWORD` to the `docker-compose.yml` file: @@ -22,3 +22,25 @@ Starts Flowise from [DockerHub Image](https://hub.docker.com/repository/docker/f 3. `docker-compose up -d` 4. Open [http://localhost:3000](http://localhost:3000) 5. You can bring the containers down by `docker-compose stop` + +## 🌱 Env Variables + +If you like to persist your data (flows, logs, apikeys), set these variables in the `.env` file inside `docker` folder: + +- DATABASE_PATH=/root/.flowise +- APIKEY_PATH=/root/.flowise +- LOG_PATH=/root/.flowise/logs + +Flowise also support different environment variables to configure your instance. Read [more](https://docs.flowiseai.com/environment-variables) + +| Variable | Description | Type | Default | +| ---------------- | ---------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | +| PORT | The HTTP port Flowise runs on | Number | 3000 | +| FLOWISE_USERNAME | Username to login | String | +| FLOWISE_PASSWORD | Password to login | String | +| DEBUG | Print logs onto terminal/console | Boolean | +| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/packages/server` | +| LOG_LEVEL | Different log levels for loggers to be saved | Enum String: `error`, `info`, `verbose`, `debug` | `info` | +| DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | +| APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | +| EXECUTION_MODE | Whether predictions run in their own process or the main process | Enum String: `child`, `main` | `main` | diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3077c43d..7f616fa5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -8,11 +8,12 @@ services: - PORT=${PORT} - FLOWISE_USERNAME=${FLOWISE_USERNAME} - FLOWISE_PASSWORD=${FLOWISE_PASSWORD} + - DEBUG=${DEBUG} - DATABASE_PATH=${DATABASE_PATH} - APIKEY_PATH=${APIKEY_PATH} - LOG_PATH=${LOG_PATH} + - LOG_LEVEL=${LOG_LEVEL} - EXECUTION_MODE=${EXECUTION_MODE} - - DEBUG=${DEBUG} ports: - '${PORT}:${PORT}' volumes: diff --git a/packages/server/.env.example b/packages/server/.env.example index d9b2da76..181ad715 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -4,6 +4,6 @@ PORT=3000 # DEBUG=true # DATABASE_PATH=/your_database_path/.flowise # APIKEY_PATH=/your_api_key_path/.flowise -# LOG_PATH=/your_log_path/logs +# LOG_PATH=/your_log_path/.flowise/logs # LOG_LEVEL=debug (error | warn | info | verbose | debug) # EXECUTION_MODE=main (child | main) diff --git a/packages/server/README.md b/packages/server/README.md index 3d039c26..e18da697 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -31,7 +31,7 @@ FLOWISE_PASSWORD=1234 ## 🌱 Env Variables -Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. +Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. Read [more](https://docs.flowiseai.com/environment-variables) | Variable | Description | Type | Default | | ---------------- | ---------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | From 05e8e33d5a226741345410056eee8ad9ee066391 Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Jul 2023 23:36:18 +0100 Subject: [PATCH 19/22] update Brave Tool logo --- .../tools/BraveSearchAPI/BraveSearchAPI.ts | 2 +- .../nodes/tools/BraveSearchAPI/brave-logo.svg | 46 ------------------- .../nodes/tools/BraveSearchAPI/brave.svg | 1 + 3 files changed, 2 insertions(+), 47 deletions(-) delete mode 100644 packages/components/nodes/tools/BraveSearchAPI/brave-logo.svg create mode 100644 packages/components/nodes/tools/BraveSearchAPI/brave.svg diff --git a/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts b/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts index 76629392..b84aaccd 100644 --- a/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts +++ b/packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts @@ -16,7 +16,7 @@ class BraveSearchAPI_Tools implements INode { this.label = 'BraveSearch API' this.name = 'braveSearchAPI' this.type = 'BraveSearchAPI' - this.icon = 'brave-logo.svg' + this.icon = 'brave.svg' this.category = 'Tools' this.description = 'Wrapper around BraveSearch API - a real-time API to access Brave search results' this.inputs = [ diff --git a/packages/components/nodes/tools/BraveSearchAPI/brave-logo.svg b/packages/components/nodes/tools/BraveSearchAPI/brave-logo.svg deleted file mode 100644 index 686b5c41..00000000 --- a/packages/components/nodes/tools/BraveSearchAPI/brave-logo.svg +++ /dev/null @@ -1,46 +0,0 @@ - - - - Logotypes/bat/logo-dark@1x - Created with Sketch. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/packages/components/nodes/tools/BraveSearchAPI/brave.svg b/packages/components/nodes/tools/BraveSearchAPI/brave.svg new file mode 100644 index 00000000..0c0c0e86 --- /dev/null +++ b/packages/components/nodes/tools/BraveSearchAPI/brave.svg @@ -0,0 +1 @@ + \ No newline at end of file From 673cc029bd0c57a921a5d0d88e00b41eb7777cb0 Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 18 Jul 2023 01:22:20 +0100 Subject: [PATCH 20/22] update conversational retrieval qa chain prompt --- .../ConversationalRetrievalQAChain.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts b/packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts index 24b40d48..eeebe1ec 100644 --- a/packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts +++ b/packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts @@ -7,14 +7,14 @@ import { BaseChatMemory, BufferMemory, ChatMessageHistory } from 'langchain/memo import { PromptTemplate } from 'langchain/prompts' import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler' -const default_qa_template = `Use the following pieces of context to answer the question at the end, in its original language. If you don't know the answer, just say that you don't know in its original language, don't try to make up an answer. +const default_qa_template = `Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. {context} Question: {question} Helpful Answer:` -const qa_template = `Use the following pieces of context to answer the question at the end, in its original language. +const qa_template = `Use the following pieces of context to answer the question at the end. {context} From 56aa5258880b30d1ec646f60e8221fc9e66b79fd Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 18 Jul 2023 01:26:52 +0100 Subject: [PATCH 21/22] update Zep version --- packages/components/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/package.json b/packages/components/package.json index d3ac06c3..8cc02235 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -18,7 +18,7 @@ "dependencies": { "@aws-sdk/client-dynamodb": "^3.360.0", "@dqbd/tiktoken": "^1.0.7", - "@getzep/zep-js": "^0.3.1", + "@getzep/zep-js": "^0.4.1", "@huggingface/inference": "^2.6.1", "@opensearch-project/opensearch": "^1.2.0", "@pinecone-database/pinecone": "^0.0.12", From ed0bbe7bb73ce158d70ecae70ef3631d40ef987b Mon Sep 17 00:00:00 2001 From: Yongtae Date: Tue, 18 Jul 2023 15:09:22 +0900 Subject: [PATCH 22/22] revert content type related to form --- packages/ui/src/views/chatflows/APICodeDialog.js | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/ui/src/views/chatflows/APICodeDialog.js b/packages/ui/src/views/chatflows/APICodeDialog.js index b28cee73..57484c80 100644 --- a/packages/ui/src/views/chatflows/APICodeDialog.js +++ b/packages/ui/src/views/chatflows/APICodeDialog.js @@ -309,10 +309,7 @@ async function query(formData) { "${baseURL}/api/v1/prediction/${dialogProps.chatflowid}", { method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded" - }, - body: new URLSearchParams(formData) + body: formData } ); const result = await response.json(); @@ -356,12 +353,9 @@ async function query(formData) { const response = await fetch( "${baseURL}/api/v1/prediction/${dialogProps.chatflowid}", { - headers: { - Authorization: "Bearer ${selectedApiKey?.apiKey}", - "Content-Type": "application/x-www-form-urlencoded" - }, + headers: { Authorization: "Bearer ${selectedApiKey?.apiKey}" }, method: "POST", - body: new URLSearchParams(formData) + body: formData } ); const result = await response.json();