changes & fixes for various issues in chatflow config (#4314)

* Chatflow config - rate limit and override config were overwriting changes.

* prevent post processing JS function from escaping as JSON...

* non-streaming follow up prompts need to be escaped twice

* prevent post processing JS function from escaping as JSON...

* Adding file mimetypes for full file upload...

* lint..

* fixing the issue with storing only filtered nodes..

* return doc store processing response without await when queue mode and request is from UI

---------

Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/src/controllers/documentstore/index.ts

@@ -201,7 +201,8 @@ const processLoader = async (req: Request, res: Response, next: NextFunction) =>
         }
         const docLoaderId = req.params.loaderId
         const body = req.body
-        const apiResponse = await documentStoreService.processLoaderMiddleware(body, docLoaderId)
+        const isInternalRequest = req.headers['x-request-from'] === 'internal'
+        const apiResponse = await documentStoreService.processLoaderMiddleware(body, docLoaderId, isInternalRequest)
         return res.json(apiResponse)
     } catch (error) {
         next(error)

packages/server/src/services/documentstore/index.ts

@@ -740,7 +740,7 @@ export const processLoader = async ({ appDataSource, componentNodes, data, docLo
     return getDocumentStoreFileChunks(appDataSource, data.storeId as string, docLoaderId)
 }
 
-const processLoaderMiddleware = async (data: IDocumentStoreLoaderForPreview, docLoaderId: string) => {
+const processLoaderMiddleware = async (data: IDocumentStoreLoaderForPreview, docLoaderId: string, isInternalRequest = false) => {
     try {
         const appServer = getRunningExpressApp()
         const appDataSource = appServer.AppDataSource
@@ -761,6 +761,12 @@ const processLoaderMiddleware = async (data: IDocumentStoreLoaderForPreview, doc
             const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
             logger.debug(`[server]: Job added to queue: ${job.id}`)
 
+            if (isInternalRequest) {
+                return {
+                    jobId: job.id
+                }
+            }
+
             const queueEvents = upsertQueue.getQueueEvents()
             const result = await job.waitUntilFinished(queueEvents)
 

packages/server/src/utils/buildChatflow.ts

@@ -14,7 +14,8 @@ import {
     mapMimeTypeToInputField,
     mapExtToInputField,
     getFileFromUpload,
-    removeSpecificFileFromUpload
+    removeSpecificFileFromUpload,
+    handleEscapeCharacters
 } from 'flowise-components'
 import { StatusCodes } from 'http-status-codes'
 import {
@@ -665,6 +666,7 @@ export const executeFlow = async ({
                     const postProcessingFunction = JSON.parse(chatflowConfig?.postProcessing?.customFunction)
                     const nodeInstanceFilePath = componentNodes['customFunction'].filePath as string
                     const nodeModule = await import(nodeInstanceFilePath)
+                    //set the outputs.output to EndingNode to prevent json escaping of content...
                     const nodeData = {
                         inputs: { javascriptFunction: postProcessingFunction },
                         outputs: { output: 'output' }
@@ -681,7 +683,13 @@ export const executeFlow = async ({
                     }
                     const customFuncNodeInstance = new nodeModule.nodeClass()
                     let moderatedResponse = await customFuncNodeInstance.init(nodeData, question, options)
-                    result.text = moderatedResponse
+                    if (typeof moderatedResponse === 'string') {
+                        result.text = handleEscapeCharacters(moderatedResponse, true)
+                    } else if (typeof moderatedResponse === 'object') {
+                        result.text = '```json\n' + JSON.stringify(moderatedResponse, null, 2) + '\n```'
+                    } else {
+                        result.text = moderatedResponse
+                    }
                     resultText = result.text
                 } catch (e) {
                     logger.log('[server]: Post Processing Error:', e)