changes & fixes for various issues in chatflow config (#4314)

* Chatflow config - rate limit and override config were overwriting changes.

* prevent post processing JS function from escaping as JSON...

* non-streaming follow up prompts need to be escaped twice

* prevent post processing JS function from escaping as JSON...

* Adding file mimetypes for full file upload...

* lint..

* fixing the issue with storing only filtered nodes..

* return doc store processing response without await when queue mode and request is from UI

---------

Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/src/controllers/documentstore/index.ts

@@ -201,7 +201,8 @@ const processLoader = async (req: Request, res: Response, next: NextFunction) =>
         }
         const docLoaderId = req.params.loaderId
         const body = req.body
-        const apiResponse = await documentStoreService.processLoaderMiddleware(body, docLoaderId)
+        const isInternalRequest = req.headers['x-request-from'] === 'internal'
+        const apiResponse = await documentStoreService.processLoaderMiddleware(body, docLoaderId, isInternalRequest)
         return res.json(apiResponse)
     } catch (error) {
         next(error)

packages/server/src/services/documentstore/index.ts

@@ -740,7 +740,7 @@ export const processLoader = async ({ appDataSource, componentNodes, data, docLo
     return getDocumentStoreFileChunks(appDataSource, data.storeId as string, docLoaderId)
 }
 
-const processLoaderMiddleware = async (data: IDocumentStoreLoaderForPreview, docLoaderId: string) => {
+const processLoaderMiddleware = async (data: IDocumentStoreLoaderForPreview, docLoaderId: string, isInternalRequest = false) => {
     try {
         const appServer = getRunningExpressApp()
         const appDataSource = appServer.AppDataSource
@@ -761,6 +761,12 @@ const processLoaderMiddleware = async (data: IDocumentStoreLoaderForPreview, doc
             const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
             logger.debug(`[server]: Job added to queue: ${job.id}`)
 
+            if (isInternalRequest) {
+                return {
+                    jobId: job.id
+                }
+            }
+
             const queueEvents = upsertQueue.getQueueEvents()
             const result = await job.waitUntilFinished(queueEvents)
 

packages/server/src/utils/buildChatflow.ts

@@ -14,7 +14,8 @@ import {
     mapMimeTypeToInputField,
     mapExtToInputField,
     getFileFromUpload,
-    removeSpecificFileFromUpload
+    removeSpecificFileFromUpload,
+    handleEscapeCharacters
 } from 'flowise-components'
 import { StatusCodes } from 'http-status-codes'
 import {
@@ -665,6 +666,7 @@ export const executeFlow = async ({
                     const postProcessingFunction = JSON.parse(chatflowConfig?.postProcessing?.customFunction)
                     const nodeInstanceFilePath = componentNodes['customFunction'].filePath as string
                     const nodeModule = await import(nodeInstanceFilePath)
+                    //set the outputs.output to EndingNode to prevent json escaping of content...
                     const nodeData = {
                         inputs: { javascriptFunction: postProcessingFunction },
                         outputs: { output: 'output' }
@@ -681,7 +683,13 @@ export const executeFlow = async ({
                     }
                     const customFuncNodeInstance = new nodeModule.nodeClass()
                     let moderatedResponse = await customFuncNodeInstance.init(nodeData, question, options)
-                    result.text = moderatedResponse
+                    if (typeof moderatedResponse === 'string') {
+                        result.text = handleEscapeCharacters(moderatedResponse, true)
+                    } else if (typeof moderatedResponse === 'object') {
+                        result.text = '```json\n' + JSON.stringify(moderatedResponse, null, 2) + '\n```'
+                    } else {
+                        result.text = moderatedResponse
+                    }
                     resultText = result.text
                 } catch (e) {
                     logger.log('[server]: Post Processing Error:', e)

packages/ui/src/ui-component/cards/FollowUpPromptsCard.jsx

@@ -12,25 +12,26 @@ const FollowUpPromptsCard = ({ isGrid, followUpPrompts, sx, onPromptClick }) =>
             className={'button-container'}
             sx={{ width: '100%', maxWidth: isGrid ? 'inherit' : '400px', p: 1.5, display: 'flex', gap: 1, ...sx }}
         >
-            {followUpPrompts.map((fp, index) => (
+            {Array.isArray(followUpPrompts) &&
-                <Chip
+                followUpPrompts.map((fp, index) => (
-                    label={fp}
+                    <Chip
-                    className={'button'}
+                        label={fp}
-                    key={index}
+                        className={'button'}
-                    onClick={(e) => onPromptClick(fp, e)}
+                        key={index}
-                    sx={{
+                        onClick={(e) => onPromptClick(fp, e)}
-                        backgroundColor: 'transparent',
+                        sx={{
-                        border: '1px solid',
+                            backgroundColor: 'transparent',
-                        boxShadow: '0px 2px 1px -1px rgba(0,0,0,0.2)',
+                            border: '1px solid',
-                        color: '#2196f3',
+                            boxShadow: '0px 2px 1px -1px rgba(0,0,0,0.2)',
-                        transition: 'all 300ms cubic-bezier(0.4, 0, 0.2, 1) 0ms',
+                            color: '#2196f3',
-                        '&:hover': {
+                            transition: 'all 300ms cubic-bezier(0.4, 0, 0.2, 1) 0ms',
-                            backgroundColor: customization.isDarkMode ? 'rgba(0, 0, 0, 0.12)' : 'rgba(0, 0, 0, 0.05)',
+                            '&:hover': {
-                            border: '1px solid'
+                                backgroundColor: customization.isDarkMode ? 'rgba(0, 0, 0, 0.12)' : 'rgba(0, 0, 0, 0.05)',
-                        }
+                                border: '1px solid'
-                    }}
+                            }
-                />
+                        }}
-            ))}
+                    />
+                ))}
         </Box>
     )
 }

packages/ui/src/ui-component/extended/FileUpload.jsx

@@ -5,7 +5,7 @@ import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackba
 import parser from 'html-react-parser'
 
 // material-ui
-import { Button, Box } from '@mui/material'
+import { Button, Box, Typography } from '@mui/material'
 import { IconX, IconBulb } from '@tabler/icons-react'
 
 // Project import
@@ -22,6 +22,18 @@ const message = `Uploaded files will be parsed as strings and sent to the LLM. I
 <br />
 Refer <a href='https://docs.flowiseai.com/using-flowise/uploads#files' target='_blank'>docs</a> for more details.`
 
+const availableFileTypes = [
+    { name: 'CSS', ext: 'text/css' },
+    { name: 'CSV', ext: 'text/csv' },
+    { name: 'HTML', ext: 'text/html' },
+    { name: 'JSON', ext: 'application/json' },
+    { name: 'Markdown', ext: 'text/markdown' },
+    { name: 'PDF', ext: 'application/pdf' },
+    { name: 'SQL', ext: 'application/sql' },
+    { name: 'Text File', ext: 'text/plain' },
+    { name: 'XML', ext: 'application/xml' }
+]
+
 const FileUpload = ({ dialogProps }) => {
     const dispatch = useDispatch()
 
@@ -31,16 +43,27 @@ const FileUpload = ({ dialogProps }) => {
     const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
 
     const [fullFileUpload, setFullFileUpload] = useState(false)
+    const [allowedFileTypes, setAllowedFileTypes] = useState([])
     const [chatbotConfig, setChatbotConfig] = useState({})
 
     const handleChange = (value) => {
         setFullFileUpload(value)
     }
 
+    const handleAllowedFileTypesChange = (event) => {
+        const { checked, value } = event.target
+        if (checked) {
+            setAllowedFileTypes((prev) => [...prev, value])
+        } else {
+            setAllowedFileTypes((prev) => prev.filter((item) => item !== value))
+        }
+    }
+
     const onSave = async () => {
         try {
             const value = {
-                status: fullFileUpload
+                status: fullFileUpload,
+                allowedUploadFileTypes: allowedFileTypes.join(',')
             }
             chatbotConfig.fullFileUpload = value
 
@@ -82,6 +105,9 @@ const FileUpload = ({ dialogProps }) => {
     }
 
     useEffect(() => {
+        /* backward compatibility - by default, allow all */
+        const allowedFileTypes = availableFileTypes.map((fileType) => fileType.ext)
+        setAllowedFileTypes(allowedFileTypes)
         if (dialogProps.chatflow) {
             if (dialogProps.chatflow.chatbotConfig) {
                 try {
@@ -90,6 +116,10 @@ const FileUpload = ({ dialogProps }) => {
                     if (chatbotConfig.fullFileUpload) {
                         setFullFileUpload(chatbotConfig.fullFileUpload.status)
                     }
+                    if (chatbotConfig.fullFileUpload?.allowedUploadFileTypes) {
+                        const allowedFileTypes = chatbotConfig.fullFileUpload.allowedUploadFileTypes.split(',')
+                        setAllowedFileTypes(allowedFileTypes)
+                    }
                 } catch (e) {
                     setChatbotConfig({})
                 }
@@ -135,8 +165,44 @@ const FileUpload = ({ dialogProps }) => {
                 </div>
                 <SwitchInput label='Enable Full File Upload' onChange={handleChange} value={fullFileUpload} />
             </Box>
-            {/* TODO: Allow selection of allowed file types*/}
+
-            <StyledButton style={{ marginBottom: 10, marginTop: 10 }} variant='contained' onClick={onSave}>
+            <Typography sx={{ fontSize: 14, fontWeight: 500, marginBottom: 1 }}>Allow Uploads of Type</Typography>
+            <div
+                style={{
+                    display: 'grid',
+                    gridTemplateColumns: 'repeat(auto-fill, minmax(250px, 1fr))',
+                    gap: 15,
+                    padding: 10,
+                    width: '100%',
+                    marginBottom: '10px'
+                }}
+            >
+                {availableFileTypes.map((fileType) => (
+                    <div
+                        key={fileType.ext}
+                        style={{
+                            display: 'flex',
+                            flexDirection: 'row',
+                            alignItems: 'center',
+                            justifyContent: 'start'
+                        }}
+                    >
+                        <input
+                            type='checkbox'
+                            id={fileType.ext}
+                            name={fileType.ext}
+                            checked={allowedFileTypes.indexOf(fileType.ext) !== -1}
+                            value={fileType.ext}
+                            disabled={!fullFileUpload}
+                            onChange={handleAllowedFileTypesChange}
+                        />
+                        <label htmlFor={fileType.ext} style={{ marginLeft: 10 }}>
+                            {fileType.name} ({fileType.ext})
+                        </label>
+                    </div>
+                ))}
+            </div>
+            <StyledButton style={{ marginBottom: 10, marginTop: 20 }} variant='contained' onClick={onSave}>
                 Save
             </StyledButton>
         </>

packages/ui/src/ui-component/extended/OverrideConfig.jsx

@@ -116,25 +116,27 @@ const OverrideConfig = ({ dialogProps }) => {
     }
 
     const formatObj = () => {
-        const obj = {
+        let apiConfig = JSON.parse(dialogProps.chatflow.apiConfig)
-            overrideConfig: { status: overrideConfigStatus }
+        if (apiConfig === null || apiConfig === undefined) {
+            apiConfig = {}
         }
 
+        let overrideConfig = { status: overrideConfigStatus }
         if (overrideConfigStatus) {
-            // loop through each key in nodeOverrides and filter out the enabled ones
             const filteredNodeOverrides = {}
             for (const key in nodeOverrides) {
                 filteredNodeOverrides[key] = nodeOverrides[key].filter((node) => node.enabled)
             }
 
-            obj.overrideConfig = {
+            overrideConfig = {
-                ...obj.overrideConfig,
+                ...overrideConfig,
                 nodes: filteredNodeOverrides,
                 variables: variableOverrides.filter((node) => node.enabled)
             }
         }
+        apiConfig.overrideConfig = overrideConfig
 
-        return obj
+        return apiConfig
     }
 
     const onNodeOverrideToggle = (node, property, status) => {
@@ -206,7 +208,7 @@ const OverrideConfig = ({ dialogProps }) => {
         if (!overrideConfigStatus) {
             setNodeOverrides(newNodeOverrides)
         } else {
-            const updatedNodeOverrides = { ...nodeOverrides }
+            const updatedNodeOverrides = { ...newNodeOverrides }
 
             Object.keys(updatedNodeOverrides).forEach((node) => {
                 if (!seenNodes.has(node)) {

packages/ui/src/ui-component/extended/RateLimit.jsx

@@ -19,7 +19,7 @@ import chatflowsApi from '@/api/chatflows'
 // utils
 import useNotifier from '@/utils/useNotifier'
 
-const RateLimit = () => {
+const RateLimit = ({ dialogProps }) => {
     const dispatch = useDispatch()
     const chatflow = useSelector((state) => state.canvas.chatflow)
     const chatflowid = chatflow.id
@@ -36,9 +36,11 @@ const RateLimit = () => {
     const [limitMsg, setLimitMsg] = useState(apiConfig?.rateLimit?.limitMsg ?? '')
 
     const formatObj = () => {
-        const obj = {
+        let apiConfig = JSON.parse(dialogProps.chatflow.apiConfig)
-            rateLimit: { status: rateLimitStatus }
+        if (apiConfig === null || apiConfig === undefined) {
+            apiConfig = {}
         }
+        let obj = { status: rateLimitStatus }
 
         if (rateLimitStatus) {
             const rateLimitValuesBoolean = [!limitMax, !limitDuration, !limitMsg]
@@ -46,16 +48,16 @@ const RateLimit = () => {
             if (rateLimitFilledValues.length >= 1 && rateLimitFilledValues.length <= 2) {
                 throw new Error('Need to fill all rate limit input fields')
             } else if (rateLimitFilledValues.length === 3) {
-                obj.rateLimit = {
+                obj = {
-                    ...obj.rateLimit,
+                    ...obj,
                     limitMax,
                     limitDuration,
                     limitMsg
                 }
             }
         }
-
+        apiConfig.rateLimit = obj
-        return obj
+        return apiConfig
     }
 
     const handleChange = (value) => {
@@ -173,7 +175,8 @@ const RateLimit = () => {
 }
 
 RateLimit.propTypes = {
-    isSessionMemory: PropTypes.bool
+    isSessionMemory: PropTypes.bool,
+    dialogProps: PropTypes.object
 }
 
 export default RateLimit

packages/ui/src/ui-component/extended/Security.jsx

@@ -12,7 +12,7 @@ const Security = ({ dialogProps }) => {
 
     return (
         <Stack direction='column' divider={<Divider sx={{ my: 0.5, borderColor: theme.palette.grey[900] + 25 }} />} spacing={4}>
-            <RateLimit />
+            <RateLimit dialogProps={dialogProps} />
             <AllowedDomains dialogProps={dialogProps} />
             <OverrideConfig dialogProps={dialogProps} />
         </Stack>

packages/ui/src/views/chatmessage/ChatMessage.jsx

@@ -200,6 +200,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview
 
     // full file upload
     const [fullFileUpload, setFullFileUpload] = useState(false)
+    const [fullFileUploadAllowedTypes, setFullFileUploadAllowedTypes] = useState('*')
 
     // feedback
     const [chatFeedbackStatus, setChatFeedbackStatus] = useState(false)
@@ -693,7 +694,11 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview
 
         if (data.followUpPrompts) {
             const followUpPrompts = JSON.parse(data.followUpPrompts)
-            setFollowUpPrompts(followUpPrompts)
+            if (typeof followUpPrompts === 'string') {
+                setFollowUpPrompts(JSON.parse(followUpPrompts))
+            } else {
+                setFollowUpPrompts(followUpPrompts)
+            }
         }
     }
 
@@ -981,7 +986,9 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview
     }
 
     const getFileUploadAllowedTypes = () => {
-        if (fullFileUpload) return '*'
+        if (fullFileUpload) {
+            return fullFileUploadAllowedTypes === '' ? '*' : fullFileUploadAllowedTypes
+        }
         return fileUploadAllowedTypes.includes('*') ? '*' : fileUploadAllowedTypes || '*'
     }
 
@@ -1118,6 +1125,9 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview
 
                 if (config.fullFileUpload) {
                     setFullFileUpload(config.fullFileUpload.status)
+                    if (config.fullFileUpload?.allowedUploadFileTypes) {
+                        setFullFileUploadAllowedTypes(config.fullFileUpload?.allowedUploadFileTypes)
+                    }
                 }
             }
         }
@@ -1198,7 +1208,13 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview
         if (followUpPromptsStatus && messages.length > 0) {
             const lastMessage = messages[messages.length - 1]
             if (lastMessage.type === 'apiMessage' && lastMessage.followUpPrompts) {
-                setFollowUpPrompts(lastMessage.followUpPrompts)
+                if (Array.isArray(lastMessage.followUpPrompts)) {
+                    setFollowUpPrompts(lastMessage.followUpPrompts)
+                }
+                if (typeof lastMessage.followUpPrompts === 'string') {
+                    const followUpPrompts = JSON.parse(lastMessage.followUpPrompts)
+                    setFollowUpPrompts(followUpPrompts)
+                }
             } else if (lastMessage.type === 'userMessage') {
                 setFollowUpPrompts([])
             }