Merge branch 'main' into feature/output-parsers

# Conflicts:
#	packages/ui/src/views/chatmessage/ChatMessage.js

packages/ui/src/views/chatmessage/ChatMessage.js

@@ -30,7 +30,7 @@ import { baseURL, maxScroll } from 'store/constant'
 
 import robotPNG from 'assets/images/robot.png'
 import userPNG from 'assets/images/account.png'
-import { isValidURL } from 'utils/genericHelper'
+import { isValidURL, removeDuplicateURL } from 'utils/genericHelper'
 
 export const ChatMessage = ({ open, chatflowid, isDialog }) => {
     const theme = useTheme()
@@ -50,9 +50,10 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
     const [isChatFlowAvailableToStream, setIsChatFlowAvailableToStream] = useState(false)
     const [sourceDialogOpen, setSourceDialogOpen] = useState(false)
     const [sourceDialogProps, setSourceDialogProps] = useState({})
+    const [chatId, setChatId] = useState(undefined)
 
     const inputRef = useRef(null)
-    const getChatmessageApi = useApi(chatmessageApi.getChatmessageFromChatflow)
+    const getChatmessageApi = useApi(chatmessageApi.getInternalChatmessageFromChatflow)
     const getIsChatflowStreamingApi = useApi(chatflowsApi.getIsChatflowStreaming)
 
     const onSourceDialogClick = (data) => {
@@ -64,21 +65,6 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
         window.open(data, '_blank')
     }
 
-    const removeDuplicateURL = (message) => {
-        const visitedURLs = []
-        const newSourceDocuments = []
-
-        message.sourceDocuments.forEach((source) => {
-            if (isValidURL(source.metadata.source) && !visitedURLs.includes(source.metadata.source)) {
-                visitedURLs.push(source.metadata.source)
-                newSourceDocuments.push(source)
-            } else if (!isValidURL(source.metadata.source)) {
-                newSourceDocuments.push(source)
-            }
-        })
-        return newSourceDocuments
-    }
-
     const scrollToBottom = () => {
         if (ps.current) {
             ps.current.scrollTo({ top: maxScroll })
@@ -87,20 +73,6 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
 
     const onChange = useCallback((e) => setUserInput(e.target.value), [setUserInput])
 
-    const addChatMessage = async (message, type, sourceDocuments) => {
-        try {
-            const newChatMessageBody = {
-                role: type,
-                content: message,
-                chatflowid: chatflowid
-            }
-            if (sourceDocuments) newChatMessageBody.sourceDocuments = JSON.stringify(sourceDocuments)
-            await chatmessageApi.createNewChatmessage(chatflowid, newChatMessageBody)
-        } catch (error) {
-            console.error(error)
-        }
-    }
-
     const updateLastMessage = (text) => {
         setMessages((prevMessages) => {
             let allMessages = [...cloneDeep(prevMessages)]
@@ -123,7 +95,6 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
     const handleError = (message = 'Oops! There seems to be an error. Please try again.') => {
         message = message.replace(`Unable to parse JSON response from chat agent.\n\n`, '')
         setMessages((prevMessages) => [...prevMessages, { message, type: 'apiMessage' }])
-        addChatMessage(message, 'apiMessage')
         setLoading(false)
         setUserInput('')
         setTimeout(() => {
@@ -141,42 +112,36 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
 
         setLoading(true)
         setMessages((prevMessages) => [...prevMessages, { message: userInput, type: 'userMessage' }])
-        // waiting for first chatmessage saved, the first chatmessage will be used in sendMessageAndGetPrediction
-        await addChatMessage(userInput, 'userMessage')
 
         // Send user question and history to API
         try {
             const params = {
                 question: userInput,
-                history: messages.filter((msg) => msg.message !== 'Hi there! How can I help?')
+                history: messages.filter((msg) => msg.message !== 'Hi there! How can I help?'),
+                chatId
             }
             if (isChatFlowAvailableToStream) params.socketIOClientId = socketIOClientId
 
             const response = await predictionApi.sendMessageAndGetPrediction(chatflowid, params)
 
             if (response.data) {
-                let data = response.data
-
-                if (typeof data === 'object' && data.text && data.sourceDocuments) {
-                    if (!isChatFlowAvailableToStream) {
-                        setMessages((prevMessages) => [
-                            ...prevMessages,
-                            { message: data.text, sourceDocuments: data.sourceDocuments, type: 'apiMessage' }
-                        ])
-                    }
-                    addChatMessage(data.text, 'apiMessage', data.sourceDocuments)
-                } else if (typeof data === 'object' && data.json) {
-                    const text = '```json\n' + JSON.stringify(data.json, null, 2)
-                    if (!isChatFlowAvailableToStream) {
-                        setMessages((prevMessages) => [...prevMessages, { message: text, type: 'apiMessage' }])
-                    }
-                    addChatMessage(text, 'apiMessage')
-                } else {
-                    if (!isChatFlowAvailableToStream) {
-                        setMessages((prevMessages) => [...prevMessages, { message: data, type: 'apiMessage' }])
-                    }
-                    addChatMessage(data, 'apiMessage')
+                const data = response.data
+                if (!chatId) {
+                    setChatId(data.chatId)
+                    localStorage.setItem(`${chatflowid}_INTERNAL`, data.chatId)
                 }
+                if (!isChatFlowAvailableToStream) {
+                    let text = ''
+                    if (data.text) text = data.text
+                    else if (data.json) text = '```json\n' + JSON.stringify(data.json, null, 2)
+                    else text = JSON.stringify(data, null, 2)
+
+                    setMessages((prevMessages) => [
+                        ...prevMessages,
+                        { message: text, sourceDocuments: data?.sourceDocuments, type: 'apiMessage' }
+                    ])
+                }
+
                 setLoading(false)
                 setUserInput('')
                 setTimeout(() => {
@@ -206,16 +171,18 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
 
     // Get chatmessages successful
     useEffect(() => {
-        if (getChatmessageApi.data) {
-            const loadedMessages = []
-            for (const message of getChatmessageApi.data) {
+        if (getChatmessageApi.data?.length) {
+            const chatId = getChatmessageApi.data[0]?.chatId
+            setChatId(chatId)
+            localStorage.setItem(`${chatflowid}_INTERNAL`, chatId)
+            const loadedMessages = getChatmessageApi.data.map((message) => {
                 const obj = {
                     message: message.content,
                     type: message.role
                 }
                 if (message.sourceDocuments) obj.sourceDocuments = JSON.parse(message.sourceDocuments)
-                loadedMessages.push(obj)
-            }
+                return obj
+            })
             setMessages((prevMessages) => [...prevMessages, ...loadedMessages])
         }