import { useCallback, useEffect, useRef, useState } from 'react' import { useSelector } from 'react-redux' import PropTypes from 'prop-types' import socketIOClient from 'socket.io-client' import { cloneDeep } from 'lodash' import rehypeMathjax from 'rehype-mathjax' import rehypeRaw from 'rehype-raw' import remarkGfm from 'remark-gfm' import remarkMath from 'remark-math' import axios from 'axios' import audioUploadSVG from 'assets/images/wave-sound.jpg' import { Box, Button, Card, CardActions, CardMedia, Chip, CircularProgress, Divider, Grid, IconButton, InputAdornment, OutlinedInput, Typography } from '@mui/material' import { useTheme } from '@mui/material/styles' import { IconDownload, IconSend, IconMicrophone, IconPhotoPlus, IconCircleDot } from '@tabler/icons' // project import import { CodeBlock } from 'ui-component/markdown/CodeBlock' import { MemoizedReactMarkdown } from 'ui-component/markdown/MemoizedReactMarkdown' import SourceDocDialog from 'ui-component/dialog/SourceDocDialog' import './ChatMessage.css' import './audio-recording.css' // api import chatmessageApi from 'api/chatmessage' import chatflowsApi from 'api/chatflows' import predictionApi from 'api/prediction' // Hooks import useApi from 'hooks/useApi' // Const import { baseURL, maxScroll } from 'store/constant' import robotPNG from 'assets/images/robot.png' import userPNG from 'assets/images/account.png' import StarterPromptsCard from '../../ui-component/cards/StarterPromptsCard' import { isValidURL, removeDuplicateURL, setLocalStorageChatflow } from 'utils/genericHelper' import DeleteIcon from '@mui/icons-material/Delete' import { cancelAudioRecording, startAudioRecording, stopAudioRecording } from './audio-recording' export const ChatMessage = ({ open, chatflowid, isDialog }) => { const theme = useTheme() const customization = useSelector((state) => state.customization) const ps = useRef() const [userInput, setUserInput] = useState('') const [loading, setLoading] = useState(false) const [messages, setMessages] = useState([ { message: 'Hi there! How can I help?', type: 'apiMessage' } ]) const [socketIOClientId, setSocketIOClientId] = useState('') const [isChatFlowAvailableToStream, setIsChatFlowAvailableToStream] = useState(false) const [sourceDialogOpen, setSourceDialogOpen] = useState(false) const [sourceDialogProps, setSourceDialogProps] = useState({}) const [chatId, setChatId] = useState(undefined) const inputRef = useRef(null) const getChatmessageApi = useApi(chatmessageApi.getInternalChatmessageFromChatflow) const getIsChatflowStreamingApi = useApi(chatflowsApi.getIsChatflowStreaming) const getChatflowConfig = useApi(chatflowsApi.getSpecificChatflow) const [starterPrompts, setStarterPrompts] = useState([]) // drag & drop and file input const fileUploadRef = useRef(null) const getAllowChatFlowUploads = useApi(chatflowsApi.getAllowChatflowUploads) const [isChatFlowAvailableForUploads, setIsChatFlowAvailableForUploads] = useState(false) const [previews, setPreviews] = useState([]) const [isDragOver, setIsDragOver] = useState(false) // recording const [isRecording, setIsRecording] = useState(false) const [recordingNotSupported, setRecordingNotSupported] = useState(false) const handleDragOver = (e) => { if (!isChatFlowAvailableForUploads) { return } e.preventDefault() } const isFileAllowedForUpload = (file) => { const constraints = getAllowChatFlowUploads.data let acceptFile = false if (constraints.allowUploads) { const fileType = file.type const sizeInMB = file.size / 1024 / 1024 constraints.allowed.map((allowed) => { if (allowed.allowedTypes.includes(fileType) && sizeInMB <= allowed.maxUploadSize) { acceptFile = true } }) } if (!acceptFile) { alert(`Cannot upload file. Kindly check the allowed file types and maximum allowed size.`) } return acceptFile } const handleDrop = async (e) => { if (!isChatFlowAvailableForUploads) { return } e.preventDefault() setIsDragOver(false) let files = [] if (e.dataTransfer.files.length > 0) { for (const file of e.dataTransfer.files) { if (isFileAllowedForUpload(file) === false) { return } const reader = new FileReader() const { name } = file files.push( new Promise((resolve) => { reader.onload = (evt) => { if (!evt?.target?.result) { return } const { result } = evt.target let previewUrl if (file.type.startsWith('audio/')) { previewUrl = audioUploadSVG } else if (file.type.startsWith('image/')) { previewUrl = URL.createObjectURL(file) } resolve({ data: result, preview: previewUrl, type: 'file', name: name, mime: file.type }) } reader.readAsDataURL(file) }) ) } const newFiles = await Promise.all(files) setPreviews((prevPreviews) => [...prevPreviews, ...newFiles]) // if (newFiles.length > 0) { // document.getElementById('messagelist').style.height = '80%' // } } if (e.dataTransfer.items) { for (const item of e.dataTransfer.items) { if (item.kind === 'string' && item.type.match('^text/uri-list')) { item.getAsString((s) => { let upload = { data: s, preview: s, type: 'url', name: s.substring(s.lastIndexOf('/') + 1) } setPreviews((prevPreviews) => [...prevPreviews, upload]) }) } else if (item.kind === 'string' && item.type.match('^text/html')) { item.getAsString((s) => { if (s.indexOf('href') === -1) return //extract href let start = s.substring(s.indexOf('href') + 6) let hrefStr = start.substring(0, start.indexOf('"')) let upload = { data: hrefStr, preview: hrefStr, type: 'url', name: hrefStr.substring(hrefStr.lastIndexOf('/') + 1) } setPreviews((prevPreviews) => [...prevPreviews, upload]) }) } } } } const handleFileChange = async (event) => { const fileObj = event.target.files && event.target.files[0] if (!fileObj) { return } let files = [] for (const file of event.target.files) { if (isFileAllowedForUpload(file) === false) { return } const reader = new FileReader() const { name } = file files.push( new Promise((resolve) => { reader.onload = (evt) => { if (!evt?.target?.result) { return } const { result } = evt.target resolve({ data: result, preview: URL.createObjectURL(file), type: 'file', name: name, mime: file.type }) } reader.readAsDataURL(file) }) ) } const newFiles = await Promise.all(files) setPreviews((prevPreviews) => [...prevPreviews, ...newFiles]) // 👇️ reset file input event.target.value = null } const addRecordingToPreviews = (blob) => { const mimeType = blob.type.substring(0, blob.type.indexOf(';')) // read blob and add to previews const reader = new FileReader() reader.readAsDataURL(blob) reader.onloadend = () => { const base64data = reader.result const upload = { data: base64data, preview: audioUploadSVG, type: 'audio', name: 'audio.wav', mime: mimeType } setPreviews((prevPreviews) => [...prevPreviews, upload]) } } const handleDragEnter = (e) => { if (isChatFlowAvailableForUploads) { e.preventDefault() setIsDragOver(true) } } const handleDragLeave = (e) => { if (isChatFlowAvailableForUploads) { e.preventDefault() if (e.originalEvent?.pageX !== 0 || e.originalEvent?.pageY !== 0) { return false } setIsDragOver(false) // Set the drag over state to false when the drag leaves } } const handleDeletePreview = (itemToDelete) => { if (itemToDelete.type === 'file') { URL.revokeObjectURL(itemToDelete.preview) // Clean up for file } setPreviews(previews.filter((item) => item !== itemToDelete)) } const handleUploadClick = () => { // 👇️ open file input box on click of another element fileUploadRef.current.click() } const previewStyle = { width: '128px', height: '64px', objectFit: 'fit' // This makes the image cover the area, cropping it if necessary } const messageImageStyle = { width: '128px', height: '128px', objectFit: 'cover' // This makes the image cover the area, cropping it if necessary } const clearPreviews = () => { // Revoke the data uris to avoid memory leaks previews.forEach((file) => URL.revokeObjectURL(file.preview)) setPreviews([]) } const onMicrophonePressed = () => { setIsRecording(true) startAudioRecording(setIsRecording, setRecordingNotSupported) } const onRecordingCancelled = () => { cancelAudioRecording() setIsRecording(false) setRecordingNotSupported(false) } const onRecordingStopped = () => { stopAudioRecording(addRecordingToPreviews) setIsRecording(false) setRecordingNotSupported(false) } const onSourceDialogClick = (data, title) => { setSourceDialogProps({ data, title }) setSourceDialogOpen(true) } const onURLClick = (data) => { window.open(data, '_blank') } const scrollToBottom = () => { if (ps.current) { ps.current.scrollTo({ top: maxScroll }) } } const onChange = useCallback((e) => setUserInput(e.target.value), [setUserInput]) const updateLastMessage = (text) => { setMessages((prevMessages) => { let allMessages = [...cloneDeep(prevMessages)] if (allMessages[allMessages.length - 1].type === 'userMessage') return allMessages allMessages[allMessages.length - 1].message += text return allMessages }) } const updateLastMessageSourceDocuments = (sourceDocuments) => { setMessages((prevMessages) => { let allMessages = [...cloneDeep(prevMessages)] if (allMessages[allMessages.length - 1].type === 'userMessage') return allMessages allMessages[allMessages.length - 1].sourceDocuments = sourceDocuments return allMessages }) } // Handle errors const handleError = (message = 'Oops! There seems to be an error. Please try again.') => { message = message.replace(`Unable to parse JSON response from chat agent.\n\n`, '') setMessages((prevMessages) => [...prevMessages, { message, type: 'apiMessage' }]) setLoading(false) setUserInput('') setTimeout(() => { inputRef.current?.focus() }, 100) } const handlePromptClick = async (promptStarterInput) => { setUserInput(promptStarterInput) handleSubmit(undefined, promptStarterInput) } // Handle form submission const handleSubmit = async (e, promptStarterInput) => { if (e) e.preventDefault() if (!promptStarterInput && userInput.trim() === '') { return } let input = userInput if (promptStarterInput !== undefined && promptStarterInput.trim() !== '') input = promptStarterInput setLoading(true) const urls = [] previews.map((item) => { urls.push({ data: item.data, type: item.type, name: item.name, mime: item.mime }) }) clearPreviews() setMessages((prevMessages) => [...prevMessages, { message: input, type: 'userMessage', fileUploads: urls }]) // Send user question and history to API try { const params = { question: input, history: messages.filter((msg) => msg.message !== 'Hi there! How can I help?'), chatId } if (urls) params.uploads = urls if (isChatFlowAvailableToStream) params.socketIOClientId = socketIOClientId const response = await predictionApi.sendMessageAndGetPrediction(chatflowid, params) if (response.data) { const data = response.data if (!chatId) setChatId(data.chatId) if (!isChatFlowAvailableToStream) { let text = '' if (data.text) text = data.text else if (data.json) text = '```json\n' + JSON.stringify(data.json, null, 2) else text = JSON.stringify(data, null, 2) setMessages((prevMessages) => [ ...prevMessages, { message: text, sourceDocuments: data?.sourceDocuments, usedTools: data?.usedTools, fileAnnotations: data?.fileAnnotations, type: 'apiMessage' } ]) } setLocalStorageChatflow(chatflowid, data.chatId, messages) setLoading(false) setUserInput('') setTimeout(() => { inputRef.current?.focus() scrollToBottom() }, 100) } } catch (error) { const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}` handleError(errorData) return } } // Prevent blank submissions and allow for multiline input const handleEnter = (e) => { // Check if IME composition is in progress const isIMEComposition = e.isComposing || e.keyCode === 229 if (e.key === 'Enter' && userInput && !isIMEComposition) { if (!e.shiftKey && userInput) { handleSubmit(e) } } else if (e.key === 'Enter') { e.preventDefault() } } const downloadFile = async (fileAnnotation) => { try { const response = await axios.post( `${baseURL}/api/v1/openai-assistants-file`, { fileName: fileAnnotation.fileName }, { responseType: 'blob' } ) const blob = new Blob([response.data], { type: response.headers['content-type'] }) const downloadUrl = window.URL.createObjectURL(blob) const link = document.createElement('a') link.href = downloadUrl link.download = fileAnnotation.fileName document.body.appendChild(link) link.click() link.remove() } catch (error) { console.error('Download failed:', error) } } // Get chatmessages successful useEffect(() => { if (getChatmessageApi.data?.length) { const chatId = getChatmessageApi.data[0]?.chatId setChatId(chatId) const loadedMessages = getChatmessageApi.data.map((message) => { const obj = { message: message.content, type: message.role } if (message.sourceDocuments) obj.sourceDocuments = JSON.parse(message.sourceDocuments) if (message.usedTools) obj.usedTools = JSON.parse(message.usedTools) if (message.fileAnnotations) obj.fileAnnotations = JSON.parse(message.fileAnnotations) if (message.fileUploads) { obj.fileUploads = JSON.parse(message.fileUploads) obj.fileUploads.forEach((file) => { if (file.type === 'stored-file') { file.data = `${baseURL}/api/v1/get-upload-file/${file.name}?chatId=${chatId}` } }) } return obj }) setMessages((prevMessages) => [...prevMessages, ...loadedMessages]) setLocalStorageChatflow(chatflowid, chatId, messages) } // eslint-disable-next-line react-hooks/exhaustive-deps }, [getChatmessageApi.data]) // Get chatflow streaming capability useEffect(() => { if (getIsChatflowStreamingApi.data) { setIsChatFlowAvailableToStream(getIsChatflowStreamingApi.data?.isStreaming ?? false) } // eslint-disable-next-line react-hooks/exhaustive-deps }, [getIsChatflowStreamingApi.data]) // Get chatflow uploads capability useEffect(() => { if (getAllowChatFlowUploads.data) { setIsChatFlowAvailableForUploads(getAllowChatFlowUploads.data?.allowUploads ?? false) } // eslint-disable-next-line react-hooks/exhaustive-deps }, [getAllowChatFlowUploads.data]) useEffect(() => { if (getChatflowConfig.data) { if (getChatflowConfig.data?.chatbotConfig && JSON.parse(getChatflowConfig.data?.chatbotConfig)) { let config = JSON.parse(getChatflowConfig.data?.chatbotConfig) if (config.starterPrompts) { let inputFields = [] Object.getOwnPropertyNames(config.starterPrompts).forEach((key) => { if (config.starterPrompts[key]) { inputFields.push(config.starterPrompts[key]) } }) setStarterPrompts(inputFields) } } } // eslint-disable-next-line react-hooks/exhaustive-deps }, [getChatflowConfig.data]) // Auto scroll chat to bottom useEffect(() => { scrollToBottom() }, [messages]) useEffect(() => { if (isDialog && inputRef) { setTimeout(() => { inputRef.current?.focus() }, 100) } }, [isDialog, inputRef]) useEffect(() => { let socket if (open && chatflowid) { getChatmessageApi.request(chatflowid) getIsChatflowStreamingApi.request(chatflowid) getAllowChatFlowUploads.request(chatflowid) getChatflowConfig.request(chatflowid) scrollToBottom() setIsRecording(false) socket = socketIOClient(baseURL) socket.on('connect', () => { setSocketIOClientId(socket.id) }) socket.on('start', () => { setMessages((prevMessages) => [...prevMessages, { message: '', type: 'apiMessage' }]) }) socket.on('sourceDocuments', updateLastMessageSourceDocuments) socket.on('token', updateLastMessage) } return () => { setUserInput('') setLoading(false) setMessages([ { message: 'Hi there! How can I help?', type: 'apiMessage' } ]) if (socket) { socket.disconnect() setSocketIOClientId('') } } // eslint-disable-next-line react-hooks/exhaustive-deps }, [open, chatflowid]) return (
00:00
To record audio, use browsers like Chrome and Firefox that support audio recording.
{children}
)
}
}}
>
{message.message}