From c609c63f44f61267ba97dba8a8924f4692320681 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Wed, 13 Dec 2023 22:10:00 +0530 Subject: [PATCH] MultiModal: start integration of audio input (live recording) for MultiModal. --- .../ui/src/views/chatmessage/ChatMessage.js | 537 +++++++++++++++++- .../src/views/chatmessage/audio-recording.css | 278 +++++++++ .../src/views/chatmessage/audio-recording.js | 433 ++++++++++++++ 3 files changed, 1232 insertions(+), 16 deletions(-) create mode 100644 packages/ui/src/views/chatmessage/audio-recording.css create mode 100644 packages/ui/src/views/chatmessage/audio-recording.js diff --git a/packages/ui/src/views/chatmessage/ChatMessage.js b/packages/ui/src/views/chatmessage/ChatMessage.js index a4a13df0..d7bbaf9e 100644 --- a/packages/ui/src/views/chatmessage/ChatMessage.js +++ b/packages/ui/src/views/chatmessage/ChatMessage.js @@ -26,13 +26,23 @@ import { Typography } from '@mui/material' import { useTheme } from '@mui/material/styles' -import { IconDownload, IconSend, IconUpload } from '@tabler/icons' +import { + IconDownload, + IconSend, + IconUpload, + IconMicrophone, + IconPhotoPlus, + IconPlayerStop, + IconPlayerRecord, + IconCircleDot +} from '@tabler/icons' // project import import { CodeBlock } from 'ui-component/markdown/CodeBlock' import { MemoizedReactMarkdown } from 'ui-component/markdown/MemoizedReactMarkdown' import SourceDocDialog from 'ui-component/dialog/SourceDocDialog' import './ChatMessage.css' +import './audio-recording.css' // api import chatmessageApi from 'api/chatmessage' @@ -477,6 +487,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { getIsChatflowStreamingApi.request(chatflowid) getAllowChatFlowUploads.request(chatflowid) scrollToBottom() + initAudioRecording() socket = socketIOClient(baseURL) @@ -519,6 +530,39 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { onDrop={handleDrop} className={`file-drop-field`} > +
+
+ +
+ +

+
+ +
+
+

+ Audio is playing. + . + . +

+
+
+
+
+

To record audio, use browsers like Chrome and Firefox that support audio recording.

+ +
+
+ {/* eslint-disable-next-line jsx-a11y/media-has-caption */} + {isDragOver && getAllowChatFlowUploads.data?.allowUploads && ( Drop here to upload @@ -750,7 +794,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { disabled={loading || !chatflowid} edge='start' > - @@ -758,20 +802,35 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { ) } endAdornment={ - - - {loading ? ( -
- -
- ) : ( - // Send icon SVG in input field - - )} - - + <> + {isChatFlowAvailableForUploads && ( + + + + + + )} + + + {loading ? ( +
+ +
+ ) : ( + // Send icon SVG in input field + + )} + + + } /> {isChatFlowAvailableForUploads && ( @@ -791,3 +850,449 @@ ChatMessage.propTypes = { chatflowid: PropTypes.string, isDialog: PropTypes.bool } + +// audio-recording.js --------------- +//View +let microphoneButton = document.getElementsByClassName('start-recording-button')[0] +let recordingControlButtonsContainer = document.getElementsByClassName('recording-control-buttons-container')[0] +let stopRecordingButton = document.getElementsByClassName('stop-recording-button')[0] +let cancelRecordingButton = document.getElementsByClassName('cancel-recording-button')[0] +let elapsedTimeTag = document.getElementsByClassName('elapsed-time')[0] +let closeBrowserNotSupportedBoxButton = document.getElementsByClassName('close-browser-not-supported-box')[0] +let overlay = document.getElementsByClassName('overlay')[0] +let audioElement = document.getElementsByClassName('audio-element')[0] +let audioElementSource = audioElement?.getElementsByTagName('source')[0] +let textIndicatorOfAudiPlaying = document.getElementsByClassName('text-indication-of-audio-playing')[0] + +const initAudioRecording = () => { + microphoneButton = document.getElementsByClassName('start-recording-button')[0] + recordingControlButtonsContainer = document.getElementsByClassName('recording-control-buttons-container')[0] + stopRecordingButton = document.getElementsByClassName('stop-recording-button')[0] + cancelRecordingButton = document.getElementsByClassName('cancel-recording-button')[0] + elapsedTimeTag = document.getElementsByClassName('elapsed-time')[0] + closeBrowserNotSupportedBoxButton = document.getElementsByClassName('close-browser-not-supported-box')[0] + overlay = document.getElementsByClassName('overlay')[0] + audioElement = document.getElementsByClassName('audio-element')[0] + audioElementSource = audioElement?.getElementsByTagName('source')[0] + textIndicatorOfAudiPlaying = document.getElementsByClassName('text-indication-of-audio-playing')[0] + //Listeners + + //Listen to start recording button + if (microphoneButton) microphoneButton.onclick = startAudioRecording + + //Listen to stop recording button + if (stopRecordingButton) stopRecordingButton.onclick = stopAudioRecording + + //Listen to cancel recording button + if (cancelRecordingButton) cancelRecordingButton.onclick = cancelAudioRecording + + //Listen to when the ok button is clicked in the browser not supporting audio recording box + if (closeBrowserNotSupportedBoxButton) closeBrowserNotSupportedBoxButton.onclick = hideBrowserNotSupportedOverlay + + //Listen to when the audio being played ends + if (audioElement) audioElement.onended = hideTextIndicatorOfAudioPlaying +} + +/** Displays recording control buttons */ +function handleDisplayingRecordingControlButtons() { + //Hide the microphone button that starts audio recording + microphoneButton.style.display = 'none' + + //Display the recording control buttons + recordingControlButtonsContainer.classList.remove('hide') + + //Handle the displaying of the elapsed recording time + handleElapsedRecordingTime() +} + +/** Hide the displayed recording control buttons */ +function handleHidingRecordingControlButtons() { + //Display the microphone button that starts audio recording + microphoneButton.style.display = 'block' + + //Hide the recording control buttons + recordingControlButtonsContainer.classList.add('hide') + + //stop interval that handles both time elapsed and the red dot + clearInterval(elapsedTimeTimer) +} + +/** Displays browser not supported info box for the user*/ +function displayBrowserNotSupportedOverlay() { + overlay.classList.remove('hide') +} + +/** Displays browser not supported info box for the user*/ +function hideBrowserNotSupportedOverlay() { + overlay.classList.add('hide') +} + +/** Creates a source element for the audio element in the HTML document*/ +function createSourceForAudioElement() { + let sourceElement = document.createElement('source') + audioElement.appendChild(sourceElement) + + audioElementSource = sourceElement +} + +/** Display the text indicator of the audio being playing in the background */ +function displayTextIndicatorOfAudioPlaying() { + textIndicatorOfAudiPlaying.classList.remove('hide') +} + +/** Hide the text indicator of the audio being playing in the background */ +function hideTextIndicatorOfAudioPlaying() { + textIndicatorOfAudiPlaying.classList.add('hide') +} + +//Controller + +/** Stores the actual start time when an audio recording begins to take place to ensure elapsed time start time is accurate*/ +let audioRecordStartTime + +/** Stores the maximum recording time in hours to stop recording once maximum recording hour has been reached */ +let maximumRecordingTimeInHours = 1 + +/** Stores the reference of the setInterval function that controls the timer in audio recording*/ +let elapsedTimeTimer + +/** Starts the audio recording*/ +function startAudioRecording() { + console.log('Recording Audio...') + + //If a previous audio recording is playing, pause it + let recorderAudioIsPlaying = !audioElement.paused // the paused property tells whether the media element is paused or not + console.log('paused?', !recorderAudioIsPlaying) + if (recorderAudioIsPlaying) { + audioElement.pause() + //also hide the audio playing indicator displayed on the screen + hideTextIndicatorOfAudioPlaying() + } + + //start recording using the audio recording API + audioRecorder + .start() + .then(() => { + //on success + + //store the recording start time to display the elapsed time according to it + audioRecordStartTime = new Date() + + //display control buttons to offer the functionality of stop and cancel + handleDisplayingRecordingControlButtons() + }) + .catch((error) => { + //on error + //No Browser Support Error + if (error.message.includes('mediaDevices API or getUserMedia method is not supported in this browser.')) { + console.log('To record audio, use browsers like Chrome and Firefox.') + displayBrowserNotSupportedOverlay() + } + + //Error handling structure + switch (error.name) { + case 'AbortError': //error from navigator.mediaDevices.getUserMedia + console.log('An AbortError has occurred.') + break + case 'NotAllowedError': //error from navigator.mediaDevices.getUserMedia + console.log('A NotAllowedError has occurred. User might have denied permission.') + break + case 'NotFoundError': //error from navigator.mediaDevices.getUserMedia + console.log('A NotFoundError has occurred.') + break + case 'NotReadableError': //error from navigator.mediaDevices.getUserMedia + console.log('A NotReadableError has occurred.') + break + case 'SecurityError': //error from navigator.mediaDevices.getUserMedia or from the MediaRecorder.start + console.log('A SecurityError has occurred.') + break + case 'TypeError': //error from navigator.mediaDevices.getUserMedia + console.log('A TypeError has occurred.') + break + case 'InvalidStateError': //error from the MediaRecorder.start + console.log('An InvalidStateError has occurred.') + break + case 'UnknownError': //error from the MediaRecorder.start + console.log('An UnknownError has occurred.') + break + default: + console.log('An error occurred with the error name ' + error.name) + } + }) +} +/** Stop the currently started audio recording & sends it + */ +function stopAudioRecording() { + console.log('Stopping Audio Recording...') + + //stop the recording using the audio recording API + audioRecorder + .stop() + .then((audioAsblob) => { + //Play recorder audio + playAudio(audioAsblob) + + //hide recording control button & return record icon + handleHidingRecordingControlButtons() + }) + .catch((error) => { + //Error handling structure + switch (error.name) { + case 'InvalidStateError': //error from the MediaRecorder.stop + console.log('An InvalidStateError has occurred.') + break + default: + console.log('An error occurred with the error name ' + error.name) + } + }) +} + +/** Cancel the currently started audio recording */ +function cancelAudioRecording() { + console.log('Canceling audio...') + + //cancel the recording using the audio recording API + audioRecorder.cancel() + + //hide recording control button & return record icon + handleHidingRecordingControlButtons() +} + +/** Plays recorded audio using the audio element in the HTML document + * @param {Blob} recorderAudioAsBlob - recorded audio as a Blob Object + */ +function playAudio(recorderAudioAsBlob) { + //read content of files (Blobs) asynchronously + let reader = new FileReader() + + //once content has been read + reader.onload = (e) => { + //store the base64 URL that represents the URL of the recording audio + let base64URL = e.target.result + + //If this is the first audio playing, create a source element + //as pre-populating the HTML with a source of empty src causes error + if (!audioElementSource) + //if it is not defined create it (happens first time only) + createSourceForAudioElement() + + //set the audio element's source using the base64 URL + audioElementSource.src = base64URL + + //set the type of the audio element based on the recorded audio's Blob type + let BlobType = recorderAudioAsBlob.type.includes(';') + ? recorderAudioAsBlob.type.substr(0, recorderAudioAsBlob.type.indexOf(';')) + : recorderAudioAsBlob.type + audioElementSource.type = BlobType + + //call the load method as it is used to update the audio element after changing the source or other settings + audioElement.load() + + //play the audio after successfully setting new src and type that corresponds to the recorded audio + console.log('Playing audio...') + audioElement.play() + + //Display text indicator of having the audio play in the background + displayTextIndicatorOfAudioPlaying() + } + + //read content and convert it to a URL (base64) + reader.readAsDataURL(recorderAudioAsBlob) +} + +/** Computes the elapsed recording time since the moment the function is called in the format h:m:s*/ +function handleElapsedRecordingTime() { + //display initial time when recording begins + displayElapsedTimeDuringAudioRecording('00:00') + + //create an interval that compute & displays elapsed time, as well as, animate red dot - every second + elapsedTimeTimer = setInterval(() => { + //compute the elapsed time every second + let elapsedTime = computeElapsedTime(audioRecordStartTime) //pass the actual record start time + //display the elapsed time + displayElapsedTimeDuringAudioRecording(elapsedTime) + }, 1000) //every second +} + +/** Display elapsed time during audio recording + * @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss + */ +function displayElapsedTimeDuringAudioRecording(elapsedTime) { + //1. display the passed elapsed time as the elapsed time in the elapsedTime HTML element + elapsedTimeTag.innerHTML = elapsedTime + + //2. Stop the recording when the max number of hours is reached + if (elapsedTimeReachedMaximumNumberOfHours(elapsedTime)) { + stopAudioRecording() + } +} + +/** + * @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss + * @returns {Boolean} whether the elapsed time reached the maximum number of hours or not + */ +function elapsedTimeReachedMaximumNumberOfHours(elapsedTime) { + //Split the elapsed time by the symbol that separates the hours, minutes and seconds : + let elapsedTimeSplit = elapsedTime.split(':') + + //Turn the maximum recording time in hours to a string and pad it with zero if less than 10 + let maximumRecordingTimeInHoursAsString = + maximumRecordingTimeInHours < 10 ? '0' + maximumRecordingTimeInHours : maximumRecordingTimeInHours.toString() + + //if the elapsed time reach hours and also reach the maximum recording time in hours return true + if (elapsedTimeSplit.length === 3 && elapsedTimeSplit[0] === maximumRecordingTimeInHoursAsString) return true + //otherwise, return false + else return false +} + +/** Computes the elapsedTime since the moment the function is called in the format mm:ss or hh:mm:ss + * @param {String} startTime - start time to compute the elapsed time since + * @returns {String} elapsed time in mm:ss format or hh:mm:ss format, if elapsed hours are 0. + */ +function computeElapsedTime(startTime) { + //record end time + let endTime = new Date() + + //time difference in ms + let timeDiff = endTime - startTime + + //convert time difference from ms to seconds + timeDiff = timeDiff / 1000 + + //extract integer seconds that don't form a minute using % + let seconds = Math.floor(timeDiff % 60) //ignoring incomplete seconds (floor) + + //pad seconds with a zero if necessary + seconds = seconds < 10 ? '0' + seconds : seconds + + //convert time difference from seconds to minutes using % + timeDiff = Math.floor(timeDiff / 60) + + //extract integer minutes that don't form an hour using % + let minutes = timeDiff % 60 //no need to floor possible incomplete minutes, because they've been handled as seconds + minutes = minutes < 10 ? '0' + minutes : minutes + + //convert time difference from minutes to hours + timeDiff = Math.floor(timeDiff / 60) + + //extract integer hours that don't form a day using % + let hours = timeDiff % 24 //no need to floor possible incomplete hours, because they've been handled as seconds + + //convert time difference from hours to days + timeDiff = Math.floor(timeDiff / 24) + + // the rest of timeDiff is number of days + let days = timeDiff //add days to hours + + let totalHours = hours + days * 24 + totalHours = totalHours < 10 ? '0' + totalHours : totalHours + + if (totalHours === '00') { + return minutes + ':' + seconds + } else { + return totalHours + ':' + minutes + ':' + seconds + } +} + +//API to handle audio recording + +const audioRecorder = { + /** Stores the recorded audio as Blob objects of audio data as the recording continues*/ + audioBlobs: [] /*of type Blob[]*/, + /** Stores the reference of the MediaRecorder instance that handles the MediaStream when recording starts*/ + mediaRecorder: null /*of type MediaRecorder*/, + /** Stores the reference to the stream currently capturing the audio*/ + streamBeingCaptured: null /*of type MediaStream*/, + /** Start recording the audio + * @returns {Promise} - returns a promise that resolves if audio recording successfully started + */ + start: function () { + //Feature Detection + if (!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia)) { + //Feature is not supported in browser + //return a custom error + return Promise.reject(new Error('mediaDevices API or getUserMedia method is not supported in this browser.')) + } else { + //Feature is supported in browser + + //create an audio stream + return ( + navigator.mediaDevices + .getUserMedia({ audio: true } /*of type MediaStreamConstraints*/) + //returns a promise that resolves to the audio stream + .then((stream) /*of type MediaStream*/ => { + //save the reference of the stream to be able to stop it when necessary + audioRecorder.streamBeingCaptured = stream + + //create a media recorder instance by passing that stream into the MediaRecorder constructor + audioRecorder.mediaRecorder = new MediaRecorder(stream) /*the MediaRecorder interface of the MediaStream Recording + API provides functionality to easily record media*/ + + //clear previously saved audio Blobs, if any + audioRecorder.audioBlobs = [] + + //add a dataavailable event listener in order to store the audio data Blobs when recording + audioRecorder.mediaRecorder.addEventListener('dataavailable', (event) => { + //store audio Blob object + audioRecorder.audioBlobs.push(event.data) + }) + + //start the recording by calling the start method on the media recorder + audioRecorder.mediaRecorder.start() + }) + ) + + /* errors are not handled in the API because if its handled and the promise is chained, the .then after the catch will be executed*/ + } + }, + /** Stop the started audio recording + * @returns {Promise} - returns a promise that resolves to the audio as a blob file + */ + stop: function () { + //return a promise that would return the blob or URL of the recording + return new Promise((resolve) => { + //save audio type to pass to set the Blob type + let mimeType = audioRecorder.mediaRecorder.mimeType + + //listen to the stop event in order to create & return a single Blob object + audioRecorder.mediaRecorder.addEventListener('stop', () => { + //create a single blob object, as we might have gathered a few Blob objects that needs to be joined as one + let audioBlob = new Blob(audioRecorder.audioBlobs, { type: mimeType }) + + //resolve promise with the single audio blob representing the recorded audio + resolve(audioBlob) + }) + audioRecorder.cancel() + }) + }, + /** Cancel audio recording*/ + cancel: function () { + //stop the recording feature + audioRecorder.mediaRecorder.stop() + + //stop all the tracks on the active stream in order to stop the stream + audioRecorder.stopStream() + + //reset API properties for next recording + audioRecorder.resetRecordingProperties() + }, + /** Stop all the tracks on the active stream in order to stop the stream and remove + * the red flashing dot showing in the tab + */ + stopStream: function () { + //stopping the capturing request by stopping all the tracks on the active stream + audioRecorder.streamBeingCaptured + .getTracks() //get all tracks from the stream + .forEach((track) /*of type MediaStreamTrack*/ => track.stop()) //stop each one + }, + /** Reset all the recording properties including the media recorder and stream being captured*/ + resetRecordingProperties: function () { + audioRecorder.mediaRecorder = null + audioRecorder.streamBeingCaptured = null + + /*No need to remove event listeners attached to mediaRecorder as + If a DOM element which is removed is reference-free (no references pointing to it), the element itself is picked + up by the garbage collector as well as any event handlers/listeners associated with it. + getEventListeners(audioRecorder.mediaRecorder) will return an empty array of events.*/ + } +} diff --git a/packages/ui/src/views/chatmessage/audio-recording.css b/packages/ui/src/views/chatmessage/audio-recording.css new file mode 100644 index 00000000..5ba0fa50 --- /dev/null +++ b/packages/ui/src/views/chatmessage/audio-recording.css @@ -0,0 +1,278 @@ +/* style.css*/ + +/* Media Queries */ + +/* Small Devices*/ + +@media (min-width: 0px) { + * { + box-sizing: border-box; + } + .audio-recording-container { + width: 100%; + /* view port height*/ + /*targeting Chrome & Safari*/ + display: -webkit-flex; + /*targeting IE10*/ + display: -ms-flex; + display: flex; + flex-direction: column; + justify-content: center; + /*horizontal centering*/ + align-items: center; + } + .start-recording-button { + font-size: 70px; + color: #435f7a; + cursor: pointer; + } + .start-recording-button:hover { + opacity: 1; + } + .recording-control-buttons-container { + /*targeting Chrome & Safari*/ + display: -webkit-flex; + /*targeting IE10*/ + display: -ms-flex; + display: flex; + justify-content: space-evenly; + /*horizontal centering*/ + align-items: center; + width: 334px; + margin-bottom: 30px; + } + .cancel-recording-button, + .stop-recording-button { + font-size: 70px; + cursor: pointer; + } + .cancel-recording-button { + color: red; + opacity: 0.7; + } + .cancel-recording-button:hover { + color: rgb(206, 4, 4); + } + .stop-recording-button { + color: #33cc33; + opacity: 0.7; + } + .stop-recording-button:hover { + color: #27a527; + } + .recording-elapsed-time { + /*targeting Chrome & Safari*/ + display: -webkit-flex; + /*targeting IE10*/ + display: -ms-flex; + display: flex; + justify-content: center; + /*horizontal centering*/ + align-items: center; + } + .red-recording-dot { + font-size: 25px; + color: red; + margin-right: 12px; + /*transitions with Firefox, IE and Opera Support browser support*/ + animation-name: flashing-recording-dot; + -webkit-animation-name: flashing-recording-dot; + -moz-animation-name: flashing-recording-dot; + -o-animation-name: flashing-recording-dot; + animation-duration: 2s; + -webkit-animation-duration: 2s; + -moz-animation-duration: 2s; + -o-animation-duration: 2s; + animation-iteration-count: infinite; + -webkit-animation-iteration-count: infinite; + -moz-animation-iteration-count: infinite; + -o-animation-iteration-count: infinite; + } + /* The animation code */ + @keyframes flashing-recording-dot { + 0% { + opacity: 1; + } + 50% { + opacity: 0; + } + 100% { + opacity: 1; + } + } + @-webkit-keyframes flashing-recording-dot { + 0% { + opacity: 1; + } + 50% { + opacity: 0; + } + 100% { + opacity: 1; + } + } + @-moz-keyframes flashing-recording-dot { + 0% { + opacity: 1; + } + 50% { + opacity: 0; + } + 100% { + opacity: 1; + } + } + @-o-keyframes flashing-recording-dot { + 0% { + opacity: 1; + } + 50% { + opacity: 0; + } + 100% { + opacity: 1; + } + } + .elapsed-time { + font-size: 32px; + } + .recording-control-buttons-container.hide { + display: none; + } + .overlay { + position: absolute; + top: 0; + width: 100%; + background-color: rgba(82, 76, 76, 0.35); + /*targeting Chrome & Safari*/ + display: -webkit-flex; + /*targeting IE10*/ + display: -ms-flex; + display: flex; + justify-content: center; + /*horizontal centering*/ + align-items: center; + } + .overlay.hide { + display: none; + } + .browser-not-supporting-audio-recording-box { + /*targeting Chrome & Safari*/ + display: -webkit-flex; + /*targeting IE10*/ + display: -ms-flex; + display: flex; + flex-direction: column; + justify-content: space-between; + /*horizontal centering*/ + align-items: center; + width: 317px; + height: 119px; + background-color: white; + border-radius: 10px; + padding: 15px; + font-size: 16px; + } + .close-browser-not-supported-box { + cursor: pointer; + background-color: #abc1c05c; + border-radius: 10px; + font-size: 16px; + border: none; + } + .close-browser-not-supported-box:hover { + background-color: #92a5a45c; + } + .close-browser-not-supported-box:focus { + outline: none; + border: none; + } + .audio-element.hide { + display: none; + } + .text-indication-of-audio-playing-container { + height: 20px; + } + .text-indication-of-audio-playing { + font-size: 20px; + } + .text-indication-of-audio-playing.hide { + display: none; + } + /* 3 Dots animation*/ + .text-indication-of-audio-playing span { + /*transitions with Firefox, IE and Opera Support browser support*/ + animation-name: blinking-dot; + -webkit-animation-name: blinking-dot; + -moz-animation-name: blinking-dot; + -o-animation-name: blinking-dot; + animation-duration: 2s; + -webkit-animation-duration: 2s; + -moz-animation-duration: 2s; + -o-animation-duration: 2s; + animation-iteration-count: infinite; + -webkit-animation-iteration-count: infinite; + -moz-animation-iteration-count: infinite; + -o-animation-iteration-count: infinite; + } + .text-indication-of-audio-playing span:nth-child(2) { + animation-delay: .4s; + -webkit-animation-delay: .4s; + -moz-animation-delay: .4s; + -o-animation-delay: .4s; + } + .text-indication-of-audio-playing span:nth-child(3) { + animation-delay: .8s; + -webkit-animation-delay: .8s; + -moz-animation-delay: .8s; + -o-animation-delay: .8s; + } + /* The animation code */ + @keyframes blinking-dot { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0; + } + } + /* The animation code */ + @-webkit-keyframes blinking-dot { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0; + } + } + /* The animation code */ + @-moz-keyframes blinking-dot { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0; + } + } + /* The animation code */ + @-o-keyframes blinking-dot { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0; + } + } +} \ No newline at end of file diff --git a/packages/ui/src/views/chatmessage/audio-recording.js b/packages/ui/src/views/chatmessage/audio-recording.js new file mode 100644 index 00000000..395443fe --- /dev/null +++ b/packages/ui/src/views/chatmessage/audio-recording.js @@ -0,0 +1,433 @@ +// audio-recording.js --------------- +//View +let microphoneButton = document.getElementsByClassName('start-recording-button')[0] +let recordingControlButtonsContainer = document.getElementsByClassName('recording-control-buttons-container')[0] +let stopRecordingButton = document.getElementsByClassName('stop-recording-button')[0] +let cancelRecordingButton = document.getElementsByClassName('cancel-recording-button')[0] +let elapsedTimeTag = document.getElementsByClassName('elapsed-time')[0] +let closeBrowserNotSupportedBoxButton = document.getElementsByClassName('close-browser-not-supported-box')[0] +let overlay = document.getElementsByClassName('overlay')[0] +let audioElement = document.getElementsByClassName('audio-element')[0] +let audioElementSource = document.getElementsByClassName('audio-element')[0].getElementsByTagName('source')[0] +let textIndicatorOfAudiPlaying = document.getElementsByClassName('text-indication-of-audio-playing')[0] + +//Listeners + +//Listen to start recording button +microphoneButton.onclick = startAudioRecording + +//Listen to stop recording button +stopRecordingButton.onclick = stopAudioRecording + +//Listen to cancel recording button +cancelRecordingButton.onclick = cancelAudioRecording + +//Listen to when the ok button is clicked in the browser not supporting audio recording box +closeBrowserNotSupportedBoxButton.onclick = hideBrowserNotSupportedOverlay + +//Listen to when the audio being played ends +audioElement.onended = hideTextIndicatorOfAudioPlaying + +/** Displays recording control buttons */ +function handleDisplayingRecordingControlButtons() { + //Hide the microphone button that starts audio recording + microphoneButton.style.display = 'none' + + //Display the recording control buttons + recordingControlButtonsContainer.classList.remove('hide') + + //Handle the displaying of the elapsed recording time + handleElapsedRecordingTime() +} + +/** Hide the displayed recording control buttons */ +function handleHidingRecordingControlButtons() { + //Display the microphone button that starts audio recording + microphoneButton.style.display = 'block' + + //Hide the recording control buttons + recordingControlButtonsContainer.classList.add('hide') + + //stop interval that handles both time elapsed and the red dot + clearInterval(elapsedTimeTimer) +} + +/** Displays browser not supported info box for the user*/ +function displayBrowserNotSupportedOverlay() { + overlay.classList.remove('hide') +} + +/** Displays browser not supported info box for the user*/ +function hideBrowserNotSupportedOverlay() { + overlay.classList.add('hide') +} + +/** Creates a source element for the audio element in the HTML document*/ +function createSourceForAudioElement() { + let sourceElement = document.createElement('source') + audioElement.appendChild(sourceElement) + + audioElementSource = sourceElement +} + +/** Display the text indicator of the audio being playing in the background */ +function displayTextIndicatorOfAudioPlaying() { + textIndicatorOfAudiPlaying.classList.remove('hide') +} + +/** Hide the text indicator of the audio being playing in the background */ +function hideTextIndicatorOfAudioPlaying() { + textIndicatorOfAudiPlaying.classList.add('hide') +} + +//Controller + +/** Stores the actual start time when an audio recording begins to take place to ensure elapsed time start time is accurate*/ +let audioRecordStartTime + +/** Stores the maximum recording time in hours to stop recording once maximum recording hour has been reached */ +let maximumRecordingTimeInHours = 1 + +/** Stores the reference of the setInterval function that controls the timer in audio recording*/ +let elapsedTimeTimer + +/** Starts the audio recording*/ +function startAudioRecording() { + console.log('Recording Audio...') + + //If a previous audio recording is playing, pause it + let recorderAudioIsPlaying = !audioElement.paused // the paused property tells whether the media element is paused or not + console.log('paused?', !recorderAudioIsPlaying) + if (recorderAudioIsPlaying) { + audioElement.pause() + //also hide the audio playing indicator displayed on the screen + hideTextIndicatorOfAudioPlaying() + } + + //start recording using the audio recording API + audioRecorder + .start() + .then(() => { + //on success + + //store the recording start time to display the elapsed time according to it + audioRecordStartTime = new Date() + + //display control buttons to offer the functionality of stop and cancel + handleDisplayingRecordingControlButtons() + }) + .catch((error) => { + //on error + //No Browser Support Error + if (error.message.includes('mediaDevices API or getUserMedia method is not supported in this browser.')) { + console.log('To record audio, use browsers like Chrome and Firefox.') + displayBrowserNotSupportedOverlay() + } + + //Error handling structure + switch (error.name) { + case 'AbortError': //error from navigator.mediaDevices.getUserMedia + console.log('An AbortError has occurred.') + break + case 'NotAllowedError': //error from navigator.mediaDevices.getUserMedia + console.log('A NotAllowedError has occurred. User might have denied permission.') + break + case 'NotFoundError': //error from navigator.mediaDevices.getUserMedia + console.log('A NotFoundError has occurred.') + break + case 'NotReadableError': //error from navigator.mediaDevices.getUserMedia + console.log('A NotReadableError has occurred.') + break + case 'SecurityError': //error from navigator.mediaDevices.getUserMedia or from the MediaRecorder.start + console.log('A SecurityError has occurred.') + break + case 'TypeError': //error from navigator.mediaDevices.getUserMedia + console.log('A TypeError has occurred.') + break + case 'InvalidStateError': //error from the MediaRecorder.start + console.log('An InvalidStateError has occurred.') + break + case 'UnknownError': //error from the MediaRecorder.start + console.log('An UnknownError has occurred.') + break + default: + console.log('An error occurred with the error name ' + error.name) + } + }) +} +/** Stop the currently started audio recording & sends it + */ +function stopAudioRecording() { + console.log('Stopping Audio Recording...') + + //stop the recording using the audio recording API + audioRecorder + .stop() + .then((audioAsblob) => { + //Play recorder audio + playAudio(audioAsblob) + + //hide recording control button & return record icon + handleHidingRecordingControlButtons() + }) + .catch((error) => { + //Error handling structure + switch (error.name) { + case 'InvalidStateError': //error from the MediaRecorder.stop + console.log('An InvalidStateError has occurred.') + break + default: + console.log('An error occurred with the error name ' + error.name) + } + }) +} + +/** Cancel the currently started audio recording */ +function cancelAudioRecording() { + console.log('Canceling audio...') + + //cancel the recording using the audio recording API + audioRecorder.cancel() + + //hide recording control button & return record icon + handleHidingRecordingControlButtons() +} + +/** Plays recorded audio using the audio element in the HTML document + * @param {Blob} recorderAudioAsBlob - recorded audio as a Blob Object + */ +function playAudio(recorderAudioAsBlob) { + //read content of files (Blobs) asynchronously + let reader = new FileReader() + + //once content has been read + reader.onload = (e) => { + //store the base64 URL that represents the URL of the recording audio + let base64URL = e.target.result + + //If this is the first audio playing, create a source element + //as pre-populating the HTML with a source of empty src causes error + if (!audioElementSource) + //if it is not defined create it (happens first time only) + createSourceForAudioElement() + + //set the audio element's source using the base64 URL + audioElementSource.src = base64URL + + //set the type of the audio element based on the recorded audio's Blob type + let BlobType = recorderAudioAsBlob.type.includes(';') + ? recorderAudioAsBlob.type.substr(0, recorderAudioAsBlob.type.indexOf(';')) + : recorderAudioAsBlob.type + audioElementSource.type = BlobType + + //call the load method as it is used to update the audio element after changing the source or other settings + audioElement.load() + + //play the audio after successfully setting new src and type that corresponds to the recorded audio + console.log('Playing audio...') + audioElement.play() + + //Display text indicator of having the audio play in the background + displayTextIndicatorOfAudioPlaying() + } + + //read content and convert it to a URL (base64) + reader.readAsDataURL(recorderAudioAsBlob) +} + +/** Computes the elapsed recording time since the moment the function is called in the format h:m:s*/ +function handleElapsedRecordingTime() { + //display initial time when recording begins + displayElapsedTimeDuringAudioRecording('00:00') + + //create an interval that compute & displays elapsed time, as well as, animate red dot - every second + elapsedTimeTimer = setInterval(() => { + //compute the elapsed time every second + let elapsedTime = computeElapsedTime(audioRecordStartTime) //pass the actual record start time + //display the elapsed time + displayElapsedTimeDuringAudioRecording(elapsedTime) + }, 1000) //every second +} + +/** Display elapsed time during audio recording + * @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss + */ +function displayElapsedTimeDuringAudioRecording(elapsedTime) { + //1. display the passed elapsed time as the elapsed time in the elapsedTime HTML element + elapsedTimeTag.innerHTML = elapsedTime + + //2. Stop the recording when the max number of hours is reached + if (elapsedTimeReachedMaximumNumberOfHours(elapsedTime)) { + stopAudioRecording() + } +} + +/** + * @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss + * @returns {Boolean} whether the elapsed time reached the maximum number of hours or not + */ +function elapsedTimeReachedMaximumNumberOfHours(elapsedTime) { + //Split the elapsed time by the symbol that separates the hours, minutes and seconds : + let elapsedTimeSplit = elapsedTime.split(':') + + //Turn the maximum recording time in hours to a string and pad it with zero if less than 10 + let maximumRecordingTimeInHoursAsString = + maximumRecordingTimeInHours < 10 ? '0' + maximumRecordingTimeInHours : maximumRecordingTimeInHours.toString() + + //if the elapsed time reach hours and also reach the maximum recording time in hours return true + if (elapsedTimeSplit.length === 3 && elapsedTimeSplit[0] === maximumRecordingTimeInHoursAsString) return true + //otherwise, return false + else return false +} + +/** Computes the elapsedTime since the moment the function is called in the format mm:ss or hh:mm:ss + * @param {String} startTime - start time to compute the elapsed time since + * @returns {String} elapsed time in mm:ss format or hh:mm:ss format, if elapsed hours are 0. + */ +function computeElapsedTime(startTime) { + //record end time + let endTime = new Date() + + //time difference in ms + let timeDiff = endTime - startTime + + //convert time difference from ms to seconds + timeDiff = timeDiff / 1000 + + //extract integer seconds that don't form a minute using % + let seconds = Math.floor(timeDiff % 60) //ignoring incomplete seconds (floor) + + //pad seconds with a zero if necessary + seconds = seconds < 10 ? '0' + seconds : seconds + + //convert time difference from seconds to minutes using % + timeDiff = Math.floor(timeDiff / 60) + + //extract integer minutes that don't form an hour using % + let minutes = timeDiff % 60 //no need to floor possible incomplete minutes, because they've been handled as seconds + minutes = minutes < 10 ? '0' + minutes : minutes + + //convert time difference from minutes to hours + timeDiff = Math.floor(timeDiff / 60) + + //extract integer hours that don't form a day using % + let hours = timeDiff % 24 //no need to floor possible incomplete hours, because they've been handled as seconds + + //convert time difference from hours to days + timeDiff = Math.floor(timeDiff / 24) + + // the rest of timeDiff is number of days + let days = timeDiff //add days to hours + + let totalHours = hours + days * 24 + totalHours = totalHours < 10 ? '0' + totalHours : totalHours + + if (totalHours === '00') { + return minutes + ':' + seconds + } else { + return totalHours + ':' + minutes + ':' + seconds + } +} + +//API to handle audio recording + +const audioRecorder = { + /** Stores the recorded audio as Blob objects of audio data as the recording continues*/ + audioBlobs: [] /*of type Blob[]*/, + /** Stores the reference of the MediaRecorder instance that handles the MediaStream when recording starts*/ + mediaRecorder: null /*of type MediaRecorder*/, + /** Stores the reference to the stream currently capturing the audio*/ + streamBeingCaptured: null /*of type MediaStream*/, + /** Start recording the audio + * @returns {Promise} - returns a promise that resolves if audio recording successfully started + */ + start: function () { + //Feature Detection + if (!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia)) { + //Feature is not supported in browser + //return a custom error + return Promise.reject(new Error('mediaDevices API or getUserMedia method is not supported in this browser.')) + } else { + //Feature is supported in browser + + //create an audio stream + return ( + navigator.mediaDevices + .getUserMedia({ audio: true } /*of type MediaStreamConstraints*/) + //returns a promise that resolves to the audio stream + .then((stream) /*of type MediaStream*/ => { + //save the reference of the stream to be able to stop it when necessary + audioRecorder.streamBeingCaptured = stream + + //create a media recorder instance by passing that stream into the MediaRecorder constructor + audioRecorder.mediaRecorder = new MediaRecorder(stream) /*the MediaRecorder interface of the MediaStream Recording + API provides functionality to easily record media*/ + + //clear previously saved audio Blobs, if any + audioRecorder.audioBlobs = [] + + //add a dataavailable event listener in order to store the audio data Blobs when recording + audioRecorder.mediaRecorder.addEventListener('dataavailable', (event) => { + //store audio Blob object + audioRecorder.audioBlobs.push(event.data) + }) + + //start the recording by calling the start method on the media recorder + audioRecorder.mediaRecorder.start() + }) + ) + + /* errors are not handled in the API because if its handled and the promise is chained, the .then after the catch will be executed*/ + } + }, + /** Stop the started audio recording + * @returns {Promise} - returns a promise that resolves to the audio as a blob file + */ + stop: function () { + //return a promise that would return the blob or URL of the recording + return new Promise((resolve) => { + //save audio type to pass to set the Blob type + let mimeType = audioRecorder.mediaRecorder.mimeType + + //listen to the stop event in order to create & return a single Blob object + audioRecorder.mediaRecorder.addEventListener('stop', () => { + //create a single blob object, as we might have gathered a few Blob objects that needs to be joined as one + let audioBlob = new Blob(audioRecorder.audioBlobs, { type: mimeType }) + + //resolve promise with the single audio blob representing the recorded audio + resolve(audioBlob) + }) + audioRecorder.cancel() + }) + }, + /** Cancel audio recording*/ + cancel: function () { + //stop the recording feature + audioRecorder.mediaRecorder.stop() + + //stop all the tracks on the active stream in order to stop the stream + audioRecorder.stopStream() + + //reset API properties for next recording + audioRecorder.resetRecordingProperties() + }, + /** Stop all the tracks on the active stream in order to stop the stream and remove + * the red flashing dot showing in the tab + */ + stopStream: function () { + //stopping the capturing request by stopping all the tracks on the active stream + audioRecorder.streamBeingCaptured + .getTracks() //get all tracks from the stream + .forEach((track) /*of type MediaStreamTrack*/ => track.stop()) //stop each one + }, + /** Reset all the recording properties including the media recorder and stream being captured*/ + resetRecordingProperties: function () { + audioRecorder.mediaRecorder = null + audioRecorder.streamBeingCaptured = null + + /*No need to remove event listeners attached to mediaRecorder as + If a DOM element which is removed is reference-free (no references pointing to it), the element itself is picked + up by the garbage collector as well as any event handlers/listeners associated with it. + getEventListeners(audioRecorder.mediaRecorder) will return an empty array of events.*/ + } +}