farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 17:01:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

MultiModal: start integration of audio input (live recording) for MultiModal.

Browse Source
This commit is contained in:
vinodkiran
2023-12-13 22:10:00 +05:30
parent 1bd1fd5828
commit c609c63f44
3 changed files with 1232 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ui/src/views/chatmessage/ChatMessage.js
+521 -16
View File
@@ -26,13 +26,23 @@ import {
Typography
} from '@mui/material'
import { useTheme } from '@mui/material/styles'
import { IconDownload, IconSend, IconUpload } from '@tabler/icons'
import {
IconDownload,
IconSend,
IconUpload,
IconMicrophone,
IconPhotoPlus,
IconPlayerStop,
IconPlayerRecord,
IconCircleDot
} from '@tabler/icons'
// project import
import { CodeBlock } from 'ui-component/markdown/CodeBlock'
import { MemoizedReactMarkdown } from 'ui-component/markdown/MemoizedReactMarkdown'
import SourceDocDialog from 'ui-component/dialog/SourceDocDialog'
import './ChatMessage.css'
import './audio-recording.css'
// api
import chatmessageApi from 'api/chatmessage'
@@ -477,6 +487,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
getIsChatflowStreamingApi.request(chatflowid)
getAllowChatFlowUploads.request(chatflowid)
scrollToBottom()
initAudioRecording()
socket = socketIOClient(baseURL)
@@ -519,6 +530,39 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
onDrop={handleDrop}
className={`file-drop-field`}
>
<div className={'audio-recording-container'}>
<div className='recording-control-buttons-container hide'>
<i className='cancel-recording-button' aria-hidden='true'>
<IconPlayerRecord />
</i>
<div className='recording-elapsed-time'>
<i className='red-recording-dot' aria-hidden='true'>
<IconCircleDot />
</i>
<p className='elapsed-time'></p>
</div>
<i className='stop-recording-button' aria-hidden='true'>
<IconPlayerStop />
</i>
</div>
<div className='text-indication-of-audio-playing-container'>
<p className='text-indication-of-audio-playing hide'>
Audio is playing<span>.</span>
<span>.</span>
<span>.</span>
</p>
</div>
</div>
<div className='overlay hide'>
<div className='browser-not-supporting-audio-recording-box'>
<p>To record audio, use browsers like Chrome and Firefox that support audio recording.</p>
<button type='button' className='close-browser-not-supported-box'>
Ok.
</button>
</div>
</div>
{/* eslint-disable-next-line jsx-a11y/media-has-caption */}
<audio controls className='audio-element hide'></audio>
{isDragOver && getAllowChatFlowUploads.data?.allowUploads && (
<Box className='drop-overlay'>
<Typography variant='h2'>Drop here to upload</Typography>
@@ -750,7 +794,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
disabled={loading || !chatflowid}
edge='start'
>
<IconUpload
<IconPhotoPlus
color={loading || !chatflowid ? '#9e9e9e' : customization.isDarkMode ? 'white' : '#1e88e5'}
/>
</IconButton>
@@ -758,20 +802,35 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
)
}
endAdornment={
<InputAdornment position='end' sx={{ padding: '15px' }}>
<IconButton type='submit' disabled={loading || !chatflowid} edge='end'>
{loading ? (
<div>
<CircularProgress color='inherit' size={20} />
</div>
) : (
// Send icon SVG in input field
<IconSend
color={loading || !chatflowid ? '#9e9e9e' : customization.isDarkMode ? 'white' : '#1e88e5'}
/>
)}
</IconButton>
</InputAdornment>
<>
{isChatFlowAvailableForUploads && (
<InputAdornment className={'start-recording-button'} position='end'>
<IconButton type='button' disabled={loading || !chatflowid} edge='end'>
<IconMicrophone
color={
loading || !chatflowid ? '#9e9e9e' : customization.isDarkMode ? 'white' : '#1e88e5'
}
/>
</IconButton>
</InputAdornment>
)}
<InputAdornment position='end' sx={{ padding: '15px' }}>
<IconButton type='submit' disabled={loading || !chatflowid} edge='end'>
{loading ? (
<div>
<CircularProgress color='inherit' size={20} />
</div>
) : (
// Send icon SVG in input field
<IconSend
color={
loading || !chatflowid ? '#9e9e9e' : customization.isDarkMode ? 'white' : '#1e88e5'
}
/>
)}
</IconButton>
</InputAdornment>
</>
}
/>
{isChatFlowAvailableForUploads && (
@@ -791,3 +850,449 @@ ChatMessage.propTypes = {
chatflowid: PropTypes.string,
isDialog: PropTypes.bool
}
// audio-recording.js ---------------
//View
let microphoneButton = document.getElementsByClassName('start-recording-button')[0]
let recordingControlButtonsContainer = document.getElementsByClassName('recording-control-buttons-container')[0]
let stopRecordingButton = document.getElementsByClassName('stop-recording-button')[0]
let cancelRecordingButton = document.getElementsByClassName('cancel-recording-button')[0]
let elapsedTimeTag = document.getElementsByClassName('elapsed-time')[0]
let closeBrowserNotSupportedBoxButton = document.getElementsByClassName('close-browser-not-supported-box')[0]
let overlay = document.getElementsByClassName('overlay')[0]
let audioElement = document.getElementsByClassName('audio-element')[0]
let audioElementSource = audioElement?.getElementsByTagName('source')[0]
let textIndicatorOfAudiPlaying = document.getElementsByClassName('text-indication-of-audio-playing')[0]
const initAudioRecording = () => {
microphoneButton = document.getElementsByClassName('start-recording-button')[0]
recordingControlButtonsContainer = document.getElementsByClassName('recording-control-buttons-container')[0]
stopRecordingButton = document.getElementsByClassName('stop-recording-button')[0]
cancelRecordingButton = document.getElementsByClassName('cancel-recording-button')[0]
elapsedTimeTag = document.getElementsByClassName('elapsed-time')[0]
closeBrowserNotSupportedBoxButton = document.getElementsByClassName('close-browser-not-supported-box')[0]
overlay = document.getElementsByClassName('overlay')[0]
audioElement = document.getElementsByClassName('audio-element')[0]
audioElementSource = audioElement?.getElementsByTagName('source')[0]
textIndicatorOfAudiPlaying = document.getElementsByClassName('text-indication-of-audio-playing')[0]
//Listeners
//Listen to start recording button
if (microphoneButton) microphoneButton.onclick = startAudioRecording
//Listen to stop recording button
if (stopRecordingButton) stopRecordingButton.onclick = stopAudioRecording
//Listen to cancel recording button
if (cancelRecordingButton) cancelRecordingButton.onclick = cancelAudioRecording
//Listen to when the ok button is clicked in the browser not supporting audio recording box
if (closeBrowserNotSupportedBoxButton) closeBrowserNotSupportedBoxButton.onclick = hideBrowserNotSupportedOverlay
//Listen to when the audio being played ends
if (audioElement) audioElement.onended = hideTextIndicatorOfAudioPlaying
}
/** Displays recording control buttons */
function handleDisplayingRecordingControlButtons() {
//Hide the microphone button that starts audio recording
microphoneButton.style.display = 'none'
//Display the recording control buttons
recordingControlButtonsContainer.classList.remove('hide')
//Handle the displaying of the elapsed recording time
handleElapsedRecordingTime()
}
/** Hide the displayed recording control buttons */
function handleHidingRecordingControlButtons() {
//Display the microphone button that starts audio recording
microphoneButton.style.display = 'block'
//Hide the recording control buttons
recordingControlButtonsContainer.classList.add('hide')
//stop interval that handles both time elapsed and the red dot
clearInterval(elapsedTimeTimer)
}
/** Displays browser not supported info box for the user*/
function displayBrowserNotSupportedOverlay() {
overlay.classList.remove('hide')
}
/** Displays browser not supported info box for the user*/
function hideBrowserNotSupportedOverlay() {
overlay.classList.add('hide')
}
/** Creates a source element for the audio element in the HTML document*/
function createSourceForAudioElement() {
let sourceElement = document.createElement('source')
audioElement.appendChild(sourceElement)
audioElementSource = sourceElement
}
/** Display the text indicator of the audio being playing in the background */
function displayTextIndicatorOfAudioPlaying() {
textIndicatorOfAudiPlaying.classList.remove('hide')
}
/** Hide the text indicator of the audio being playing in the background */
function hideTextIndicatorOfAudioPlaying() {
textIndicatorOfAudiPlaying.classList.add('hide')
}
//Controller
/** Stores the actual start time when an audio recording begins to take place to ensure elapsed time start time is accurate*/
let audioRecordStartTime
/** Stores the maximum recording time in hours to stop recording once maximum recording hour has been reached */
let maximumRecordingTimeInHours = 1
/** Stores the reference of the setInterval function that controls the timer in audio recording*/
let elapsedTimeTimer
/** Starts the audio recording*/
function startAudioRecording() {
console.log('Recording Audio...')
//If a previous audio recording is playing, pause it
let recorderAudioIsPlaying = !audioElement.paused // the paused property tells whether the media element is paused or not
console.log('paused?', !recorderAudioIsPlaying)
if (recorderAudioIsPlaying) {
audioElement.pause()
//also hide the audio playing indicator displayed on the screen
hideTextIndicatorOfAudioPlaying()
}
//start recording using the audio recording API
audioRecorder
.start()
.then(() => {
//on success
//store the recording start time to display the elapsed time according to it
audioRecordStartTime = new Date()
//display control buttons to offer the functionality of stop and cancel
handleDisplayingRecordingControlButtons()
})
.catch((error) => {
//on error
//No Browser Support Error
if (error.message.includes('mediaDevices API or getUserMedia method is not supported in this browser.')) {
console.log('To record audio, use browsers like Chrome and Firefox.')
displayBrowserNotSupportedOverlay()
}
//Error handling structure
switch (error.name) {
case 'AbortError': //error from navigator.mediaDevices.getUserMedia
console.log('An AbortError has occurred.')
break
case 'NotAllowedError': //error from navigator.mediaDevices.getUserMedia
console.log('A NotAllowedError has occurred. User might have denied permission.')
break
case 'NotFoundError': //error from navigator.mediaDevices.getUserMedia
console.log('A NotFoundError has occurred.')
break
case 'NotReadableError': //error from navigator.mediaDevices.getUserMedia
console.log('A NotReadableError has occurred.')
break
case 'SecurityError': //error from navigator.mediaDevices.getUserMedia or from the MediaRecorder.start
console.log('A SecurityError has occurred.')
break
case 'TypeError': //error from navigator.mediaDevices.getUserMedia
console.log('A TypeError has occurred.')
break
case 'InvalidStateError': //error from the MediaRecorder.start
console.log('An InvalidStateError has occurred.')
break
case 'UnknownError': //error from the MediaRecorder.start
console.log('An UnknownError has occurred.')
break
default:
console.log('An error occurred with the error name ' + error.name)
}
})
}
/** Stop the currently started audio recording & sends it
*/
function stopAudioRecording() {
console.log('Stopping Audio Recording...')
//stop the recording using the audio recording API
audioRecorder
.stop()
.then((audioAsblob) => {
//Play recorder audio
playAudio(audioAsblob)
//hide recording control button & return record icon
handleHidingRecordingControlButtons()
})
.catch((error) => {
//Error handling structure
switch (error.name) {
case 'InvalidStateError': //error from the MediaRecorder.stop
console.log('An InvalidStateError has occurred.')
break
default:
console.log('An error occurred with the error name ' + error.name)
}
})
}
/** Cancel the currently started audio recording */
function cancelAudioRecording() {
console.log('Canceling audio...')
//cancel the recording using the audio recording API
audioRecorder.cancel()
//hide recording control button & return record icon
handleHidingRecordingControlButtons()
}
/** Plays recorded audio using the audio element in the HTML document
* @param {Blob} recorderAudioAsBlob - recorded audio as a Blob Object
*/
function playAudio(recorderAudioAsBlob) {
//read content of files (Blobs) asynchronously
let reader = new FileReader()
//once content has been read
reader.onload = (e) => {
//store the base64 URL that represents the URL of the recording audio
let base64URL = e.target.result
//If this is the first audio playing, create a source element
//as pre-populating the HTML with a source of empty src causes error
if (!audioElementSource)
//if it is not defined create it (happens first time only)
createSourceForAudioElement()
//set the audio element's source using the base64 URL
audioElementSource.src = base64URL
//set the type of the audio element based on the recorded audio's Blob type
let BlobType = recorderAudioAsBlob.type.includes(';')
? recorderAudioAsBlob.type.substr(0, recorderAudioAsBlob.type.indexOf(';'))
: recorderAudioAsBlob.type
audioElementSource.type = BlobType
//call the load method as it is used to update the audio element after changing the source or other settings
audioElement.load()
//play the audio after successfully setting new src and type that corresponds to the recorded audio
console.log('Playing audio...')
audioElement.play()
//Display text indicator of having the audio play in the background
displayTextIndicatorOfAudioPlaying()
}
//read content and convert it to a URL (base64)
reader.readAsDataURL(recorderAudioAsBlob)
}
/** Computes the elapsed recording time since the moment the function is called in the format h:m:s*/
function handleElapsedRecordingTime() {
//display initial time when recording begins
displayElapsedTimeDuringAudioRecording('00:00')
//create an interval that compute & displays elapsed time, as well as, animate red dot - every second
elapsedTimeTimer = setInterval(() => {
//compute the elapsed time every second
let elapsedTime = computeElapsedTime(audioRecordStartTime) //pass the actual record start time
//display the elapsed time
displayElapsedTimeDuringAudioRecording(elapsedTime)
}, 1000) //every second
}
/** Display elapsed time during audio recording
* @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss
*/
function displayElapsedTimeDuringAudioRecording(elapsedTime) {
//1. display the passed elapsed time as the elapsed time in the elapsedTime HTML element
elapsedTimeTag.innerHTML = elapsedTime
//2. Stop the recording when the max number of hours is reached
if (elapsedTimeReachedMaximumNumberOfHours(elapsedTime)) {
stopAudioRecording()
}
}
/**
* @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss
* @returns {Boolean} whether the elapsed time reached the maximum number of hours or not
*/
function elapsedTimeReachedMaximumNumberOfHours(elapsedTime) {
//Split the elapsed time by the symbol that separates the hours, minutes and seconds :
let elapsedTimeSplit = elapsedTime.split(':')
//Turn the maximum recording time in hours to a string and pad it with zero if less than 10
let maximumRecordingTimeInHoursAsString =
maximumRecordingTimeInHours < 10 ? '0' + maximumRecordingTimeInHours : maximumRecordingTimeInHours.toString()
//if the elapsed time reach hours and also reach the maximum recording time in hours return true
if (elapsedTimeSplit.length === 3 && elapsedTimeSplit[0] === maximumRecordingTimeInHoursAsString) return true
//otherwise, return false
else return false
}
/** Computes the elapsedTime since the moment the function is called in the format mm:ss or hh:mm:ss
* @param {String} startTime - start time to compute the elapsed time since
* @returns {String} elapsed time in mm:ss format or hh:mm:ss format, if elapsed hours are 0.
*/
function computeElapsedTime(startTime) {
//record end time
let endTime = new Date()
//time difference in ms
let timeDiff = endTime - startTime
//convert time difference from ms to seconds
timeDiff = timeDiff / 1000
//extract integer seconds that don't form a minute using %
let seconds = Math.floor(timeDiff % 60) //ignoring incomplete seconds (floor)
//pad seconds with a zero if necessary
seconds = seconds < 10 ? '0' + seconds : seconds
//convert time difference from seconds to minutes using %
timeDiff = Math.floor(timeDiff / 60)
//extract integer minutes that don't form an hour using %
let minutes = timeDiff % 60 //no need to floor possible incomplete minutes, because they've been handled as seconds
minutes = minutes < 10 ? '0' + minutes : minutes
//convert time difference from minutes to hours
timeDiff = Math.floor(timeDiff / 60)
//extract integer hours that don't form a day using %
let hours = timeDiff % 24 //no need to floor possible incomplete hours, because they've been handled as seconds
//convert time difference from hours to days
timeDiff = Math.floor(timeDiff / 24)
// the rest of timeDiff is number of days
let days = timeDiff //add days to hours
let totalHours = hours + days * 24
totalHours = totalHours < 10 ? '0' + totalHours : totalHours
if (totalHours === '00') {
return minutes + ':' + seconds
} else {
return totalHours + ':' + minutes + ':' + seconds
}
}
//API to handle audio recording
const audioRecorder = {
/** Stores the recorded audio as Blob objects of audio data as the recording continues*/
audioBlobs: [] /*of type Blob[]*/,
/** Stores the reference of the MediaRecorder instance that handles the MediaStream when recording starts*/
mediaRecorder: null /*of type MediaRecorder*/,
/** Stores the reference to the stream currently capturing the audio*/
streamBeingCaptured: null /*of type MediaStream*/,
/** Start recording the audio
* @returns {Promise} - returns a promise that resolves if audio recording successfully started
*/
start: function () {
//Feature Detection
if (!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia)) {
//Feature is not supported in browser
//return a custom error
return Promise.reject(new Error('mediaDevices API or getUserMedia method is not supported in this browser.'))
} else {
//Feature is supported in browser
//create an audio stream
return (
navigator.mediaDevices
.getUserMedia({ audio: true } /*of type MediaStreamConstraints*/)
//returns a promise that resolves to the audio stream
.then((stream) /*of type MediaStream*/ => {
//save the reference of the stream to be able to stop it when necessary
audioRecorder.streamBeingCaptured = stream
//create a media recorder instance by passing that stream into the MediaRecorder constructor
audioRecorder.mediaRecorder = new MediaRecorder(stream) /*the MediaRecorder interface of the MediaStream Recording
API provides functionality to easily record media*/
//clear previously saved audio Blobs, if any
audioRecorder.audioBlobs = []
//add a dataavailable event listener in order to store the audio data Blobs when recording
audioRecorder.mediaRecorder.addEventListener('dataavailable', (event) => {
//store audio Blob object
audioRecorder.audioBlobs.push(event.data)
})
//start the recording by calling the start method on the media recorder
audioRecorder.mediaRecorder.start()
})
)
/* errors are not handled in the API because if its handled and the promise is chained, the .then after the catch will be executed*/
}
},
/** Stop the started audio recording
* @returns {Promise} - returns a promise that resolves to the audio as a blob file
*/
stop: function () {
//return a promise that would return the blob or URL of the recording
return new Promise((resolve) => {
//save audio type to pass to set the Blob type
let mimeType = audioRecorder.mediaRecorder.mimeType
//listen to the stop event in order to create & return a single Blob object
audioRecorder.mediaRecorder.addEventListener('stop', () => {
//create a single blob object, as we might have gathered a few Blob objects that needs to be joined as one
let audioBlob = new Blob(audioRecorder.audioBlobs, { type: mimeType })
//resolve promise with the single audio blob representing the recorded audio
resolve(audioBlob)
})
audioRecorder.cancel()
})
},
/** Cancel audio recording*/
cancel: function () {
//stop the recording feature
audioRecorder.mediaRecorder.stop()
//stop all the tracks on the active stream in order to stop the stream
audioRecorder.stopStream()
//reset API properties for next recording
audioRecorder.resetRecordingProperties()
},
/** Stop all the tracks on the active stream in order to stop the stream and remove
* the red flashing dot showing in the tab
*/
stopStream: function () {
//stopping the capturing request by stopping all the tracks on the active stream
audioRecorder.streamBeingCaptured
.getTracks() //get all tracks from the stream
.forEach((track) /*of type MediaStreamTrack*/ => track.stop()) //stop each one
},
/** Reset all the recording properties including the media recorder and stream being captured*/
resetRecordingProperties: function () {
audioRecorder.mediaRecorder = null
audioRecorder.streamBeingCaptured = null
/*No need to remove event listeners attached to mediaRecorder as
If a DOM element which is removed is reference-free (no references pointing to it), the element itself is picked
up by the garbage collector as well as any event handlers/listeners associated with it.
getEventListeners(audioRecorder.mediaRecorder) will return an empty array of events.*/
}
}
packages/ui/src/views/chatmessage/audio-recording.css
+278
View File
@@ -0,0 +1,278 @@
/* style.css*/
/* Media Queries */
/* Small Devices*/
@media (min-width: 0px) {
* {
box-sizing: border-box;
}
.audio-recording-container {
width: 100%;
/* view port height*/
/*targeting Chrome & Safari*/
display: -webkit-flex;
/*targeting IE10*/
display: -ms-flex;
display: flex;
flex-direction: column;
justify-content: center;
/*horizontal centering*/
align-items: center;
}
.start-recording-button {
font-size: 70px;
color: #435f7a;
cursor: pointer;
}
.start-recording-button:hover {
opacity: 1;
}
.recording-control-buttons-container {
/*targeting Chrome & Safari*/
display: -webkit-flex;
/*targeting IE10*/
display: -ms-flex;
display: flex;
justify-content: space-evenly;
/*horizontal centering*/
align-items: center;
width: 334px;
margin-bottom: 30px;
}
.cancel-recording-button,
.stop-recording-button {
font-size: 70px;
cursor: pointer;
}
.cancel-recording-button {
color: red;
opacity: 0.7;
}
.cancel-recording-button:hover {
color: rgb(206, 4, 4);
}
.stop-recording-button {
color: #33cc33;
opacity: 0.7;
}
.stop-recording-button:hover {
color: #27a527;
}
.recording-elapsed-time {
/*targeting Chrome & Safari*/
display: -webkit-flex;
/*targeting IE10*/
display: -ms-flex;
display: flex;
justify-content: center;
/*horizontal centering*/
align-items: center;
}
.red-recording-dot {
font-size: 25px;
color: red;
margin-right: 12px;
/*transitions with Firefox, IE and Opera Support browser support*/
animation-name: flashing-recording-dot;
-webkit-animation-name: flashing-recording-dot;
-moz-animation-name: flashing-recording-dot;
-o-animation-name: flashing-recording-dot;
animation-duration: 2s;
-webkit-animation-duration: 2s;
-moz-animation-duration: 2s;
-o-animation-duration: 2s;
animation-iteration-count: infinite;
-webkit-animation-iteration-count: infinite;
-moz-animation-iteration-count: infinite;
-o-animation-iteration-count: infinite;
}
/* The animation code */
@keyframes flashing-recording-dot {
0% {
opacity: 1;
}
50% {
opacity: 0;
}
100% {
opacity: 1;
}
}
@-webkit-keyframes flashing-recording-dot {
0% {
opacity: 1;
}
50% {
opacity: 0;
}
100% {
opacity: 1;
}
}
@-moz-keyframes flashing-recording-dot {
0% {
opacity: 1;
}
50% {
opacity: 0;
}
100% {
opacity: 1;
}
}
@-o-keyframes flashing-recording-dot {
0% {
opacity: 1;
}
50% {
opacity: 0;
}
100% {
opacity: 1;
}
}
.elapsed-time {
font-size: 32px;
}
.recording-control-buttons-container.hide {
display: none;
}
.overlay {
position: absolute;
top: 0;
width: 100%;
background-color: rgba(82, 76, 76, 0.35);
/*targeting Chrome & Safari*/
display: -webkit-flex;
/*targeting IE10*/
display: -ms-flex;
display: flex;
justify-content: center;
/*horizontal centering*/
align-items: center;
}
.overlay.hide {
display: none;
}
.browser-not-supporting-audio-recording-box {
/*targeting Chrome & Safari*/
display: -webkit-flex;
/*targeting IE10*/
display: -ms-flex;
display: flex;
flex-direction: column;
justify-content: space-between;
/*horizontal centering*/
align-items: center;
width: 317px;
height: 119px;
background-color: white;
border-radius: 10px;
padding: 15px;
font-size: 16px;
}
.close-browser-not-supported-box {
cursor: pointer;
background-color: #abc1c05c;
border-radius: 10px;
font-size: 16px;
border: none;
}
.close-browser-not-supported-box:hover {
background-color: #92a5a45c;
}
.close-browser-not-supported-box:focus {
outline: none;
border: none;
}
.audio-element.hide {
display: none;
}
.text-indication-of-audio-playing-container {
height: 20px;
}
.text-indication-of-audio-playing {
font-size: 20px;
}
.text-indication-of-audio-playing.hide {
display: none;
}
/* 3 Dots animation*/
.text-indication-of-audio-playing span {
/*transitions with Firefox, IE and Opera Support browser support*/
animation-name: blinking-dot;
-webkit-animation-name: blinking-dot;
-moz-animation-name: blinking-dot;
-o-animation-name: blinking-dot;
animation-duration: 2s;
-webkit-animation-duration: 2s;
-moz-animation-duration: 2s;
-o-animation-duration: 2s;
animation-iteration-count: infinite;
-webkit-animation-iteration-count: infinite;
-moz-animation-iteration-count: infinite;
-o-animation-iteration-count: infinite;
}
.text-indication-of-audio-playing span:nth-child(2) {
animation-delay: .4s;
-webkit-animation-delay: .4s;
-moz-animation-delay: .4s;
-o-animation-delay: .4s;
}
.text-indication-of-audio-playing span:nth-child(3) {
animation-delay: .8s;
-webkit-animation-delay: .8s;
-moz-animation-delay: .8s;
-o-animation-delay: .8s;
}
/* The animation code */
@keyframes blinking-dot {
0% {
opacity: 0;
}
50% {
opacity: 1;
}
100% {
opacity: 0;
}
}
/* The animation code */
@-webkit-keyframes blinking-dot {
0% {
opacity: 0;
}
50% {
opacity: 1;
}
100% {
opacity: 0;
}
}
/* The animation code */
@-moz-keyframes blinking-dot {
0% {
opacity: 0;
}
50% {
opacity: 1;
}
100% {
opacity: 0;
}
}
/* The animation code */
@-o-keyframes blinking-dot {
0% {
opacity: 0;
}
50% {
opacity: 1;
}
100% {
opacity: 0;
}
}
}
packages/ui/src/views/chatmessage/audio-recording.js
+433
View File
@@ -0,0 +1,433 @@
// audio-recording.js ---------------
//View
let microphoneButton = document.getElementsByClassName('start-recording-button')[0]
let recordingControlButtonsContainer = document.getElementsByClassName('recording-control-buttons-container')[0]
let stopRecordingButton = document.getElementsByClassName('stop-recording-button')[0]
let cancelRecordingButton = document.getElementsByClassName('cancel-recording-button')[0]
let elapsedTimeTag = document.getElementsByClassName('elapsed-time')[0]
let closeBrowserNotSupportedBoxButton = document.getElementsByClassName('close-browser-not-supported-box')[0]
let overlay = document.getElementsByClassName('overlay')[0]
let audioElement = document.getElementsByClassName('audio-element')[0]
let audioElementSource = document.getElementsByClassName('audio-element')[0].getElementsByTagName('source')[0]
let textIndicatorOfAudiPlaying = document.getElementsByClassName('text-indication-of-audio-playing')[0]
//Listeners
//Listen to start recording button
microphoneButton.onclick = startAudioRecording
//Listen to stop recording button
stopRecordingButton.onclick = stopAudioRecording
//Listen to cancel recording button
cancelRecordingButton.onclick = cancelAudioRecording
//Listen to when the ok button is clicked in the browser not supporting audio recording box
closeBrowserNotSupportedBoxButton.onclick = hideBrowserNotSupportedOverlay
//Listen to when the audio being played ends
audioElement.onended = hideTextIndicatorOfAudioPlaying
/** Displays recording control buttons */
function handleDisplayingRecordingControlButtons() {
//Hide the microphone button that starts audio recording
microphoneButton.style.display = 'none'
//Display the recording control buttons
recordingControlButtonsContainer.classList.remove('hide')
//Handle the displaying of the elapsed recording time
handleElapsedRecordingTime()
}
/** Hide the displayed recording control buttons */
function handleHidingRecordingControlButtons() {
//Display the microphone button that starts audio recording
microphoneButton.style.display = 'block'
//Hide the recording control buttons
recordingControlButtonsContainer.classList.add('hide')
//stop interval that handles both time elapsed and the red dot
clearInterval(elapsedTimeTimer)
}
/** Displays browser not supported info box for the user*/
function displayBrowserNotSupportedOverlay() {
overlay.classList.remove('hide')
}
/** Displays browser not supported info box for the user*/
function hideBrowserNotSupportedOverlay() {
overlay.classList.add('hide')
}
/** Creates a source element for the audio element in the HTML document*/
function createSourceForAudioElement() {
let sourceElement = document.createElement('source')
audioElement.appendChild(sourceElement)
audioElementSource = sourceElement
}
/** Display the text indicator of the audio being playing in the background */
function displayTextIndicatorOfAudioPlaying() {
textIndicatorOfAudiPlaying.classList.remove('hide')
}
/** Hide the text indicator of the audio being playing in the background */
function hideTextIndicatorOfAudioPlaying() {
textIndicatorOfAudiPlaying.classList.add('hide')
}
//Controller
/** Stores the actual start time when an audio recording begins to take place to ensure elapsed time start time is accurate*/
let audioRecordStartTime
/** Stores the maximum recording time in hours to stop recording once maximum recording hour has been reached */
let maximumRecordingTimeInHours = 1
/** Stores the reference of the setInterval function that controls the timer in audio recording*/
let elapsedTimeTimer
/** Starts the audio recording*/
function startAudioRecording() {
console.log('Recording Audio...')
//If a previous audio recording is playing, pause it
let recorderAudioIsPlaying = !audioElement.paused // the paused property tells whether the media element is paused or not
console.log('paused?', !recorderAudioIsPlaying)
if (recorderAudioIsPlaying) {
audioElement.pause()
//also hide the audio playing indicator displayed on the screen
hideTextIndicatorOfAudioPlaying()
}
//start recording using the audio recording API
audioRecorder
.start()
.then(() => {
//on success
//store the recording start time to display the elapsed time according to it
audioRecordStartTime = new Date()
//display control buttons to offer the functionality of stop and cancel
handleDisplayingRecordingControlButtons()
})
.catch((error) => {
//on error
//No Browser Support Error
if (error.message.includes('mediaDevices API or getUserMedia method is not supported in this browser.')) {
console.log('To record audio, use browsers like Chrome and Firefox.')
displayBrowserNotSupportedOverlay()
}
//Error handling structure
switch (error.name) {
case 'AbortError': //error from navigator.mediaDevices.getUserMedia
console.log('An AbortError has occurred.')
break
case 'NotAllowedError': //error from navigator.mediaDevices.getUserMedia
console.log('A NotAllowedError has occurred. User might have denied permission.')
break
case 'NotFoundError': //error from navigator.mediaDevices.getUserMedia
console.log('A NotFoundError has occurred.')
break
case 'NotReadableError': //error from navigator.mediaDevices.getUserMedia
console.log('A NotReadableError has occurred.')
break
case 'SecurityError': //error from navigator.mediaDevices.getUserMedia or from the MediaRecorder.start
console.log('A SecurityError has occurred.')
break
case 'TypeError': //error from navigator.mediaDevices.getUserMedia
console.log('A TypeError has occurred.')
break
case 'InvalidStateError': //error from the MediaRecorder.start
console.log('An InvalidStateError has occurred.')
break
case 'UnknownError': //error from the MediaRecorder.start
console.log('An UnknownError has occurred.')
break
default:
console.log('An error occurred with the error name ' + error.name)
}
})
}
/** Stop the currently started audio recording & sends it
*/
function stopAudioRecording() {
console.log('Stopping Audio Recording...')
//stop the recording using the audio recording API
audioRecorder
.stop()
.then((audioAsblob) => {
//Play recorder audio
playAudio(audioAsblob)
//hide recording control button & return record icon
handleHidingRecordingControlButtons()
})
.catch((error) => {
//Error handling structure
switch (error.name) {
case 'InvalidStateError': //error from the MediaRecorder.stop
console.log('An InvalidStateError has occurred.')
break
default:
console.log('An error occurred with the error name ' + error.name)
}
})
}
/** Cancel the currently started audio recording */
function cancelAudioRecording() {
console.log('Canceling audio...')
//cancel the recording using the audio recording API
audioRecorder.cancel()
//hide recording control button & return record icon
handleHidingRecordingControlButtons()
}
/** Plays recorded audio using the audio element in the HTML document
* @param {Blob} recorderAudioAsBlob - recorded audio as a Blob Object
*/
function playAudio(recorderAudioAsBlob) {
//read content of files (Blobs) asynchronously
let reader = new FileReader()
//once content has been read
reader.onload = (e) => {
//store the base64 URL that represents the URL of the recording audio
let base64URL = e.target.result
//If this is the first audio playing, create a source element
//as pre-populating the HTML with a source of empty src causes error
if (!audioElementSource)
//if it is not defined create it (happens first time only)
createSourceForAudioElement()
//set the audio element's source using the base64 URL
audioElementSource.src = base64URL
//set the type of the audio element based on the recorded audio's Blob type
let BlobType = recorderAudioAsBlob.type.includes(';')
? recorderAudioAsBlob.type.substr(0, recorderAudioAsBlob.type.indexOf(';'))
: recorderAudioAsBlob.type
audioElementSource.type = BlobType
//call the load method as it is used to update the audio element after changing the source or other settings
audioElement.load()
//play the audio after successfully setting new src and type that corresponds to the recorded audio
console.log('Playing audio...')
audioElement.play()
//Display text indicator of having the audio play in the background
displayTextIndicatorOfAudioPlaying()
}
//read content and convert it to a URL (base64)
reader.readAsDataURL(recorderAudioAsBlob)
}
/** Computes the elapsed recording time since the moment the function is called in the format h:m:s*/
function handleElapsedRecordingTime() {
//display initial time when recording begins
displayElapsedTimeDuringAudioRecording('00:00')
//create an interval that compute & displays elapsed time, as well as, animate red dot - every second
elapsedTimeTimer = setInterval(() => {
//compute the elapsed time every second
let elapsedTime = computeElapsedTime(audioRecordStartTime) //pass the actual record start time
//display the elapsed time
displayElapsedTimeDuringAudioRecording(elapsedTime)
}, 1000) //every second
}
/** Display elapsed time during audio recording
* @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss
*/
function displayElapsedTimeDuringAudioRecording(elapsedTime) {
//1. display the passed elapsed time as the elapsed time in the elapsedTime HTML element
elapsedTimeTag.innerHTML = elapsedTime
//2. Stop the recording when the max number of hours is reached
if (elapsedTimeReachedMaximumNumberOfHours(elapsedTime)) {
stopAudioRecording()
}
}
/**
* @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss
* @returns {Boolean} whether the elapsed time reached the maximum number of hours or not
*/
function elapsedTimeReachedMaximumNumberOfHours(elapsedTime) {
//Split the elapsed time by the symbol that separates the hours, minutes and seconds :
let elapsedTimeSplit = elapsedTime.split(':')
//Turn the maximum recording time in hours to a string and pad it with zero if less than 10
let maximumRecordingTimeInHoursAsString =
maximumRecordingTimeInHours < 10 ? '0' + maximumRecordingTimeInHours : maximumRecordingTimeInHours.toString()
//if the elapsed time reach hours and also reach the maximum recording time in hours return true
if (elapsedTimeSplit.length === 3 && elapsedTimeSplit[0] === maximumRecordingTimeInHoursAsString) return true
//otherwise, return false
else return false
}
/** Computes the elapsedTime since the moment the function is called in the format mm:ss or hh:mm:ss
* @param {String} startTime - start time to compute the elapsed time since
* @returns {String} elapsed time in mm:ss format or hh:mm:ss format, if elapsed hours are 0.
*/
function computeElapsedTime(startTime) {
//record end time
let endTime = new Date()
//time difference in ms
let timeDiff = endTime - startTime
//convert time difference from ms to seconds
timeDiff = timeDiff / 1000
//extract integer seconds that don't form a minute using %
let seconds = Math.floor(timeDiff % 60) //ignoring incomplete seconds (floor)
//pad seconds with a zero if necessary
seconds = seconds < 10 ? '0' + seconds : seconds
//convert time difference from seconds to minutes using %
timeDiff = Math.floor(timeDiff / 60)
//extract integer minutes that don't form an hour using %
let minutes = timeDiff % 60 //no need to floor possible incomplete minutes, because they've been handled as seconds
minutes = minutes < 10 ? '0' + minutes : minutes
//convert time difference from minutes to hours
timeDiff = Math.floor(timeDiff / 60)
//extract integer hours that don't form a day using %
let hours = timeDiff % 24 //no need to floor possible incomplete hours, because they've been handled as seconds
//convert time difference from hours to days
timeDiff = Math.floor(timeDiff / 24)
// the rest of timeDiff is number of days
let days = timeDiff //add days to hours
let totalHours = hours + days * 24
totalHours = totalHours < 10 ? '0' + totalHours : totalHours
if (totalHours === '00') {
return minutes + ':' + seconds
} else {
return totalHours + ':' + minutes + ':' + seconds
}
}
//API to handle audio recording
const audioRecorder = {
/** Stores the recorded audio as Blob objects of audio data as the recording continues*/
audioBlobs: [] /*of type Blob[]*/,
/** Stores the reference of the MediaRecorder instance that handles the MediaStream when recording starts*/
mediaRecorder: null /*of type MediaRecorder*/,
/** Stores the reference to the stream currently capturing the audio*/
streamBeingCaptured: null /*of type MediaStream*/,
/** Start recording the audio
* @returns {Promise} - returns a promise that resolves if audio recording successfully started
*/
start: function () {
//Feature Detection
if (!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia)) {
//Feature is not supported in browser
//return a custom error
return Promise.reject(new Error('mediaDevices API or getUserMedia method is not supported in this browser.'))
} else {
//Feature is supported in browser
//create an audio stream
return (
navigator.mediaDevices
.getUserMedia({ audio: true } /*of type MediaStreamConstraints*/)
//returns a promise that resolves to the audio stream
.then((stream) /*of type MediaStream*/ => {
//save the reference of the stream to be able to stop it when necessary
audioRecorder.streamBeingCaptured = stream
//create a media recorder instance by passing that stream into the MediaRecorder constructor
audioRecorder.mediaRecorder = new MediaRecorder(stream) /*the MediaRecorder interface of the MediaStream Recording
API provides functionality to easily record media*/
//clear previously saved audio Blobs, if any
audioRecorder.audioBlobs = []
//add a dataavailable event listener in order to store the audio data Blobs when recording
audioRecorder.mediaRecorder.addEventListener('dataavailable', (event) => {
//store audio Blob object
audioRecorder.audioBlobs.push(event.data)
})
//start the recording by calling the start method on the media recorder
audioRecorder.mediaRecorder.start()
})
)
/* errors are not handled in the API because if its handled and the promise is chained, the .then after the catch will be executed*/
}
},
/** Stop the started audio recording
* @returns {Promise} - returns a promise that resolves to the audio as a blob file
*/
stop: function () {
//return a promise that would return the blob or URL of the recording
return new Promise((resolve) => {
//save audio type to pass to set the Blob type
let mimeType = audioRecorder.mediaRecorder.mimeType
//listen to the stop event in order to create & return a single Blob object
audioRecorder.mediaRecorder.addEventListener('stop', () => {
//create a single blob object, as we might have gathered a few Blob objects that needs to be joined as one
let audioBlob = new Blob(audioRecorder.audioBlobs, { type: mimeType })
//resolve promise with the single audio blob representing the recorded audio
resolve(audioBlob)
})
audioRecorder.cancel()
})
},
/** Cancel audio recording*/
cancel: function () {
//stop the recording feature
audioRecorder.mediaRecorder.stop()
//stop all the tracks on the active stream in order to stop the stream
audioRecorder.stopStream()
//reset API properties for next recording
audioRecorder.resetRecordingProperties()
},
/** Stop all the tracks on the active stream in order to stop the stream and remove
* the red flashing dot showing in the tab
*/
stopStream: function () {
//stopping the capturing request by stopping all the tracks on the active stream
audioRecorder.streamBeingCaptured
.getTracks() //get all tracks from the stream
.forEach((track) /*of type MediaStreamTrack*/ => track.stop()) //stop each one
},
/** Reset all the recording properties including the media recorder and stream being captured*/
resetRecordingProperties: function () {
audioRecorder.mediaRecorder = null
audioRecorder.streamBeingCaptured = null
/*No need to remove event listeners attached to mediaRecorder as
If a DOM element which is removed is reference-free (no references pointing to it), the element itself is picked
up by the garbage collector as well as any event handlers/listeners associated with it.
getEventListeners(audioRecorder.mediaRecorder) will return an empty array of events.*/
}
}