mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 17:01:00 +03:00
Text to speech (#5062)
* Add tts UI * Add tts backend * Add description to eleven labs credentials * Fix issue with fetching eleven labs voices * Fix issue with text to speech tab not showing correct saved voice * Add option to autoplay tts audio after prediction completes * Fix crash issue when first changing tts provider * Set up streaming response for text to speech audio * Update controllers - fix issue with sse client getting removed before tts events are sent * Use existing sse streamer to stream tts audio before sse client is removed * Add tts sse to redis publisher * Fix issues with TTS - openai voices, streaming audio, rate limiting, speed of speech * Refactor * Refactor TTS - fix issues with tts loading and stop audio buttons * Abort TTS SSE when clicking the stop button * Update SSE handling for TTS * Fix issue with test voice feature * Fix issue with tts voices not loading * Update generate tts endpoint and its usage in internal chat * Whitelist tts generate endpoint * Refactor Text-to-Speech Provider Selection and Enhance UI Components - Updated the text-to-speech controller to select the active provider based on status instead of the first available provider - Added audio waveform controls and test audio functionality in the TextToSpeech component, allowing users to play and pause test audio - Integrated Autocomplete for voice selection in the TextToSpeech component - Implemented TTS action management in ChatMessage to prevent auto-scrolling during TTS actions * - Implemented stopAllTTS function calls to halt existing TTS audio before playing new audio or starting a new TTS stream * Updated the condition for enabling TTS providers to exclude the 'none' provider, ensuring only valid providers are considered for text-to-speech functionality. * Remove unnecessary code * Add ability to abort audio streaming in TTS and release lock on chat input * Remove logger * Fix tts audio not playing when clicking speaker button * update * TTS abort controller * Fix abort not working for TTS autoplay * Send metadata event when aborting autoplay TTS * Fix UI issue * Remove elevenlabs sdk from root package.json * Remove redundant condition for tts autoplay in chatflow --------- Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Ilango
@@ -0,0 +1,16 @@
|
||||
import client from './client'
|
||||
|
||||
const abortTTS = (body) => client.post('/text-to-speech/abort', body)
|
||||
|
||||
const generateVoice = (body) =>
|
||||
client.post('/text-to-speech/generate', body, {
|
||||
responseType: 'arraybuffer'
|
||||
})
|
||||
|
||||
const listVoices = (params) => client.get('/text-to-speech/voices', { params })
|
||||
|
||||
export default {
|
||||
abortTTS,
|
||||
generateVoice,
|
||||
listVoices
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect width="24" height="24" rx="4" fill="#000000"/>
|
||||
<path d="M6 8h3v8H6V8zm5-2h3v12h-3V6zm5 4h3v4h-3v-4z" fill="#ffffff"/>
|
||||
<circle cx="7.5" cy="12" r="1" fill="#00ff88"/>
|
||||
<circle cx="12.5" cy="12" r="1" fill="#00ff88"/>
|
||||
<circle cx="17.5" cy="12" r="1" fill="#00ff88"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 383 B |
@@ -4,6 +4,7 @@ import { createPortal } from 'react-dom'
|
||||
import { Box, Dialog, DialogContent, DialogTitle, Tabs, Tab } from '@mui/material'
|
||||
import { tabsClasses } from '@mui/material/Tabs'
|
||||
import SpeechToText from '@/ui-component/extended/SpeechToText'
|
||||
import TextToSpeech from '@/ui-component/extended/TextToSpeech'
|
||||
import Security from '@/ui-component/extended/Security'
|
||||
import ChatFeedback from '@/ui-component/extended/ChatFeedback'
|
||||
import AnalyseFlow from '@/ui-component/extended/AnalyseFlow'
|
||||
@@ -30,6 +31,10 @@ const CHATFLOW_CONFIGURATION_TABS = [
|
||||
label: 'Speech to Text',
|
||||
id: 'speechToText'
|
||||
},
|
||||
{
|
||||
label: 'Text to Speech',
|
||||
id: 'textToSpeech'
|
||||
},
|
||||
{
|
||||
label: 'Chat Feedback',
|
||||
id: 'chatFeedback'
|
||||
@@ -125,18 +130,19 @@ const ChatflowConfigurationDialog = ({ show, isAgentCanvas, dialogProps, onCance
|
||||
alignItems: 'center',
|
||||
mb: 1
|
||||
}}
|
||||
key={index}
|
||||
key={item.id}
|
||||
label={item.label}
|
||||
{...a11yProps(index)}
|
||||
></Tab>
|
||||
))}
|
||||
</Tabs>
|
||||
{filteredTabs.map((item, index) => (
|
||||
<TabPanel key={index} value={tabValue} index={index}>
|
||||
<TabPanel key={item.id} value={tabValue} index={index}>
|
||||
{item.id === 'security' && <Security dialogProps={dialogProps} />}
|
||||
{item.id === 'conversationStarters' ? <StarterPrompts dialogProps={dialogProps} /> : null}
|
||||
{item.id === 'followUpPrompts' ? <FollowUpPrompts dialogProps={dialogProps} /> : null}
|
||||
{item.id === 'speechToText' ? <SpeechToText dialogProps={dialogProps} /> : null}
|
||||
{item.id === 'textToSpeech' ? <TextToSpeech dialogProps={dialogProps} /> : null}
|
||||
{item.id === 'chatFeedback' ? <ChatFeedback dialogProps={dialogProps} /> : null}
|
||||
{item.id === 'analyseChatflow' ? <AnalyseFlow dialogProps={dialogProps} /> : null}
|
||||
{item.id === 'leads' ? <Leads dialogProps={dialogProps} /> : null}
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
import { useRef, useEffect, useState, useCallback } from 'react'
|
||||
import PropTypes from 'prop-types'
|
||||
import { Box, IconButton, CircularProgress } from '@mui/material'
|
||||
import { IconPlayerPlay, IconPlayerPause } from '@tabler/icons-react'
|
||||
import { useTheme } from '@mui/material/styles'
|
||||
|
||||
const AudioWaveform = ({
|
||||
audioSrc,
|
||||
onPlay,
|
||||
onPause,
|
||||
onEnded,
|
||||
isPlaying = false,
|
||||
duration: _duration = 0,
|
||||
isGenerating = false,
|
||||
disabled = false,
|
||||
externalAudioRef = null,
|
||||
resetProgress = false
|
||||
}) => {
|
||||
const canvasRef = useRef(null)
|
||||
const audioRef = useRef(null)
|
||||
const animationRef = useRef(null)
|
||||
const theme = useTheme()
|
||||
|
||||
const [progress, setProgress] = useState(0)
|
||||
const [_audioBuffer, setAudioBuffer] = useState(null)
|
||||
const [waveformData, setWaveformData] = useState([])
|
||||
|
||||
// Generate waveform visualization data
|
||||
const generateWaveform = useCallback((buffer) => {
|
||||
if (!buffer) return []
|
||||
|
||||
const rawData = buffer.getChannelData(0)
|
||||
const samples = 200 // More bars for smoother appearance like reference
|
||||
const blockSize = Math.floor(rawData.length / samples)
|
||||
const filteredData = []
|
||||
|
||||
for (let i = 0; i < samples; i++) {
|
||||
let blockStart = blockSize * i
|
||||
let sum = 0
|
||||
for (let j = 0; j < blockSize; j++) {
|
||||
sum += Math.abs(rawData[blockStart + j])
|
||||
}
|
||||
filteredData.push(sum / blockSize)
|
||||
}
|
||||
|
||||
// Normalize the data
|
||||
const maxValue = Math.max(...filteredData)
|
||||
return filteredData.map((value) => (value / maxValue) * 100)
|
||||
}, [])
|
||||
|
||||
// Generate realistic placeholder waveform like in reference
|
||||
const generatePlaceholderWaveform = useCallback(() => {
|
||||
const samples = 200
|
||||
const waveform = []
|
||||
|
||||
for (let i = 0; i < samples; i++) {
|
||||
// Create a more realistic waveform pattern
|
||||
const position = i / samples
|
||||
const baseHeight = 20 + Math.sin(position * Math.PI * 4) * 15
|
||||
const variation = Math.random() * 40 + 10
|
||||
const envelope = Math.sin(position * Math.PI) * 0.8 + 0.2
|
||||
|
||||
waveform.push((baseHeight + variation) * envelope)
|
||||
}
|
||||
|
||||
return waveform
|
||||
}, [])
|
||||
|
||||
// Draw waveform on canvas
|
||||
const drawWaveform = useCallback(() => {
|
||||
const canvas = canvasRef.current
|
||||
if (!canvas || waveformData.length === 0) return
|
||||
|
||||
const ctx = canvas.getContext('2d')
|
||||
|
||||
// Handle high DPI displays for crisp rendering
|
||||
const dpr = window.devicePixelRatio || 1
|
||||
const rect = canvas.getBoundingClientRect()
|
||||
|
||||
canvas.width = rect.width * dpr
|
||||
canvas.height = rect.height * dpr
|
||||
ctx.scale(dpr, dpr)
|
||||
|
||||
canvas.style.width = rect.width + 'px'
|
||||
canvas.style.height = rect.height + 'px'
|
||||
|
||||
ctx.clearRect(0, 0, rect.width, rect.height)
|
||||
|
||||
// More bars for smoother appearance like the reference
|
||||
const totalBars = waveformData.length
|
||||
const barWidth = 2 // Fixed thin bar width like in reference
|
||||
const barSpacing = 1 // Small gap between bars
|
||||
const totalWidth = rect.width
|
||||
const startX = (totalWidth - totalBars * (barWidth + barSpacing)) / 2
|
||||
const centerY = rect.height / 2
|
||||
|
||||
waveformData.forEach((value, index) => {
|
||||
const barHeight = Math.max(2, (value / 100) * (rect.height * 0.8))
|
||||
const x = startX + index * (barWidth + barSpacing)
|
||||
|
||||
// Determine color based on playback progress
|
||||
const progressIndex = Math.floor((progress / 100) * waveformData.length)
|
||||
const isPlayed = index <= progressIndex
|
||||
|
||||
ctx.fillStyle = isPlayed ? theme.palette.primary.main : theme.palette.mode === 'dark' ? '#444' : '#ccc'
|
||||
|
||||
// Draw thin vertical bars like in reference
|
||||
ctx.fillRect(x, centerY - barHeight / 2, barWidth, barHeight)
|
||||
})
|
||||
}, [waveformData, progress, theme])
|
||||
|
||||
// Load and decode audio for waveform generation
|
||||
useEffect(() => {
|
||||
if (audioSrc && audioSrc.startsWith('blob:')) {
|
||||
const loadAudioBuffer = async () => {
|
||||
try {
|
||||
const response = await fetch(audioSrc)
|
||||
const arrayBuffer = await response.arrayBuffer()
|
||||
const audioContext = new (window.AudioContext || window.webkitAudioContext)()
|
||||
const buffer = await audioContext.decodeAudioData(arrayBuffer)
|
||||
setAudioBuffer(buffer)
|
||||
const waveform = generateWaveform(buffer)
|
||||
setWaveformData(waveform)
|
||||
} catch (error) {
|
||||
console.error('Error loading audio buffer:', error)
|
||||
// Generate placeholder waveform
|
||||
const placeholder = generatePlaceholderWaveform()
|
||||
setWaveformData(placeholder)
|
||||
}
|
||||
}
|
||||
loadAudioBuffer()
|
||||
} else {
|
||||
// Always show placeholder waveform when no audio source
|
||||
const placeholder = generatePlaceholderWaveform()
|
||||
setWaveformData(placeholder)
|
||||
}
|
||||
}, [audioSrc, generateWaveform, generatePlaceholderWaveform])
|
||||
|
||||
// Reset progress when resetProgress prop is true
|
||||
useEffect(() => {
|
||||
if (resetProgress) {
|
||||
setProgress(0)
|
||||
}
|
||||
}, [resetProgress])
|
||||
|
||||
// Draw waveform when data changes or progress updates
|
||||
useEffect(() => {
|
||||
drawWaveform()
|
||||
}, [drawWaveform, progress])
|
||||
|
||||
// Update progress during playback
|
||||
useEffect(() => {
|
||||
const activeAudioRef = externalAudioRef || audioRef.current
|
||||
if (isPlaying && activeAudioRef && audioSrc) {
|
||||
const updateProgress = () => {
|
||||
const audio = externalAudioRef || audioRef.current
|
||||
if (audio && audio.duration && !isNaN(audio.duration)) {
|
||||
const currentProgress = (audio.currentTime / audio.duration) * 100
|
||||
setProgress(currentProgress)
|
||||
}
|
||||
if (isPlaying && audio && !audio.paused) {
|
||||
animationRef.current = requestAnimationFrame(updateProgress)
|
||||
}
|
||||
}
|
||||
|
||||
// Start the update loop
|
||||
animationRef.current = requestAnimationFrame(updateProgress)
|
||||
} else {
|
||||
if (animationRef.current) {
|
||||
cancelAnimationFrame(animationRef.current)
|
||||
}
|
||||
}
|
||||
|
||||
return () => {
|
||||
if (animationRef.current) {
|
||||
cancelAnimationFrame(animationRef.current)
|
||||
}
|
||||
}
|
||||
}, [isPlaying, audioSrc, externalAudioRef])
|
||||
|
||||
const handlePlayPause = () => {
|
||||
if (isPlaying) {
|
||||
onPause?.()
|
||||
} else {
|
||||
onPlay?.()
|
||||
}
|
||||
}
|
||||
|
||||
// Handle canvas click for seeking
|
||||
const handleCanvasClick = (event) => {
|
||||
const activeAudio = externalAudioRef || audioRef.current
|
||||
if (!activeAudio || !activeAudio.duration || disabled || isGenerating) return
|
||||
|
||||
const canvas = canvasRef.current
|
||||
const rect = canvas.getBoundingClientRect()
|
||||
const clickX = event.clientX - rect.left
|
||||
|
||||
// Use the actual canvas display width for more accurate clicking
|
||||
const clickProgress = Math.max(0, Math.min(100, (clickX / rect.width) * 100))
|
||||
const seekTime = (clickProgress / 100) * activeAudio.duration
|
||||
|
||||
activeAudio.currentTime = seekTime
|
||||
setProgress(clickProgress)
|
||||
}
|
||||
|
||||
return (
|
||||
<Box sx={{ width: '100%' }}>
|
||||
{/* Hidden audio element for duration and seeking - only if no external ref */}
|
||||
{audioSrc && !externalAudioRef && (
|
||||
<audio
|
||||
ref={audioRef}
|
||||
src={audioSrc}
|
||||
onLoadedMetadata={() => {
|
||||
if (audioRef.current) {
|
||||
setProgress(0)
|
||||
}
|
||||
}}
|
||||
onTimeUpdate={() => {
|
||||
// Additional progress update on timeupdate event
|
||||
const audio = audioRef.current
|
||||
if (audio && audio.duration && !isNaN(audio.duration)) {
|
||||
const currentProgress = (audio.currentTime / audio.duration) * 100
|
||||
setProgress(currentProgress)
|
||||
}
|
||||
}}
|
||||
onEnded={() => {
|
||||
setProgress(0)
|
||||
onEnded?.()
|
||||
}}
|
||||
style={{ display: 'none' }}
|
||||
>
|
||||
<track kind='captions' />
|
||||
</audio>
|
||||
)}
|
||||
|
||||
{/* Play button and Waveform side by side */}
|
||||
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
|
||||
{/* Play/Pause Button */}
|
||||
<IconButton
|
||||
onClick={handlePlayPause}
|
||||
disabled={disabled || isGenerating}
|
||||
size='small'
|
||||
sx={{
|
||||
width: 32,
|
||||
height: 32,
|
||||
flexShrink: 0,
|
||||
backgroundColor: isPlaying ? 'transparent' : theme.palette.primary.main,
|
||||
color: isPlaying ? theme.palette.primary.main : 'white',
|
||||
border: isPlaying ? `1px solid ${theme.palette.primary.main}` : 'none',
|
||||
'&:hover': {
|
||||
backgroundColor: isPlaying ? theme.palette.primary.main : theme.palette.primary.dark,
|
||||
color: 'white'
|
||||
},
|
||||
'&:disabled': {
|
||||
backgroundColor: theme.palette.action.disabled,
|
||||
color: theme.palette.action.disabled,
|
||||
border: 'none'
|
||||
}
|
||||
}}
|
||||
>
|
||||
{isGenerating ? (
|
||||
<CircularProgress size={16} />
|
||||
) : isPlaying ? (
|
||||
<IconPlayerPause size={16} />
|
||||
) : (
|
||||
<IconPlayerPlay size={16} />
|
||||
)}
|
||||
</IconButton>
|
||||
|
||||
{/* Waveform Canvas */}
|
||||
<Box
|
||||
sx={{
|
||||
flex: 1,
|
||||
cursor: !disabled && !isGenerating && audioSrc ? 'pointer' : 'default',
|
||||
display: 'flex',
|
||||
alignItems: 'center'
|
||||
}}
|
||||
>
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
width={400}
|
||||
height={32}
|
||||
onClick={handleCanvasClick}
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '32px',
|
||||
backgroundColor: 'transparent',
|
||||
opacity: disabled ? 0.6 : 1,
|
||||
display: 'block'
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
</Box>
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
|
||||
AudioWaveform.propTypes = {
|
||||
audioSrc: PropTypes.string,
|
||||
onPlay: PropTypes.func,
|
||||
onPause: PropTypes.func,
|
||||
onEnded: PropTypes.func,
|
||||
isPlaying: PropTypes.bool,
|
||||
duration: PropTypes.number,
|
||||
isGenerating: PropTypes.bool,
|
||||
disabled: PropTypes.bool,
|
||||
externalAudioRef: PropTypes.object,
|
||||
resetProgress: PropTypes.bool
|
||||
}
|
||||
|
||||
export default AudioWaveform
|
||||
@@ -402,7 +402,15 @@ const SpeechToText = ({ dialogProps }) => {
|
||||
sx={{ ml: 1 }}
|
||||
primary={speechToTextProviders[selectedProvider].label}
|
||||
secondary={
|
||||
<a target='_blank' rel='noreferrer' href={speechToTextProviders[selectedProvider].url}>
|
||||
<a
|
||||
target='_blank'
|
||||
rel='noreferrer'
|
||||
href={speechToTextProviders[selectedProvider].url}
|
||||
style={{
|
||||
color: theme?.customization?.isDarkMode ? '#90caf9' : '#1976d2',
|
||||
textDecoration: 'underline'
|
||||
}}
|
||||
>
|
||||
{speechToTextProviders[selectedProvider].url}
|
||||
</a>
|
||||
}
|
||||
|
||||
@@ -0,0 +1,660 @@
|
||||
import { useDispatch } from 'react-redux'
|
||||
import { useState, useEffect } from 'react'
|
||||
import PropTypes from 'prop-types'
|
||||
import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction, SET_CHATFLOW } from '@/store/actions'
|
||||
|
||||
// material-ui
|
||||
import {
|
||||
Typography,
|
||||
Box,
|
||||
Button,
|
||||
FormControl,
|
||||
ListItem,
|
||||
ListItemAvatar,
|
||||
ListItemText,
|
||||
MenuItem,
|
||||
Select,
|
||||
CircularProgress,
|
||||
Autocomplete,
|
||||
TextField
|
||||
} from '@mui/material'
|
||||
import { IconX, IconVolume } from '@tabler/icons-react'
|
||||
import { useTheme } from '@mui/material/styles'
|
||||
|
||||
// Project import
|
||||
import CredentialInputHandler from '@/views/canvas/CredentialInputHandler'
|
||||
import { TooltipWithParser } from '@/ui-component/tooltip/TooltipWithParser'
|
||||
import { SwitchInput } from '@/ui-component/switch/Switch'
|
||||
import { Input } from '@/ui-component/input/Input'
|
||||
import { StyledButton } from '@/ui-component/button/StyledButton'
|
||||
import { Dropdown } from '@/ui-component/dropdown/Dropdown'
|
||||
import AudioWaveform from '@/ui-component/extended/AudioWaveform'
|
||||
import openAISVG from '@/assets/images/openai.svg'
|
||||
import elevenLabsSVG from '@/assets/images/elevenlabs.svg'
|
||||
|
||||
// store
|
||||
import useNotifier from '@/utils/useNotifier'
|
||||
|
||||
// API
|
||||
import chatflowsApi from '@/api/chatflows'
|
||||
import ttsApi from '@/api/tts'
|
||||
|
||||
const TextToSpeechType = {
|
||||
OPENAI_TTS: 'openai',
|
||||
ELEVEN_LABS_TTS: 'elevenlabs'
|
||||
}
|
||||
|
||||
// Weird quirk - the key must match the name property value.
|
||||
const textToSpeechProviders = {
|
||||
[TextToSpeechType.OPENAI_TTS]: {
|
||||
label: 'OpenAI TTS',
|
||||
name: TextToSpeechType.OPENAI_TTS,
|
||||
icon: openAISVG,
|
||||
url: 'https://platform.openai.com/docs/guides/text-to-speech',
|
||||
inputs: [
|
||||
{
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['openAIApi']
|
||||
},
|
||||
{
|
||||
label: 'Voice',
|
||||
name: 'voice',
|
||||
type: 'voice_select',
|
||||
description: 'The voice to use when generating the audio',
|
||||
default: 'alloy',
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
},
|
||||
[TextToSpeechType.ELEVEN_LABS_TTS]: {
|
||||
label: 'Eleven Labs TTS',
|
||||
name: TextToSpeechType.ELEVEN_LABS_TTS,
|
||||
icon: elevenLabsSVG,
|
||||
url: 'https://elevenlabs.io/',
|
||||
inputs: [
|
||||
{
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['elevenLabsApi']
|
||||
},
|
||||
{
|
||||
label: 'Voice',
|
||||
name: 'voice',
|
||||
type: 'voice_select',
|
||||
description: 'The voice to use for text-to-speech',
|
||||
default: '21m00Tcm4TlvDq8ikWAM',
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
const TextToSpeech = ({ dialogProps }) => {
|
||||
const dispatch = useDispatch()
|
||||
|
||||
useNotifier()
|
||||
const theme = useTheme()
|
||||
|
||||
const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
|
||||
const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
|
||||
|
||||
const [textToSpeech, setTextToSpeech] = useState(null)
|
||||
const [selectedProvider, setSelectedProvider] = useState('none')
|
||||
const [voices, setVoices] = useState([])
|
||||
const [loadingVoices, setLoadingVoices] = useState(false)
|
||||
const [testAudioSrc, setTestAudioSrc] = useState(null)
|
||||
const [isTestPlaying, setIsTestPlaying] = useState(false)
|
||||
const [testAudioRef, setTestAudioRef] = useState(null)
|
||||
const [isGeneratingTest, setIsGeneratingTest] = useState(false)
|
||||
const [resetWaveform, setResetWaveform] = useState(false)
|
||||
|
||||
const resetTestAudio = () => {
|
||||
if (testAudioSrc) {
|
||||
URL.revokeObjectURL(testAudioSrc)
|
||||
setTestAudioSrc(null)
|
||||
}
|
||||
setIsTestPlaying(false)
|
||||
setResetWaveform(true)
|
||||
setTimeout(() => setResetWaveform(false), 100)
|
||||
}
|
||||
|
||||
const onSave = async () => {
|
||||
const textToSpeechConfig = setValue(true, selectedProvider, 'status')
|
||||
try {
|
||||
const saveResp = await chatflowsApi.updateChatflow(dialogProps.chatflow.id, {
|
||||
textToSpeech: JSON.stringify(textToSpeechConfig)
|
||||
})
|
||||
if (saveResp.data) {
|
||||
enqueueSnackbar({
|
||||
message: 'Text To Speech Configuration Saved',
|
||||
options: {
|
||||
key: Date.now() + Math.random(),
|
||||
variant: 'success',
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
dispatch({ type: SET_CHATFLOW, chatflow: saveResp.data })
|
||||
}
|
||||
} catch (error) {
|
||||
enqueueSnackbar({
|
||||
message: `Failed to save Text To Speech Configuration: ${
|
||||
typeof error.response.data === 'object' ? error.response.data.message : error.response.data
|
||||
}`,
|
||||
options: {
|
||||
key: Date.now() + Math.random(),
|
||||
variant: 'error',
|
||||
persist: true,
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const setValue = (value, providerName, inputParamName) => {
|
||||
let newVal = {}
|
||||
if (!textToSpeech || !Object.hasOwn(textToSpeech, providerName)) {
|
||||
newVal = { ...(textToSpeech || {}), [providerName]: {} }
|
||||
} else {
|
||||
newVal = { ...textToSpeech }
|
||||
}
|
||||
|
||||
newVal[providerName][inputParamName] = value
|
||||
if (inputParamName === 'status' && value === true) {
|
||||
// ensure that the others are turned off
|
||||
Object.keys(textToSpeechProviders).forEach((key) => {
|
||||
const provider = textToSpeechProviders[key]
|
||||
if (provider.name !== providerName) {
|
||||
newVal[provider.name] = { ...(textToSpeech?.[provider.name] || {}), status: false }
|
||||
}
|
||||
})
|
||||
if (providerName !== 'none' && newVal['none']) {
|
||||
newVal['none'].status = false
|
||||
}
|
||||
}
|
||||
|
||||
// Reset test audio when voice or credential is changed
|
||||
if ((inputParamName === 'voice' || inputParamName === 'credentialId') && providerName === selectedProvider) {
|
||||
resetTestAudio()
|
||||
}
|
||||
|
||||
setTextToSpeech(newVal)
|
||||
return newVal
|
||||
}
|
||||
|
||||
const handleProviderChange = (provider, configOverride = null) => {
|
||||
setSelectedProvider(provider)
|
||||
setVoices([])
|
||||
resetTestAudio()
|
||||
|
||||
if (provider !== 'none') {
|
||||
const config = configOverride || textToSpeech
|
||||
const credentialId = config?.[provider]?.credentialId
|
||||
if (credentialId) {
|
||||
loadVoicesForProvider(provider, credentialId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const loadVoicesForProvider = async (provider, credentialId) => {
|
||||
if (provider === 'none' || !credentialId) return
|
||||
|
||||
setLoadingVoices(true)
|
||||
try {
|
||||
const params = new URLSearchParams({ provider })
|
||||
params.append('credentialId', credentialId)
|
||||
|
||||
const response = await ttsApi.listVoices(params)
|
||||
|
||||
if (response.data) {
|
||||
const voicesData = await response.data
|
||||
setVoices(voicesData)
|
||||
} else {
|
||||
setVoices([])
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error loading voices:', error)
|
||||
setVoices([])
|
||||
} finally {
|
||||
setLoadingVoices(false)
|
||||
}
|
||||
}
|
||||
|
||||
const testTTS = async () => {
|
||||
if (selectedProvider === 'none' || !textToSpeech?.[selectedProvider]?.credentialId) {
|
||||
enqueueSnackbar({
|
||||
message: 'Please select a provider and configure credentials first',
|
||||
options: { variant: 'warning' }
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
setIsGeneratingTest(true)
|
||||
|
||||
try {
|
||||
const providerConfig = textToSpeech?.[selectedProvider] || {}
|
||||
const body = {
|
||||
text: 'Today is a wonderful day to build something with Flowise!',
|
||||
provider: selectedProvider,
|
||||
credentialId: providerConfig.credentialId,
|
||||
voice: providerConfig.voice,
|
||||
model: providerConfig.model
|
||||
}
|
||||
|
||||
const response = await fetch('/api/v1/text-to-speech/generate', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-request-from': 'internal'
|
||||
},
|
||||
credentials: 'include',
|
||||
body: JSON.stringify(body)
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`)
|
||||
}
|
||||
|
||||
const audioChunks = []
|
||||
const reader = response.body.getReader()
|
||||
let buffer = ''
|
||||
|
||||
let done = false
|
||||
while (!done) {
|
||||
const result = await reader.read()
|
||||
done = result.done
|
||||
if (done) break
|
||||
|
||||
const chunk = new TextDecoder().decode(result.value, { stream: true })
|
||||
buffer += chunk
|
||||
const lines = buffer.split('\n\n')
|
||||
buffer = lines.pop() || ''
|
||||
|
||||
for (const eventBlock of lines) {
|
||||
if (eventBlock.trim()) {
|
||||
const event = parseSSEEvent(eventBlock)
|
||||
if (event && event.event === 'tts_data' && event.data?.audioChunk) {
|
||||
const audioBuffer = Uint8Array.from(atob(event.data.audioChunk), (c) => c.charCodeAt(0))
|
||||
audioChunks.push(audioBuffer)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (audioChunks.length > 0) {
|
||||
// Combine all chunks into a single blob
|
||||
const totalLength = audioChunks.reduce((sum, chunk) => sum + chunk.length, 0)
|
||||
const combinedBuffer = new Uint8Array(totalLength)
|
||||
let offset = 0
|
||||
|
||||
for (const chunk of audioChunks) {
|
||||
combinedBuffer.set(chunk, offset)
|
||||
offset += chunk.length
|
||||
}
|
||||
|
||||
const audioBlob = new Blob([combinedBuffer], { type: 'audio/mpeg' })
|
||||
const audioUrl = URL.createObjectURL(audioBlob)
|
||||
|
||||
// Clean up previous audio
|
||||
if (testAudioSrc) {
|
||||
URL.revokeObjectURL(testAudioSrc)
|
||||
}
|
||||
|
||||
setTestAudioSrc(audioUrl)
|
||||
} else {
|
||||
throw new Error('No audio data received')
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error testing TTS:', error)
|
||||
enqueueSnackbar({
|
||||
message: `TTS test failed: ${error.message}`,
|
||||
options: { variant: 'error' }
|
||||
})
|
||||
} finally {
|
||||
setIsGeneratingTest(false)
|
||||
}
|
||||
}
|
||||
|
||||
const parseSSEEvent = (eventBlock) => {
|
||||
const lines = eventBlock.trim().split('\n')
|
||||
const event = { event: null, data: null }
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('event:')) {
|
||||
event.event = line.substring(6).trim()
|
||||
} else if (line.startsWith('data:')) {
|
||||
const dataStr = line.substring(5).trim()
|
||||
try {
|
||||
const parsed = JSON.parse(dataStr)
|
||||
if (parsed.data) {
|
||||
event.data = parsed.data
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing SSE data:', e)
|
||||
}
|
||||
}
|
||||
}
|
||||
return event.event ? event : null
|
||||
}
|
||||
|
||||
// Audio control functions for waveform component
|
||||
const handleTestPlay = async () => {
|
||||
// If audio already exists, just play it
|
||||
if (testAudioRef && testAudioSrc) {
|
||||
testAudioRef.play()
|
||||
setIsTestPlaying(true)
|
||||
return
|
||||
}
|
||||
|
||||
// If no audio exists, generate it first
|
||||
if (!testAudioSrc) {
|
||||
await testTTS()
|
||||
// testTTS will set the audio source, and we'll play it in the next useEffect
|
||||
}
|
||||
}
|
||||
|
||||
const handleTestPause = () => {
|
||||
if (testAudioRef) {
|
||||
testAudioRef.pause()
|
||||
setIsTestPlaying(false)
|
||||
}
|
||||
}
|
||||
|
||||
const handleTestEnded = () => {
|
||||
setIsTestPlaying(false)
|
||||
}
|
||||
|
||||
// Auto-play when audio is generated (if user clicked play)
|
||||
useEffect(() => {
|
||||
if (testAudioSrc && testAudioRef && !isTestPlaying) {
|
||||
// Small delay to ensure audio element is ready
|
||||
setTimeout(() => {
|
||||
testAudioRef.play()
|
||||
setIsTestPlaying(true)
|
||||
}, 100)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [testAudioSrc, testAudioRef])
|
||||
|
||||
useEffect(() => {
|
||||
if (dialogProps.chatflow && dialogProps.chatflow.textToSpeech) {
|
||||
try {
|
||||
const textToSpeechConfig = JSON.parse(dialogProps.chatflow.textToSpeech)
|
||||
let selectedProvider = 'none'
|
||||
Object.keys(textToSpeechProviders).forEach((key) => {
|
||||
const providerConfig = textToSpeechConfig[key]
|
||||
if (providerConfig && providerConfig.status) {
|
||||
selectedProvider = key
|
||||
}
|
||||
})
|
||||
setSelectedProvider(selectedProvider)
|
||||
setTextToSpeech(textToSpeechConfig)
|
||||
handleProviderChange(selectedProvider, textToSpeechConfig)
|
||||
} catch {
|
||||
setTextToSpeech(null)
|
||||
setSelectedProvider('none')
|
||||
}
|
||||
}
|
||||
|
||||
return () => {
|
||||
setTextToSpeech(null)
|
||||
setSelectedProvider('none')
|
||||
setVoices([])
|
||||
resetTestAudio()
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [dialogProps])
|
||||
|
||||
return (
|
||||
<>
|
||||
<Box fullWidth sx={{ mb: 1, display: 'flex', flexDirection: 'column', gap: 1 }}>
|
||||
<Typography>Providers</Typography>
|
||||
<FormControl fullWidth>
|
||||
<Select
|
||||
size='small'
|
||||
value={selectedProvider}
|
||||
onChange={(event) => handleProviderChange(event.target.value)}
|
||||
sx={{
|
||||
'& .MuiSvgIcon-root': {
|
||||
color: theme?.customization?.isDarkMode ? '#fff' : 'inherit'
|
||||
}
|
||||
}}
|
||||
>
|
||||
<MenuItem value='none'>None</MenuItem>
|
||||
{Object.values(textToSpeechProviders).map((provider) => (
|
||||
<MenuItem key={provider.name} value={provider.name}>
|
||||
{provider.label}
|
||||
</MenuItem>
|
||||
))}
|
||||
</Select>
|
||||
</FormControl>
|
||||
</Box>
|
||||
{selectedProvider !== 'none' && (
|
||||
<>
|
||||
<ListItem sx={{ mt: 3 }} alignItems='center'>
|
||||
<ListItemAvatar>
|
||||
<div
|
||||
style={{
|
||||
width: 50,
|
||||
height: 50,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'white',
|
||||
flexShrink: 0,
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center'
|
||||
}}
|
||||
>
|
||||
<img
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
padding: 10,
|
||||
objectFit: 'contain'
|
||||
}}
|
||||
alt='TTS Provider'
|
||||
src={textToSpeechProviders[selectedProvider].icon}
|
||||
/>
|
||||
</div>
|
||||
</ListItemAvatar>
|
||||
<ListItemText
|
||||
sx={{ ml: 1 }}
|
||||
primary={textToSpeechProviders[selectedProvider].label}
|
||||
secondary={
|
||||
<a
|
||||
target='_blank'
|
||||
rel='noreferrer'
|
||||
href={textToSpeechProviders[selectedProvider].url}
|
||||
style={{
|
||||
color: theme?.customization?.isDarkMode ? '#90caf9' : '#1976d2',
|
||||
textDecoration: 'underline'
|
||||
}}
|
||||
>
|
||||
{textToSpeechProviders[selectedProvider].url}
|
||||
</a>
|
||||
}
|
||||
/>
|
||||
</ListItem>
|
||||
{textToSpeechProviders[selectedProvider].inputs.map((inputParam) => (
|
||||
<Box key={`${selectedProvider}-${inputParam.name}`} sx={{ p: 2 }}>
|
||||
<div style={{ display: 'flex', flexDirection: 'row' }}>
|
||||
<Typography>
|
||||
{inputParam.label}
|
||||
{!inputParam.optional && <span style={{ color: 'red' }}> *</span>}
|
||||
{inputParam.description && (
|
||||
<TooltipWithParser style={{ marginLeft: 10 }} title={inputParam.description} />
|
||||
)}
|
||||
</Typography>
|
||||
</div>
|
||||
{inputParam.type === 'credential' && (
|
||||
<CredentialInputHandler
|
||||
key={textToSpeech?.[selectedProvider]?.credentialId}
|
||||
data={
|
||||
textToSpeech?.[selectedProvider]?.credentialId
|
||||
? { credential: textToSpeech?.[selectedProvider]?.credentialId }
|
||||
: {}
|
||||
}
|
||||
inputParam={inputParam}
|
||||
onSelect={(newValue) => {
|
||||
setValue(newValue, selectedProvider, 'credentialId')
|
||||
// Load voices when credential is updated
|
||||
if (newValue && selectedProvider !== 'none') {
|
||||
setTimeout(() => loadVoicesForProvider(selectedProvider, newValue), 100)
|
||||
}
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
{inputParam.type === 'boolean' && (
|
||||
<SwitchInput
|
||||
onChange={(newValue) => setValue(newValue, selectedProvider, inputParam.name)}
|
||||
value={
|
||||
textToSpeech?.[selectedProvider]
|
||||
? textToSpeech[selectedProvider][inputParam.name]
|
||||
: inputParam.default ?? false
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{(inputParam.type === 'string' || inputParam.type === 'password' || inputParam.type === 'number') && (
|
||||
<Input
|
||||
inputParam={inputParam}
|
||||
onChange={(newValue) => setValue(newValue, selectedProvider, inputParam.name)}
|
||||
value={
|
||||
textToSpeech?.[selectedProvider]
|
||||
? textToSpeech[selectedProvider][inputParam.name]
|
||||
: inputParam.default ?? ''
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{inputParam.type === 'options' && (
|
||||
<Dropdown
|
||||
name={inputParam.name}
|
||||
options={inputParam.options}
|
||||
onSelect={(newValue) => setValue(newValue, selectedProvider, inputParam.name)}
|
||||
value={
|
||||
textToSpeech?.[selectedProvider]
|
||||
? textToSpeech[selectedProvider][inputParam.name]
|
||||
: inputParam.default ?? 'choose an option'
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{inputParam.type === 'voice_select' && (
|
||||
<Autocomplete
|
||||
size='small'
|
||||
sx={{ mt: 1 }}
|
||||
options={voices}
|
||||
loading={loadingVoices}
|
||||
getOptionLabel={(option) => option.name || ''}
|
||||
value={
|
||||
voices.find(
|
||||
(voice) =>
|
||||
voice.id === (textToSpeech?.[selectedProvider]?.[inputParam.name] || inputParam.default)
|
||||
) || null
|
||||
}
|
||||
onChange={(event, newValue) => {
|
||||
setValue(newValue ? newValue.id : '', selectedProvider, inputParam.name)
|
||||
}}
|
||||
renderInput={(params) => (
|
||||
<TextField
|
||||
{...params}
|
||||
placeholder={loadingVoices ? 'Loading voices...' : 'Choose a voice'}
|
||||
InputProps={{
|
||||
...params.InputProps,
|
||||
endAdornment: (
|
||||
<>
|
||||
{loadingVoices ? <CircularProgress color='inherit' size={20} /> : null}
|
||||
{params.InputProps.endAdornment}
|
||||
</>
|
||||
)
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
disabled={loadingVoices || !textToSpeech?.[selectedProvider]?.credentialId}
|
||||
/>
|
||||
)}
|
||||
</Box>
|
||||
))}
|
||||
|
||||
{/* Auto-play Toggle */}
|
||||
<Box sx={{ p: 2 }}>
|
||||
<div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center' }}>
|
||||
<Typography>
|
||||
Automatically play audio
|
||||
<TooltipWithParser
|
||||
style={{ marginLeft: 10 }}
|
||||
title='When enabled, bot responses will be automatically converted to speech and played'
|
||||
/>
|
||||
</Typography>
|
||||
</div>
|
||||
<SwitchInput
|
||||
onChange={(newValue) => setValue(newValue, selectedProvider, 'autoPlay')}
|
||||
value={textToSpeech?.[selectedProvider] ? textToSpeech[selectedProvider].autoPlay ?? false : false}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
{/* Test Voice Section */}
|
||||
<Box sx={{ p: 2 }}>
|
||||
<Typography variant='h6' sx={{ mb: 2, display: 'flex', alignItems: 'center', gap: 1 }}>
|
||||
<IconVolume size={20} />
|
||||
Test Voice
|
||||
</Typography>
|
||||
|
||||
<Typography variant='body2' color='textSecondary' sx={{ mb: 2 }}>
|
||||
Test text: "Today is a wonderful day to build something with Flowise!"
|
||||
</Typography>
|
||||
|
||||
<AudioWaveform
|
||||
audioSrc={testAudioSrc}
|
||||
onPlay={handleTestPlay}
|
||||
onPause={handleTestPause}
|
||||
onEnded={handleTestEnded}
|
||||
isPlaying={isTestPlaying}
|
||||
isGenerating={isGeneratingTest}
|
||||
disabled={!textToSpeech?.[selectedProvider]?.credentialId}
|
||||
externalAudioRef={testAudioRef}
|
||||
resetProgress={resetWaveform}
|
||||
/>
|
||||
|
||||
{/* Hidden audio element for waveform control */}
|
||||
{testAudioSrc && (
|
||||
<audio
|
||||
ref={(ref) => setTestAudioRef(ref)}
|
||||
src={testAudioSrc}
|
||||
onPlay={() => setIsTestPlaying(true)}
|
||||
onPause={() => setIsTestPlaying(false)}
|
||||
onEnded={handleTestEnded}
|
||||
style={{ display: 'none' }}
|
||||
>
|
||||
<track kind='captions' />
|
||||
</audio>
|
||||
)}
|
||||
</Box>
|
||||
</>
|
||||
)}
|
||||
<StyledButton
|
||||
style={{ marginBottom: 10, marginTop: 10 }}
|
||||
disabled={selectedProvider !== 'none' && !textToSpeech?.[selectedProvider]?.credentialId}
|
||||
variant='contained'
|
||||
onClick={onSave}
|
||||
>
|
||||
Save
|
||||
</StyledButton>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
TextToSpeech.propTypes = {
|
||||
dialogProps: PropTypes.object
|
||||
}
|
||||
|
||||
export default TextToSpeech
|
||||
@@ -38,7 +38,8 @@ import {
|
||||
IconSquareFilled,
|
||||
IconCheck,
|
||||
IconPaperclip,
|
||||
IconSparkles
|
||||
IconSparkles,
|
||||
IconVolume
|
||||
} from '@tabler/icons-react'
|
||||
import robotPNG from '@/assets/images/robot.png'
|
||||
import userPNG from '@/assets/images/account.png'
|
||||
@@ -72,6 +73,7 @@ import attachmentsApi from '@/api/attachments'
|
||||
import chatmessagefeedbackApi from '@/api/chatmessagefeedback'
|
||||
import leadsApi from '@/api/lead'
|
||||
import executionsApi from '@/api/executions'
|
||||
import ttsApi from '@/api/tts'
|
||||
|
||||
// Hooks
|
||||
import useApi from '@/hooks/useApi'
|
||||
@@ -251,6 +253,27 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
|
||||
const [isConfigLoading, setIsConfigLoading] = useState(true)
|
||||
|
||||
// TTS state
|
||||
const [isTTSLoading, setIsTTSLoading] = useState({})
|
||||
const [isTTSPlaying, setIsTTSPlaying] = useState({})
|
||||
const [ttsAudio, setTtsAudio] = useState({})
|
||||
const [isTTSEnabled, setIsTTSEnabled] = useState(false)
|
||||
|
||||
// TTS streaming state
|
||||
const [ttsStreamingState, setTtsStreamingState] = useState({
|
||||
mediaSource: null,
|
||||
sourceBuffer: null,
|
||||
audio: null,
|
||||
chunkQueue: [],
|
||||
isBuffering: false,
|
||||
audioFormat: null,
|
||||
abortController: null
|
||||
})
|
||||
|
||||
// Ref to prevent auto-scroll during TTS actions (using ref to avoid re-renders)
|
||||
const isTTSActionRef = useRef(false)
|
||||
const ttsTimeoutRef = useRef(null)
|
||||
|
||||
const isFileAllowedForUpload = (file) => {
|
||||
const constraints = getAllowChatFlowUploads.data
|
||||
/**
|
||||
@@ -463,7 +486,12 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
const handleAbort = async () => {
|
||||
setIsMessageStopping(true)
|
||||
try {
|
||||
// Stop all TTS streams first
|
||||
await handleTTSAbortAll()
|
||||
stopAllTTS()
|
||||
|
||||
await chatmessageApi.abortMessage(chatflowid, chatId)
|
||||
setIsMessageStopping(false)
|
||||
} catch (error) {
|
||||
setIsMessageStopping(false)
|
||||
enqueueSnackbar({
|
||||
@@ -536,6 +564,22 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to manage TTS action flag
|
||||
const setTTSAction = (isActive) => {
|
||||
isTTSActionRef.current = isActive
|
||||
if (ttsTimeoutRef.current) {
|
||||
clearTimeout(ttsTimeoutRef.current)
|
||||
ttsTimeoutRef.current = null
|
||||
}
|
||||
if (isActive) {
|
||||
// Reset the flag after a longer delay to ensure all state changes are complete
|
||||
ttsTimeoutRef.current = setTimeout(() => {
|
||||
isTTSActionRef.current = false
|
||||
ttsTimeoutRef.current = null
|
||||
}, 300)
|
||||
}
|
||||
}
|
||||
|
||||
const onChange = useCallback((e) => setUserInput(e.target.value), [setUserInput])
|
||||
|
||||
const updateLastMessage = (text) => {
|
||||
@@ -949,6 +993,7 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
setLoading(false)
|
||||
setUserInput('')
|
||||
setUploadedFiles([])
|
||||
|
||||
setTimeout(() => {
|
||||
inputRef.current?.focus()
|
||||
scrollToBottom()
|
||||
@@ -1027,6 +1072,18 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
abortMessage(payload.data)
|
||||
closeResponse()
|
||||
break
|
||||
case 'tts_start':
|
||||
handleTTSStart(payload.data)
|
||||
break
|
||||
case 'tts_data':
|
||||
handleTTSDataChunk(payload.data.audioChunk)
|
||||
break
|
||||
case 'tts_end':
|
||||
handleTTSEnd()
|
||||
break
|
||||
case 'tts_abort':
|
||||
handleTTSAbort(payload.data)
|
||||
break
|
||||
case 'end':
|
||||
setLocalStorageChatflow(chatflowid, chatId)
|
||||
closeResponse()
|
||||
@@ -1293,6 +1350,30 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if TTS is configured
|
||||
if (getChatflowConfig.data && getChatflowConfig.data.textToSpeech) {
|
||||
try {
|
||||
const ttsConfig =
|
||||
typeof getChatflowConfig.data.textToSpeech === 'string'
|
||||
? JSON.parse(getChatflowConfig.data.textToSpeech)
|
||||
: getChatflowConfig.data.textToSpeech
|
||||
|
||||
let isEnabled = false
|
||||
if (ttsConfig) {
|
||||
Object.keys(ttsConfig).forEach((provider) => {
|
||||
if (provider !== 'none' && ttsConfig?.[provider]?.status) {
|
||||
isEnabled = true
|
||||
}
|
||||
})
|
||||
}
|
||||
setIsTTSEnabled(isEnabled)
|
||||
} catch (error) {
|
||||
setIsTTSEnabled(false)
|
||||
}
|
||||
} else {
|
||||
setIsTTSEnabled(false)
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getChatflowConfig.data])
|
||||
|
||||
@@ -1313,9 +1394,11 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
}
|
||||
}, [isChatFlowAvailableForRAGFileUploads, fullFileUpload])
|
||||
|
||||
// Auto scroll chat to bottom
|
||||
// Auto scroll chat to bottom (but not during TTS actions)
|
||||
useEffect(() => {
|
||||
scrollToBottom()
|
||||
if (!isTTSActionRef.current) {
|
||||
scrollToBottom()
|
||||
}
|
||||
}, [messages])
|
||||
|
||||
useEffect(() => {
|
||||
@@ -1497,9 +1580,451 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
return allMessages
|
||||
})
|
||||
}
|
||||
|
||||
setIsLeadSaving(false)
|
||||
}
|
||||
|
||||
const cleanupTTSForMessage = (messageId) => {
|
||||
if (ttsAudio[messageId]) {
|
||||
ttsAudio[messageId].pause()
|
||||
ttsAudio[messageId].currentTime = 0
|
||||
setTtsAudio((prev) => {
|
||||
const newState = { ...prev }
|
||||
delete newState[messageId]
|
||||
return newState
|
||||
})
|
||||
}
|
||||
|
||||
if (ttsStreamingState.audio) {
|
||||
ttsStreamingState.audio.pause()
|
||||
cleanupTTSStreaming()
|
||||
}
|
||||
|
||||
setIsTTSPlaying((prev) => {
|
||||
const newState = { ...prev }
|
||||
delete newState[messageId]
|
||||
return newState
|
||||
})
|
||||
|
||||
setIsTTSLoading((prev) => {
|
||||
const newState = { ...prev }
|
||||
delete newState[messageId]
|
||||
return newState
|
||||
})
|
||||
}
|
||||
|
||||
const handleTTSStop = async (messageId) => {
|
||||
setTTSAction(true)
|
||||
await ttsApi.abortTTS({ chatflowId: chatflowid, chatId, chatMessageId: messageId })
|
||||
cleanupTTSForMessage(messageId)
|
||||
setIsMessageStopping(false)
|
||||
}
|
||||
|
||||
const stopAllTTS = () => {
|
||||
Object.keys(ttsAudio).forEach((messageId) => {
|
||||
if (ttsAudio[messageId]) {
|
||||
ttsAudio[messageId].pause()
|
||||
ttsAudio[messageId].currentTime = 0
|
||||
}
|
||||
})
|
||||
setTtsAudio({})
|
||||
|
||||
if (ttsStreamingState.abortController) {
|
||||
ttsStreamingState.abortController.abort()
|
||||
}
|
||||
|
||||
if (ttsStreamingState.audio) {
|
||||
ttsStreamingState.audio.pause()
|
||||
cleanupTTSStreaming()
|
||||
}
|
||||
|
||||
setIsTTSPlaying({})
|
||||
setIsTTSLoading({})
|
||||
}
|
||||
|
||||
const handleTTSClick = async (messageId, messageText) => {
|
||||
if (isTTSLoading[messageId]) return
|
||||
|
||||
if (isTTSPlaying[messageId] || ttsAudio[messageId]) {
|
||||
handleTTSStop(messageId)
|
||||
return
|
||||
}
|
||||
|
||||
setTTSAction(true)
|
||||
|
||||
// abort all ongoing streams and clear audio sources
|
||||
await handleTTSAbortAll()
|
||||
stopAllTTS()
|
||||
|
||||
handleTTSStart({ chatMessageId: messageId, format: 'mp3' })
|
||||
try {
|
||||
const abortController = new AbortController()
|
||||
setTtsStreamingState((prev) => ({ ...prev, abortController }))
|
||||
|
||||
const response = await fetch('/api/v1/text-to-speech/generate', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-request-from': 'internal'
|
||||
},
|
||||
credentials: 'include',
|
||||
signal: abortController.signal,
|
||||
body: JSON.stringify({
|
||||
chatflowId: chatflowid,
|
||||
chatId: chatId,
|
||||
chatMessageId: messageId,
|
||||
text: messageText
|
||||
})
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`TTS request failed: ${response.status}`)
|
||||
}
|
||||
|
||||
const reader = response.body.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ''
|
||||
|
||||
let done = false
|
||||
while (!done) {
|
||||
if (abortController.signal.aborted) {
|
||||
break
|
||||
}
|
||||
|
||||
const result = await reader.read()
|
||||
done = result.done
|
||||
if (done) {
|
||||
break
|
||||
}
|
||||
const value = result.value
|
||||
const chunk = decoder.decode(value, { stream: true })
|
||||
buffer += chunk
|
||||
|
||||
const lines = buffer.split('\n\n')
|
||||
buffer = lines.pop() || ''
|
||||
|
||||
for (const eventBlock of lines) {
|
||||
if (eventBlock.trim()) {
|
||||
const event = parseSSEEvent(eventBlock)
|
||||
if (event) {
|
||||
switch (event.event) {
|
||||
case 'tts_start':
|
||||
break
|
||||
case 'tts_data':
|
||||
if (!abortController.signal.aborted) {
|
||||
handleTTSDataChunk(event.data.audioChunk)
|
||||
}
|
||||
break
|
||||
case 'tts_end':
|
||||
if (!abortController.signal.aborted) {
|
||||
handleTTSEnd()
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.name === 'AbortError') {
|
||||
console.error('TTS request was aborted')
|
||||
} else {
|
||||
console.error('Error with TTS:', error)
|
||||
enqueueSnackbar({
|
||||
message: `TTS failed: ${error.message}`,
|
||||
options: { variant: 'error' }
|
||||
})
|
||||
}
|
||||
} finally {
|
||||
setIsTTSLoading((prev) => {
|
||||
const newState = { ...prev }
|
||||
delete newState[messageId]
|
||||
return newState
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const parseSSEEvent = (eventBlock) => {
|
||||
const lines = eventBlock.split('\n')
|
||||
const event = {}
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('event:')) {
|
||||
event.event = line.substring(6).trim()
|
||||
} else if (line.startsWith('data:')) {
|
||||
const dataStr = line.substring(5).trim()
|
||||
try {
|
||||
const parsed = JSON.parse(dataStr)
|
||||
if (parsed.data) {
|
||||
event.data = parsed.data
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing SSE data:', e, 'Raw data:', dataStr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return event.event ? event : null
|
||||
}
|
||||
|
||||
const initializeTTSStreaming = (data) => {
|
||||
try {
|
||||
const mediaSource = new MediaSource()
|
||||
const audio = new Audio()
|
||||
audio.src = URL.createObjectURL(mediaSource)
|
||||
|
||||
mediaSource.addEventListener('sourceopen', () => {
|
||||
try {
|
||||
const mimeType = data.format === 'mp3' ? 'audio/mpeg' : 'audio/mpeg'
|
||||
const sourceBuffer = mediaSource.addSourceBuffer(mimeType)
|
||||
|
||||
setTtsStreamingState((prevState) => ({
|
||||
...prevState,
|
||||
mediaSource,
|
||||
sourceBuffer,
|
||||
audio
|
||||
}))
|
||||
|
||||
audio.play().catch((playError) => {
|
||||
console.error('Error starting audio playback:', playError)
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error setting up source buffer:', error)
|
||||
console.error('MediaSource readyState:', mediaSource.readyState)
|
||||
console.error('Requested MIME type:', mimeType)
|
||||
}
|
||||
})
|
||||
|
||||
audio.addEventListener('playing', () => {
|
||||
setIsTTSLoading((prevState) => {
|
||||
const newState = { ...prevState }
|
||||
delete newState[data.chatMessageId]
|
||||
return newState
|
||||
})
|
||||
setIsTTSPlaying((prevState) => ({
|
||||
...prevState,
|
||||
[data.chatMessageId]: true
|
||||
}))
|
||||
})
|
||||
|
||||
audio.addEventListener('ended', () => {
|
||||
setIsTTSPlaying((prevState) => {
|
||||
const newState = { ...prevState }
|
||||
delete newState[data.chatMessageId]
|
||||
return newState
|
||||
})
|
||||
cleanupTTSStreaming()
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error initializing TTS streaming:', error)
|
||||
}
|
||||
}
|
||||
|
||||
const cleanupTTSStreaming = () => {
|
||||
setTtsStreamingState((prevState) => {
|
||||
if (prevState.abortController) {
|
||||
prevState.abortController.abort()
|
||||
}
|
||||
|
||||
if (prevState.audio) {
|
||||
prevState.audio.pause()
|
||||
prevState.audio.removeAttribute('src')
|
||||
if (prevState.audio.src) {
|
||||
URL.revokeObjectURL(prevState.audio.src)
|
||||
}
|
||||
}
|
||||
|
||||
if (prevState.mediaSource) {
|
||||
if (prevState.mediaSource.readyState === 'open') {
|
||||
try {
|
||||
prevState.mediaSource.endOfStream()
|
||||
} catch (e) {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
}
|
||||
prevState.mediaSource.removeEventListener('sourceopen', () => {})
|
||||
}
|
||||
|
||||
return {
|
||||
mediaSource: null,
|
||||
sourceBuffer: null,
|
||||
audio: null,
|
||||
chunkQueue: [],
|
||||
isBuffering: false,
|
||||
audioFormat: null,
|
||||
abortController: null
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const processChunkQueue = () => {
|
||||
setTtsStreamingState((prevState) => {
|
||||
if (!prevState.sourceBuffer || prevState.sourceBuffer.updating || prevState.chunkQueue.length === 0) {
|
||||
return prevState
|
||||
}
|
||||
|
||||
const chunk = prevState.chunkQueue.shift()
|
||||
|
||||
try {
|
||||
prevState.sourceBuffer.appendBuffer(chunk)
|
||||
return {
|
||||
...prevState,
|
||||
chunkQueue: [...prevState.chunkQueue],
|
||||
isBuffering: true
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error appending chunk to buffer:', error)
|
||||
return prevState
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const handleTTSStart = (data) => {
|
||||
setTTSAction(true)
|
||||
|
||||
// Stop all existing TTS audio before starting new stream
|
||||
stopAllTTS()
|
||||
|
||||
setIsTTSLoading((prevState) => ({
|
||||
...prevState,
|
||||
[data.chatMessageId]: true
|
||||
}))
|
||||
setMessages((prevMessages) => {
|
||||
const allMessages = [...cloneDeep(prevMessages)]
|
||||
const lastMessage = allMessages[allMessages.length - 1]
|
||||
if (lastMessage.type === 'userMessage') return allMessages
|
||||
if (lastMessage.id) return allMessages
|
||||
allMessages[allMessages.length - 1].id = data.chatMessageId
|
||||
return allMessages
|
||||
})
|
||||
setTtsStreamingState({
|
||||
mediaSource: null,
|
||||
sourceBuffer: null,
|
||||
audio: null,
|
||||
chunkQueue: [],
|
||||
isBuffering: false,
|
||||
audioFormat: data.format,
|
||||
abortController: null
|
||||
})
|
||||
|
||||
setTimeout(() => initializeTTSStreaming(data), 0)
|
||||
}
|
||||
|
||||
const handleTTSDataChunk = (base64Data) => {
|
||||
try {
|
||||
const audioBuffer = Uint8Array.from(atob(base64Data), (c) => c.charCodeAt(0))
|
||||
|
||||
setTtsStreamingState((prevState) => {
|
||||
const newState = {
|
||||
...prevState,
|
||||
chunkQueue: [...prevState.chunkQueue, audioBuffer]
|
||||
}
|
||||
|
||||
if (prevState.sourceBuffer && !prevState.sourceBuffer.updating) {
|
||||
setTimeout(() => processChunkQueue(), 0)
|
||||
}
|
||||
|
||||
return newState
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error handling TTS data chunk:', error)
|
||||
}
|
||||
}
|
||||
|
||||
const handleTTSEnd = () => {
|
||||
setTtsStreamingState((prevState) => {
|
||||
if (prevState.mediaSource && prevState.mediaSource.readyState === 'open') {
|
||||
try {
|
||||
if (prevState.sourceBuffer && prevState.chunkQueue.length > 0 && !prevState.sourceBuffer.updating) {
|
||||
const remainingChunks = [...prevState.chunkQueue]
|
||||
remainingChunks.forEach((chunk, index) => {
|
||||
setTimeout(() => {
|
||||
if (prevState.sourceBuffer && !prevState.sourceBuffer.updating) {
|
||||
try {
|
||||
prevState.sourceBuffer.appendBuffer(chunk)
|
||||
if (index === remainingChunks.length - 1) {
|
||||
setTimeout(() => {
|
||||
if (prevState.mediaSource && prevState.mediaSource.readyState === 'open') {
|
||||
prevState.mediaSource.endOfStream()
|
||||
}
|
||||
}, 100)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error appending remaining chunk:', error)
|
||||
}
|
||||
}
|
||||
}, index * 50)
|
||||
})
|
||||
return {
|
||||
...prevState,
|
||||
chunkQueue: []
|
||||
}
|
||||
}
|
||||
|
||||
if (prevState.sourceBuffer && !prevState.sourceBuffer.updating) {
|
||||
prevState.mediaSource.endOfStream()
|
||||
} else if (prevState.sourceBuffer) {
|
||||
prevState.sourceBuffer.addEventListener(
|
||||
'updateend',
|
||||
() => {
|
||||
if (prevState.mediaSource && prevState.mediaSource.readyState === 'open') {
|
||||
prevState.mediaSource.endOfStream()
|
||||
}
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error ending TTS stream:', error)
|
||||
}
|
||||
}
|
||||
return prevState
|
||||
})
|
||||
}
|
||||
|
||||
const handleTTSAbort = (data) => {
|
||||
const messageId = data.chatMessageId
|
||||
cleanupTTSForMessage(messageId)
|
||||
}
|
||||
|
||||
const handleTTSAbortAll = async () => {
|
||||
const activeTTSMessages = Object.keys(isTTSLoading).concat(Object.keys(isTTSPlaying))
|
||||
for (const messageId of activeTTSMessages) {
|
||||
await ttsApi.abortTTS({ chatflowId: chatflowid, chatId, chatMessageId: messageId })
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (ttsStreamingState.sourceBuffer) {
|
||||
const sourceBuffer = ttsStreamingState.sourceBuffer
|
||||
|
||||
const handleUpdateEnd = () => {
|
||||
setTtsStreamingState((prevState) => ({
|
||||
...prevState,
|
||||
isBuffering: false
|
||||
}))
|
||||
setTimeout(() => processChunkQueue(), 0)
|
||||
}
|
||||
|
||||
sourceBuffer.addEventListener('updateend', handleUpdateEnd)
|
||||
|
||||
return () => {
|
||||
sourceBuffer.removeEventListener('updateend', handleUpdateEnd)
|
||||
}
|
||||
}
|
||||
}, [ttsStreamingState.sourceBuffer])
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
cleanupTTSStreaming()
|
||||
// Cleanup TTS timeout on unmount
|
||||
if (ttsTimeoutRef.current) {
|
||||
clearTimeout(ttsTimeoutRef.current)
|
||||
ttsTimeoutRef.current = null
|
||||
}
|
||||
}
|
||||
}, [])
|
||||
|
||||
const getInputDisabled = () => {
|
||||
return (
|
||||
loading ||
|
||||
@@ -2151,7 +2676,7 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{message.type === 'apiMessage' && message.id && chatFeedbackStatus ? (
|
||||
{message.type === 'apiMessage' && message.id ? (
|
||||
<>
|
||||
<Box
|
||||
sx={{
|
||||
@@ -2161,25 +2686,62 @@ const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setP
|
||||
gap: 1
|
||||
}}
|
||||
>
|
||||
<CopyToClipboardButton onClick={() => copyMessageToClipboard(message.message)} />
|
||||
{!message.feedback ||
|
||||
message.feedback.rating === '' ||
|
||||
message.feedback.rating === 'THUMBS_UP' ? (
|
||||
<ThumbsUpButton
|
||||
isDisabled={message.feedback && message.feedback.rating === 'THUMBS_UP'}
|
||||
rating={message.feedback ? message.feedback.rating : ''}
|
||||
onClick={() => onThumbsUpClick(message.id)}
|
||||
/>
|
||||
) : null}
|
||||
{!message.feedback ||
|
||||
message.feedback.rating === '' ||
|
||||
message.feedback.rating === 'THUMBS_DOWN' ? (
|
||||
<ThumbsDownButton
|
||||
isDisabled={message.feedback && message.feedback.rating === 'THUMBS_DOWN'}
|
||||
rating={message.feedback ? message.feedback.rating : ''}
|
||||
onClick={() => onThumbsDownClick(message.id)}
|
||||
/>
|
||||
) : null}
|
||||
{isTTSEnabled && (
|
||||
<IconButton
|
||||
size='small'
|
||||
onClick={() =>
|
||||
isTTSPlaying[message.id]
|
||||
? handleTTSStop(message.id)
|
||||
: handleTTSClick(message.id, message.message)
|
||||
}
|
||||
disabled={isTTSLoading[message.id]}
|
||||
sx={{
|
||||
backgroundColor: ttsAudio[message.id] ? 'primary.main' : 'transparent',
|
||||
color: ttsAudio[message.id] ? 'white' : 'inherit',
|
||||
'&:hover': {
|
||||
backgroundColor: ttsAudio[message.id] ? 'primary.dark' : 'action.hover'
|
||||
}
|
||||
}}
|
||||
>
|
||||
{isTTSLoading[message.id] ? (
|
||||
<CircularProgress size={16} />
|
||||
) : isTTSPlaying[message.id] ? (
|
||||
<IconCircleDot style={{ width: '20px', height: '20px' }} color={'red'} />
|
||||
) : (
|
||||
<IconVolume
|
||||
style={{ width: '20px', height: '20px' }}
|
||||
color={customization.isDarkMode ? 'white' : '#1e88e5'}
|
||||
/>
|
||||
)}
|
||||
</IconButton>
|
||||
)}
|
||||
{chatFeedbackStatus && (
|
||||
<>
|
||||
<CopyToClipboardButton
|
||||
onClick={() => copyMessageToClipboard(message.message)}
|
||||
/>
|
||||
{!message.feedback ||
|
||||
message.feedback.rating === '' ||
|
||||
message.feedback.rating === 'THUMBS_UP' ? (
|
||||
<ThumbsUpButton
|
||||
isDisabled={message.feedback && message.feedback.rating === 'THUMBS_UP'}
|
||||
rating={message.feedback ? message.feedback.rating : ''}
|
||||
onClick={() => onThumbsUpClick(message.id)}
|
||||
/>
|
||||
) : null}
|
||||
{!message.feedback ||
|
||||
message.feedback.rating === '' ||
|
||||
message.feedback.rating === 'THUMBS_DOWN' ? (
|
||||
<ThumbsDownButton
|
||||
isDisabled={
|
||||
message.feedback && message.feedback.rating === 'THUMBS_DOWN'
|
||||
}
|
||||
rating={message.feedback ? message.feedback.rating : ''}
|
||||
onClick={() => onThumbsDownClick(message.id)}
|
||||
/>
|
||||
) : null}
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
</>
|
||||
) : null}
|
||||
|
||||
Reference in New Issue
Block a user