SpeechToText: Adding SpeechToText at the Chatflow level.

This commit is contained in:
vinodkiran
2024-01-27 20:26:46 +05:30
parent 3ce22d0dde
commit d61e3d53ec
12 changed files with 418 additions and 5 deletions
@@ -31,6 +31,9 @@ export class ChatFlow implements IChatFlow {
@Column({ nullable: true, type: 'text' })
analytic?: string
@Column({ nullable: true, type: 'text' })
speechToText?: string
@CreateDateColumn()
createdDate: Date
@@ -0,0 +1,12 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddSpeechToText1706364937060 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const columnExists = await queryRunner.hasColumn('chat_flow', 'speechToText')
if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_flow\` ADD COLUMN \`speechToText\` TEXT;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE \`chat_flow\` DROP COLUMN \`speechToText\`;`)
}
}
@@ -12,6 +12,7 @@ import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryT
import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
export const mysqlMigrations = [
Init1693840429259,
@@ -27,5 +28,6 @@ export const mysqlMigrations = [
AddCategoryToChatFlow1699900910291,
AddFileAnnotationsToChatMessage1700271021237,
AddFileUploadsToChatMessage1701788586491,
AddVariableEntity1699325775451
AddVariableEntity1699325775451,
AddSpeechToText1706364937060
]
@@ -0,0 +1,11 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddSpeechToText1706364937060 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN IF NOT EXISTS "speechToText" TEXT;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "speechToText";`)
}
}
@@ -12,6 +12,7 @@ import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryT
import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
export const postgresMigrations = [
Init1693891895163,
@@ -27,5 +28,6 @@ export const postgresMigrations = [
AddCategoryToChatFlow1699900910291,
AddFileAnnotationsToChatMessage1700271021237,
AddFileUploadsToChatMessage1701788586491,
AddVariableEntity1699325775451
AddVariableEntity1699325775451,
AddSpeechToText1706364937060
]
@@ -0,0 +1,11 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddSpeechToText1706364937060 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN "speechToText" TEXT;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "speechToText";`)
}
}
@@ -12,6 +12,7 @@ import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryT
import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
export const sqliteMigrations = [
Init1693835579790,
@@ -27,5 +28,6 @@ export const sqliteMigrations = [
AddCategoryToChatFlow1699900910291,
AddFileAnnotationsToChatMessage1700271021237,
AddFileUploadsToChatMessage1701788586491,
AddVariableEntity1699325775451
AddVariableEntity1699325775451,
AddSpeechToText1706364937060
]
Binary file not shown.

After

Width:  |  Height:  |  Size: 8.5 KiB

+8
View File
@@ -0,0 +1,8 @@
<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M14.9814 25.2665C15.353 26.2672 16.0645 27.1054 16.9914 27.6347C17.9183 28.164 19.0018 28.3507 20.0524 28.1622C21.103 27.9737 22.054 27.422 22.7391 26.6034C23.4242 25.7849 23.7998 24.7517 23.8004 23.6842V17.5533C23.8004 17.1909 23.6043 16.8569 23.2879 16.6802L15.9995 12.6108" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M7.17701 19.5848C6.49568 20.4069 6.12505 21.4424 6.12993 22.5101C6.13481 23.5779 6.51489 24.6099 7.2037 25.4258C7.89252 26.2416 8.84622 26.7893 9.89802 26.9732C10.9498 27.157 12.0328 26.9653 12.9575 26.4314L18.1044 23.4263C18.4114 23.247 18.6002 22.9182 18.6002 22.5627V14.106" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M8.19877 9.98459C6.39026 9.67775 4.57524 10.4982 3.60403 12.1806C3.00524 13.2178 2.84295 14.4504 3.15284 15.6073C3.46273 16.7642 4.21943 17.7507 5.25652 18.3498L10.3049 21.3269C10.6109 21.5074 10.9898 21.5119 11.3001 21.3388L18.6 17.2655" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M17.0172 6.06585C16.6456 5.06522 15.9342 4.227 15.0072 3.6977C14.0803 3.1684 12.9969 2.98168 11.9462 3.17018C10.8956 3.35869 9.94464 3.91042 9.25954 4.72895C8.57444 5.54747 8.19879 6.58074 8.19824 7.64814V13.6575C8.19824 14.0154 8.38951 14.346 8.69977 14.5244L15.9992 18.7215" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M24.8216 11.7476C25.5029 10.9255 25.8735 9.89004 25.8687 8.8223C25.8638 7.75457 25.4837 6.72253 24.7949 5.90667C24.1061 5.09082 23.1524 4.54308 22.1006 4.35924C21.0488 4.17541 19.9658 4.36718 19.0411 4.90101L13.8942 7.90613C13.5872 8.08539 13.3984 8.41418 13.3984 8.76971V17.2265" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M23.7997 21.2595C25.6082 21.5663 27.4232 20.7459 28.3944 19.0635C28.9932 18.0263 29.1555 16.7937 28.8456 15.6368C28.5357 14.4799 27.779 13.4934 26.7419 12.8943L21.6409 9.91752C21.3316 9.73703 20.9494 9.7357 20.6388 9.91405L17.2696 11.849L13.3984 14.0723" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
</svg>

After

Width:  |  Height:  |  Size: 2.2 KiB

+18 -2
View File
@@ -1,8 +1,17 @@
// assets
import { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff } from '@tabler/icons'
import {
IconTrash,
IconFileUpload,
IconFileExport,
IconCopy,
IconSearch,
IconMessage,
IconPictureInPictureOff,
IconMicrophone
} from '@tabler/icons'
// constant
const icons = { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff }
const icons = { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff, IconMicrophone }
// ==============================|| SETTINGS MENU ITEMS ||============================== //
@@ -25,6 +34,13 @@ const settings = {
url: '',
icon: icons.IconMessage
},
{
id: 'enableSpeechToText',
title: 'Enable Speech to Text',
type: 'item',
url: '',
icon: icons.IconMicrophone
},
{
id: 'duplicateChatflow',
title: 'Duplicate Chatflow',
@@ -0,0 +1,332 @@
import { createPortal } from 'react-dom'
import { useDispatch } from 'react-redux'
import { useState, useEffect } from 'react'
import PropTypes from 'prop-types'
import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction, SET_CHATFLOW } from 'store/actions'
// material-ui
import {
Typography,
Box,
Button,
Dialog,
DialogContent,
DialogTitle,
DialogActions,
Accordion,
AccordionSummary,
AccordionDetails,
ListItem,
ListItemAvatar,
ListItemText
} from '@mui/material'
import ExpandMoreIcon from '@mui/icons-material/ExpandMore'
import { IconX } from '@tabler/icons'
// Project import
import CredentialInputHandler from 'views/canvas/CredentialInputHandler'
import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser'
import { SwitchInput } from 'ui-component/switch/Switch'
import { Input } from 'ui-component/input/Input'
import { StyledButton } from 'ui-component/button/StyledButton'
import openAISVG from 'assets/images/openai.svg'
import assemblyAIPng from 'assets/images/assemblyai.png'
// store
import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from 'store/actions'
import useNotifier from 'utils/useNotifier'
// API
import chatflowsApi from 'api/chatflows'
const speechToTextProviders = [
{
label: 'OpenAI Wisper',
name: 'openAIWisper',
icon: openAISVG,
url: 'https://platform.openai.com/docs/guides/speech-to-text',
inputs: [
{
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['openAIApi']
},
{
label: 'On/Off',
name: 'status',
type: 'boolean',
optional: true
}
]
},
{
label: 'Assembly AI',
name: 'assemblyAiTranscribe',
icon: assemblyAIPng,
url: 'https://www.assemblyai.com/',
inputs: [
{
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['assemblyAiApi']
},
{
label: 'On/Off',
name: 'status',
type: 'boolean',
optional: true
}
]
}
]
const SpeechToTextDialog = ({ show, dialogProps, onCancel }) => {
const portalElement = document.getElementById('portal')
const dispatch = useDispatch()
useNotifier()
const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
const [speechToText, setSpeechToText] = useState({})
const [providerExpanded, setProviderExpanded] = useState({})
const onSave = async () => {
try {
const saveResp = await chatflowsApi.updateChatflow(dialogProps.chatflow.id, {
speechToText: JSON.stringify(speechToText)
})
if (saveResp.data) {
enqueueSnackbar({
message: 'Analytic Configuration Saved',
options: {
key: new Date().getTime() + Math.random(),
variant: 'success',
action: (key) => (
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
<IconX />
</Button>
)
}
})
dispatch({ type: SET_CHATFLOW, chatflow: saveResp.data })
}
onCancel()
} catch (error) {
const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
enqueueSnackbar({
message: `Failed to save Analytic Configuration: ${errorData}`,
options: {
key: new Date().getTime() + Math.random(),
variant: 'error',
persist: true,
action: (key) => (
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
<IconX />
</Button>
)
}
})
}
}
const setValue = (value, providerName, inputParamName) => {
let newVal = {}
if (!Object.prototype.hasOwnProperty.call(speechToText, providerName)) {
newVal = { ...speechToText, [providerName]: {} }
} else {
newVal = { ...speechToText }
}
newVal[providerName][inputParamName] = value
if (inputParamName === 'status' && value === true) {
//ensure that the others are turned off
speechToTextProviders.forEach((provider) => {
if (provider.name !== providerName) {
newVal[provider.name] = { ...speechToText[provider.name], status: false }
}
})
}
setSpeechToText(newVal)
}
const handleAccordionChange = (providerName) => (event, isExpanded) => {
const accordionProviders = { ...providerExpanded }
accordionProviders[providerName] = isExpanded
setProviderExpanded(accordionProviders)
}
useEffect(() => {
if (dialogProps.chatflow && dialogProps.chatflow.speechToText) {
try {
setSpeechToText(JSON.parse(dialogProps.chatflow.speechToText))
} catch (e) {
setSpeechToText({})
console.error(e)
}
}
return () => {
setSpeechToText({})
setProviderExpanded({})
}
}, [dialogProps])
useEffect(() => {
if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
else dispatch({ type: HIDE_CANVAS_DIALOG })
return () => dispatch({ type: HIDE_CANVAS_DIALOG })
}, [show, dispatch])
const component = show ? (
<Dialog
onClose={onCancel}
open={show}
fullWidth
maxWidth='sm'
aria-labelledby='alert-dialog-title'
aria-describedby='alert-dialog-description'
>
<DialogTitle sx={{ fontSize: '1rem' }} id='alert-dialog-title'>
Speech To Text Configuration
</DialogTitle>
<DialogContent>
{speechToTextProviders.map((provider, index) => (
<Accordion
expanded={providerExpanded[provider.name] || false}
onChange={handleAccordionChange(provider.name)}
disableGutters
key={index}
>
<AccordionSummary expandIcon={<ExpandMoreIcon />} aria-controls={provider.name} id={provider.name}>
<ListItem style={{ padding: 0, margin: 0 }} alignItems='center'>
<ListItemAvatar>
<div
style={{
width: 50,
height: 50,
borderRadius: '50%',
backgroundColor: 'white'
}}
>
<img
style={{
width: '100%',
height: '100%',
padding: 10,
objectFit: 'contain'
}}
alt='AI'
src={provider.icon}
/>
</div>
</ListItemAvatar>
<ListItemText
sx={{ ml: 1 }}
primary={provider.label}
secondary={
<a target='_blank' rel='noreferrer' href={provider.url}>
{provider.url}
</a>
}
/>
{speechToText[provider.name] && speechToText[provider.name].status && (
<div
style={{
display: 'flex',
flexDirection: 'row',
alignContent: 'center',
alignItems: 'center',
background: '#d8f3dc',
borderRadius: 15,
padding: 5,
paddingLeft: 7,
paddingRight: 7,
marginRight: 10
}}
>
<div
style={{
width: 15,
height: 15,
borderRadius: '50%',
backgroundColor: '#70e000'
}}
/>
<span style={{ color: '#006400', marginLeft: 10 }}>ON</span>
</div>
)}
</ListItem>
</AccordionSummary>
<AccordionDetails>
{provider.inputs.map((inputParam, index) => (
<Box key={index} sx={{ p: 2 }}>
<div style={{ display: 'flex', flexDirection: 'row' }}>
<Typography>
{inputParam.label}
{!inputParam.optional && <span style={{ color: 'red' }}>&nbsp;*</span>}
{inputParam.description && (
<TooltipWithParser style={{ marginLeft: 10 }} title={inputParam.description} />
)}
</Typography>
</div>
{providerExpanded[provider.name] && inputParam.type === 'credential' && (
<CredentialInputHandler
data={
speechToText[provider.name] ? { credential: speechToText[provider.name].credentialId } : {}
}
inputParam={inputParam}
onSelect={(newValue) => setValue(newValue, provider.name, 'credentialId')}
/>
)}
{inputParam.type === 'boolean' && (
<SwitchInput
onChange={(newValue) => setValue(newValue, provider.name, inputParam.name)}
value={
speechToText[provider.name]
? speechToText[provider.name][inputParam.name]
: inputParam.default ?? false
}
/>
)}
{providerExpanded[provider.name] &&
(inputParam.type === 'string' ||
inputParam.type === 'password' ||
inputParam.type === 'number') && (
<Input
inputParam={inputParam}
onChange={(newValue) => setValue(newValue, provider.name, inputParam.name)}
value={
speechToText[provider.name]
? speechToText[provider.name][inputParam.name]
: inputParam.default ?? ''
}
/>
)}
</Box>
))}
</AccordionDetails>
</Accordion>
))}
</DialogContent>
<DialogActions>
<StyledButton variant='contained' onClick={onSave}>
Save
</StyledButton>
</DialogActions>
</Dialog>
) : null
return createPortal(component, portalElement)
}
SpeechToTextDialog.propTypes = {
show: PropTypes.bool,
dialogProps: PropTypes.object,
onCancel: PropTypes.func
}
export default SpeechToTextDialog
@@ -28,6 +28,7 @@ import useApi from 'hooks/useApi'
import { generateExportFlowData } from 'utils/genericHelper'
import { uiBaseURL } from 'store/constant'
import { SET_CHATFLOW } from 'store/actions'
import SpeechToTextDialog from '../../ui-component/dialog/SpeechToTextDialog'
// ==============================|| CANVAS HEADER ||============================== //
@@ -46,6 +47,8 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
const [apiDialogProps, setAPIDialogProps] = useState({})
const [analyseDialogOpen, setAnalyseDialogOpen] = useState(false)
const [analyseDialogProps, setAnalyseDialogProps] = useState({})
const [speechToAudioDialogOpen, setSpeechToAudioDialogOpen] = useState(false)
const [speechToAudioDialogProps, setSpeechToAudioialogProps] = useState({})
const [conversationStartersDialogOpen, setConversationStartersDialogOpen] = useState(false)
const [conversationStartersDialogProps, setConversationStartersDialogProps] = useState({})
const [viewMessagesDialogOpen, setViewMessagesDialogOpen] = useState(false)
@@ -71,6 +74,12 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
chatflow: chatflow
})
setAnalyseDialogOpen(true)
} else if (setting === 'enableSpeechToText') {
setSpeechToAudioialogProps({
title: 'Speech to Text',
chatflow: chatflow
})
setSpeechToAudioDialogOpen(true)
} else if (setting === 'viewMessages') {
setViewMessagesDialogProps({
title: 'View Messages',
@@ -385,6 +394,11 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
/>
<APICodeDialog show={apiDialogOpen} dialogProps={apiDialogProps} onCancel={() => setAPIDialogOpen(false)} />
<AnalyseFlowDialog show={analyseDialogOpen} dialogProps={analyseDialogProps} onCancel={() => setAnalyseDialogOpen(false)} />
<SpeechToTextDialog
show={speechToAudioDialogOpen}
dialogProps={speechToAudioDialogProps}
onCancel={() => setSpeechToAudioDialogOpen(false)}
/>
<StarterPromptsDialog
show={conversationStartersDialogOpen}
dialogProps={conversationStartersDialogProps}