diff --git a/packages/server/src/database/entities/ChatFlow.ts b/packages/server/src/database/entities/ChatFlow.ts index b3131c2e..b9048bad 100644 --- a/packages/server/src/database/entities/ChatFlow.ts +++ b/packages/server/src/database/entities/ChatFlow.ts @@ -31,6 +31,9 @@ export class ChatFlow implements IChatFlow { @Column({ nullable: true, type: 'text' }) analytic?: string + @Column({ nullable: true, type: 'text' }) + speechToText?: string + @CreateDateColumn() createdDate: Date diff --git a/packages/server/src/database/migrations/mysql/1706364937060-AddSpeechToText.ts b/packages/server/src/database/migrations/mysql/1706364937060-AddSpeechToText.ts new file mode 100644 index 00000000..ac11be89 --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1706364937060-AddSpeechToText.ts @@ -0,0 +1,12 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddSpeechToText1706364937060 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + const columnExists = await queryRunner.hasColumn('chat_flow', 'speechToText') + if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_flow\` ADD COLUMN \`speechToText\` TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`chat_flow\` DROP COLUMN \`speechToText\`;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index ad5f103d..549742a1 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -12,6 +12,7 @@ import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryT import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage' import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage' import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity' +import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText' export const mysqlMigrations = [ Init1693840429259, @@ -27,5 +28,6 @@ export const mysqlMigrations = [ AddCategoryToChatFlow1699900910291, AddFileAnnotationsToChatMessage1700271021237, AddFileUploadsToChatMessage1701788586491, - AddVariableEntity1699325775451 + AddVariableEntity1699325775451, + AddSpeechToText1706364937060 ] diff --git a/packages/server/src/database/migrations/postgres/1706364937060-AddSpeechToText.ts b/packages/server/src/database/migrations/postgres/1706364937060-AddSpeechToText.ts new file mode 100644 index 00000000..8ce5b672 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1706364937060-AddSpeechToText.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddSpeechToText1706364937060 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN IF NOT EXISTS "speechToText" TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "speechToText";`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts index 984bac66..bd631903 100644 --- a/packages/server/src/database/migrations/postgres/index.ts +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -12,6 +12,7 @@ import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryT import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage' import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage' import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity' +import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText' export const postgresMigrations = [ Init1693891895163, @@ -27,5 +28,6 @@ export const postgresMigrations = [ AddCategoryToChatFlow1699900910291, AddFileAnnotationsToChatMessage1700271021237, AddFileUploadsToChatMessage1701788586491, - AddVariableEntity1699325775451 + AddVariableEntity1699325775451, + AddSpeechToText1706364937060 ] diff --git a/packages/server/src/database/migrations/sqlite/1706364937060-AddSpeechToText.ts b/packages/server/src/database/migrations/sqlite/1706364937060-AddSpeechToText.ts new file mode 100644 index 00000000..1d77ffea --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1706364937060-AddSpeechToText.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddSpeechToText1706364937060 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN "speechToText" TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "speechToText";`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index 19b122d8..a50b0792 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -12,6 +12,7 @@ import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryT import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage' import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage' import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity' +import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText' export const sqliteMigrations = [ Init1693835579790, @@ -27,5 +28,6 @@ export const sqliteMigrations = [ AddCategoryToChatFlow1699900910291, AddFileAnnotationsToChatMessage1700271021237, AddFileUploadsToChatMessage1701788586491, - AddVariableEntity1699325775451 + AddVariableEntity1699325775451, + AddSpeechToText1706364937060 ] diff --git a/packages/ui/src/assets/images/assemblyai.png b/packages/ui/src/assets/images/assemblyai.png new file mode 100644 index 00000000..8919cb18 Binary files /dev/null and b/packages/ui/src/assets/images/assemblyai.png differ diff --git a/packages/ui/src/assets/images/openai.svg b/packages/ui/src/assets/images/openai.svg new file mode 100644 index 00000000..5c20398a --- /dev/null +++ b/packages/ui/src/assets/images/openai.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/packages/ui/src/menu-items/settings.js b/packages/ui/src/menu-items/settings.js index 1e0f58dd..ed610582 100644 --- a/packages/ui/src/menu-items/settings.js +++ b/packages/ui/src/menu-items/settings.js @@ -1,8 +1,17 @@ // assets -import { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff } from '@tabler/icons' +import { + IconTrash, + IconFileUpload, + IconFileExport, + IconCopy, + IconSearch, + IconMessage, + IconPictureInPictureOff, + IconMicrophone +} from '@tabler/icons' // constant -const icons = { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff } +const icons = { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff, IconMicrophone } // ==============================|| SETTINGS MENU ITEMS ||============================== // @@ -25,6 +34,13 @@ const settings = { url: '', icon: icons.IconMessage }, + { + id: 'enableSpeechToText', + title: 'Enable Speech to Text', + type: 'item', + url: '', + icon: icons.IconMicrophone + }, { id: 'duplicateChatflow', title: 'Duplicate Chatflow', diff --git a/packages/ui/src/ui-component/dialog/SpeechToTextDialog.js b/packages/ui/src/ui-component/dialog/SpeechToTextDialog.js new file mode 100644 index 00000000..fa2b7a78 --- /dev/null +++ b/packages/ui/src/ui-component/dialog/SpeechToTextDialog.js @@ -0,0 +1,332 @@ +import { createPortal } from 'react-dom' +import { useDispatch } from 'react-redux' +import { useState, useEffect } from 'react' +import PropTypes from 'prop-types' +import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction, SET_CHATFLOW } from 'store/actions' + +// material-ui +import { + Typography, + Box, + Button, + Dialog, + DialogContent, + DialogTitle, + DialogActions, + Accordion, + AccordionSummary, + AccordionDetails, + ListItem, + ListItemAvatar, + ListItemText +} from '@mui/material' +import ExpandMoreIcon from '@mui/icons-material/ExpandMore' +import { IconX } from '@tabler/icons' + +// Project import +import CredentialInputHandler from 'views/canvas/CredentialInputHandler' +import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser' +import { SwitchInput } from 'ui-component/switch/Switch' +import { Input } from 'ui-component/input/Input' +import { StyledButton } from 'ui-component/button/StyledButton' +import openAISVG from 'assets/images/openai.svg' +import assemblyAIPng from 'assets/images/assemblyai.png' + +// store +import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from 'store/actions' +import useNotifier from 'utils/useNotifier' + +// API +import chatflowsApi from 'api/chatflows' + +const speechToTextProviders = [ + { + label: 'OpenAI Wisper', + name: 'openAIWisper', + icon: openAISVG, + url: 'https://platform.openai.com/docs/guides/speech-to-text', + inputs: [ + { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['openAIApi'] + }, + { + label: 'On/Off', + name: 'status', + type: 'boolean', + optional: true + } + ] + }, + { + label: 'Assembly AI', + name: 'assemblyAiTranscribe', + icon: assemblyAIPng, + url: 'https://www.assemblyai.com/', + inputs: [ + { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['assemblyAiApi'] + }, + { + label: 'On/Off', + name: 'status', + type: 'boolean', + optional: true + } + ] + } +] + +const SpeechToTextDialog = ({ show, dialogProps, onCancel }) => { + const portalElement = document.getElementById('portal') + const dispatch = useDispatch() + + useNotifier() + + const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) + const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args)) + + const [speechToText, setSpeechToText] = useState({}) + const [providerExpanded, setProviderExpanded] = useState({}) + + const onSave = async () => { + try { + const saveResp = await chatflowsApi.updateChatflow(dialogProps.chatflow.id, { + speechToText: JSON.stringify(speechToText) + }) + if (saveResp.data) { + enqueueSnackbar({ + message: 'Analytic Configuration Saved', + options: { + key: new Date().getTime() + Math.random(), + variant: 'success', + action: (key) => ( + + ) + } + }) + dispatch({ type: SET_CHATFLOW, chatflow: saveResp.data }) + } + onCancel() + } catch (error) { + const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}` + enqueueSnackbar({ + message: `Failed to save Analytic Configuration: ${errorData}`, + options: { + key: new Date().getTime() + Math.random(), + variant: 'error', + persist: true, + action: (key) => ( + + ) + } + }) + } + } + + const setValue = (value, providerName, inputParamName) => { + let newVal = {} + if (!Object.prototype.hasOwnProperty.call(speechToText, providerName)) { + newVal = { ...speechToText, [providerName]: {} } + } else { + newVal = { ...speechToText } + } + + newVal[providerName][inputParamName] = value + if (inputParamName === 'status' && value === true) { + //ensure that the others are turned off + speechToTextProviders.forEach((provider) => { + if (provider.name !== providerName) { + newVal[provider.name] = { ...speechToText[provider.name], status: false } + } + }) + } + setSpeechToText(newVal) + } + + const handleAccordionChange = (providerName) => (event, isExpanded) => { + const accordionProviders = { ...providerExpanded } + accordionProviders[providerName] = isExpanded + setProviderExpanded(accordionProviders) + } + + useEffect(() => { + if (dialogProps.chatflow && dialogProps.chatflow.speechToText) { + try { + setSpeechToText(JSON.parse(dialogProps.chatflow.speechToText)) + } catch (e) { + setSpeechToText({}) + console.error(e) + } + } + + return () => { + setSpeechToText({}) + setProviderExpanded({}) + } + }, [dialogProps]) + + useEffect(() => { + if (show) dispatch({ type: SHOW_CANVAS_DIALOG }) + else dispatch({ type: HIDE_CANVAS_DIALOG }) + return () => dispatch({ type: HIDE_CANVAS_DIALOG }) + }, [show, dispatch]) + + const component = show ? ( + + + Speech To Text Configuration + + + {speechToTextProviders.map((provider, index) => ( + + } aria-controls={provider.name} id={provider.name}> + + +
+ AI +
+ + + {provider.url} + + } + /> + {speechToText[provider.name] && speechToText[provider.name].status && ( +
+
+ ON +
+ )} + + + + {provider.inputs.map((inputParam, index) => ( + +
+ + {inputParam.label} + {!inputParam.optional &&  *} + {inputParam.description && ( + + )} + +
+ {providerExpanded[provider.name] && inputParam.type === 'credential' && ( + setValue(newValue, provider.name, 'credentialId')} + /> + )} + {inputParam.type === 'boolean' && ( + setValue(newValue, provider.name, inputParam.name)} + value={ + speechToText[provider.name] + ? speechToText[provider.name][inputParam.name] + : inputParam.default ?? false + } + /> + )} + {providerExpanded[provider.name] && + (inputParam.type === 'string' || + inputParam.type === 'password' || + inputParam.type === 'number') && ( + setValue(newValue, provider.name, inputParam.name)} + value={ + speechToText[provider.name] + ? speechToText[provider.name][inputParam.name] + : inputParam.default ?? '' + } + /> + )} + + ))} + + + ))} + + + + Save + + +
+ ) : null + + return createPortal(component, portalElement) +} + +SpeechToTextDialog.propTypes = { + show: PropTypes.bool, + dialogProps: PropTypes.object, + onCancel: PropTypes.func +} + +export default SpeechToTextDialog diff --git a/packages/ui/src/views/canvas/CanvasHeader.js b/packages/ui/src/views/canvas/CanvasHeader.js index 85408cd8..a8589f48 100644 --- a/packages/ui/src/views/canvas/CanvasHeader.js +++ b/packages/ui/src/views/canvas/CanvasHeader.js @@ -28,6 +28,7 @@ import useApi from 'hooks/useApi' import { generateExportFlowData } from 'utils/genericHelper' import { uiBaseURL } from 'store/constant' import { SET_CHATFLOW } from 'store/actions' +import SpeechToTextDialog from '../../ui-component/dialog/SpeechToTextDialog' // ==============================|| CANVAS HEADER ||============================== // @@ -46,6 +47,8 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl const [apiDialogProps, setAPIDialogProps] = useState({}) const [analyseDialogOpen, setAnalyseDialogOpen] = useState(false) const [analyseDialogProps, setAnalyseDialogProps] = useState({}) + const [speechToAudioDialogOpen, setSpeechToAudioDialogOpen] = useState(false) + const [speechToAudioDialogProps, setSpeechToAudioialogProps] = useState({}) const [conversationStartersDialogOpen, setConversationStartersDialogOpen] = useState(false) const [conversationStartersDialogProps, setConversationStartersDialogProps] = useState({}) const [viewMessagesDialogOpen, setViewMessagesDialogOpen] = useState(false) @@ -71,6 +74,12 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl chatflow: chatflow }) setAnalyseDialogOpen(true) + } else if (setting === 'enableSpeechToText') { + setSpeechToAudioialogProps({ + title: 'Speech to Text', + chatflow: chatflow + }) + setSpeechToAudioDialogOpen(true) } else if (setting === 'viewMessages') { setViewMessagesDialogProps({ title: 'View Messages', @@ -385,6 +394,11 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl /> setAPIDialogOpen(false)} /> setAnalyseDialogOpen(false)} /> + setSpeechToAudioDialogOpen(false)} + />