Evaluations for Agentflows v2 & Assistants (#4589)

* New Feature: Evaluations for AgentFlow v2

* New Feature: Evaluations for Assistants and minor tweaks on other evaluations.

* do not store messages during evaluation for agent flows.

* common cost formatting

* moving the category names to description (in create dialog) and adjusting the side drawer label

* lint fixes

* Enhancement: Add auto-refresh toggle for evaluations with 5-second interval and adjust grid item size for metrics display.

* 1) chatflow types are stored in additional config
2) messages are now stored with type "Evaluations"
3) Message Dialog has a new Type in the ChatType Filter Dropdown
4) Chatflow badges on the view page, have the right canvas URL
5) outdated API returns chatflow type along with the stale indicator.
6) UI - Flow Indicator Icons are shown in the Chatflows Used chips & side drawer

* Refactor JWT error handling to return 401 status for expired refresh tokens. Update chat message ID assignment to remove UUID fallback. Enhance ViewMessagesDialog to set default chat type filters and implement a new method for determining chat type sources. Modify EvalsResultDialog to open links in a new tab and adjust icon sizes for better consistency. Clean up unused imports in EvaluationResultSideDrawer.

* handling on Click for deleted flows and minor code cleanup

* evals ui fix

* Refactor evaluation service to improve error handling and data parsing. Update additionalConfig handling to default to an empty object if not present. Enhance type definitions for better clarity. Adjust MetricsItemCard to prevent overflow and improve layout consistency.

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Vinod Kiran
2025-06-10 20:41:22 +05:30
committed by GitHub
parent f644c47251
commit e17994d8fe
13 changed files with 766 additions and 224 deletions
@@ -149,7 +149,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
const [sourceDialogProps, setSourceDialogProps] = useState({})
const [hardDeleteDialogOpen, setHardDeleteDialogOpen] = useState(false)
const [hardDeleteDialogProps, setHardDeleteDialogProps] = useState({})
const [chatTypeFilter, setChatTypeFilter] = useState([])
const [chatTypeFilter, setChatTypeFilter] = useState(['INTERNAL', 'EXTERNAL'])
const [feedbackTypeFilter, setFeedbackTypeFilter] = useState([])
const [startDate, setStartDate] = useState(new Date(new Date().setMonth(new Date().getMonth() - 1)))
const [endDate, setEndDate] = useState(new Date())
@@ -310,6 +310,15 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
}
}
const getChatType = (chatType) => {
if (chatType === 'INTERNAL') {
return 'UI'
} else if (chatType === 'EVALUATION') {
return 'Evaluation'
}
return 'API/Embed'
}
const exportMessages = async () => {
if (!storagePath && getStoragePathFromServer.data) {
storagePath = getStoragePathFromServer.data.storagePath
@@ -356,7 +365,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) {
obj[chatPK] = {
id: chatmsg.chatId,
source: chatmsg.chatType === 'INTERNAL' ? 'UI' : 'API/Embed',
source: getChatType(chatmsg.chatType),
sessionId: chatmsg.sessionId ?? null,
memoryType: chatmsg.memoryType ?? null,
email: chatmsg.leadEmail ?? null,
@@ -716,7 +725,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
setChatLogs([])
setAllChatLogs([])
setChatMessages([])
setChatTypeFilter([])
setChatTypeFilter(['INTERNAL', 'EXTERNAL'])
setFeedbackTypeFilter([])
setSelectedMessageIndex(0)
setSelectedChatId('')
@@ -880,6 +889,10 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
{
label: 'API/Embed',
name: 'EXTERNAL'
},
{
label: 'Evaluations',
name: 'EVALUATION'
}
]}
onSelect={(newValue) => onChatTypeSelected(newValue)}
@@ -1016,7 +1029,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
)}
{chatMessages[1].chatType && (
<div>
Source:&nbsp;<b>{chatMessages[1].chatType === 'INTERNAL' ? 'UI' : 'API/Embed'}</b>
Source:&nbsp;<b>{getChatType(chatMessages[1].chatType)}</b>
</div>
)}
{chatMessages[1].memoryType && (
@@ -21,7 +21,8 @@ import {
Switch,
StepLabel,
IconButton,
FormControlLabel
FormControlLabel,
Checkbox
} from '@mui/material'
import { useTheme } from '@mui/material/styles'
@@ -42,6 +43,7 @@ import useApi from '@/hooks/useApi'
import datasetsApi from '@/api/dataset'
import evaluatorsApi from '@/api/evaluators'
import nodesApi from '@/api/nodes'
import assistantsApi from '@/api/assistants'
// utils
import useNotifier from '@/utils/useNotifier'
@@ -57,14 +59,18 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
useNotifier()
const getAllChatflowsApi = useApi(chatflowsApi.getAllChatflows)
const getAllAgentflowsApi = useApi(chatflowsApi.getAllAgentflows)
const getAllDatasetsApi = useApi(datasetsApi.getAllDatasets)
const getAllEvaluatorsApi = useApi(evaluatorsApi.getAllEvaluators)
const getNodesByCategoryApi = useApi(nodesApi.getNodesByCategory)
const getModelsApi = useApi(nodesApi.executeNodeLoadMethod)
const getAssistantsApi = useApi(assistantsApi.getAllAssistants)
const [chatflow, setChatflow] = useState([])
const [dataset, setDataset] = useState('')
const [datasetAsOneConversation, setDatasetAsOneConversation] = useState(false)
const [flowTypes, setFlowTypes] = useState([])
const [flows, setFlows] = useState([])
const [datasets, setDatasets] = useState([])
@@ -163,6 +169,10 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
for (let i = 0; i < selectedChatflows.length; i += 1) {
selectedChatflowNames.push(flows.find((f) => f.name === selectedChatflows[i])?.label)
}
const selectedChatflowTypes = []
for (let i = 0; i < selectedChatflows.length; i += 1) {
selectedChatflowTypes.push(flows.find((f) => f.name === selectedChatflows[i])?.type)
}
const chatflowName = JSON.stringify(selectedChatflowNames)
const datasetName = datasets.find((f) => f.name === dataset)?.label
const obj = {
@@ -173,6 +183,7 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
datasetName: datasetName,
chatflowId: chatflow,
chatflowName: chatflowName,
chatflowType: JSON.stringify(selectedChatflowTypes),
selectedSimpleEvaluators: selectedSimpleEvaluators,
selectedLLMEvaluators: selectedLLMEvaluators,
model: selectedModel,
@@ -216,6 +227,8 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
getNodesByCategoryApi.request('Chat Models')
if (flows.length === 0) {
getAllChatflowsApi.request()
getAssistantsApi.request('CUSTOM')
getAllAgentflowsApi.request('AGENTFLOW')
}
if (datasets.length === 0) {
getAllDatasetsApi.request()
@@ -225,23 +238,18 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
}, [])
useEffect(() => {
if (getAllChatflowsApi.data) {
if (getAllAgentflowsApi.data && getAllChatflowsApi.data && getAssistantsApi.data) {
try {
const chatflows = getAllChatflowsApi.data
let flowNames = []
for (let i = 0; i < chatflows.length; i += 1) {
const flow = chatflows[i]
flowNames.push({
label: flow.name,
name: flow.id
})
}
setFlows(flowNames)
const agentFlows = populateFlowNames(getAllAgentflowsApi.data, 'Agentflow v2')
const chatFlows = populateFlowNames(getAllChatflowsApi.data, 'Chatflow')
const assistants = populateAssistants(getAssistantsApi.data)
setFlows([...agentFlows, ...chatFlows, ...assistants])
setFlowTypes(['Agentflow v2', 'Chatflow', 'Custom Assistant'])
} catch (e) {
console.error(e)
}
}
}, [getAllChatflowsApi.data])
}, [getAllAgentflowsApi.data, getAllChatflowsApi.data, getAssistantsApi.data])
useEffect(() => {
if (getNodesByCategoryApi.data) {
@@ -337,6 +345,44 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
if (llm !== 'no_grading') getModelsApi.request(llm, { loadMethod: 'listModels' })
}
const onChangeFlowType = (flowType) => {
const selected = flowType.target.checked
const flowTypeValue = flowType.target.value
if (selected) {
setFlowTypes([...flowTypes, flowTypeValue])
} else {
setFlowTypes(flowTypes.filter((f) => f !== flowTypeValue))
}
}
const populateFlowNames = (data, type) => {
let flowNames = []
for (let i = 0; i < data.length; i += 1) {
const flow = data[i]
flowNames.push({
label: flow.name,
name: flow.id,
type: type,
description: type
})
}
return flowNames
}
const populateAssistants = (assistants) => {
let assistantNames = []
for (let i = 0; i < assistants.length; i += 1) {
const assistant = assistants[i]
assistantNames.push({
label: JSON.parse(assistant.details).name || '',
name: assistant.id,
type: 'Custom Assistant',
description: 'Custom Assistant'
})
}
return assistantNames
}
const component = show ? (
<Dialog
fullWidth
@@ -476,18 +522,42 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
Treat all dataset rows as one conversation ?
</Typography>
<FormControlLabel
label=''
control={<Switch />}
value={datasetAsOneConversation}
onChange={() => setDatasetAsOneConversation(!datasetAsOneConversation)}
/>
</Box>
<Box>
<Typography variant='overline'>
Chatflow(s) to Evaluate<span style={{ color: 'red' }}>&nbsp;*</span>
</Typography>
<div style={{ display: 'flex', justifyContent: 'space-between' }}>
<Typography variant='overline'>
Select your flows to Evaluate
<span style={{ color: 'red' }}>&nbsp;*</span>
</Typography>
<Typography variant='overline'>
<Checkbox defaultChecked size='small' label='All' value='Chatflow' onChange={onChangeFlowType} />{' '}
Chatflows
<Checkbox
defaultChecked
size='small'
label='All'
value='Agentflow v2'
onChange={onChangeFlowType}
/>{' '}
Agentflows (v2)
<Checkbox
defaultChecked
size='small'
label='All'
value='Custom Assistant'
onChange={onChangeFlowType}
/>{' '}
Custom Assistants
</Typography>
</div>
<MultiDropdown
name={'chatflow1'}
options={flows}
options={flows.filter((f) => flowTypes.includes(f.type))}
onSelect={(newValue) => setChatflow(newValue)}
value={chatflow ?? chatflow ?? 'choose an option'}
/>
@@ -2,7 +2,6 @@ import React from 'react'
import { createPortal } from 'react-dom'
import PropTypes from 'prop-types'
import { useSelector } from 'react-redux'
import { useNavigate } from 'react-router-dom'
// Material
import {
@@ -36,7 +35,6 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) =
const portalElement = document.getElementById('portal')
const customization = useSelector((state) => state.customization)
const theme = useTheme()
const navigate = useNavigate()
const getColSpan = (evaluationsShown, llmEvaluations) => {
let colSpan = 1
@@ -45,6 +43,23 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) =
return colSpan
}
const getOpenLink = (index) => {
if (index === undefined) {
return ''
}
if (dialogProps.data?.additionalConfig?.chatflowTypes) {
switch (dialogProps.data.additionalConfig.chatflowTypes[index]) {
case 'Chatflow':
return '/canvas/' + dialogProps.data.evaluation.chatflowId[index]
case 'Custom Assistant':
return '/assistants/custom/' + dialogProps.data.evaluation.chatflowId[index]
case 'Agentflow v2':
return '/v2/agentcanvas/' + dialogProps.data.evaluation.chatflowId[index]
}
}
return '/canvas/' + dialogProps.data.evaluation.chatflowId[index]
}
const component = show ? (
<Dialog fullScreen open={show} onClose={onCancel} aria-labelledby='alert-dialog-title' aria-describedby='alert-dialog-description'>
<DialogTitle id='alert-dialog-title'>
@@ -65,7 +80,7 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) =
}}
>
<IconVectorBezier2 style={{ marginRight: 5 }} size={17} />
Chatflows Used:
Flows Used:
</div>
{(dialogProps.data.evaluation.chatflowName || []).map((chatflowUsed, index) => (
<Chip
@@ -79,7 +94,7 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) =
: '0 2px 14px 0 rgb(32 40 45 / 10%)'
}}
label={chatflowUsed}
onClick={() => navigate('/canvas/' + dialogProps.data.evaluation.chatflowId[index])}
onClick={() => window.open(getOpenLink(index), '_blank')}
></Chip>
))}
</Stack>
@@ -25,6 +25,7 @@ import {
import { useTheme } from '@mui/material/styles'
import moment from 'moment'
import PaidIcon from '@mui/icons-material/Paid'
import { IconHierarchy, IconUsersGroup, IconRobot } from '@tabler/icons-react'
import LLMIcon from '@mui/icons-material/ModelTraining'
import AlarmIcon from '@mui/icons-material/AlarmOn'
import TokensIcon from '@mui/icons-material/AutoAwesomeMotion'
@@ -116,10 +117,13 @@ const EvalEvaluationRows = () => {
const [expandTableProps, setExpandTableProps] = useState({})
const [isTableLoading, setTableLoading] = useState(false)
const [additionalConfig, setAdditionalConfig] = useState({})
const openDetailsDrawer = (item) => {
setSideDrawerDialogProps({
type: 'View',
data: item,
additionalConfig: additionalConfig,
evaluationType: evaluation.evaluationType,
evaluationChatflows: evaluation.chatflowName
})
@@ -169,7 +173,8 @@ const EvalEvaluationRows = () => {
showCustomEvals,
showTokenMetrics,
showLatencyMetrics,
showCostMetrics
showCostMetrics,
additionalConfig
}
})
setShowExpandTableDialog(true)
@@ -239,6 +244,9 @@ const EvalEvaluationRows = () => {
const data = getEvaluation.data
setSelectedEvaluationName(data.name)
getIsOutdatedApi.request(data.id)
if (data.additionalConfig) {
setAdditionalConfig(JSON.parse(data.additionalConfig))
}
data.chatflowId = typeof data.chatflowId === 'object' ? data.chatflowId : JSON.parse(data.chatflowId)
data.chatflowName = typeof data.chatflowName === 'object' ? data.chatflowName : JSON.parse(data.chatflowName)
const rows = getEvaluation.data.rows
@@ -314,6 +322,51 @@ const EvalEvaluationRows = () => {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [getEvaluation.data])
const getOpenLink = (index) => {
if (index === undefined) {
return undefined
}
const id = evaluation.chatflowId[index]
// this is to check if the evaluation is deleted!
if (outdated?.errors?.length > 0 && outdated.errors.find((e) => e.id === id)) {
return undefined
}
if (additionalConfig.chatflowTypes) {
switch (additionalConfig.chatflowTypes[index]) {
case 'Chatflow':
return '/canvas/' + evaluation.chatflowId[index]
case 'Custom Assistant':
return '/assistants/custom/' + evaluation.chatflowId[index]
case 'Agentflow v2':
return '/v2/agentcanvas/' + evaluation.chatflowId[index]
}
}
return '/canvas/' + evaluation.chatflowId[index]
}
const openFlow = (index) => {
const url = getOpenLink(index)
if (url) {
window.open(getOpenLink(index), '_blank')
}
}
const getFlowIcon = (index) => {
if (index === undefined) {
return <IconHierarchy size={17} />
}
if (additionalConfig.chatflowTypes) {
switch (additionalConfig.chatflowTypes[index]) {
case 'Chatflow':
return <IconHierarchy size={17} />
case 'Custom Assistant':
return <IconRobot size={17} />
case 'Agentflow v2':
return <IconUsersGroup size={17} />
}
}
return <IconHierarchy />
}
return (
<>
<MainCard>
@@ -405,14 +458,14 @@ const EvalEvaluationRows = () => {
}}
variant='outlined'
label={outdated.dataset.name}
onClick={() => navigate(`/dataset_rows/${outdated.dataset.id}`)}
onClick={() => window.open(`/dataset_rows/${outdated.dataset.id}`, '_blank')}
></Chip>
</>
)}
{outdated.chatflows && outdated?.errors?.length === 0 && outdated.chatflows.length > 0 && (
<>
<br />
<b style={{ color: 'rgb(116,66,16)' }}>Chatflows:</b>
<b style={{ color: 'rgb(116,66,16)' }}>Flows:</b>
<Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
{outdated.chatflows.map((chatflow, index) => (
<Chip
@@ -429,14 +482,23 @@ const EvalEvaluationRows = () => {
}}
variant='outlined'
label={chatflow.chatflowName}
onClick={() => navigate(`/canvas/${chatflow.chatflowId}`)}
onClick={() =>
window.open(
chatflow.chatflowType === 'Chatflow'
? '/canvas/' + chatflow.chatflowId
: chatflow.chatflowType === 'Custom Assistant'
? '/assistants/custom/' + chatflow.chatflowId
: '/v2/agentcanvas/' + chatflow.chatflowId,
'_blank'
)
}
></Chip>
))}
</Stack>
</>
)}
{outdated.errors.length > 0 &&
outdated.errors.map((error, index) => <ListItem key={index}>{error}</ListItem>)}
outdated.errors.map((error, index) => <ListItem key={index}>{error.message}</ListItem>)}
<IconButton
style={{ position: 'absolute', top: 10, right: 10 }}
size='small'
@@ -501,7 +563,7 @@ const EvalEvaluationRows = () => {
{showCharts && (
<Grid container={true} spacing={2}>
{customEvalsDefined && (
<Grid item={true} xs={12} sm={6} md={4} lg={4}>
<Grid item={true} xs={12} sm={12} md={4} lg={4}>
<MetricsItemCard
data={{
header: 'PASS RATE',
@@ -566,11 +628,12 @@ const EvalEvaluationRows = () => {
}}
>
<IconVectorBezier2 style={{ marginRight: 5 }} size={17} />
Chatflows Used:
Flows Used:
</div>
{(evaluation.chatflowName || []).map((chatflowUsed, index) => (
<Chip
key={index}
icon={getFlowIcon(index)}
clickable
style={{
width: 'max-content',
@@ -580,7 +643,7 @@ const EvalEvaluationRows = () => {
: '0 2px 14px 0 rgb(32 40 45 / 10%)'
}}
label={chatflowUsed}
onClick={() => navigate('/canvas/' + evaluation.chatflowId[index])}
onClick={() => openFlow(index)}
></Chip>
))}
</Stack>
@@ -1,8 +1,25 @@
import PropTypes from 'prop-types'
import { CardContent, Card, Box, SwipeableDrawer, Stack, Button, Chip, Divider, Typography } from '@mui/material'
import {
CardContent,
Card,
Box,
SwipeableDrawer,
Stack,
Button,
Chip,
Divider,
Typography,
Table,
TableHead,
TableRow,
TableBody
} from '@mui/material'
import { IconHierarchy, IconUsersGroup, IconRobot } from '@tabler/icons-react'
import { useSelector } from 'react-redux'
import { IconSquareRoundedChevronsRight } from '@tabler/icons-react'
import { evaluators as evaluatorsOptions, numericOperators } from '../evaluators/evaluatorConstant'
import TableCell from '@mui/material/TableCell'
import { Close } from '@mui/icons-material'
const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => {
const onOpen = () => {}
@@ -19,12 +36,32 @@ const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => {
return ''
}
const getFlowIcon = (index) => {
if (index === undefined) {
return <IconHierarchy size={24} />
}
if (dialogProps.additionalConfig.chatflowTypes) {
switch (dialogProps.additionalConfig.chatflowTypes[index]) {
case 'Chatflow':
return <IconHierarchy size={20} />
case 'Custom Assistant':
return <IconRobot size={20} />
case 'Agentflow v2':
return <IconUsersGroup size={20} />
}
}
return <IconHierarchy />
}
return (
<SwipeableDrawer sx={{ zIndex: 2000 }} anchor='right' open={show} onClose={() => onClickFunction()} onOpen={onOpen}>
<Button startIcon={<IconSquareRoundedChevronsRight />} onClick={() => onClickFunction()}>
Close
</Button>
<Box sx={{ width: 450, p: 3 }} role='presentation'>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', borderBottom: '1px solid #ccc' }}>
<Typography variant='overline' sx={{ margin: 1, fontWeight: 'bold' }}>
Evaluation Details
</Typography>
<Button endIcon={<Close />} onClick={() => onClickFunction()} />
</div>
<Box sx={{ width: 600, p: 2 }} role='presentation'>
<Box>
<Typography variant='overline' sx={{ fontWeight: 'bold' }}>
Evaluation Id
@@ -61,13 +98,19 @@ const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => {
<CardContent>
{dialogProps.evaluationChatflows?.length > 0 && (
<>
<Box>
<Typography variant='overline' sx={{ fontWeight: 'bold' }}>
Chatflow
<div
style={{
display: 'flex',
justifyContent: 'start',
alignItems: 'center',
marginBottom: 5
}}
>
{getFlowIcon(index)}
<Typography variant='overline' sx={{ fontWeight: 'bold', fontSize: '1.1rem', marginLeft: 1 }}>
{dialogProps.evaluationChatflows[index]}
</Typography>
<Typography variant='body2'>{dialogProps.evaluationChatflows[index]}</Typography>
</Box>
<br />
</div>
<Divider />
</>
)}
@@ -153,79 +196,222 @@ const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => {
<br />
<Divider />
<br />
<Box>
<Typography variant='overline' style={{ fontWeight: 'bold' }}>
Tokens
</Typography>
<Typography variant='body2'>
<Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.totalTokens
? 'Total: ' + dialogProps.data.metrics[index]?.totalTokens
: 'Total: N/A'
}
/>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.promptTokens
? 'Prompt: ' + dialogProps.data.metrics[index]?.promptTokens
: 'Completion: N/A'
}
/>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.completionTokens
? 'Completion: ' + dialogProps.data.metrics[index]?.completionTokens
: 'Completion: N/A'
}
/>
</Stack>
</Typography>
</Box>
{dialogProps.data.metrics[index]?.nested_metrics ? (
<Box>
<Typography variant='overline' style={{ fontWeight: 'bold' }}>
Tokens
</Typography>
<Table size='small' style={{ border: '1px solid #ccc' }}>
<TableHead>
<TableRow>
<TableCell align='left' style={{ fontSize: '11px', fontWeight: 'bold' }}>
Node
</TableCell>
<TableCell align='left' style={{ fontSize: '11px', fontWeight: 'bold' }}>
Provider & Model
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold', width: '15%' }}>
Input
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold', width: '15%' }}>
Output
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold', width: '15%' }}>
Total
</TableCell>
</TableRow>
</TableHead>
<TableBody style={{ fontSize: '8px' }}>
{dialogProps.data.metrics[index]?.nested_metrics?.map((metric, index) => (
<TableRow key={index}>
<TableCell component='th' scope='row' style={{ fontSize: '11px' }}>
{metric.nodeLabel}
</TableCell>
<TableCell component='th' scope='row' style={{ fontSize: '11px' }}>
{metric.provider}
<br />
{metric.model}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px' }}>
{metric.promptTokens}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px' }}>
{metric.completionTokens}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px' }}>
{metric.totalTokens}
</TableCell>
</TableRow>
))}
<TableRow key={index}>
<TableCell
align='right'
style={{ fontSize: '11px', fontWeight: 'bold' }}
component='th'
scope='row'
colspan={2}
>
Total
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
{dialogProps.data.metrics[index].promptTokens}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
{dialogProps.data.metrics[index].completionTokens}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
{dialogProps.data.metrics[index].totalTokens}
</TableCell>
</TableRow>
</TableBody>
</Table>
</Box>
) : (
<Box>
<Typography variant='overline' style={{ fontWeight: 'bold' }}>
Tokens
</Typography>
<Typography variant='body2'>
<Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.totalTokens
? 'Total: ' + dialogProps.data.metrics[index]?.totalTokens
: 'Total: N/A'
}
/>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.promptTokens
? 'Prompt: ' + dialogProps.data.metrics[index]?.promptTokens
: 'Prompt: N/A'
}
/>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.completionTokens
? 'Completion: ' + dialogProps.data.metrics[index]?.completionTokens
: 'Completion: N/A'
}
/>
</Stack>
</Typography>
</Box>
)}
<br />
<Box>
<Typography variant='overline' style={{ fontWeight: 'bold' }}>
Cost
</Typography>
<Typography variant='body2'>
<Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.totalCost
? 'Total: ' + dialogProps.data.metrics[index]?.totalCost
: 'Total: N/A'
}
/>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.promptCost
? 'Prompt: ' + dialogProps.data.metrics[index]?.promptCost
: 'Completion: N/A'
}
/>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.completionCost
? 'Completion: ' + dialogProps.data.metrics[index]?.completionCost
: 'Completion: N/A'
}
/>
</Stack>
</Typography>
</Box>
{dialogProps.data.metrics[index]?.nested_metrics ? (
<Box>
<Typography variant='overline' style={{ fontWeight: 'bold' }}>
Cost
</Typography>
<Table size='small' style={{ border: '1px solid #ccc' }}>
<TableHead>
<TableRow>
<TableCell align='left' style={{ fontSize: '11px', fontWeight: 'bold' }}>
Node
</TableCell>
<TableCell align='left' style={{ fontSize: '11px', fontWeight: 'bold' }}>
Provider & Model
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', width: '15%', fontWeight: 'bold' }}>
Input
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', width: '15%', fontWeight: 'bold' }}>
Output
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', width: '15%', fontWeight: 'bold' }}>
Total
</TableCell>
</TableRow>
</TableHead>
<TableBody style={{ fontSize: '8px' }}>
{dialogProps.data.metrics[index]?.nested_metrics?.map((metric, index) => (
<TableRow key={index}>
<TableCell component='th' scope='row' style={{ fontSize: '11px' }}>
{metric.nodeLabel}
</TableCell>
<TableCell component='th' scope='row' style={{ fontSize: '11px' }}>
{metric.provider} <br />
{metric.model}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px' }}>
{metric.promptCost}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px' }}>
{metric.completionCost}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px' }}>
{metric.totalCost}
</TableCell>
</TableRow>
))}
<TableRow key={index}>
<TableCell
align='right'
style={{ fontSize: '11px', fontWeight: 'bold' }}
component='th'
scope='row'
colspan={2}
>
Total
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
{dialogProps.data.metrics[index].promptCost}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
{dialogProps.data.metrics[index].completionCost}
</TableCell>
<TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
{dialogProps.data.metrics[index].totalCost}
</TableCell>
</TableRow>
</TableBody>
</Table>
</Box>
) : (
<Box>
<Typography variant='overline' style={{ fontWeight: 'bold' }}>
Cost
</Typography>
<Typography variant='body2'>
<Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.totalCost
? 'Total: ' + dialogProps.data.metrics[index]?.totalCost
: 'Total: N/A'
}
/>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.promptCost
? 'Prompt: ' + dialogProps.data.metrics[index]?.promptCost
: 'Completion: N/A'
}
/>
<Chip
variant='outlined'
size='small'
label={
dialogProps.data.metrics[index]?.completionCost
? 'Completion: ' + dialogProps.data.metrics[index]?.completionCost
: 'Completion: N/A'
}
/>
</Stack>
</Typography>
</Box>
)}
<br />
<Divider />
<br />
@@ -11,7 +11,7 @@ import SkeletonChatflowCard from '@/ui-component/cards/Skeleton/ChatflowCard'
const CardWrapper = styled(MainCard)(({ theme }) => ({
background: theme.palette.card.main,
color: theme.darkTextPrimary,
overflow: 'auto',
overflow: 'hidden',
position: 'relative',
boxShadow: '0 2px 14px 0 rgb(32 40 45 / 8%)',
cursor: 'pointer',
+78 -26
View File
@@ -1,4 +1,4 @@
import React, { useEffect, useState } from 'react'
import React, { useEffect, useState, useCallback } from 'react'
import * as PropTypes from 'prop-types'
import moment from 'moment/moment'
import { useNavigate } from 'react-router-dom'
@@ -20,7 +20,8 @@ import {
TableBody,
TableContainer,
TableHead,
TableRow
TableRow,
ToggleButton
} from '@mui/material'
import { useTheme } from '@mui/material/styles'
import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions'
@@ -35,7 +36,6 @@ import useNotifier from '@/utils/useNotifier'
// project
import MainCard from '@/ui-component/cards/MainCard'
import { StyledButton } from '@/ui-component/button/StyledButton'
import { BackdropLoader } from '@/ui-component/loading/BackdropLoader'
import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog'
import ErrorBoundary from '@/ErrorBoundary'
@@ -53,7 +53,9 @@ import {
IconTrash,
IconX,
IconChevronsUp,
IconChevronsDown
IconChevronsDown,
IconPlayerPlay,
IconPlayerPause
} from '@tabler/icons-react'
import empty_evalSVG from '@/assets/images/empty_evals.svg'
@@ -79,6 +81,7 @@ const EvalsEvaluation = () => {
const [loading, setLoading] = useState(false)
const [isTableLoading, setTableLoading] = useState(false)
const [selected, setSelected] = useState([])
const [autoRefresh, setAutoRefresh] = useState(false)
const onSelectAllClick = (event) => {
if (event.target.checked) {
@@ -240,14 +243,34 @@ const EvalsEvaluation = () => {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [createNewEvaluation.error])
const onRefresh = () => {
const onRefresh = useCallback(() => {
getAllEvaluations.request()
}
}, [getAllEvaluations])
useEffect(() => {
setTableLoading(getAllEvaluations.loading)
}, [getAllEvaluations.loading])
useEffect(() => {
let intervalId = null
if (autoRefresh) {
intervalId = setInterval(() => {
onRefresh()
}, 5000)
}
return () => {
if (intervalId) {
clearInterval(intervalId)
}
}
}, [autoRefresh, onRefresh])
const toggleAutoRefresh = () => {
setAutoRefresh(!autoRefresh)
}
return (
<>
<MainCard>
@@ -256,15 +279,52 @@ const EvalsEvaluation = () => {
) : (
<Stack flexDirection='column' sx={{ gap: 3 }}>
<ViewHeader isBackButton={false} isEditButton={false} search={false} title={'Evaluations'} description=''>
<StyledButton
color='secondary'
variant='outlined'
sx={{ borderRadius: 2, height: '100%' }}
onClick={onRefresh}
startIcon={<IconRefresh />}
<ToggleButton
value='auto-refresh'
selected={autoRefresh}
onChange={toggleAutoRefresh}
size='small'
sx={{
borderRadius: 2,
height: '100%',
backgroundColor: 'transparent',
color: autoRefresh ? '#ff9800' : '#4caf50',
border: '1px solid transparent',
'&:hover': {
backgroundColor: 'rgba(0, 0, 0, 0.04)',
color: autoRefresh ? '#f57c00' : '#388e3c',
border: '1px solid transparent'
},
'&.Mui-selected': {
backgroundColor: 'transparent',
color: '#ff9800',
border: '1px solid transparent',
'&:hover': {
backgroundColor: 'rgba(0, 0, 0, 0.04)',
color: '#f57c00',
border: '1px solid transparent'
}
}
}}
title={autoRefresh ? 'Disable auto-refresh' : 'Enable auto-refresh (every 5s)'}
>
Refresh
</StyledButton>
{autoRefresh ? <IconPlayerPause /> : <IconPlayerPlay />}
</ToggleButton>
<IconButton
sx={{
borderRadius: 2,
height: '100%',
color: theme.palette.secondary.main,
'&:hover': {
backgroundColor: 'rgba(0, 0, 0, 0.04)',
color: theme.palette.secondary.dark
}
}}
onClick={onRefresh}
title='Refresh'
>
<IconRefresh />
</IconButton>
<StyledPermissionButton
permissionId={'evaluations:create'}
sx={{ borderRadius: 2, height: '100%' }}
@@ -327,7 +387,7 @@ const EvalsEvaluation = () => {
<TableCell>Latest Version</TableCell>
<TableCell>Average Metrics</TableCell>
<TableCell>Last Evaluated</TableCell>
<TableCell>Chatflow(s)</TableCell>
<TableCell>Flow(s)</TableCell>
<TableCell>Dataset</TableCell>
<TableCell> </TableCell>
</TableRow>
@@ -438,7 +498,7 @@ function EvaluationRunRow(props) {
}
const goToDataset = (id) => {
navigate(`/dataset_rows/${id}`)
window.open(`/dataset_rows/${id}`, '_blank')
}
const onSelectAllChildClick = (event) => {
@@ -513,10 +573,6 @@ function EvaluationRunRow(props) {
}
}
const goToCanvas = (id) => {
navigate(`/canvas/${id}`)
}
const getStatusColor = (status) => {
switch (status) {
case 'pending':
@@ -619,16 +675,11 @@ function EvaluationRunRow(props) {
{props.item?.usedFlows?.map((usedFlow, index) => (
<Chip
key={index}
clickable
style={{
width: 'max-content',
borderRadius: '25px',
boxShadow: props.customization.isDarkMode
? '0 2px 14px 0 rgb(255 255 255 / 10%)'
: '0 2px 14px 0 rgb(32 40 45 / 10%)'
borderRadius: '25px'
}}
label={usedFlow}
onClick={() => goToCanvas(props.item.chatIds[index])}
></Chip>
))}
</Stack>
@@ -637,6 +688,7 @@ function EvaluationRunRow(props) {
<Chip
clickable
style={{
border: 'none',
width: 'max-content',
borderRadius: '25px',
boxShadow: props.customization.isDarkMode