mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 09:00:52 +03:00
Feature/OpenAI Response API (#5014)
* - Added support for built-in OpenAI tools including web search, code interpreter, and image generation. - Enhanced file handling by extracting artifacts and file annotations from response metadata. - Implemented download functionality for file annotations in the UI. - Updated chat history management to include additional kwargs for artifacts, file annotations, and used tools. - Improved UI components to display used tools and file annotations effectively. * remove redundant currentContainerId * update comment
This commit is contained in:
Henry Heng
@@ -61,3 +61,13 @@
|
||||
line-height: 1.6;
|
||||
margin: 0.5em 0;
|
||||
}
|
||||
|
||||
.react-markdown img {
|
||||
max-width: 100%;
|
||||
max-height: 400px;
|
||||
height: auto;
|
||||
object-fit: contain;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
|
||||
margin: 10px 0;
|
||||
}
|
||||
|
||||
@@ -244,6 +244,63 @@ export const updateOutdatedNodeData = (newComponentNodeData, existingComponentNo
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle loadConfig parameters - preserve configuration objects
|
||||
if (existingComponentNodeData.inputs && initNewComponentNodeData.inputParams) {
|
||||
// Find parameters with loadConfig: true
|
||||
const loadConfigParams = initNewComponentNodeData.inputParams.filter((param) => param.loadConfig === true)
|
||||
|
||||
for (const param of loadConfigParams) {
|
||||
const configKey = `${param.name}Config`
|
||||
|
||||
// Preserve top-level config objects (e.g., agentModelConfig)
|
||||
if (existingComponentNodeData.inputs[configKey]) {
|
||||
initNewComponentNodeData.inputs[configKey] = existingComponentNodeData.inputs[configKey]
|
||||
}
|
||||
}
|
||||
|
||||
// Handle array parameters that might contain loadConfig items
|
||||
const arrayParams = initNewComponentNodeData.inputParams.filter((param) => param.type === 'array' && param.array)
|
||||
|
||||
for (const arrayParam of arrayParams) {
|
||||
if (existingComponentNodeData.inputs[arrayParam.name] && Array.isArray(existingComponentNodeData.inputs[arrayParam.name])) {
|
||||
const existingArray = existingComponentNodeData.inputs[arrayParam.name]
|
||||
|
||||
// Find loadConfig parameters within the array definition
|
||||
const arrayLoadConfigParams = arrayParam.array.filter((subParam) => subParam.loadConfig === true)
|
||||
|
||||
if (arrayLoadConfigParams.length > 0) {
|
||||
// Process each array item to preserve config objects
|
||||
const updatedArray = existingArray.map((existingItem) => {
|
||||
if (typeof existingItem === 'object' && existingItem !== null) {
|
||||
const updatedItem = { ...existingItem }
|
||||
|
||||
// Preserve config objects for each loadConfig parameter in the array
|
||||
for (const loadConfigParam of arrayLoadConfigParams) {
|
||||
const configKey = `${loadConfigParam.name}Config`
|
||||
if (existingItem[configKey]) {
|
||||
updatedItem[configKey] = existingItem[configKey]
|
||||
}
|
||||
}
|
||||
|
||||
return updatedItem
|
||||
}
|
||||
return existingItem
|
||||
})
|
||||
|
||||
initNewComponentNodeData.inputs[arrayParam.name] = updatedArray
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also preserve any config keys that exist in the existing data but might not be explicitly handled above
|
||||
// This catches edge cases where config keys exist but don't follow the expected pattern
|
||||
for (const key in existingComponentNodeData.inputs) {
|
||||
if (key.endsWith('Config') && !initNewComponentNodeData.inputs[key]) {
|
||||
initNewComponentNodeData.inputs[key] = existingComponentNodeData.inputs[key]
|
||||
}
|
||||
}
|
||||
}
|
||||
// Check for tabs
|
||||
const inputParamsWithTabIdentifiers = initNewComponentNodeData.inputParams.filter((param) => param.tabIdentifier) || []
|
||||
|
||||
@@ -268,7 +325,7 @@ export const updateOutdatedNodeData = (newComponentNodeData, existingComponentNo
|
||||
initNewComponentNodeData.label = existingComponentNodeData.label
|
||||
}
|
||||
|
||||
// Special case for Condition node to update outputAnchors
|
||||
// Special case for Sequential Condition node to update outputAnchors
|
||||
if (initNewComponentNodeData.name.includes('seqCondition')) {
|
||||
const options = existingComponentNodeData.outputAnchors[0].options || []
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useState } from 'react'
|
||||
import { useSelector } from 'react-redux'
|
||||
import PropTypes from 'prop-types'
|
||||
import axios from 'axios'
|
||||
|
||||
// MUI
|
||||
import {
|
||||
@@ -24,7 +25,7 @@ import {
|
||||
} from '@mui/material'
|
||||
import { useTheme, darken } from '@mui/material/styles'
|
||||
import { useSnackbar } from 'notistack'
|
||||
import { IconCoins, IconClock, IconChevronDown } from '@tabler/icons-react'
|
||||
import { IconCoins, IconClock, IconChevronDown, IconDownload, IconTool } from '@tabler/icons-react'
|
||||
import toolSVG from '@/assets/images/tool.svg'
|
||||
|
||||
// Project imports
|
||||
@@ -34,6 +35,7 @@ import { AGENTFLOW_ICONS, baseURL } from '@/store/constant'
|
||||
import { JSONViewer } from '@/ui-component/json/JsonViewer'
|
||||
import ReactJson from 'flowise-react-json-view'
|
||||
import { CodeEditor } from '@/ui-component/editor/CodeEditor'
|
||||
import SourceDocDialog from '@/ui-component/dialog/SourceDocDialog'
|
||||
|
||||
import predictionApi from '@/api/prediction'
|
||||
|
||||
@@ -44,6 +46,8 @@ export const NodeExecutionDetails = ({ data, label, status, metadata, isPublic,
|
||||
const [feedbackType, setFeedbackType] = useState('')
|
||||
const [isLoading, setIsLoading] = useState(false)
|
||||
const [loadingMessage, setLoadingMessage] = useState('')
|
||||
const [sourceDialogOpen, setSourceDialogOpen] = useState(false)
|
||||
const [sourceDialogProps, setSourceDialogProps] = useState({})
|
||||
const customization = useSelector((state) => state.customization)
|
||||
const theme = useTheme()
|
||||
const { enqueueSnackbar } = useSnackbar()
|
||||
@@ -160,6 +164,11 @@ export const NodeExecutionDetails = ({ data, label, status, metadata, isPublic,
|
||||
}
|
||||
}
|
||||
|
||||
const onUsedToolClick = (data, title) => {
|
||||
setSourceDialogProps({ data, title })
|
||||
setSourceDialogOpen(true)
|
||||
}
|
||||
|
||||
const handleSubmitFeedback = () => {
|
||||
onSubmitResponse(feedbackType, feedback)
|
||||
setOpenFeedbackDialog(false)
|
||||
@@ -167,6 +176,26 @@ export const NodeExecutionDetails = ({ data, label, status, metadata, isPublic,
|
||||
setFeedbackType('')
|
||||
}
|
||||
|
||||
const downloadFile = async (fileAnnotation) => {
|
||||
try {
|
||||
const response = await axios.post(
|
||||
`${baseURL}/api/v1/openai-assistants-file/download`,
|
||||
{ fileName: fileAnnotation.fileName, chatflowId: metadata?.agentflowId, chatId: metadata?.sessionId },
|
||||
{ responseType: 'blob' }
|
||||
)
|
||||
const blob = new Blob([response.data], { type: response.headers['content-type'] })
|
||||
const downloadUrl = window.URL.createObjectURL(blob)
|
||||
const link = document.createElement('a')
|
||||
link.href = downloadUrl
|
||||
link.download = fileAnnotation.fileName
|
||||
document.body.appendChild(link)
|
||||
link.click()
|
||||
link.remove()
|
||||
} catch (error) {
|
||||
console.error('Download failed:', error)
|
||||
}
|
||||
}
|
||||
|
||||
const renderFullfilledConditions = (conditions) => {
|
||||
const fullfilledConditions = conditions.filter((condition) => condition.isFulfilled)
|
||||
return fullfilledConditions.map((condition, index) => {
|
||||
@@ -661,6 +690,35 @@ export const NodeExecutionDetails = ({ data, label, status, metadata, isPublic,
|
||||
</Typography>
|
||||
</Box>
|
||||
)}
|
||||
{message.additional_kwargs?.usedTools && message.additional_kwargs.usedTools.length > 0 && (
|
||||
<div
|
||||
style={{
|
||||
display: 'block',
|
||||
flexDirection: 'row',
|
||||
width: '100%',
|
||||
marginTop: '10px'
|
||||
}}
|
||||
>
|
||||
{message.additional_kwargs.usedTools.map((tool, index) => {
|
||||
return tool ? (
|
||||
<Chip
|
||||
size='small'
|
||||
key={index}
|
||||
label={tool.tool}
|
||||
sx={{
|
||||
mr: 1,
|
||||
mt: 1,
|
||||
borderColor: tool.error ? 'error.main' : undefined,
|
||||
color: tool.error ? 'error.main' : undefined
|
||||
}}
|
||||
variant='outlined'
|
||||
icon={<IconTool size={15} color={tool.error ? theme.palette.error.main : undefined} />}
|
||||
onClick={() => onUsedToolClick(tool, 'Used Tools')}
|
||||
/>
|
||||
) : null
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{message.additional_kwargs?.artifacts && message.additional_kwargs.artifacts.length > 0 && (
|
||||
<Box sx={{ mt: 2, mb: 1 }}>
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
||||
@@ -691,7 +749,7 @@ export const NodeExecutionDetails = ({ data, label, status, metadata, isPublic,
|
||||
)}`
|
||||
: artifact.data
|
||||
}
|
||||
sx={{ height: 'auto', maxHeight: '500px' }}
|
||||
sx={{ height: 'auto', maxHeight: '500px', objectFit: 'contain' }}
|
||||
alt={`artifact-${artifactIndex}`}
|
||||
/>
|
||||
</Card>
|
||||
@@ -797,6 +855,36 @@ export const NodeExecutionDetails = ({ data, label, status, metadata, isPublic,
|
||||
return <MemoizedReactMarkdown>{`*No data*`}</MemoizedReactMarkdown>
|
||||
}
|
||||
})()}
|
||||
{message.additional_kwargs?.fileAnnotations && message.additional_kwargs.fileAnnotations.length > 0 && (
|
||||
<div
|
||||
style={{
|
||||
display: 'block',
|
||||
flexDirection: 'row',
|
||||
width: '100%',
|
||||
marginTop: '16px',
|
||||
marginBottom: '8px'
|
||||
}}
|
||||
>
|
||||
{message.additional_kwargs.fileAnnotations.map((fileAnnotation, index) => {
|
||||
return (
|
||||
<Button
|
||||
sx={{
|
||||
fontSize: '0.85rem',
|
||||
textTransform: 'none',
|
||||
mb: 1,
|
||||
mr: 1
|
||||
}}
|
||||
key={index}
|
||||
variant='outlined'
|
||||
onClick={() => downloadFile(fileAnnotation)}
|
||||
endIcon={<IconDownload color={theme.palette.primary.main} />}
|
||||
>
|
||||
{fileAnnotation.fileName}
|
||||
</Button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</Box>
|
||||
))
|
||||
) : data?.input?.form || data?.input?.http || data?.input?.conditions ? (
|
||||
@@ -862,6 +950,106 @@ export const NodeExecutionDetails = ({ data, label, status, metadata, isPublic,
|
||||
backgroundColor: theme.palette.background.default
|
||||
}}
|
||||
>
|
||||
{data.output?.usedTools && data.output.usedTools.length > 0 && (
|
||||
<div
|
||||
style={{
|
||||
display: 'block',
|
||||
flexDirection: 'row',
|
||||
width: '100%'
|
||||
}}
|
||||
>
|
||||
{data.output.usedTools.map((tool, index) => {
|
||||
return tool ? (
|
||||
<Chip
|
||||
size='small'
|
||||
key={index}
|
||||
label={tool.tool}
|
||||
sx={{
|
||||
mr: 1,
|
||||
mt: 1,
|
||||
borderColor: tool.error ? 'error.main' : undefined,
|
||||
color: tool.error ? 'error.main' : undefined
|
||||
}}
|
||||
variant='outlined'
|
||||
icon={<IconTool size={15} color={tool.error ? theme.palette.error.main : undefined} />}
|
||||
onClick={() => onUsedToolClick(tool, 'Used Tools')}
|
||||
/>
|
||||
) : null
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{data.output?.artifacts && data.output.artifacts.length > 0 && (
|
||||
<Box sx={{ mt: 2, mb: 1 }}>
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
||||
{data.output.artifacts.map((artifact, artifactIndex) => {
|
||||
if (artifact.type === 'png' || artifact.type === 'jpeg' || artifact.type === 'jpg') {
|
||||
return (
|
||||
<Card
|
||||
key={`artifact-${artifactIndex}`}
|
||||
sx={{
|
||||
p: 0,
|
||||
m: 0,
|
||||
flex: '0 0 auto',
|
||||
border: 1,
|
||||
borderColor: 'divider',
|
||||
borderRadius: 1,
|
||||
overflow: 'hidden'
|
||||
}}
|
||||
>
|
||||
<CardMedia
|
||||
component='img'
|
||||
image={
|
||||
artifact.data.startsWith('FILE-STORAGE::')
|
||||
? `${baseURL}/api/v1/get-upload-file?chatflowId=${
|
||||
metadata?.agentflowId
|
||||
}&chatId=${metadata?.sessionId}&fileName=${artifact.data.replace(
|
||||
'FILE-STORAGE::',
|
||||
''
|
||||
)}`
|
||||
: artifact.data
|
||||
}
|
||||
sx={{ height: 'auto', maxHeight: '500px', objectFit: 'contain' }}
|
||||
alt={`artifact-${artifactIndex}`}
|
||||
/>
|
||||
</Card>
|
||||
)
|
||||
} else if (artifact.type === 'html') {
|
||||
return (
|
||||
<Box
|
||||
key={`artifact-${artifactIndex}`}
|
||||
sx={{
|
||||
mt: 1,
|
||||
border: 1,
|
||||
borderColor: 'divider',
|
||||
borderRadius: 1,
|
||||
p: 2,
|
||||
backgroundColor: theme.palette.background.paper
|
||||
}}
|
||||
>
|
||||
<SafeHTML html={artifact.data} />
|
||||
</Box>
|
||||
)
|
||||
} else {
|
||||
return (
|
||||
<Box
|
||||
key={`artifact-${artifactIndex}`}
|
||||
sx={{
|
||||
mt: 1,
|
||||
border: 1,
|
||||
borderColor: 'divider',
|
||||
borderRadius: 1,
|
||||
p: 2,
|
||||
backgroundColor: theme.palette.background.paper
|
||||
}}
|
||||
>
|
||||
<MemoizedReactMarkdown>{artifact.data}</MemoizedReactMarkdown>
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
})}
|
||||
</Box>
|
||||
</Box>
|
||||
)}
|
||||
{(() => {
|
||||
// Check if the content is a stringified JSON or array
|
||||
if (data?.output?.content) {
|
||||
@@ -882,6 +1070,36 @@ export const NodeExecutionDetails = ({ data, label, status, metadata, isPublic,
|
||||
return <MemoizedReactMarkdown>{`*No data*`}</MemoizedReactMarkdown>
|
||||
}
|
||||
})()}
|
||||
{data.output?.fileAnnotations && data.output.fileAnnotations.length > 0 && (
|
||||
<div
|
||||
style={{
|
||||
display: 'block',
|
||||
flexDirection: 'row',
|
||||
width: '100%',
|
||||
marginTop: '16px',
|
||||
marginBottom: '8px'
|
||||
}}
|
||||
>
|
||||
{data.output.fileAnnotations.map((fileAnnotation, index) => {
|
||||
return (
|
||||
<Button
|
||||
sx={{
|
||||
fontSize: '0.85rem',
|
||||
textTransform: 'none',
|
||||
mb: 1,
|
||||
mr: 1
|
||||
}}
|
||||
key={index}
|
||||
variant='outlined'
|
||||
onClick={() => downloadFile(fileAnnotation)}
|
||||
endIcon={<IconDownload color={theme.palette.primary.main} />}
|
||||
>
|
||||
{fileAnnotation.fileName}
|
||||
</Button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
{data.error && (
|
||||
@@ -1020,6 +1238,7 @@ export const NodeExecutionDetails = ({ data, label, status, metadata, isPublic,
|
||||
</Dialog>
|
||||
</>
|
||||
)}
|
||||
<SourceDocDialog show={sourceDialogOpen} dialogProps={sourceDialogProps} onCancel={() => setSourceDialogOpen(false)} />
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -21,7 +21,10 @@ import {
|
||||
IconTrash,
|
||||
IconInfoCircle,
|
||||
IconLoader,
|
||||
IconAlertCircleFilled
|
||||
IconAlertCircleFilled,
|
||||
IconCode,
|
||||
IconWorldWww,
|
||||
IconPhoto
|
||||
} from '@tabler/icons-react'
|
||||
import StopCircleIcon from '@mui/icons-material/StopCircle'
|
||||
import CancelIcon from '@mui/icons-material/Cancel'
|
||||
@@ -126,6 +129,19 @@ const AgentFlowNode = ({ data }) => {
|
||||
return <foundIcon.icon size={24} color={'white'} />
|
||||
}
|
||||
|
||||
const getBuiltInOpenAIToolIcon = (toolName) => {
|
||||
switch (toolName) {
|
||||
case 'web_search_preview':
|
||||
return <IconWorldWww size={14} color={'white'} />
|
||||
case 'code_interpreter':
|
||||
return <IconCode size={14} color={'white'} />
|
||||
case 'image_generation':
|
||||
return <IconPhoto size={14} color={'white'} />
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (ref.current) {
|
||||
setTimeout(() => {
|
||||
@@ -407,7 +423,17 @@ const AgentFlowNode = ({ data }) => {
|
||||
: [],
|
||||
toolProperty: ['selectedTool', 'toolAgentflowSelectedTool']
|
||||
},
|
||||
{ tools: data.inputs?.agentKnowledgeVSEmbeddings, toolProperty: ['vectorStore', 'embeddingModel'] }
|
||||
{ tools: data.inputs?.agentKnowledgeVSEmbeddings, toolProperty: ['vectorStore', 'embeddingModel'] },
|
||||
{
|
||||
tools: data.inputs?.agentToolsBuiltInOpenAI
|
||||
? (typeof data.inputs.agentToolsBuiltInOpenAI === 'string'
|
||||
? JSON.parse(data.inputs.agentToolsBuiltInOpenAI)
|
||||
: data.inputs.agentToolsBuiltInOpenAI
|
||||
).map((tool) => ({ builtInTool: tool }))
|
||||
: [],
|
||||
toolProperty: 'builtInTool',
|
||||
isBuiltInOpenAI: true
|
||||
}
|
||||
]
|
||||
|
||||
// Filter out undefined tools and render each valid collection
|
||||
@@ -441,6 +467,32 @@ const AgentFlowNode = ({ data }) => {
|
||||
const toolName = tool[config.toolProperty]
|
||||
if (!toolName) return []
|
||||
|
||||
// Handle built-in OpenAI tools with icons
|
||||
if (config.isBuiltInOpenAI) {
|
||||
const icon = getBuiltInOpenAIToolIcon(toolName)
|
||||
if (!icon) return []
|
||||
|
||||
return [
|
||||
<Box
|
||||
key={`tool-${configIndex}-${toolIndex}`}
|
||||
sx={{
|
||||
width: 20,
|
||||
height: 20,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: customization.isDarkMode
|
||||
? darken(data.color, 0.5)
|
||||
: darken(data.color, 0.2),
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
padding: 0.2
|
||||
}}
|
||||
>
|
||||
{icon}
|
||||
</Box>
|
||||
]
|
||||
}
|
||||
|
||||
return [
|
||||
<Box
|
||||
key={`tool-${configIndex}-${toolIndex}`}
|
||||
|
||||
Reference in New Issue
Block a user