mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 15:00:57 +03:00
Feature/OpenAI Response API (#5014)
* - Added support for built-in OpenAI tools including web search, code interpreter, and image generation. - Enhanced file handling by extracting artifacts and file annotations from response metadata. - Implemented download functionality for file annotations in the UI. - Updated chat history management to include additional kwargs for artifacts, file annotations, and used tools. - Improved UI components to display used tools and file annotations effectively. * remove redundant currentContainerId * update comment
This commit is contained in:
Henry Heng
@@ -28,6 +28,9 @@ import {
|
||||
replaceBase64ImagesWithFileReferences,
|
||||
updateFlowState
|
||||
} from '../utils'
|
||||
import { convertMultiOptionsToStringArray, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { addSingleFileToStorage } from '../../../src/storageUtils'
|
||||
import fetch from 'node-fetch'
|
||||
|
||||
interface ITool {
|
||||
agentSelectedTool: string
|
||||
@@ -78,7 +81,7 @@ class Agent_Agentflow implements INode {
|
||||
constructor() {
|
||||
this.label = 'Agent'
|
||||
this.name = 'agentAgentflow'
|
||||
this.version = 1.0
|
||||
this.version = 2.0
|
||||
this.type = 'Agent'
|
||||
this.category = 'Agent Flows'
|
||||
this.description = 'Dynamically choose and utilize tools during runtime, enabling multi-step reasoning'
|
||||
@@ -132,6 +135,32 @@ class Agent_Agentflow implements INode {
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
label: 'OpenAI Built-in Tools',
|
||||
name: 'agentToolsBuiltInOpenAI',
|
||||
type: 'multiOptions',
|
||||
optional: true,
|
||||
options: [
|
||||
{
|
||||
label: 'Web Search',
|
||||
name: 'web_search_preview',
|
||||
description: 'Search the web for the latest information'
|
||||
},
|
||||
{
|
||||
label: 'Code Interpreter',
|
||||
name: 'code_interpreter',
|
||||
description: 'Write and run Python code in a sandboxed environment'
|
||||
},
|
||||
{
|
||||
label: 'Image Generation',
|
||||
name: 'image_generation',
|
||||
description: 'Generate images based on a text prompt'
|
||||
}
|
||||
],
|
||||
show: {
|
||||
agentModel: 'chatOpenAI'
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Tools',
|
||||
name: 'agentTools',
|
||||
@@ -716,6 +745,26 @@ class Agent_Agentflow implements INode {
|
||||
const llmWithoutToolsBind = (await newLLMNodeInstance.init(newNodeData, '', options)) as BaseChatModel
|
||||
let llmNodeInstance = llmWithoutToolsBind
|
||||
|
||||
const agentToolsBuiltInOpenAI = convertMultiOptionsToStringArray(nodeData.inputs?.agentToolsBuiltInOpenAI)
|
||||
if (agentToolsBuiltInOpenAI && agentToolsBuiltInOpenAI.length > 0) {
|
||||
for (const tool of agentToolsBuiltInOpenAI) {
|
||||
const builtInTool: ICommonObject = {
|
||||
type: tool
|
||||
}
|
||||
if (tool === 'code_interpreter') {
|
||||
builtInTool.container = { type: 'auto' }
|
||||
}
|
||||
;(toolsInstance as any).push(builtInTool)
|
||||
;(availableTools as any).push({
|
||||
name: tool,
|
||||
toolNode: {
|
||||
label: tool,
|
||||
name: tool
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (llmNodeInstance && toolsInstance.length > 0) {
|
||||
if (llmNodeInstance.bindTools === undefined) {
|
||||
throw new Error(`Agent needs to have a function calling capable models.`)
|
||||
@@ -814,6 +863,7 @@ class Agent_Agentflow implements INode {
|
||||
let usedTools: IUsedTool[] = []
|
||||
let sourceDocuments: Array<any> = []
|
||||
let artifacts: any[] = []
|
||||
let fileAnnotations: any[] = []
|
||||
let additionalTokens = 0
|
||||
let isWaitingForHumanInput = false
|
||||
|
||||
@@ -879,6 +929,9 @@ class Agent_Agentflow implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
// Address built in tools (after artifacts are processed)
|
||||
const builtInUsedTools: IUsedTool[] = await this.extractBuiltInUsedTools(response, [])
|
||||
|
||||
if (!humanInput && response.tool_calls && response.tool_calls.length > 0) {
|
||||
const result = await this.handleToolCalls({
|
||||
response,
|
||||
@@ -954,6 +1007,46 @@ class Agent_Agentflow implements INode {
|
||||
} else {
|
||||
finalResponse = JSON.stringify(response, null, 2)
|
||||
}
|
||||
|
||||
// Address built in tools
|
||||
const additionalBuiltInUsedTools: IUsedTool[] = await this.extractBuiltInUsedTools(response, builtInUsedTools)
|
||||
if (additionalBuiltInUsedTools.length > 0) {
|
||||
usedTools = [...new Set([...usedTools, ...additionalBuiltInUsedTools])]
|
||||
|
||||
// Stream used tools if this is the last node
|
||||
if (isLastNode && sseStreamer) {
|
||||
sseStreamer.streamUsedToolsEvent(chatId, flatten(usedTools))
|
||||
}
|
||||
}
|
||||
|
||||
// Extract artifacts from annotations in response metadata
|
||||
if (response.response_metadata) {
|
||||
const { artifacts: extractedArtifacts, fileAnnotations: extractedFileAnnotations } =
|
||||
await this.extractArtifactsFromResponse(response.response_metadata, newNodeData, options)
|
||||
if (extractedArtifacts.length > 0) {
|
||||
artifacts = [...artifacts, ...extractedArtifacts]
|
||||
|
||||
// Stream artifacts if this is the last node
|
||||
if (isLastNode && sseStreamer) {
|
||||
sseStreamer.streamArtifactsEvent(chatId, extractedArtifacts)
|
||||
}
|
||||
}
|
||||
|
||||
if (extractedFileAnnotations.length > 0) {
|
||||
fileAnnotations = [...fileAnnotations, ...extractedFileAnnotations]
|
||||
|
||||
// Stream file annotations if this is the last node
|
||||
if (isLastNode && sseStreamer) {
|
||||
sseStreamer.streamFileAnnotationsEvent(chatId, fileAnnotations)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Replace sandbox links with proper download URLs. Example: [Download the script](sandbox:/mnt/data/dummy_bar_graph.py)
|
||||
if (finalResponse.includes('sandbox:/')) {
|
||||
finalResponse = await this.processSandboxLinks(finalResponse, options.baseURL, options.chatflowid, chatId)
|
||||
}
|
||||
|
||||
const output = this.prepareOutputObject(
|
||||
response,
|
||||
availableTools,
|
||||
@@ -965,7 +1058,8 @@ class Agent_Agentflow implements INode {
|
||||
sourceDocuments,
|
||||
artifacts,
|
||||
additionalTokens,
|
||||
isWaitingForHumanInput
|
||||
isWaitingForHumanInput,
|
||||
fileAnnotations
|
||||
)
|
||||
|
||||
// End analytics tracking
|
||||
@@ -978,6 +1072,11 @@ class Agent_Agentflow implements INode {
|
||||
this.sendStreamingEvents(options, chatId, response)
|
||||
}
|
||||
|
||||
// Stream file annotations if any were extracted
|
||||
if (fileAnnotations.length > 0 && isLastNode && sseStreamer) {
|
||||
sseStreamer.streamFileAnnotationsEvent(chatId, fileAnnotations)
|
||||
}
|
||||
|
||||
// Process template variables in state
|
||||
if (newState && Object.keys(newState).length > 0) {
|
||||
for (const key in newState) {
|
||||
@@ -1043,7 +1142,16 @@ class Agent_Agentflow implements INode {
|
||||
{
|
||||
role: returnRole,
|
||||
content: finalResponse,
|
||||
name: nodeData?.label ? nodeData?.label.toLowerCase().replace(/\s/g, '_').trim() : nodeData?.id
|
||||
name: nodeData?.label ? nodeData?.label.toLowerCase().replace(/\s/g, '_').trim() : nodeData?.id,
|
||||
...(((artifacts && artifacts.length > 0) ||
|
||||
(fileAnnotations && fileAnnotations.length > 0) ||
|
||||
(usedTools && usedTools.length > 0)) && {
|
||||
additional_kwargs: {
|
||||
...(artifacts && artifacts.length > 0 && { artifacts }),
|
||||
...(fileAnnotations && fileAnnotations.length > 0 && { fileAnnotations }),
|
||||
...(usedTools && usedTools.length > 0 && { usedTools })
|
||||
}
|
||||
})
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1059,6 +1167,105 @@ class Agent_Agentflow implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts built-in used tools from response metadata and processes image generation results
|
||||
*/
|
||||
private async extractBuiltInUsedTools(response: AIMessageChunk, builtInUsedTools: IUsedTool[] = []): Promise<IUsedTool[]> {
|
||||
if (!response.response_metadata) {
|
||||
return builtInUsedTools
|
||||
}
|
||||
|
||||
const { output, tools } = response.response_metadata
|
||||
|
||||
if (!output || !Array.isArray(output) || output.length === 0 || !tools || !Array.isArray(tools) || tools.length === 0) {
|
||||
return builtInUsedTools
|
||||
}
|
||||
|
||||
for (const outputItem of output) {
|
||||
if (outputItem.type && outputItem.type.endsWith('_call')) {
|
||||
let toolInput = outputItem.action ?? outputItem.code
|
||||
let toolOutput = outputItem.status === 'completed' ? 'Success' : outputItem.status
|
||||
|
||||
// Handle image generation calls specially
|
||||
if (outputItem.type === 'image_generation_call') {
|
||||
// Create input summary for image generation
|
||||
toolInput = {
|
||||
prompt: outputItem.revised_prompt || 'Image generation request',
|
||||
size: outputItem.size || '1024x1024',
|
||||
quality: outputItem.quality || 'standard',
|
||||
output_format: outputItem.output_format || 'png'
|
||||
}
|
||||
|
||||
// Check if image has been processed (base64 replaced with file path)
|
||||
if (outputItem.result && !outputItem.result.startsWith('data:') && !outputItem.result.includes('base64')) {
|
||||
toolOutput = `Image generated and saved`
|
||||
} else {
|
||||
toolOutput = `Image generated (base64)`
|
||||
}
|
||||
}
|
||||
|
||||
// Remove "_call" suffix to get the base tool name
|
||||
const baseToolName = outputItem.type.replace('_call', '')
|
||||
|
||||
// Find matching tool that includes the base name in its type
|
||||
const matchingTool = tools.find((tool) => tool.type && tool.type.includes(baseToolName))
|
||||
|
||||
if (matchingTool) {
|
||||
// Check for duplicates
|
||||
if (builtInUsedTools.find((tool) => tool.tool === matchingTool.type)) {
|
||||
continue
|
||||
}
|
||||
|
||||
builtInUsedTools.push({
|
||||
tool: matchingTool.type,
|
||||
toolInput,
|
||||
toolOutput
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return builtInUsedTools
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves base64 image data to storage and returns file information
|
||||
*/
|
||||
private async saveBase64Image(
|
||||
outputItem: any,
|
||||
options: ICommonObject
|
||||
): Promise<{ filePath: string; fileName: string; totalSize: number } | null> {
|
||||
try {
|
||||
if (!outputItem.result) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Extract base64 data and create buffer
|
||||
const base64Data = outputItem.result
|
||||
const imageBuffer = Buffer.from(base64Data, 'base64')
|
||||
|
||||
// Determine file extension and MIME type
|
||||
const outputFormat = outputItem.output_format || 'png'
|
||||
const fileName = `generated_image_${outputItem.id || Date.now()}.${outputFormat}`
|
||||
const mimeType = outputFormat === 'png' ? 'image/png' : 'image/jpeg'
|
||||
|
||||
// Save the image using the existing storage utility
|
||||
const { path, totalSize } = await addSingleFileToStorage(
|
||||
mimeType,
|
||||
imageBuffer,
|
||||
fileName,
|
||||
options.orgId,
|
||||
options.chatflowid,
|
||||
options.chatId
|
||||
)
|
||||
|
||||
return { filePath: path, fileName, totalSize }
|
||||
} catch (error) {
|
||||
console.error('Error saving base64 image:', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles memory management based on the specified memory type
|
||||
*/
|
||||
@@ -1265,7 +1472,8 @@ class Agent_Agentflow implements INode {
|
||||
sourceDocuments: Array<any>,
|
||||
artifacts: any[],
|
||||
additionalTokens: number = 0,
|
||||
isWaitingForHumanInput: boolean = false
|
||||
isWaitingForHumanInput: boolean = false,
|
||||
fileAnnotations: any[] = []
|
||||
): any {
|
||||
const output: any = {
|
||||
content: finalResponse,
|
||||
@@ -1296,6 +1504,10 @@ class Agent_Agentflow implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
if (response.response_metadata) {
|
||||
output.responseMetadata = response.response_metadata
|
||||
}
|
||||
|
||||
// Add used tools, source documents and artifacts to output
|
||||
if (usedTools && usedTools.length > 0) {
|
||||
output.usedTools = flatten(usedTools)
|
||||
@@ -1317,6 +1529,10 @@ class Agent_Agentflow implements INode {
|
||||
output.isWaitingForHumanInput = isWaitingForHumanInput
|
||||
}
|
||||
|
||||
if (fileAnnotations && fileAnnotations.length > 0) {
|
||||
output.fileAnnotations = fileAnnotations
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
@@ -1808,6 +2024,10 @@ class Agent_Agentflow implements INode {
|
||||
// Get LLM response after tool calls
|
||||
let newResponse: AIMessageChunk
|
||||
|
||||
if (llmNodeInstance && (llmNodeInstance as any).builtInTools && (llmNodeInstance as any).builtInTools.length > 0) {
|
||||
toolsInstance.push(...(llmNodeInstance as any).builtInTools)
|
||||
}
|
||||
|
||||
if (llmNodeInstance && toolsInstance.length > 0) {
|
||||
if (llmNodeInstance.bindTools === undefined) {
|
||||
throw new Error(`Agent needs to have a function calling capable models.`)
|
||||
@@ -1872,6 +2092,224 @@ class Agent_Agentflow implements INode {
|
||||
|
||||
return { response: newResponse, usedTools, sourceDocuments, artifacts, totalTokens, isWaitingForHumanInput }
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts artifacts from response metadata (both annotations and built-in tools)
|
||||
*/
|
||||
private async extractArtifactsFromResponse(
|
||||
responseMetadata: any,
|
||||
modelNodeData: INodeData,
|
||||
options: ICommonObject
|
||||
): Promise<{ artifacts: any[]; fileAnnotations: any[] }> {
|
||||
const artifacts: any[] = []
|
||||
const fileAnnotations: any[] = []
|
||||
|
||||
if (!responseMetadata?.output || !Array.isArray(responseMetadata.output)) {
|
||||
return { artifacts, fileAnnotations }
|
||||
}
|
||||
|
||||
for (const outputItem of responseMetadata.output) {
|
||||
// Handle container file citations from annotations
|
||||
if (outputItem.type === 'message' && outputItem.content && Array.isArray(outputItem.content)) {
|
||||
for (const contentItem of outputItem.content) {
|
||||
if (contentItem.annotations && Array.isArray(contentItem.annotations)) {
|
||||
for (const annotation of contentItem.annotations) {
|
||||
if (annotation.type === 'container_file_citation' && annotation.file_id && annotation.filename) {
|
||||
try {
|
||||
// Download and store the file content
|
||||
const downloadResult = await this.downloadContainerFile(
|
||||
annotation.container_id,
|
||||
annotation.file_id,
|
||||
annotation.filename,
|
||||
modelNodeData,
|
||||
options
|
||||
)
|
||||
|
||||
if (downloadResult) {
|
||||
const fileType = this.getArtifactTypeFromFilename(annotation.filename)
|
||||
|
||||
if (fileType === 'png' || fileType === 'jpeg' || fileType === 'jpg') {
|
||||
const artifact = {
|
||||
type: fileType,
|
||||
data: downloadResult.filePath
|
||||
}
|
||||
|
||||
artifacts.push(artifact)
|
||||
} else {
|
||||
fileAnnotations.push({
|
||||
filePath: downloadResult.filePath,
|
||||
fileName: annotation.filename
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error processing annotation:', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle built-in tool artifacts (like image generation)
|
||||
if (outputItem.type === 'image_generation_call' && outputItem.result) {
|
||||
try {
|
||||
const savedImageResult = await this.saveBase64Image(outputItem, options)
|
||||
if (savedImageResult) {
|
||||
// Replace the base64 result with the file path in the response metadata
|
||||
outputItem.result = savedImageResult.filePath
|
||||
|
||||
// Create artifact in the same format as other image artifacts
|
||||
const fileType = this.getArtifactTypeFromFilename(savedImageResult.fileName)
|
||||
artifacts.push({
|
||||
type: fileType,
|
||||
data: savedImageResult.filePath
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error processing image generation artifact:', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { artifacts, fileAnnotations }
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads file content from container file citation
|
||||
*/
|
||||
private async downloadContainerFile(
|
||||
containerId: string,
|
||||
fileId: string,
|
||||
filename: string,
|
||||
modelNodeData: INodeData,
|
||||
options: ICommonObject
|
||||
): Promise<{ filePath: string; totalSize: number } | null> {
|
||||
try {
|
||||
const credentialData = await getCredentialData(modelNodeData.credential ?? '', options)
|
||||
const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, modelNodeData)
|
||||
|
||||
if (!openAIApiKey) {
|
||||
console.warn('No OpenAI API key available for downloading container file')
|
||||
return null
|
||||
}
|
||||
|
||||
// Download the file using OpenAI Container API
|
||||
const response = await fetch(`https://api.openai.com/v1/containers/${containerId}/files/${fileId}/content`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: '*/*',
|
||||
Authorization: `Bearer ${openAIApiKey}`
|
||||
}
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
console.warn(
|
||||
`Failed to download container file ${fileId} from container ${containerId}: ${response.status} ${response.statusText}`
|
||||
)
|
||||
return null
|
||||
}
|
||||
|
||||
// Extract the binary data from the Response object
|
||||
const data = await response.arrayBuffer()
|
||||
const dataBuffer = Buffer.from(data)
|
||||
const mimeType = this.getMimeTypeFromFilename(filename)
|
||||
|
||||
// Store the file using the same storage utility as OpenAIAssistant
|
||||
const { path, totalSize } = await addSingleFileToStorage(
|
||||
mimeType,
|
||||
dataBuffer,
|
||||
filename,
|
||||
options.orgId,
|
||||
options.chatflowid,
|
||||
options.chatId
|
||||
)
|
||||
|
||||
return { filePath: path, totalSize }
|
||||
} catch (error) {
|
||||
console.error('Error downloading container file:', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets MIME type from filename extension
|
||||
*/
|
||||
private getMimeTypeFromFilename(filename: string): string {
|
||||
const extension = filename.toLowerCase().split('.').pop()
|
||||
const mimeTypes: { [key: string]: string } = {
|
||||
png: 'image/png',
|
||||
jpg: 'image/jpeg',
|
||||
jpeg: 'image/jpeg',
|
||||
gif: 'image/gif',
|
||||
pdf: 'application/pdf',
|
||||
txt: 'text/plain',
|
||||
csv: 'text/csv',
|
||||
json: 'application/json',
|
||||
html: 'text/html',
|
||||
xml: 'application/xml'
|
||||
}
|
||||
return mimeTypes[extension || ''] || 'application/octet-stream'
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets artifact type from filename extension for UI rendering
|
||||
*/
|
||||
private getArtifactTypeFromFilename(filename: string): string {
|
||||
const extension = filename.toLowerCase().split('.').pop()
|
||||
const artifactTypes: { [key: string]: string } = {
|
||||
png: 'png',
|
||||
jpg: 'jpeg',
|
||||
jpeg: 'jpeg',
|
||||
html: 'html',
|
||||
htm: 'html',
|
||||
md: 'markdown',
|
||||
markdown: 'markdown',
|
||||
json: 'json',
|
||||
js: 'javascript',
|
||||
javascript: 'javascript',
|
||||
tex: 'latex',
|
||||
latex: 'latex',
|
||||
txt: 'text',
|
||||
csv: 'text',
|
||||
pdf: 'text'
|
||||
}
|
||||
return artifactTypes[extension || ''] || 'text'
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes sandbox links in the response text and converts them to file annotations
|
||||
*/
|
||||
private async processSandboxLinks(text: string, baseURL: string, chatflowId: string, chatId: string): Promise<string> {
|
||||
let processedResponse = text
|
||||
|
||||
// Regex to match sandbox links: [text](sandbox:/path/to/file)
|
||||
const sandboxLinkRegex = /\[([^\]]+)\]\(sandbox:\/([^)]+)\)/g
|
||||
const matches = Array.from(text.matchAll(sandboxLinkRegex))
|
||||
|
||||
for (const match of matches) {
|
||||
const fullMatch = match[0]
|
||||
const linkText = match[1]
|
||||
const filePath = match[2]
|
||||
|
||||
try {
|
||||
// Extract filename from the file path
|
||||
const fileName = filePath.split('/').pop() || filePath
|
||||
|
||||
// Replace sandbox link with proper download URL
|
||||
const downloadUrl = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatflowId}&chatId=${chatId}&fileName=${fileName}&download=true`
|
||||
const newLink = `[${linkText}](${downloadUrl})`
|
||||
|
||||
processedResponse = processedResponse.replace(fullMatch, newLink)
|
||||
} catch (error) {
|
||||
console.error('Error processing sandbox link:', error)
|
||||
// If there's an error, remove the sandbox link as fallback
|
||||
processedResponse = processedResponse.replace(fullMatch, linkText)
|
||||
}
|
||||
}
|
||||
|
||||
return processedResponse
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Agent_Agentflow }
|
||||
|
||||
@@ -313,6 +313,9 @@ export const getPastChatHistoryImageMessages = async (
|
||||
if (message.additional_kwargs && message.additional_kwargs.fileUploads) {
|
||||
// example: [{"type":"stored-file","name":"0_DiXc4ZklSTo3M8J4.jpg","mime":"image/jpeg"}]
|
||||
const fileUploads = message.additional_kwargs.fileUploads
|
||||
const artifacts = message.additional_kwargs.artifacts
|
||||
const fileAnnotations = message.additional_kwargs.fileAnnotations
|
||||
const usedTools = message.additional_kwargs.usedTools
|
||||
try {
|
||||
let messageWithFileUploads = ''
|
||||
const uploads: IFileUpload[] = typeof fileUploads === 'string' ? JSON.parse(fileUploads) : fileUploads
|
||||
@@ -358,22 +361,83 @@ export const getPastChatHistoryImageMessages = async (
|
||||
}
|
||||
}
|
||||
const messageContent = messageWithFileUploads ? `${messageWithFileUploads}\n\n${message.content}` : message.content
|
||||
const hasArtifacts = artifacts && Array.isArray(artifacts) && artifacts.length > 0
|
||||
const hasFileAnnotations = fileAnnotations && Array.isArray(fileAnnotations) && fileAnnotations.length > 0
|
||||
const hasUsedTools = usedTools && Array.isArray(usedTools) && usedTools.length > 0
|
||||
|
||||
if (imageContents.length > 0) {
|
||||
chatHistory.push({
|
||||
const imageMessage: any = {
|
||||
role: messageRole,
|
||||
content: imageContents
|
||||
})
|
||||
}
|
||||
if (hasArtifacts || hasFileAnnotations || hasUsedTools) {
|
||||
imageMessage.additional_kwargs = {}
|
||||
if (hasArtifacts) imageMessage.additional_kwargs.artifacts = artifacts
|
||||
if (hasFileAnnotations) imageMessage.additional_kwargs.fileAnnotations = fileAnnotations
|
||||
if (hasUsedTools) imageMessage.additional_kwargs.usedTools = usedTools
|
||||
}
|
||||
chatHistory.push(imageMessage)
|
||||
transformedPastMessages.push({
|
||||
role: messageRole,
|
||||
content: [...JSON.parse((pastChatHistory[i] as any).additional_kwargs.fileUploads)]
|
||||
})
|
||||
}
|
||||
chatHistory.push({
|
||||
|
||||
const contentMessage: any = {
|
||||
role: messageRole,
|
||||
content: messageContent
|
||||
})
|
||||
}
|
||||
if (hasArtifacts || hasFileAnnotations || hasUsedTools) {
|
||||
contentMessage.additional_kwargs = {}
|
||||
if (hasArtifacts) contentMessage.additional_kwargs.artifacts = artifacts
|
||||
if (hasFileAnnotations) contentMessage.additional_kwargs.fileAnnotations = fileAnnotations
|
||||
if (hasUsedTools) contentMessage.additional_kwargs.usedTools = usedTools
|
||||
}
|
||||
chatHistory.push(contentMessage)
|
||||
} catch (e) {
|
||||
// failed to parse fileUploads, continue with text only
|
||||
const hasArtifacts = artifacts && Array.isArray(artifacts) && artifacts.length > 0
|
||||
const hasFileAnnotations = fileAnnotations && Array.isArray(fileAnnotations) && fileAnnotations.length > 0
|
||||
const hasUsedTools = usedTools && Array.isArray(usedTools) && usedTools.length > 0
|
||||
|
||||
const errorMessage: any = {
|
||||
role: messageRole,
|
||||
content: message.content
|
||||
}
|
||||
if (hasArtifacts || hasFileAnnotations || hasUsedTools) {
|
||||
errorMessage.additional_kwargs = {}
|
||||
if (hasArtifacts) errorMessage.additional_kwargs.artifacts = artifacts
|
||||
if (hasFileAnnotations) errorMessage.additional_kwargs.fileAnnotations = fileAnnotations
|
||||
if (hasUsedTools) errorMessage.additional_kwargs.usedTools = usedTools
|
||||
}
|
||||
chatHistory.push(errorMessage)
|
||||
}
|
||||
} else if (message.additional_kwargs) {
|
||||
const hasArtifacts =
|
||||
message.additional_kwargs.artifacts &&
|
||||
Array.isArray(message.additional_kwargs.artifacts) &&
|
||||
message.additional_kwargs.artifacts.length > 0
|
||||
const hasFileAnnotations =
|
||||
message.additional_kwargs.fileAnnotations &&
|
||||
Array.isArray(message.additional_kwargs.fileAnnotations) &&
|
||||
message.additional_kwargs.fileAnnotations.length > 0
|
||||
const hasUsedTools =
|
||||
message.additional_kwargs.usedTools &&
|
||||
Array.isArray(message.additional_kwargs.usedTools) &&
|
||||
message.additional_kwargs.usedTools.length > 0
|
||||
|
||||
if (hasArtifacts || hasFileAnnotations || hasUsedTools) {
|
||||
const messageAdditionalKwargs: any = {}
|
||||
if (hasArtifacts) messageAdditionalKwargs.artifacts = message.additional_kwargs.artifacts
|
||||
if (hasFileAnnotations) messageAdditionalKwargs.fileAnnotations = message.additional_kwargs.fileAnnotations
|
||||
if (hasUsedTools) messageAdditionalKwargs.usedTools = message.additional_kwargs.usedTools
|
||||
|
||||
chatHistory.push({
|
||||
role: messageRole,
|
||||
content: message.content,
|
||||
additional_kwargs: messageAdditionalKwargs
|
||||
})
|
||||
} else {
|
||||
chatHistory.push({
|
||||
role: messageRole,
|
||||
content: message.content
|
||||
|
||||
Reference in New Issue
Block a user