Feature/OpenAI Assistant V2 (#2258)

* add gpt4 turbo to assistant * OpenAI Assistant V2 * update langfuse handler
2026-06-28 15:00:57 +03:00 · 2024-04-25 20:14:04 +01:00
parent 4782c0f6fc
commit 7360d1d9a6
25 changed files with 23422 additions and 17637 deletions
@@ -1,12 +1,10 @@
 import OpenAI from 'openai'
-import path from 'path'
-import * as fs from 'fs'
 import { StatusCodes } from 'http-status-codes'
-import { uniqWith, isEqual } from 'lodash'
+import { uniqWith, isEqual, cloneDeep } from 'lodash'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { Assistant } from '../../database/entities/Assistant'
 import { Credential } from '../../database/entities/Credential'
-import { getUserHome, decryptCredentialData, getAppVersion } from '../../utils'
+import { decryptCredentialData, getAppVersion } from '../../utils'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { getErrorMessage } from '../../errors/utils'

@@ -34,6 +32,7 @@ const createAssistant = async (requestBody: any): Promise<any> => {
            }
            const openai = new OpenAI({ apiKey: openAIApiKey })

+            // Prepare tools
            let tools = []
            if (assistantDetails.tools) {
                for (const tool of assistantDetails.tools ?? []) {
@@ -43,40 +42,25 @@ const createAssistant = async (requestBody: any): Promise<any> => {
                }
            }

-            if (assistantDetails.uploadFiles) {
-                // Base64 strings
-                let files: string[] = []
-                const fileBase64 = assistantDetails.uploadFiles
-                if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
-                    files = JSON.parse(fileBase64)
-                } else {
-                    files = [fileBase64]
-                }
+            // Save tool_resources to be stored later into database
+            const savedToolResources = cloneDeep(assistantDetails.tool_resources)

-                const uploadedFiles = []
-                for (const file of files) {
-                    const splitDataURI = file.split(',')
-                    const filename = splitDataURI.pop()?.split(':')[1] ?? ''
-                    const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-                    const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename)
-                    if (!fs.existsSync(path.join(getUserHome(), '.flowise', 'openai-assistant'))) {
-                        fs.mkdirSync(path.dirname(filePath), { recursive: true })
+            // Cleanup tool_resources for creating assistant
+            if (assistantDetails.tool_resources) {
+                for (const toolResource in assistantDetails.tool_resources) {
+                    if (toolResource === 'file_search') {
+                        assistantDetails.tool_resources['file_search'] = {
+                            vector_store_ids: assistantDetails.tool_resources['file_search'].vector_store_ids
+                        }
+                    } else if (toolResource === 'code_interpreter') {
+                        assistantDetails.tool_resources['code_interpreter'] = {
+                            file_ids: assistantDetails.tool_resources['code_interpreter'].file_ids
+                        }
                    }
-                    if (!fs.existsSync(filePath)) {
-                        fs.writeFileSync(filePath, bf)
-                    }
-
-                    const createdFile = await openai.files.create({
-                        file: fs.createReadStream(filePath),
-                        purpose: 'assistants'
-                    })
-                    uploadedFiles.push(createdFile)
-
-                    fs.unlinkSync(filePath)
                }
-                assistantDetails.files = [...assistantDetails.files, ...uploadedFiles]
            }

+            // If the assistant doesn't exist, create a new one
            if (!assistantDetails.id) {
                const newAssistant = await openai.beta.assistants.create({
                    name: assistantDetails.name,
@@ -84,12 +68,15 @@ const createAssistant = async (requestBody: any): Promise<any> => {
                    instructions: assistantDetails.instructions,
                    model: assistantDetails.model,
                    tools,
-                    file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
+                    tool_resources: assistantDetails.tool_resources,
+                    temperature: assistantDetails.temperature,
+                    top_p: assistantDetails.top_p
                })
                assistantDetails.id = newAssistant.id
            } else {
                const retrievedAssistant = await openai.beta.assistants.retrieve(assistantDetails.id)
-                let filteredTools = uniqWith([...retrievedAssistant.tools, ...tools], isEqual)
+                let filteredTools = uniqWith([...retrievedAssistant.tools.filter((tool) => tool.type === 'function'), ...tools], isEqual)
+                // Remove empty functions
                filteredTools = filteredTools.filter((tool) => !(tool.type === 'function' && !(tool as any).function))

                await openai.beta.assistants.update(assistantDetails.id, {
@@ -98,17 +85,16 @@ const createAssistant = async (requestBody: any): Promise<any> => {
                    instructions: assistantDetails.instructions ?? '',
                    model: assistantDetails.model,
                    tools: filteredTools,
-                    file_ids: uniqWith(
-                        [...retrievedAssistant.file_ids, ...(assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)],
-                        isEqual
-                    )
+                    tool_resources: assistantDetails.tool_resources,
+                    temperature: assistantDetails.temperature,
+                    top_p: assistantDetails.top_p
                })
            }

            const newAssistantDetails = {
                ...assistantDetails
            }
-            if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles
+            if (savedToolResources) newAssistantDetails.tool_resources = savedToolResources

            requestBody.details = JSON.stringify(newAssistantDetails)
        } catch (error) {
@@ -117,7 +103,7 @@ const createAssistant = async (requestBody: any): Promise<any> => {
        const newAssistant = new Assistant()
        Object.assign(newAssistant, requestBody)

-        const assistant = await appServer.AppDataSource.getRepository(Assistant).create(newAssistant)
+        const assistant = appServer.AppDataSource.getRepository(Assistant).create(newAssistant)
        const dbResponse = await appServer.AppDataSource.getRepository(Assistant).save(assistant)

        await appServer.telemetry.sendTelemetry('assistant_created', {
@@ -249,42 +235,26 @@ const updateAssistant = async (assistantId: string, requestBody: any): Promise<a
                }
            }

-            if (assistantDetails.uploadFiles) {
-                // Base64 strings
-                let files: string[] = []
-                const fileBase64 = assistantDetails.uploadFiles
-                if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
-                    files = JSON.parse(fileBase64)
-                } else {
-                    files = [fileBase64]
-                }
+            // Save tool_resources to be stored later into database
+            const savedToolResources = cloneDeep(assistantDetails.tool_resources)

-                const uploadedFiles = []
-                for (const file of files) {
-                    const splitDataURI = file.split(',')
-                    const filename = splitDataURI.pop()?.split(':')[1] ?? ''
-                    const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-                    const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename)
-                    if (!fs.existsSync(path.join(getUserHome(), '.flowise', 'openai-assistant'))) {
-                        fs.mkdirSync(path.dirname(filePath), { recursive: true })
+            // Cleanup tool_resources before updating
+            if (assistantDetails.tool_resources) {
+                for (const toolResource in assistantDetails.tool_resources) {
+                    if (toolResource === 'file_search') {
+                        assistantDetails.tool_resources['file_search'] = {
+                            vector_store_ids: assistantDetails.tool_resources['file_search'].vector_store_ids
+                        }
+                    } else if (toolResource === 'code_interpreter') {
+                        assistantDetails.tool_resources['code_interpreter'] = {
+                            file_ids: assistantDetails.tool_resources['code_interpreter'].file_ids
+                        }
                    }
-                    if (!fs.existsSync(filePath)) {
-                        fs.writeFileSync(filePath, bf)
-                    }
-
-                    const createdFile = await openai.files.create({
-                        file: fs.createReadStream(filePath),
-                        purpose: 'assistants'
-                    })
-                    uploadedFiles.push(createdFile)
-
-                    fs.unlinkSync(filePath)
                }
-                assistantDetails.files = [...assistantDetails.files, ...uploadedFiles]
            }

            const retrievedAssistant = await openai.beta.assistants.retrieve(openAIAssistantId)
-            let filteredTools = uniqWith([...retrievedAssistant.tools, ...tools], isEqual)
+            let filteredTools = uniqWith([...retrievedAssistant.tools.filter((tool) => tool.type === 'function'), ...tools], isEqual)
            filteredTools = filteredTools.filter((tool) => !(tool.type === 'function' && !(tool as any).function))

            await openai.beta.assistants.update(openAIAssistantId, {
@@ -293,23 +263,22 @@ const updateAssistant = async (assistantId: string, requestBody: any): Promise<a
                instructions: assistantDetails.instructions,
                model: assistantDetails.model,
                tools: filteredTools,
-                file_ids: uniqWith(
-                    [...retrievedAssistant.file_ids, ...(assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)],
-                    isEqual
-                )
+                tool_resources: assistantDetails.tool_resources,
+                temperature: assistantDetails.temperature,
+                top_p: assistantDetails.top_p
            })

            const newAssistantDetails = {
                ...assistantDetails,
                id: openAIAssistantId
            }
-            if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles
+            if (savedToolResources) newAssistantDetails.tool_resources = savedToolResources

            const updateAssistant = new Assistant()
            body.details = JSON.stringify(newAssistantDetails)
            Object.assign(updateAssistant, body)

-            await appServer.AppDataSource.getRepository(Assistant).merge(assistant, updateAssistant)
+            appServer.AppDataSource.getRepository(Assistant).merge(assistant, updateAssistant)
            const dbResponse = await appServer.AppDataSource.getRepository(Assistant).save(assistant)
            return dbResponse
        } catch (error) {
@@ -0,0 +1,257 @@
+import OpenAI from 'openai'
+import { StatusCodes } from 'http-status-codes'
+import fs from 'fs'
+import { Credential } from '../../database/entities/Credential'
+import { InternalFlowiseError } from '../../errors/internalFlowiseError'
+import { getErrorMessage } from '../../errors/utils'
+import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
+import { decryptCredentialData } from '../../utils'
+
+const getAssistantVectorStore = async (credentialId: string, vectorStoreId: string) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const credential = await appServer.AppDataSource.getRepository(Credential).findOneBy({
+            id: credentialId
+        })
+        if (!credential) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Credential ${credentialId} not found in the database!`)
+        }
+        // Decrpyt credentialData
+        const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
+        const openAIApiKey = decryptedCredentialData['openAIApiKey']
+        if (!openAIApiKey) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `OpenAI ApiKey not found`)
+        }
+
+        const openai = new OpenAI({ apiKey: openAIApiKey })
+        const dbResponse = await openai.beta.vectorStores.retrieve(vectorStoreId)
+        return dbResponse
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: openaiAssistantsVectorStoreService.getAssistantVectorStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const listAssistantVectorStore = async (credentialId: string) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const credential = await appServer.AppDataSource.getRepository(Credential).findOneBy({
+            id: credentialId
+        })
+        if (!credential) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Credential ${credentialId} not found in the database!`)
+        }
+        // Decrpyt credentialData
+        const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
+        const openAIApiKey = decryptedCredentialData['openAIApiKey']
+        if (!openAIApiKey) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `OpenAI ApiKey not found`)
+        }
+
+        const openai = new OpenAI({ apiKey: openAIApiKey })
+        const dbResponse = await openai.beta.vectorStores.list()
+        return dbResponse.data
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: openaiAssistantsVectorStoreService.listAssistantVectorStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const createAssistantVectorStore = async (credentialId: string, obj: OpenAI.Beta.VectorStores.VectorStoreCreateParams) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const credential = await appServer.AppDataSource.getRepository(Credential).findOneBy({
+            id: credentialId
+        })
+        if (!credential) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Credential ${credentialId} not found in the database!`)
+        }
+        // Decrpyt credentialData
+        const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
+        const openAIApiKey = decryptedCredentialData['openAIApiKey']
+        if (!openAIApiKey) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `OpenAI ApiKey not found`)
+        }
+
+        const openai = new OpenAI({ apiKey: openAIApiKey })
+        const dbResponse = await openai.beta.vectorStores.create(obj)
+        return dbResponse
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: openaiAssistantsVectorStoreService.createAssistantVectorStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const updateAssistantVectorStore = async (
+    credentialId: string,
+    vectorStoreId: string,
+    obj: OpenAI.Beta.VectorStores.VectorStoreUpdateParams
+) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const credential = await appServer.AppDataSource.getRepository(Credential).findOneBy({
+            id: credentialId
+        })
+        if (!credential) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Credential ${credentialId} not found in the database!`)
+        }
+        // Decrpyt credentialData
+        const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
+        const openAIApiKey = decryptedCredentialData['openAIApiKey']
+        if (!openAIApiKey) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `OpenAI ApiKey not found`)
+        }
+
+        const openai = new OpenAI({ apiKey: openAIApiKey })
+        const dbResponse = await openai.beta.vectorStores.update(vectorStoreId, obj)
+        const vectorStoreFiles = await openai.beta.vectorStores.files.list(vectorStoreId)
+        if (vectorStoreFiles.data?.length) {
+            const files = []
+            for (const file of vectorStoreFiles.data) {
+                const fileData = await openai.files.retrieve(file.id)
+                files.push(fileData)
+            }
+            ;(dbResponse as any).files = files
+        }
+        return dbResponse
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: openaiAssistantsVectorStoreService.updateAssistantVectorStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const deleteAssistantVectorStore = async (credentialId: string, vectorStoreId: string) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const credential = await appServer.AppDataSource.getRepository(Credential).findOneBy({
+            id: credentialId
+        })
+        if (!credential) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Credential ${credentialId} not found in the database!`)
+        }
+        // Decrpyt credentialData
+        const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
+        const openAIApiKey = decryptedCredentialData['openAIApiKey']
+        if (!openAIApiKey) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `OpenAI ApiKey not found`)
+        }
+
+        const openai = new OpenAI({ apiKey: openAIApiKey })
+        const dbResponse = await openai.beta.vectorStores.del(vectorStoreId)
+        return dbResponse
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: openaiAssistantsVectorStoreService.deleteAssistantVectorStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const uploadFilesToAssistantVectorStore = async (
+    credentialId: string,
+    vectorStoreId: string,
+    files: { filePath: string; fileName: string }[]
+): Promise<any> => {
+    try {
+        const appServer = getRunningExpressApp()
+        const credential = await appServer.AppDataSource.getRepository(Credential).findOneBy({
+            id: credentialId
+        })
+        if (!credential) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Credential ${credentialId} not found in the database!`)
+        }
+        // Decrpyt credentialData
+        const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
+        const openAIApiKey = decryptedCredentialData['openAIApiKey']
+        if (!openAIApiKey) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `OpenAI ApiKey not found`)
+        }
+
+        const openai = new OpenAI({ apiKey: openAIApiKey })
+        const uploadedFiles = []
+        for (const file of files) {
+            const createdFile = await openai.files.create({
+                file: new File([new Blob([fs.readFileSync(file.filePath)])], file.fileName),
+                purpose: 'assistants'
+            })
+            uploadedFiles.push(createdFile)
+            fs.unlinkSync(file.filePath)
+        }
+
+        const file_ids = [...uploadedFiles.map((file) => file.id)]
+
+        const res = await openai.beta.vectorStores.fileBatches.createAndPoll(vectorStoreId, {
+            file_ids
+        })
+        if (res.status === 'completed' && res.file_counts.completed === uploadedFiles.length) return uploadedFiles
+        else if (res.status === 'failed')
+            throw new InternalFlowiseError(
+                StatusCodes.INTERNAL_SERVER_ERROR,
+                'Error: openaiAssistantsVectorStoreService.uploadFilesToAssistantVectorStore - Upload failed!'
+            )
+        else
+            throw new InternalFlowiseError(
+                StatusCodes.INTERNAL_SERVER_ERROR,
+                'Error: openaiAssistantsVectorStoreService.uploadFilesToAssistantVectorStore - Upload cancelled!'
+            )
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: openaiAssistantsVectorStoreService.uploadFilesToAssistantVectorStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const deleteFilesFromAssistantVectorStore = async (credentialId: string, vectorStoreId: string, file_ids: string[]) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const credential = await appServer.AppDataSource.getRepository(Credential).findOneBy({
+            id: credentialId
+        })
+        if (!credential) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Credential ${credentialId} not found in the database!`)
+        }
+        // Decrpyt credentialData
+        const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
+        const openAIApiKey = decryptedCredentialData['openAIApiKey']
+        if (!openAIApiKey) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `OpenAI ApiKey not found`)
+        }
+
+        const openai = new OpenAI({ apiKey: openAIApiKey })
+        const deletedFileIds = []
+        let count = 0
+        for (const file of file_ids) {
+            const res = await openai.beta.vectorStores.files.del(vectorStoreId, file)
+            if (res.deleted) {
+                deletedFileIds.push(file)
+                count += 1
+            }
+        }
+
+        return { deletedFileIds, count }
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: openaiAssistantsVectorStoreService.uploadFilesToAssistantVectorStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+export default {
+    getAssistantVectorStore,
+    listAssistantVectorStore,
+    createAssistantVectorStore,
+    updateAssistantVectorStore,
+    deleteAssistantVectorStore,
+    uploadFilesToAssistantVectorStore,
+    deleteFilesFromAssistantVectorStore
+}
@@ -1,4 +1,5 @@
 import OpenAI from 'openai'
+import fs from 'fs'
 import { StatusCodes } from 'http-status-codes'
 import { decryptCredentialData } from '../../utils'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
@@ -59,8 +60,18 @@ const getSingleOpenaiAssistant = async (credentialId: string, assistantId: strin
        const dbResponse = await openai.beta.assistants.retrieve(assistantId)
        const resp = await openai.files.list()
        const existingFiles = resp.data ?? []
-        if (dbResponse.file_ids && dbResponse.file_ids.length) {
-            ;(dbResponse as any).files = existingFiles.filter((file) => dbResponse.file_ids.includes(file.id))
+        if (dbResponse.tool_resources?.code_interpreter?.file_ids?.length) {
+            ;(dbResponse.tool_resources.code_interpreter as any).files = [
+                ...existingFiles.filter((file) => dbResponse.tool_resources?.code_interpreter?.file_ids?.includes(file.id))
+            ]
+        }
+        if (dbResponse.tool_resources?.file_search?.vector_store_ids?.length) {
+            // Since there can only be 1 vector store per assistant
+            const vectorStoreId = dbResponse.tool_resources.file_search.vector_store_ids[0]
+            const vectorStoreFiles = await openai.beta.vectorStores.files.list(vectorStoreId)
+            const fileIds = vectorStoreFiles.data?.map((file) => file.id) ?? []
+            ;(dbResponse.tool_resources.file_search as any).files = [...existingFiles.filter((file) => fileIds.includes(file.id))]
+            ;(dbResponse.tool_resources.file_search as any).vector_store_object = await openai.beta.vectorStores.retrieve(vectorStoreId)
        }
        return dbResponse
    } catch (error) {
@@ -71,7 +82,38 @@ const getSingleOpenaiAssistant = async (credentialId: string, assistantId: strin
    }
 }

+const uploadFilesToAssistant = async (credentialId: string, files: { filePath: string; fileName: string }[]) => {
+    const appServer = getRunningExpressApp()
+    const credential = await appServer.AppDataSource.getRepository(Credential).findOneBy({
+        id: credentialId
+    })
+    if (!credential) {
+        throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Credential ${credentialId} not found in the database!`)
+    }
+    // Decrpyt credentialData
+    const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
+    const openAIApiKey = decryptedCredentialData['openAIApiKey']
+    if (!openAIApiKey) {
+        throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `OpenAI ApiKey not found`)
+    }
+
+    const openai = new OpenAI({ apiKey: openAIApiKey })
+    const uploadedFiles = []
+
+    for (const file of files) {
+        const createdFile = await openai.files.create({
+            file: new File([new Blob([fs.readFileSync(file.filePath)])], file.fileName),
+            purpose: 'assistants'
+        })
+        uploadedFiles.push(createdFile)
+        fs.unlinkSync(file.filePath)
+    }
+
+    return uploadedFiles
+}
+
 export default {
    getAllOpenaiAssistants,
-    getSingleOpenaiAssistant
+    getSingleOpenaiAssistant,
+    uploadFilesToAssistant
 }