farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

GPT Vision: Initial implementation of the OpenAI Vision API

Browse Source
This commit is contained in:
vinodkiran
2023-12-06 12:31:33 +05:30
parent c96572e10f
commit 73f7046316
15 changed files with 447 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/Interface.ts
+2
View File
@@ -31,6 +31,7 @@ export interface IChatMessage {
sourceDocuments?: string
usedTools?: string
fileAnnotations?: string
fileUploads?: string
chatType: string
chatId: string
memoryType?: string
@@ -167,6 +168,7 @@ export interface IncomingInput {
socketIOClientId?: string
chatId?: string
stopNodeId?: string
uploads?: string
}
export interface IActiveChatflows {
packages/server/src/database/entities/ChatMessage.ts
+3
View File
@@ -26,6 +26,9 @@ export class ChatMessage implements IChatMessage {
@Column({ nullable: true, type: 'text' })
fileAnnotations?: string
@Column({ nullable: true, type: 'text' })
fileUploads?: string
@Column()
chatType: string
packages/server/src/database/migrations/mysql/1701788586491-AddFileUploadsToChatMessage.ts
+12
View File
@@ -0,0 +1,12 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddFileUploadsToChatMessage1701788586491 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const columnExists = await queryRunner.hasColumn('chat_message', 'fileUploads')
if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`fileUploads\` TEXT;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`fileUploads\`;`)
}
}
packages/server/src/database/migrations/mysql/index.ts
+3 -1
View File
@@ -10,6 +10,7 @@ import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEnt
import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow'
import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
export const mysqlMigrations = [
Init1693840429259,
@@ -23,5 +24,6 @@ export const mysqlMigrations = [
AddAssistantEntity1699325775451,
AddUsedToolsToChatMessage1699481607341,
AddCategoryToChatFlow1699900910291,
AddFileAnnotationsToChatMessage1700271021237
AddFileAnnotationsToChatMessage1700271021237,
AddFileUploadsToChatMessage1701788586491
]
packages/server/src/database/migrations/postgres/1701788586491-AddFileUploadsToChatMessage.ts
+11
View File
@@ -0,0 +1,11 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddFileUploadsToChatMessage1701788586491 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN IF NOT EXISTS "fileUploads" TEXT;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "fileUploads";`)
}
}
packages/server/src/database/migrations/postgres/index.ts
+3 -1
View File
@@ -10,6 +10,7 @@ import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEnt
import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow'
import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
export const postgresMigrations = [
Init1693891895163,
@@ -23,5 +24,6 @@ export const postgresMigrations = [
AddAssistantEntity1699325775451,
AddUsedToolsToChatMessage1699481607341,
AddCategoryToChatFlow1699900910291,
AddFileAnnotationsToChatMessage1700271021237
AddFileAnnotationsToChatMessage1700271021237,
AddFileUploadsToChatMessage1701788586491
]
packages/server/src/database/migrations/sqlite/1701788586491-AddFileUploadsToChatMessage.ts
+20
View File
@@ -0,0 +1,20 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddFileUploadsToChatMessage1701788586491 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "usedTools" text, "fileAnnotations" text, "fileUploads" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', "chatId" VARCHAR NOT NULL, "memoryType" VARCHAR, "sessionId" VARCHAR);`
)
await queryRunner.query(
`INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "fileAnnotations", "usedTools", "createdDate", "chatType", "chatId", "memoryType", "sessionId") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "usedTools", "fileAnnotations", "createdDate", "chatType", "chatId", "memoryType", "sessionId" FROM "chat_message";`
)
await queryRunner.query(`DROP TABLE "chat_message";`)
await queryRunner.query(`ALTER TABLE "temp_chat_message" RENAME TO "chat_message";`)
await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE IF EXISTS "temp_chat_message";`)
await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "fileUploads";`)
}
}
packages/server/src/database/migrations/sqlite/index.ts
+3 -1
View File
@@ -10,6 +10,7 @@ import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEnt
import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow'
import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
export const sqliteMigrations = [
Init1693835579790,
@@ -23,5 +24,6 @@ export const sqliteMigrations = [
AddAssistantEntity1699325775451,
AddUsedToolsToChatMessage1699481607341,
AddCategoryToChatFlow1699900910291,
AddFileAnnotationsToChatMessage1700271021237
AddFileAnnotationsToChatMessage1700271021237,
AddFileUploadsToChatMessage1701788586491
]
packages/server/src/index.ts
+38 -6
View File
@@ -410,9 +410,7 @@ export class App {
})
if (!chatflow) return res.status(404).send(`Chatflow ${req.params.id} not found`)
const obj = {
allowUploads: this.shouldAllowUploads(chatflow)
}
const obj = this.shouldAllowUploads(chatflow)
return res.json(obj)
})
@@ -1255,16 +1253,30 @@ export class App {
}
private uploadAllowedNodes = ['OpenAIVisionChain']
private shouldAllowUploads(result: ChatFlow): boolean {
private shouldAllowUploads(result: ChatFlow): any {
const flowObj = JSON.parse(result.flowData)
let allowUploads = false
let allowedTypes: string[] = []
let maxUploadSize: number = -1
flowObj.nodes.forEach((node: IReactFlowNode) => {
if (this.uploadAllowedNodes.indexOf(node.data.type) > -1) {
logger.debug(`[server]: Found Eligible Node ${node.data.type}, Allowing Uploads.`)
allowUploads = true
node.data.inputParams.map((param: any) => {
if (param.name === 'allowedUploadTypes') {
allowedTypes = param.default.split(';')
}
if (param.name === 'maxUploadSize') {
maxUploadSize = parseInt(param.default ? param.default : '0')
}
})
}
})
return allowUploads
return {
allowUploads,
allowedTypes,
maxUploadSize
}
}
/**
@@ -1392,6 +1404,23 @@ export class App {
if (!isKeyValidated) return res.status(401).send('Unauthorized')
}
if (incomingInput.uploads) {
// @ts-ignore
;(incomingInput.uploads as any[]).forEach((url: any) => {
if (url.type === 'file') {
const filename = url.name
const bf = url.data
const filePath = path.join(getUserHome(), '.flowise', 'gptvision', filename)
if (!fs.existsSync(path.join(getUserHome(), '.flowise', 'gptvision'))) {
fs.mkdirSync(path.dirname(filePath), { recursive: true })
}
if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf)
fs.unlinkSync(filePath)
url.data = bf.toString('base64')
}
})
}
let isStreamValid = false
const files = (req.files as any[]) || []
@@ -1534,6 +1563,7 @@ export class App {
let result = isStreamValid
? await nodeInstance.run(nodeToExecuteData, incomingInput.question, {
uploads: incomingInput.uploads,
chatHistory: incomingInput.history,
socketIO,
socketIOClientId: incomingInput.socketIOClientId,
@@ -1544,6 +1574,7 @@ export class App {
chatId
})
: await nodeInstance.run(nodeToExecuteData, incomingInput.question, {
uploads: incomingInput.uploads,
chatHistory: incomingInput.history,
logger,
appDataSource: this.AppDataSource,
@@ -1567,7 +1598,8 @@ export class App {
chatId,
memoryType,
sessionId,
createdDate: userMessageDateTime
createdDate: userMessageDateTime,
fileUploads: incomingInput.uploads ? JSON.stringify(incomingInput.uploads) : ''
}
await this.addChatMessage(userMessage)