farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into FEATURE/Vision

Browse Source 
# Conflicts:
#	packages/components/nodes/chains/ConversationChain/ConversationChain.ts
#	packages/server/src/index.ts
#	packages/server/src/utils/index.ts
This commit is contained in:
Henry
2024-02-02 02:54:06 +00:00
parent eab8c19f8c 37828de664
commit 9cd0362f24
136 changed files with 5054 additions and 2019 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts
+23 -11
View File
@@ -1,4 +1,4 @@
import { ChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
import { BaseChatModelParams } from 'langchain/chat_models/base'
import type { ClientOptions } from 'openai'
import type { LegacyOpenAIInput } from '@langchain/openai/dist/types'
@@ -6,47 +6,59 @@ import { BaseLanguageModelInput } from 'langchain/base_language'
import { ChatOpenAICallOptions } from '@langchain/openai/dist/chat_models'
import { BaseMessageChunk, BaseMessageLike, HumanMessage, LLMResult } from 'langchain/schema'
import { Callbacks } from '@langchain/core/callbacks/manager'
import { ICommonObject, INodeData } from '../../../src'
import { addImagesToMessages } from '../../../src/MultiModalUtils'
import { ICommonObject, IMultiModalOption, INodeData } from '../../../src'
import { addImagesToMessages } from '../../../src/multiModalUtils'
export class FlowiseChatOpenAI extends ChatOpenAI {
multiModal: {}
export class ChatOpenAI extends LangchainChatOpenAI {
//TODO: Should be class variables and not static
public static chainNodeData: INodeData
public static chainNodeOptions: ICommonObject
configuredModel: string
configuredMaxToken?: number
multiModalOption?: IMultiModalOption
constructor(
fields?: Partial<OpenAIChatInput> & BaseChatModelParams & { openAIApiKey?: string },
fields?: Partial<OpenAIChatInput> & BaseChatModelParams & { openAIApiKey?: string; multiModalOption?: IMultiModalOption },
/** @deprecated */
configuration?: ClientOptions & LegacyOpenAIInput
) {
super(fields)
this.multiModalOption = fields?.multiModalOption
this.configuredModel = fields?.modelName ?? 'gpt-3.5-turbo'
this.configuredMaxToken = fields?.maxTokens
}
async invoke(input: BaseLanguageModelInput, options?: ChatOpenAICallOptions): Promise<BaseMessageChunk> {
//input.messages
return super.invoke(input, options)
}
async generate(messages: BaseMessageLike[][], options?: string[] | ChatOpenAICallOptions, callbacks?: Callbacks): Promise<LLMResult> {
//messages
await this.injectMultiModalMessages(messages)
return super.generate(messages, options, callbacks)
}
private async injectMultiModalMessages(messages: BaseMessageLike[][]) {
const nodeData = FlowiseChatOpenAI.chainNodeData
const optionsData = FlowiseChatOpenAI.chainNodeOptions
const messageContent = addImagesToMessages(nodeData, optionsData)
const nodeData = ChatOpenAI.chainNodeData
const optionsData = ChatOpenAI.chainNodeOptions
const messageContent = addImagesToMessages(nodeData, optionsData, this.multiModalOption)
if (messageContent?.length) {
if (messages[0].length > 0 && messages[0][messages[0].length - 1] instanceof HumanMessage) {
const lastMessage = messages[0].pop()
if (lastMessage instanceof HumanMessage) {
lastMessage.content = messageContent
// Change model to gpt-4-vision
this.modelName = 'gpt-4-vision-preview'
// Change default max token to higher when using gpt-4-vision
this.maxTokens = 1024
}
messages[0].push(lastMessage as HumanMessage)
}
} else {
// revert to previous values if image upload is empty
this.modelName = this.configuredModel
this.maxTokens = this.configuredMaxToken
}
}
}