farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into FEATURE/Vision

Browse Source 
# Conflicts:
#	packages/components/nodes/chains/ConversationChain/ConversationChain.ts
#	packages/server/src/index.ts
#	packages/server/src/utils/index.ts
This commit is contained in:
Henry
2024-02-02 02:54:06 +00:00
parent eab8c19f8c 37828de664
commit 9cd0362f24
136 changed files with 5054 additions and 2019 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts
+90 -3
View File
@@ -1,7 +1,9 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { convertMultiOptionsToStringArray, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { BaseCache } from 'langchain/schema'
import { ChatGoogleGenerativeAI } from '@langchain/google-genai'
import { ChatGoogleGenerativeAI, GoogleGenerativeAIChatInput } from '@langchain/google-genai'
import { HarmBlockThreshold, HarmCategory } from '@google/generative-ai'
import type { SafetySetting } from '@google/generative-ai'
class GoogleGenerativeAI_ChatModels implements INode {
label: string
@@ -74,6 +76,73 @@ class GoogleGenerativeAI_ChatModels implements INode {
step: 0.1,
optional: true,
additionalParams: true
},
{
label: 'Top Next Highest Probability Tokens',
name: 'topK',
type: 'number',
description: `Decode using top-k sampling: consider the set of top_k most probable tokens. Must be positive`,
step: 1,
optional: true,
additionalParams: true
},
{
label: 'Harm Category',
name: 'harmCategory',
type: 'multiOptions',
description:
'Refer to <a target="_blank" href="https://cloud.google.com/vertex-ai/docs/generative-ai/multimodal/configure-safety-attributes#safety_attribute_definitions">official guide</a> on how to use Harm Category',
options: [
{
label: 'Dangerous',
name: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT
},
{
label: 'Harassment',
name: HarmCategory.HARM_CATEGORY_HARASSMENT
},
{
label: 'Hate Speech',
name: HarmCategory.HARM_CATEGORY_HATE_SPEECH
},
{
label: 'Sexually Explicit',
name: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT
}
],
optional: true,
additionalParams: true
},
{
label: 'Harm Block Threshold',
name: 'harmBlockThreshold',
type: 'multiOptions',
description:
'Refer to <a target="_blank" href="https://cloud.google.com/vertex-ai/docs/generative-ai/multimodal/configure-safety-attributes#safety_setting_thresholds">official guide</a> on how to use Harm Block Threshold',
options: [
{
label: 'Low and Above',
name: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE
},
{
label: 'Medium and Above',
name: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE
},
{
label: 'None',
name: HarmBlockThreshold.BLOCK_NONE
},
{
label: 'Only High',
name: HarmBlockThreshold.BLOCK_ONLY_HIGH
},
{
label: 'Threshold Unspecified',
name: HarmBlockThreshold.HARM_BLOCK_THRESHOLD_UNSPECIFIED
}
],
optional: true,
additionalParams: true
}
]
}
@@ -86,9 +155,12 @@ class GoogleGenerativeAI_ChatModels implements INode {
const modelName = nodeData.inputs?.modelName as string
const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string
const topP = nodeData.inputs?.topP as string
const topK = nodeData.inputs?.topK as string
const harmCategory = nodeData.inputs?.harmCategory as string
const harmBlockThreshold = nodeData.inputs?.harmBlockThreshold as string
const cache = nodeData.inputs?.cache as BaseCache
const obj = {
const obj: Partial<GoogleGenerativeAIChatInput> = {
apiKey: apiKey,
modelName: modelName,
maxOutputTokens: 2048
@@ -98,8 +170,23 @@ class GoogleGenerativeAI_ChatModels implements INode {
const model = new ChatGoogleGenerativeAI(obj)
if (topP) model.topP = parseFloat(topP)
if (topK) model.topK = parseFloat(topK)
if (cache) model.cache = cache
if (temperature) model.temperature = parseFloat(temperature)
// Safety Settings
let harmCategories: string[] = convertMultiOptionsToStringArray(harmCategory)
let harmBlockThresholds: string[] = convertMultiOptionsToStringArray(harmBlockThreshold)
if (harmCategories.length != harmBlockThresholds.length)
throw new Error(`Harm Category & Harm Block Threshold are not the same length`)
const safetySettings: SafetySetting[] = harmCategories.map((harmCategory, index) => {
return {
category: harmCategory as HarmCategory,
threshold: harmBlockThresholds[index] as HarmBlockThreshold
}
})
if (safetySettings.length > 0) model.safetySettings = safetySettings
return model
}
}
packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts
+26 -11
View File
@@ -1,9 +1,9 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { ChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
import { BaseCache } from 'langchain/schema'
import { BaseLLMParams } from 'langchain/llms/base'
import { FlowiseChatOpenAI } from './FlowiseChatOpenAI'
import { ChatOpenAI } from './FlowiseChatOpenAI'
class ChatOpenAI_ChatModels implements INode {
label: string
@@ -20,12 +20,12 @@ class ChatOpenAI_ChatModels implements INode {
constructor() {
this.label = 'ChatOpenAI'
this.name = 'chatOpenAI'
this.version = 3.0
this.version = 4.0
this.type = 'ChatOpenAI'
this.icon = 'openai.svg'
this.category = 'Chat Models'
this.description = 'Wrapper around OpenAI large language models that use the Chat endpoint'
this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)]
this.baseClasses = [this.type, ...getBaseClasses(LangchainChatOpenAI)]
this.credential = {
label: 'Connect Credential',
name: 'credential',
@@ -48,6 +48,14 @@ class ChatOpenAI_ChatModels implements INode {
label: 'gpt-4',
name: 'gpt-4'
},
{
label: 'gpt-4-turbo-preview',
name: 'gpt-4-turbo-preview'
},
{
label: 'gpt-4-0125-preview',
name: 'gpt-4-0125-preview'
},
{
label: 'gpt-4-1106-preview',
name: 'gpt-4-1106-preview'
@@ -72,6 +80,10 @@ class ChatOpenAI_ChatModels implements INode {
label: 'gpt-3.5-turbo',
name: 'gpt-3.5-turbo'
},
{
label: 'gpt-3.5-turbo-0125',
name: 'gpt-3.5-turbo-0125'
},
{
label: 'gpt-3.5-turbo-1106',
name: 'gpt-3.5-turbo-1106'
@@ -158,7 +170,7 @@ class ChatOpenAI_ChatModels implements INode {
label: 'Allow Image Uploads',
name: 'allowImageUploads',
type: 'boolean',
description: 'Enabling this option, would default the model to gpt-4-vision-preview',
description: 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat',
default: false,
optional: true
},
@@ -231,16 +243,19 @@ class ChatOpenAI_ChatModels implements INode {
throw new Error("Invalid JSON in the ChatOpenAI's BaseOptions: " + exception)
}
}
const model = new FlowiseChatOpenAI(obj, {
const model = new ChatOpenAI(obj, {
baseURL: basePath,
baseOptions: parsedBaseOptions
})
const multiModal = {
allowImageUploads: allowImageUploads ?? false,
imageResolution
const multiModalOption: IMultiModalOption = {
image: {
allowImageUploads: allowImageUploads ?? false,
imageResolution
}
}
model.multiModal = multiModal
model.multiModalOption = multiModalOption
return model
}
}
packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts
+23 -11
View File
@@ -1,4 +1,4 @@
import { ChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
import { BaseChatModelParams } from 'langchain/chat_models/base'
import type { ClientOptions } from 'openai'
import type { LegacyOpenAIInput } from '@langchain/openai/dist/types'
@@ -6,47 +6,59 @@ import { BaseLanguageModelInput } from 'langchain/base_language'
import { ChatOpenAICallOptions } from '@langchain/openai/dist/chat_models'
import { BaseMessageChunk, BaseMessageLike, HumanMessage, LLMResult } from 'langchain/schema'
import { Callbacks } from '@langchain/core/callbacks/manager'
import { ICommonObject, INodeData } from '../../../src'
import { addImagesToMessages } from '../../../src/MultiModalUtils'
import { ICommonObject, IMultiModalOption, INodeData } from '../../../src'
import { addImagesToMessages } from '../../../src/multiModalUtils'
export class FlowiseChatOpenAI extends ChatOpenAI {
multiModal: {}
export class ChatOpenAI extends LangchainChatOpenAI {
//TODO: Should be class variables and not static
public static chainNodeData: INodeData
public static chainNodeOptions: ICommonObject
configuredModel: string
configuredMaxToken?: number
multiModalOption?: IMultiModalOption
constructor(
fields?: Partial<OpenAIChatInput> & BaseChatModelParams & { openAIApiKey?: string },
fields?: Partial<OpenAIChatInput> & BaseChatModelParams & { openAIApiKey?: string; multiModalOption?: IMultiModalOption },
/** @deprecated */
configuration?: ClientOptions & LegacyOpenAIInput
) {
super(fields)
this.multiModalOption = fields?.multiModalOption
this.configuredModel = fields?.modelName ?? 'gpt-3.5-turbo'
this.configuredMaxToken = fields?.maxTokens
}
async invoke(input: BaseLanguageModelInput, options?: ChatOpenAICallOptions): Promise<BaseMessageChunk> {
//input.messages
return super.invoke(input, options)
}
async generate(messages: BaseMessageLike[][], options?: string[] | ChatOpenAICallOptions, callbacks?: Callbacks): Promise<LLMResult> {
//messages
await this.injectMultiModalMessages(messages)
return super.generate(messages, options, callbacks)
}
private async injectMultiModalMessages(messages: BaseMessageLike[][]) {
const nodeData = FlowiseChatOpenAI.chainNodeData
const optionsData = FlowiseChatOpenAI.chainNodeOptions
const messageContent = addImagesToMessages(nodeData, optionsData)
const nodeData = ChatOpenAI.chainNodeData
const optionsData = ChatOpenAI.chainNodeOptions
const messageContent = addImagesToMessages(nodeData, optionsData, this.multiModalOption)
if (messageContent?.length) {
if (messages[0].length > 0 && messages[0][messages[0].length - 1] instanceof HumanMessage) {
const lastMessage = messages[0].pop()
if (lastMessage instanceof HumanMessage) {
lastMessage.content = messageContent
// Change model to gpt-4-vision
this.modelName = 'gpt-4-vision-preview'
// Change default max token to higher when using gpt-4-vision
this.maxTokens = 1024
}
messages[0].push(lastMessage as HumanMessage)
}
} else {
// revert to previous values if image upload is empty
this.modelName = this.configuredModel
this.maxTokens = this.configuredMaxToken
}
}
}