From 9efb70e04c6152076308cfcf7b9927dff10b7835 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Thu, 26 Jun 2025 12:54:47 +0100 Subject: [PATCH] Feature/Safety settings to google genai (#4737) * add safety settings to google genai * add safety settings to google genai --- packages/components/nodes/agentflow/utils.ts | 4 +- .../ChatGoogleGenerativeAI.ts | 156 +++++++++++------- packages/components/src/utils.ts | 2 +- .../src/ui-component/array/ArrayRenderer.jsx | 55 ++++-- .../views/docstore/DocStoreInputHandler.jsx | 4 + 5 files changed, 139 insertions(+), 82 deletions(-) diff --git a/packages/components/nodes/agentflow/utils.ts b/packages/components/nodes/agentflow/utils.ts index 5d731789..5e2b4a84 100644 --- a/packages/components/nodes/agentflow/utils.ts +++ b/packages/components/nodes/agentflow/utils.ts @@ -4,7 +4,7 @@ import { getFileFromStorage } from '../../src/storageUtils' import { ICommonObject, IFileUpload } from '../../src/Interface' import { BaseMessageLike } from '@langchain/core/messages' import { IFlowState } from './Interface.Agentflow' -import { mapMimeTypeToInputField } from '../../src/utils' +import { handleEscapeCharacters, mapMimeTypeToInputField } from '../../src/utils' export const addImagesToMessages = async ( options: ICommonObject, @@ -354,7 +354,7 @@ export const getPastChatHistoryImageMessages = async ( } } const documents: string = await fileLoaderNodeInstance.init(nodeData, '', nodeOptions) - messageWithFileUploads += `${documents}\n\n` + messageWithFileUploads += `${handleEscapeCharacters(documents, true)}\n\n` } } const messageContent = messageWithFileUploads ? `${messageWithFileUploads}\n\n${message.content}` : message.content diff --git a/packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts b/packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts index 58b78b34..40549ae1 100644 --- a/packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts +++ b/packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts @@ -2,7 +2,7 @@ import { HarmBlockThreshold, HarmCategory } from '@google/generative-ai' import type { SafetySetting } from '@google/generative-ai' import { BaseCache } from '@langchain/core/caches' import { ICommonObject, IMultiModalOption, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface' -import { convertMultiOptionsToStringArray, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { getModels, MODEL_TYPE } from '../../../src/modelLoader' import { ChatGoogleGenerativeAI } from './FlowiseChatGoogleGenerativeAI' import { GoogleGenerativeAIChatInput } from '@langchain/google-genai' @@ -22,7 +22,7 @@ class GoogleGenerativeAI_ChatModels implements INode { constructor() { this.label = 'ChatGoogleGenerativeAI' this.name = 'chatGoogleGenerativeAI' - this.version = 3.0 + this.version = 3.1 this.type = 'ChatGoogleGenerativeAI' this.icon = 'GoogleGemini.svg' this.category = 'Chat Models' @@ -101,58 +101,75 @@ class GoogleGenerativeAI_ChatModels implements INode { additionalParams: true }, { - label: 'Harm Category', - name: 'harmCategory', - type: 'multiOptions', + label: 'Safety Settings', + name: 'safetySettings', + type: 'array', description: - 'Refer to official guide on how to use Harm Category', - options: [ + 'Safety settings for the model. Refer to the official guide on how to use Safety Settings', + array: [ { - label: 'Dangerous', - name: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT + label: 'Harm Category', + name: 'harmCategory', + type: 'options', + options: [ + { + label: 'Dangerous', + name: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + description: 'Promotes, facilitates, or encourages harmful acts.' + }, + { + label: 'Harassment', + name: HarmCategory.HARM_CATEGORY_HARASSMENT, + description: 'Negative or harmful comments targeting identity and/or protected attributes.' + }, + { + label: 'Hate Speech', + name: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + description: 'Content that is rude, disrespectful, or profane.' + }, + { + label: 'Sexually Explicit', + name: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + description: 'Contains references to sexual acts or other lewd content.' + }, + { + label: 'Civic Integrity', + name: HarmCategory.HARM_CATEGORY_CIVIC_INTEGRITY, + description: 'Election-related queries.' + } + ] }, { - label: 'Harassment', - name: HarmCategory.HARM_CATEGORY_HARASSMENT - }, - { - label: 'Hate Speech', - name: HarmCategory.HARM_CATEGORY_HATE_SPEECH - }, - { - label: 'Sexually Explicit', - name: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT - } - ], - optional: true, - additionalParams: true - }, - { - label: 'Harm Block Threshold', - name: 'harmBlockThreshold', - type: 'multiOptions', - description: - 'Refer to official guide on how to use Harm Block Threshold', - options: [ - { - label: 'Low and Above', - name: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE - }, - { - label: 'Medium and Above', - name: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE - }, - { - label: 'None', - name: HarmBlockThreshold.BLOCK_NONE - }, - { - label: 'Only High', - name: HarmBlockThreshold.BLOCK_ONLY_HIGH - }, - { - label: 'Threshold Unspecified', - name: HarmBlockThreshold.HARM_BLOCK_THRESHOLD_UNSPECIFIED + label: 'Harm Block Threshold', + name: 'harmBlockThreshold', + type: 'options', + options: [ + { + label: 'None', + name: HarmBlockThreshold.BLOCK_NONE, + description: 'Always show regardless of probability of unsafe content' + }, + { + label: 'Only High', + name: HarmBlockThreshold.BLOCK_ONLY_HIGH, + description: 'Block when high probability of unsafe content' + }, + { + label: 'Medium and Above', + name: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE, + description: 'Block when medium or high probability of unsafe content' + }, + { + label: 'Low and Above', + name: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + description: 'Block when low, medium or high probability of unsafe content' + }, + { + label: 'Threshold Unspecified (Default Threshold)', + name: HarmBlockThreshold.HARM_BLOCK_THRESHOLD_UNSPECIFIED, + description: 'Threshold is unspecified, block using default threshold' + } + ] } ], optional: true, @@ -195,8 +212,8 @@ class GoogleGenerativeAI_ChatModels implements INode { const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string - const harmCategory = nodeData.inputs?.harmCategory as string - const harmBlockThreshold = nodeData.inputs?.harmBlockThreshold as string + const _safetySettings = nodeData.inputs?.safetySettings as string + const cache = nodeData.inputs?.cache as BaseCache const streaming = nodeData.inputs?.streaming as boolean const baseUrl = nodeData.inputs?.baseUrl as string | undefined @@ -220,17 +237,32 @@ class GoogleGenerativeAI_ChatModels implements INode { if (temperature) obj.temperature = parseFloat(temperature) if (baseUrl) obj.baseUrl = baseUrl - // Safety Settings - let harmCategories: string[] = convertMultiOptionsToStringArray(harmCategory) - let harmBlockThresholds: string[] = convertMultiOptionsToStringArray(harmBlockThreshold) - if (harmCategories.length != harmBlockThresholds.length) - throw new Error(`Harm Category & Harm Block Threshold are not the same length`) - const safetySettings: SafetySetting[] = harmCategories.map((harmCategory, index) => { - return { - category: harmCategory as HarmCategory, - threshold: harmBlockThresholds[index] as HarmBlockThreshold + let safetySettings: SafetySetting[] = [] + if (_safetySettings) { + try { + const parsedSafetySettings = typeof _safetySettings === 'string' ? JSON.parse(_safetySettings) : _safetySettings + if (Array.isArray(parsedSafetySettings)) { + const validSettings = parsedSafetySettings + .filter((setting: any) => setting.harmCategory && setting.harmBlockThreshold) + .map((setting: any) => ({ + category: setting.harmCategory as HarmCategory, + threshold: setting.harmBlockThreshold as HarmBlockThreshold + })) + + // Remove duplicates by keeping only the first occurrence of each harm category + const seenCategories = new Set() + safetySettings = validSettings.filter((setting) => { + if (seenCategories.has(setting.category)) { + return false + } + seenCategories.add(setting.category) + return true + }) + } + } catch (error) { + console.warn('Failed to parse safety settings:', error) } - }) + } if (safetySettings.length > 0) obj.safetySettings = safetySettings const multiModalOption: IMultiModalOption = { diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index 50d2465c..e811dd15 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -758,7 +758,7 @@ export const mapChatMessageToBaseMessage = async (chatmessages: any[] = [], orgI } } const documents: string = await fileLoaderNodeInstance.init(nodeData, '', options) - messageWithFileUploads += `${documents}\n\n` + messageWithFileUploads += `${handleEscapeCharacters(documents, true)}\n\n` } } const messageContent = messageWithFileUploads ? `${messageWithFileUploads}\n\n${message.content}` : message.content diff --git a/packages/ui/src/ui-component/array/ArrayRenderer.jsx b/packages/ui/src/ui-component/array/ArrayRenderer.jsx index c3849954..79a5d027 100644 --- a/packages/ui/src/ui-component/array/ArrayRenderer.jsx +++ b/packages/ui/src/ui-component/array/ArrayRenderer.jsx @@ -5,16 +5,18 @@ import { Chip, Box, Button, IconButton } from '@mui/material' import { useTheme } from '@mui/material/styles' import { IconTrash, IconPlus } from '@tabler/icons-react' import NodeInputHandler from '@/views/canvas/NodeInputHandler' +import DocStoreInputHandler from '@/views/docstore/DocStoreInputHandler' import { showHideInputs } from '@/utils/genericHelper' import { cloneDeep } from 'lodash' import { flowContext } from '@/store/context/ReactFlowContext' -export const ArrayRenderer = ({ inputParam, data, disabled }) => { +export const ArrayRenderer = ({ inputParam, data, disabled, isDocStore = false }) => { const [arrayItems, setArrayItems] = useState([]) // these are the actual values. Ex: [{name: 'John', age: 30}, {name: 'Jane', age: 25}] const [itemParameters, setItemParameters] = useState([]) // these are the input parameters for each array item. Ex: [{label: 'Name', type: 'string', display: true}, {label: 'age', type: 'number', display: false}] const theme = useTheme() const customization = useSelector((state) => state.customization) - const { reactFlowInstance } = useContext(flowContext) + const flowContextValue = useContext(flowContext) + const { reactFlowInstance } = flowContextValue || {} // Handler for when input values change within array items const handleItemInputChange = ({ inputParam: changedParam, newValue }, itemIndex) => { @@ -70,6 +72,9 @@ export const ArrayRenderer = ({ inputParam, data, disabled }) => { }, [data, inputParam]) const updateOutputAnchors = (items, type, indexToDelete) => { + // Skip output anchor updates for DocStore context + if (isDocStore || !reactFlowInstance) return + if (data.name !== 'conditionAgentflow' && data.name !== 'conditionAgentAgentflow') return const updatedOutputs = items.map((_, i) => ({ @@ -221,20 +226,35 @@ export const ArrayRenderer = ({ inputParam, data, disabled }) => { {/* Render input fields for array item */} {itemParameters[index] .filter((param) => param.display !== false) - .map((param, _index) => ( - { - handleItemInputChange({ inputParam, newValue }, index) - }} - /> - ))} + .map((param, _index) => { + if (isDocStore) { + return ( + { + handleItemInputChange({ inputParam, newValue }, index) + }} + /> + ) + } + return ( + { + handleItemInputChange({ inputParam, newValue }, index) + }} + /> + ) + })} ) })} @@ -257,5 +277,6 @@ export const ArrayRenderer = ({ inputParam, data, disabled }) => { ArrayRenderer.propTypes = { inputParam: PropTypes.object.isRequired, data: PropTypes.object.isRequired, - disabled: PropTypes.bool + disabled: PropTypes.bool, + isDocStore: PropTypes.bool } diff --git a/packages/ui/src/views/docstore/DocStoreInputHandler.jsx b/packages/ui/src/views/docstore/DocStoreInputHandler.jsx index 8f315f96..dc67f446 100644 --- a/packages/ui/src/views/docstore/DocStoreInputHandler.jsx +++ b/packages/ui/src/views/docstore/DocStoreInputHandler.jsx @@ -17,6 +17,7 @@ import { SwitchInput } from '@/ui-component/switch/Switch' import { JsonEditorInput } from '@/ui-component/json/JsonEditor' import { TooltipWithParser } from '@/ui-component/tooltip/TooltipWithParser' import { CodeEditor } from '@/ui-component/editor/CodeEditor' +import { ArrayRenderer } from '@/ui-component/array/ArrayRenderer' import ExpandTextDialog from '@/ui-component/dialog/ExpandTextDialog' import ManageScrapedLinksDialog from '@/ui-component/dialog/ManageScrapedLinksDialog' import CredentialInputHandler from '@/views/canvas/CredentialInputHandler' @@ -259,6 +260,9 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false, onNodeDataCh )} + {inputParam.type === 'array' && ( + + )} {(data.name === 'cheerioWebScraper' || data.name === 'puppeteerWebScraper' || data.name === 'playwrightWebScraper') &&