Feature/Safety settings to google genai (#4737)

* add safety settings to google genai

* add safety settings to google genai

packages/components/nodes/agentflow/utils.ts

@@ -4,7 +4,7 @@ import { getFileFromStorage } from '../../src/storageUtils'
 import { ICommonObject, IFileUpload } from '../../src/Interface'
 import { BaseMessageLike } from '@langchain/core/messages'
 import { IFlowState } from './Interface.Agentflow'
-import { mapMimeTypeToInputField } from '../../src/utils'
+import { handleEscapeCharacters, mapMimeTypeToInputField } from '../../src/utils'
 
 export const addImagesToMessages = async (
     options: ICommonObject,
@@ -354,7 +354,7 @@ export const getPastChatHistoryImageMessages = async (
                             }
                         }
                         const documents: string = await fileLoaderNodeInstance.init(nodeData, '', nodeOptions)
-                        messageWithFileUploads += `<doc name='${upload.name}'>${documents}</doc>\n\n`
+                        messageWithFileUploads += `<doc name='${upload.name}'>${handleEscapeCharacters(documents, true)}</doc>\n\n`
                     }
                 }
                 const messageContent = messageWithFileUploads ? `${messageWithFileUploads}\n\n${message.content}` : message.content

packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts

@@ -2,7 +2,7 @@ import { HarmBlockThreshold, HarmCategory } from '@google/generative-ai'
 import type { SafetySetting } from '@google/generative-ai'
 import { BaseCache } from '@langchain/core/caches'
 import { ICommonObject, IMultiModalOption, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
-import { convertMultiOptionsToStringArray, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
 import { ChatGoogleGenerativeAI } from './FlowiseChatGoogleGenerativeAI'
 import { GoogleGenerativeAIChatInput } from '@langchain/google-genai'
@@ -22,7 +22,7 @@ class GoogleGenerativeAI_ChatModels implements INode {
     constructor() {
         this.label = 'ChatGoogleGenerativeAI'
         this.name = 'chatGoogleGenerativeAI'
-        this.version = 3.0
+        this.version = 3.1
         this.type = 'ChatGoogleGenerativeAI'
         this.icon = 'GoogleGemini.svg'
         this.category = 'Chat Models'
@@ -101,58 +101,75 @@ class GoogleGenerativeAI_ChatModels implements INode {
                 additionalParams: true
             },
             {
-                label: 'Harm Category',
+                label: 'Safety Settings',
-                name: 'harmCategory',
+                name: 'safetySettings',
-                type: 'multiOptions',
+                type: 'array',
                 description:
-                    'Refer to <a target="_blank" href="https://cloud.google.com/vertex-ai/docs/generative-ai/multimodal/configure-safety-attributes#safety_attribute_definitions">official guide</a> on how to use Harm Category',
+                    'Safety settings for the model. Refer to the <a href="https://ai.google.dev/gemini-api/docs/safety-settings">official guide</a> on how to use Safety Settings',
-                options: [
+                array: [
                     {
-                        label: 'Dangerous',
+                        label: 'Harm Category',
-                        name: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT
+                        name: 'harmCategory',
+                        type: 'options',
+                        options: [
+                            {
+                                label: 'Dangerous',
+                                name: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                                description: 'Promotes, facilitates, or encourages harmful acts.'
+                            },
+                            {
+                                label: 'Harassment',
+                                name: HarmCategory.HARM_CATEGORY_HARASSMENT,
+                                description: 'Negative or harmful comments targeting identity and/or protected attributes.'
+                            },
+                            {
+                                label: 'Hate Speech',
+                                name: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                                description: 'Content that is rude, disrespectful, or profane.'
+                            },
+                            {
+                                label: 'Sexually Explicit',
+                                name: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                                description: 'Contains references to sexual acts or other lewd content.'
+                            },
+                            {
+                                label: 'Civic Integrity',
+                                name: HarmCategory.HARM_CATEGORY_CIVIC_INTEGRITY,
+                                description: 'Election-related queries.'
+                            }
+                        ]
                     },
                     {
-                        label: 'Harassment',
+                        label: 'Harm Block Threshold',
-                        name: HarmCategory.HARM_CATEGORY_HARASSMENT
+                        name: 'harmBlockThreshold',
-                    },
+                        type: 'options',
-                    {
+                        options: [
-                        label: 'Hate Speech',
+                            {
-                        name: HarmCategory.HARM_CATEGORY_HATE_SPEECH
+                                label: 'None',
-                    },
+                                name: HarmBlockThreshold.BLOCK_NONE,
-                    {
+                                description: 'Always show regardless of probability of unsafe content'
-                        label: 'Sexually Explicit',
+                            },
-                        name: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT
+                            {
-                    }
+                                label: 'Only High',
-                ],
+                                name: HarmBlockThreshold.BLOCK_ONLY_HIGH,
-                optional: true,
+                                description: 'Block when high probability of unsafe content'
-                additionalParams: true
+                            },
-            },
+                            {
-            {
+                                label: 'Medium and Above',
-                label: 'Harm Block Threshold',
+                                name: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
-                name: 'harmBlockThreshold',
+                                description: 'Block when medium or high probability of unsafe content'
-                type: 'multiOptions',
+                            },
-                description:
+                            {
-                    'Refer to <a target="_blank" href="https://cloud.google.com/vertex-ai/docs/generative-ai/multimodal/configure-safety-attributes#safety_setting_thresholds">official guide</a> on how to use Harm Block Threshold',
+                                label: 'Low and Above',
-                options: [
+                                name: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE,
-                    {
+                                description: 'Block when low, medium or high probability of unsafe content'
-                        label: 'Low and Above',
+                            },
-                        name: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE
+                            {
-                    },
+                                label: 'Threshold Unspecified (Default Threshold)',
-                    {
+                                name: HarmBlockThreshold.HARM_BLOCK_THRESHOLD_UNSPECIFIED,
-                        label: 'Medium and Above',
+                                description: 'Threshold is unspecified, block using default threshold'
-                        name: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE
+                            }
-                    },
+                        ]
-                    {
-                        label: 'None',
-                        name: HarmBlockThreshold.BLOCK_NONE
-                    },
-                    {
-                        label: 'Only High',
-                        name: HarmBlockThreshold.BLOCK_ONLY_HIGH
-                    },
-                    {
-                        label: 'Threshold Unspecified',
-                        name: HarmBlockThreshold.HARM_BLOCK_THRESHOLD_UNSPECIFIED
                     }
                 ],
                 optional: true,
@@ -195,8 +212,8 @@ class GoogleGenerativeAI_ChatModels implements INode {
         const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string
         const topP = nodeData.inputs?.topP as string
         const topK = nodeData.inputs?.topK as string
-        const harmCategory = nodeData.inputs?.harmCategory as string
+        const _safetySettings = nodeData.inputs?.safetySettings as string
-        const harmBlockThreshold = nodeData.inputs?.harmBlockThreshold as string
+
         const cache = nodeData.inputs?.cache as BaseCache
         const streaming = nodeData.inputs?.streaming as boolean
         const baseUrl = nodeData.inputs?.baseUrl as string | undefined
@@ -220,17 +237,32 @@ class GoogleGenerativeAI_ChatModels implements INode {
         if (temperature) obj.temperature = parseFloat(temperature)
         if (baseUrl) obj.baseUrl = baseUrl
 
-        // Safety Settings
+        let safetySettings: SafetySetting[] = []
-        let harmCategories: string[] = convertMultiOptionsToStringArray(harmCategory)
+        if (_safetySettings) {
-        let harmBlockThresholds: string[] = convertMultiOptionsToStringArray(harmBlockThreshold)
+            try {
-        if (harmCategories.length != harmBlockThresholds.length)
+                const parsedSafetySettings = typeof _safetySettings === 'string' ? JSON.parse(_safetySettings) : _safetySettings
-            throw new Error(`Harm Category & Harm Block Threshold are not the same length`)
+                if (Array.isArray(parsedSafetySettings)) {
-        const safetySettings: SafetySetting[] = harmCategories.map((harmCategory, index) => {
+                    const validSettings = parsedSafetySettings
-            return {
+                        .filter((setting: any) => setting.harmCategory && setting.harmBlockThreshold)
-                category: harmCategory as HarmCategory,
+                        .map((setting: any) => ({
-                threshold: harmBlockThresholds[index] as HarmBlockThreshold
+                            category: setting.harmCategory as HarmCategory,
+                            threshold: setting.harmBlockThreshold as HarmBlockThreshold
+                        }))
+
+                    // Remove duplicates by keeping only the first occurrence of each harm category
+                    const seenCategories = new Set<HarmCategory>()
+                    safetySettings = validSettings.filter((setting) => {
+                        if (seenCategories.has(setting.category)) {
+                            return false
+                        }
+                        seenCategories.add(setting.category)
+                        return true
+                    })
+                }
+            } catch (error) {
+                console.warn('Failed to parse safety settings:', error)
             }
-        })
+        }
         if (safetySettings.length > 0) obj.safetySettings = safetySettings
 
         const multiModalOption: IMultiModalOption = {

packages/components/src/utils.ts

@@ -758,7 +758,7 @@ export const mapChatMessageToBaseMessage = async (chatmessages: any[] = [], orgI
                                 }
                             }
                             const documents: string = await fileLoaderNodeInstance.init(nodeData, '', options)
-                            messageWithFileUploads += `<doc name='${upload.name}'>${documents}</doc>\n\n`
+                            messageWithFileUploads += `<doc name='${upload.name}'>${handleEscapeCharacters(documents, true)}</doc>\n\n`
                         }
                     }
                     const messageContent = messageWithFileUploads ? `${messageWithFileUploads}\n\n${message.content}` : message.content

packages/ui/src/ui-component/array/ArrayRenderer.jsx

@@ -5,16 +5,18 @@ import { Chip, Box, Button, IconButton } from '@mui/material'
 import { useTheme } from '@mui/material/styles'
 import { IconTrash, IconPlus } from '@tabler/icons-react'
 import NodeInputHandler from '@/views/canvas/NodeInputHandler'
+import DocStoreInputHandler from '@/views/docstore/DocStoreInputHandler'
 import { showHideInputs } from '@/utils/genericHelper'
 import { cloneDeep } from 'lodash'
 import { flowContext } from '@/store/context/ReactFlowContext'
 
-export const ArrayRenderer = ({ inputParam, data, disabled }) => {
+export const ArrayRenderer = ({ inputParam, data, disabled, isDocStore = false }) => {
     const [arrayItems, setArrayItems] = useState([]) // these are the actual values. Ex: [{name: 'John', age: 30}, {name: 'Jane', age: 25}]
     const [itemParameters, setItemParameters] = useState([]) // these are the input parameters for each array item. Ex: [{label: 'Name', type: 'string', display: true}, {label: 'age', type: 'number', display: false}]
     const theme = useTheme()
     const customization = useSelector((state) => state.customization)
-    const { reactFlowInstance } = useContext(flowContext)
+    const flowContextValue = useContext(flowContext)
+    const { reactFlowInstance } = flowContextValue || {}
 
     // Handler for when input values change within array items
     const handleItemInputChange = ({ inputParam: changedParam, newValue }, itemIndex) => {
@@ -70,6 +72,9 @@ export const ArrayRenderer = ({ inputParam, data, disabled }) => {
     }, [data, inputParam])
 
     const updateOutputAnchors = (items, type, indexToDelete) => {
+        // Skip output anchor updates for DocStore context
+        if (isDocStore || !reactFlowInstance) return
+
         if (data.name !== 'conditionAgentflow' && data.name !== 'conditionAgentAgentflow') return
 
         const updatedOutputs = items.map((_, i) => ({
@@ -221,20 +226,35 @@ export const ArrayRenderer = ({ inputParam, data, disabled }) => {
                         {/* Render input fields for array item */}
                         {itemParameters[index]
                             .filter((param) => param.display !== false)
-                            .map((param, _index) => (
+                            .map((param, _index) => {
-                                <NodeInputHandler
+                                if (isDocStore) {
-                                    disabled={disabled}
+                                    return (
-                                    key={_index}
+                                        <DocStoreInputHandler
-                                    inputParam={param}
+                                            disabled={disabled}
-                                    data={itemData}
+                                            key={_index}
-                                    isAdditionalParams={true}
+                                            inputParam={param}
-                                    parentParamForArray={inputParam}
+                                            data={itemData}
-                                    arrayIndex={index}
+                                            onNodeDataChange={({ inputParam, newValue }) => {
-                                    onCustomDataChange={({ inputParam, newValue }) => {
+                                                handleItemInputChange({ inputParam, newValue }, index)
-                                        handleItemInputChange({ inputParam, newValue }, index)
+                                            }}
-                                    }}
+                                        />
-                                />
+                                    )
-                            ))}
+                                }
+                                return (
+                                    <NodeInputHandler
+                                        disabled={disabled}
+                                        key={_index}
+                                        inputParam={param}
+                                        data={itemData}
+                                        isAdditionalParams={true}
+                                        parentParamForArray={inputParam}
+                                        arrayIndex={index}
+                                        onCustomDataChange={({ inputParam, newValue }) => {
+                                            handleItemInputChange({ inputParam, newValue }, index)
+                                        }}
+                                    />
+                                )
+                            })}
                     </Box>
                 )
             })}
@@ -257,5 +277,6 @@ export const ArrayRenderer = ({ inputParam, data, disabled }) => {
 ArrayRenderer.propTypes = {
     inputParam: PropTypes.object.isRequired,
     data: PropTypes.object.isRequired,
-    disabled: PropTypes.bool
+    disabled: PropTypes.bool,
+    isDocStore: PropTypes.bool
 }

packages/ui/src/views/docstore/DocStoreInputHandler.jsx

@@ -17,6 +17,7 @@ import { SwitchInput } from '@/ui-component/switch/Switch'
 import { JsonEditorInput } from '@/ui-component/json/JsonEditor'
 import { TooltipWithParser } from '@/ui-component/tooltip/TooltipWithParser'
 import { CodeEditor } from '@/ui-component/editor/CodeEditor'
+import { ArrayRenderer } from '@/ui-component/array/ArrayRenderer'
 import ExpandTextDialog from '@/ui-component/dialog/ExpandTextDialog'
 import ManageScrapedLinksDialog from '@/ui-component/dialog/ManageScrapedLinksDialog'
 import CredentialInputHandler from '@/views/canvas/CredentialInputHandler'
@@ -259,6 +260,9 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false, onNodeDataCh
                                 </div>
                             </>
                         )}
+                        {inputParam.type === 'array' && (
+                            <ArrayRenderer inputParam={inputParam} data={data} disabled={disabled} isDocStore={true} />
+                        )}
                         {(data.name === 'cheerioWebScraper' ||
                             data.name === 'puppeteerWebScraper' ||
                             data.name === 'playwrightWebScraper') &&