Merge pull request #1397 from FlowiseAI/feature/Utilities

Feature/Utilities

packages/components/nodes/tools/CustomTool/core.ts

@@ -2,37 +2,7 @@ import { z } from 'zod'
 import { CallbackManagerForToolRun } from 'langchain/callbacks'
 import { StructuredTool, ToolParams } from 'langchain/tools'
 import { NodeVM } from 'vm2'
-
-/*
- * List of dependencies allowed to be import in vm2
- */
-const availableDependencies = [
-    '@dqbd/tiktoken',
-    '@getzep/zep-js',
-    '@huggingface/inference',
-    '@pinecone-database/pinecone',
-    '@supabase/supabase-js',
-    'axios',
-    'cheerio',
-    'chromadb',
-    'cohere-ai',
-    'd3-dsv',
-    'form-data',
-    'graphql',
-    'html-to-text',
-    'langchain',
-    'linkifyjs',
-    'mammoth',
-    'moment',
-    'node-fetch',
-    'pdf-parse',
-    'pdfjs-dist',
-    'playwright',
-    'puppeteer',
-    'srt-parser-2',
-    'typeorm',
-    'weaviate-ts-client'
-]
+import { availableDependencies } from '../../../src/utils'
 
 export interface BaseDynamicToolInput extends ToolParams {
     name: string

packages/components/nodes/utilities/CustomFunction/CustomFunction.ts

@@ -0,0 +1,124 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { NodeVM } from 'vm2'
+import { availableDependencies, handleEscapeCharacters } from '../../../src/utils'
+
+class CustomFunction_Utilities implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Custom JS Function'
+        this.name = 'customFunction'
+        this.version = 1.0
+        this.type = 'CustomFunction'
+        this.icon = 'customfunction.svg'
+        this.category = 'Utilities'
+        this.description = `Execute custom javascript function`
+        this.baseClasses = [this.type, 'Utilities']
+        this.inputs = [
+            {
+                label: 'Input Variables',
+                name: 'functionInputVariables',
+                description: 'Input variables can be used in the function with prefix $. For example: $var',
+                type: 'json',
+                optional: true,
+                acceptVariable: true,
+                list: true
+            },
+            {
+                label: 'Function Name',
+                name: 'functionName',
+                type: 'string',
+                optional: true,
+                placeholder: 'My Function'
+            },
+            {
+                label: 'Javascript Function',
+                name: 'javascriptFunction',
+                type: 'code'
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Output',
+                name: 'output',
+                baseClasses: ['string', 'number', 'boolean', 'json', 'array']
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, input: string): Promise<any> {
+        const javascriptFunction = nodeData.inputs?.javascriptFunction as string
+        const functionInputVariablesRaw = nodeData.inputs?.functionInputVariables
+
+        let inputVars: ICommonObject = {}
+        if (functionInputVariablesRaw) {
+            try {
+                inputVars =
+                    typeof functionInputVariablesRaw === 'object' ? functionInputVariablesRaw : JSON.parse(functionInputVariablesRaw)
+            } catch (exception) {
+                throw new Error("Invalid JSON in the PromptTemplate's promptValues: " + exception)
+            }
+        }
+
+        let sandbox: any = { $input: input }
+
+        if (Object.keys(inputVars).length) {
+            for (const item in inputVars) {
+                sandbox[`$${item}`] = inputVars[item]
+            }
+        }
+
+        const defaultAllowBuiltInDep = [
+            'assert',
+            'buffer',
+            'crypto',
+            'events',
+            'http',
+            'https',
+            'net',
+            'path',
+            'querystring',
+            'timers',
+            'tls',
+            'url',
+            'zlib'
+        ]
+
+        const builtinDeps = process.env.TOOL_FUNCTION_BUILTIN_DEP
+            ? defaultAllowBuiltInDep.concat(process.env.TOOL_FUNCTION_BUILTIN_DEP.split(','))
+            : defaultAllowBuiltInDep
+        const externalDeps = process.env.TOOL_FUNCTION_EXTERNAL_DEP ? process.env.TOOL_FUNCTION_EXTERNAL_DEP.split(',') : []
+        const deps = availableDependencies.concat(externalDeps)
+
+        const nodeVMOptions = {
+            console: 'inherit',
+            sandbox,
+            require: {
+                external: { modules: deps },
+                builtin: builtinDeps
+            }
+        } as any
+
+        const vm = new NodeVM(nodeVMOptions)
+        try {
+            const response = await vm.run(`module.exports = async function() {${javascriptFunction}}()`, __dirname)
+            if (typeof response === 'string') {
+                return handleEscapeCharacters(response, false)
+            }
+            return response
+        } catch (e) {
+            throw new Error(e)
+        }
+    }
+}
+
+module.exports = { nodeClass: CustomFunction_Utilities }

packages/components/nodes/utilities/CustomFunction/customfunction.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-function" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M4 4m0 2.667a2.667 2.667 0 0 1 2.667 -2.667h10.666a2.667 2.667 0 0 1 2.667 2.667v10.666a2.667 2.667 0 0 1 -2.667 2.667h-10.666a2.667 2.667 0 0 1 -2.667 -2.667z" /><path d="M9 15.5v.25c0 .69 .56 1.25 1.25 1.25c.71 0 1.304 -.538 1.374 -1.244l.752 -7.512a1.381 1.381 0 0 1 1.374 -1.244c.69 0 1.25 .56 1.25 1.25v.25" /><path d="M9 12h6" /></svg>

packages/components/nodes/utilities/GetVariable/GetVariable.ts

@@ -0,0 +1,52 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+
+class GetVariable_Utilities implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Get Variable'
+        this.name = 'getVariable'
+        this.version = 1.0
+        this.type = 'GetVariable'
+        this.icon = 'getvar.svg'
+        this.category = 'Utilities'
+        this.description = `Get variable that was saved using Set Variable node`
+        this.baseClasses = [this.type, 'Utilities']
+        this.inputs = [
+            {
+                label: 'Variable Name',
+                name: 'variableName',
+                type: 'string',
+                placeholder: 'var1'
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Output',
+                name: 'output',
+                baseClasses: ['string', 'number', 'boolean', 'json', 'array']
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const variableName = nodeData.inputs?.variableName as string
+        const dynamicVars = options.dynamicVariables as Record<string, unknown>
+
+        if (Object.prototype.hasOwnProperty.call(dynamicVars, variableName)) {
+            return dynamicVars[variableName]
+        }
+        return undefined
+    }
+}
+
+module.exports = { nodeClass: GetVariable_Utilities }

packages/components/nodes/utilities/GetVariable/getvar.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-book-download" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M12 20h-6a2 2 0 0 1 -2 -2v-12a2 2 0 0 1 2 -2h12v5" /><path d="M13 16h-7a2 2 0 0 0 -2 2" /><path d="M15 19l3 3l3 -3" /><path d="M18 22v-9" /></svg>

packages/components/nodes/utilities/SetVariable/SetVariable.ts

@@ -0,0 +1,56 @@
+import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+
+class SetVariable_Utilities implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Set Variable'
+        this.name = 'setVariable'
+        this.version = 1.0
+        this.type = 'SetVariable'
+        this.icon = 'setvar.svg'
+        this.category = 'Utilities'
+        this.description = `Set variable which can be retrieved at a later stage. Variable is only available during runtime.`
+        this.baseClasses = [this.type, 'Utilities']
+        this.inputs = [
+            {
+                label: 'Input',
+                name: 'input',
+                type: 'string | number | boolean | json | array',
+                optional: true,
+                list: true
+            },
+            {
+                label: 'Variable Name',
+                name: 'variableName',
+                type: 'string',
+                placeholder: 'var1'
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Output',
+                name: 'output',
+                baseClasses: ['string', 'number', 'boolean', 'json', 'array']
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData): Promise<any> {
+        const inputRaw = nodeData.inputs?.input
+        const variableName = nodeData.inputs?.variableName as string
+
+        return { output: inputRaw, dynamicVariables: { [variableName]: inputRaw } }
+    }
+}
+
+module.exports = { nodeClass: SetVariable_Utilities }

packages/components/nodes/utilities/SetVariable/setvar.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-book-upload" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M14 20h-8a2 2 0 0 1 -2 -2v-12a2 2 0 0 1 2 -2h12v5" /><path d="M11 16h-5a2 2 0 0 0 -2 2" /><path d="M15 16l3 -3l3 3" /><path d="M18 13v9" /></svg>

packages/components/src/utils.ts

@@ -12,6 +12,63 @@ import { AIMessage, HumanMessage } from 'langchain/schema'
 
 export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
 export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank
+/*
+ * List of dependencies allowed to be import in vm2
+ */
+export const availableDependencies = [
+    '@aws-sdk/client-bedrock-runtime',
+    '@aws-sdk/client-dynamodb',
+    '@aws-sdk/client-s3',
+    '@elastic/elasticsearch',
+    '@dqbd/tiktoken',
+    '@getzep/zep-js',
+    '@gomomento/sdk',
+    '@gomomento/sdk-core',
+    '@google-ai/generativelanguage',
+    '@huggingface/inference',
+    '@notionhq/client',
+    '@opensearch-project/opensearch',
+    '@pinecone-database/pinecone',
+    '@qdrant/js-client-rest',
+    '@supabase/supabase-js',
+    '@upstash/redis',
+    '@zilliz/milvus2-sdk-node',
+    'apify-client',
+    'axios',
+    'cheerio',
+    'chromadb',
+    'cohere-ai',
+    'd3-dsv',
+    'faiss-node',
+    'form-data',
+    'google-auth-library',
+    'graphql',
+    'html-to-text',
+    'ioredis',
+    'langchain',
+    'langfuse',
+    'langsmith',
+    'linkifyjs',
+    'llmonitor',
+    'mammoth',
+    'moment',
+    'mongodb',
+    'mysql2',
+    'node-fetch',
+    'node-html-markdown',
+    'notion-to-md',
+    'openai',
+    'pdf-parse',
+    'pdfjs-dist',
+    'pg',
+    'playwright',
+    'puppeteer',
+    'redis',
+    'replicate',
+    'srt-parser-2',
+    'typeorm',
+    'weaviate-ts-client'
+]
 
 /**
  * Get base classes of components