Feature/agentflow v2 (#4298)

* agent flow v2

* chat message background

* conditon agent flow

* add sticky note

* update human input dynamic prompt

* add HTTP node

* add default tool icon

* fix export duplicate agentflow v2

* add agentflow v2 marketplaces

* refractor memoization, add iteration nodes

* add agentflow v2 templates

* add agentflow generator

* add migration scripts for mysql, mariadb, posrgres and fix date filters for executions

* update agentflow chat history config

* fix get all flows error after deletion and rename

* add previous nodes from parent node

* update generator prompt

* update run time state when using iteration nodes

* prevent looping connection, prevent duplication of start node, add executeflow node, add nodes agentflow, chat history variable

* update embed

* convert form input to string

* bump openai version

* add react rewards

* add prompt generator to prediction queue

* add array schema to overrideconfig

* UI touchup

* update embedded chat version

* fix node info dialog

* update start node and loop default iteration

* update UI fixes for agentflow v2

* fix async drop down

* add export import to agentflowsv2, executions, fix UI bugs

* add default empty object to flowlisttable

* add ability to share trace link publicly, allow MCP tool use for Agent and Assistant

* add runtime message length to variable, display conditions on UI

* fix array validation

* add ability to add knowledge from vector store and embeddings for agent

* add agent tool require human input

* add ephemeral memory to start node

* update agent flow node to show vs and embeddings icons

* feat: add import chat data functionality for AgentFlowV2

* feat: set chatMessage.executionId to null if not found in import JSON file or database

* fix: MariaDB execution migration script to utf8mb4_unicode_520_ci

---------

Co-authored-by: Ong Chung Yau <33013947+chungyau97@users.noreply.github.com>
Co-authored-by: chungyau97 <chungyau97@gmail.com>

packages/server/src/database/entities/ChatMessage.ts

@@ -1,6 +1,7 @@
 /* eslint-disable */
-import { Entity, Column, CreateDateColumn, PrimaryGeneratedColumn, Index } from 'typeorm'
+import { Entity, Column, CreateDateColumn, PrimaryGeneratedColumn, Index, JoinColumn, OneToOne } from 'typeorm'
 import { IChatMessage, MessageType } from '../../Interface'
+import { Execution } from './Execution'
 
 @Entity()
 export class ChatMessage implements IChatMessage {
@@ -14,6 +15,13 @@ export class ChatMessage implements IChatMessage {
     @Column({ type: 'uuid' })
     chatflowid: string
 
+    @Column({ nullable: true, type: 'uuid' })
+    executionId?: string
+
+    @OneToOne(() => Execution)
+    @JoinColumn({ name: 'executionId' })
+    execution: Execution
+
     @Column({ type: 'text' })
     content: string
 

packages/server/src/database/entities/Execution.ts

@@ -0,0 +1,44 @@
+import { Entity, Column, Index, PrimaryGeneratedColumn, CreateDateColumn, UpdateDateColumn, ManyToOne, JoinColumn } from 'typeorm'
+import { IExecution, ExecutionState } from '../../Interface'
+import { ChatFlow } from './ChatFlow'
+
+@Entity()
+export class Execution implements IExecution {
+    @PrimaryGeneratedColumn('uuid')
+    id: string
+
+    @Column({ type: 'text' })
+    executionData: string
+
+    @Column()
+    state: ExecutionState
+
+    @Index()
+    @Column({ type: 'uuid' })
+    agentflowId: string
+
+    @Index()
+    @Column({ type: 'uuid' })
+    sessionId: string
+
+    @Column({ nullable: true, type: 'text' })
+    action?: string
+
+    @Column({ nullable: true })
+    isPublic?: boolean
+
+    @Column({ type: 'timestamp' })
+    @CreateDateColumn()
+    createdDate: Date
+
+    @Column({ type: 'timestamp' })
+    @UpdateDateColumn()
+    updatedDate: Date
+
+    @Column()
+    stoppedDate: Date
+
+    @ManyToOne(() => ChatFlow)
+    @JoinColumn({ name: 'agentflowId' })
+    agentflow: ChatFlow
+}

packages/server/src/database/entities/index.ts

@@ -11,6 +11,7 @@ import { Lead } from './Lead'
 import { UpsertHistory } from './UpsertHistory'
 import { ApiKey } from './ApiKey'
 import { CustomTemplate } from './CustomTemplate'
+import { Execution } from './Execution'
 
 export const entities = {
     ChatFlow,
@@ -25,5 +26,6 @@ export const entities = {
     Lead,
     UpsertHistory,
     ApiKey,
-    CustomTemplate
+    CustomTemplate,
+    Execution
 }

packages/server/src/database/migrations/mariadb/1738090872625-AddExecutionEntity.ts

@@ -0,0 +1,31 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddExecutionEntity1738090872625 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`execution\` (
+                \`id\` varchar(36) NOT NULL,
+                \`executionData\` text NOT NULL,
+                \`action\` text,
+                \`state\` varchar(255) NOT NULL,
+                \`agentflowId\` varchar(255) NOT NULL,
+                \`sessionId\` varchar(255) NOT NULL,
+                \`isPublic\` boolean,
+                \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+                \`stoppedDate\` datetime(6),
+                PRIMARY KEY (\`id\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;`
+        )
+
+        const columnExists = await queryRunner.hasColumn('chat_message', 'executionId')
+        if (!columnExists) {
+            await queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`executionId\` TEXT;`)
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS \`execution\``)
+        await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`executionId\`;`)
+    }
+}

packages/server/src/database/migrations/mariadb/index.ts

@@ -28,6 +28,7 @@ import { AddCustomTemplate1725629836652 } from './1725629836652-AddCustomTemplat
 import { AddArtifactsToChatMessage1726156258465 } from './1726156258465-AddArtifactsToChatMessage'
 import { AddFollowUpPrompts1726666318346 } from './1726666318346-AddFollowUpPrompts'
 import { AddTypeToAssistant1733011290987 } from './1733011290987-AddTypeToAssistant'
+import { AddExecutionEntity1738090872625 } from './1738090872625-AddExecutionEntity'
 
 export const mariadbMigrations = [
     Init1693840429259,
@@ -59,5 +60,6 @@ export const mariadbMigrations = [
     AddCustomTemplate1725629836652,
     AddArtifactsToChatMessage1726156258465,
     AddFollowUpPrompts1726666318346,
-    AddTypeToAssistant1733011290987
+    AddTypeToAssistant1733011290987,
+    AddExecutionEntity1738090872625
 ]

packages/server/src/database/migrations/mysql/1738090872625-AddExecutionEntity.ts

@@ -0,0 +1,31 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddExecutionEntity1738090872625 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`execution\` (
+                \`id\` varchar(36) NOT NULL,
+                \`executionData\` text NOT NULL,
+                \`action\` text,
+                \`state\` varchar(255) NOT NULL,
+                \`agentflowId\` varchar(255) NOT NULL,
+                \`sessionId\` varchar(255) NOT NULL,
+                \`isPublic\` boolean,
+                \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+                \`stoppedDate\` datetime(6),
+                PRIMARY KEY (\`id\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+
+        const columnExists = await queryRunner.hasColumn('chat_message', 'executionId')
+        if (!columnExists) {
+            await queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`executionId\` TEXT;`)
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS \`execution\``)
+        await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`executionId\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -28,6 +28,7 @@ import { AddCustomTemplate1725629836652 } from './1725629836652-AddCustomTemplat
 import { AddArtifactsToChatMessage1726156258465 } from './1726156258465-AddArtifactsToChatMessage'
 import { AddFollowUpPrompts1726666302024 } from './1726666302024-AddFollowUpPrompts'
 import { AddTypeToAssistant1733011290987 } from './1733011290987-AddTypeToAssistant'
+import { AddExecutionEntity1738090872625 } from './1738090872625-AddExecutionEntity'
 
 export const mysqlMigrations = [
     Init1693840429259,
@@ -59,5 +60,6 @@ export const mysqlMigrations = [
     AddCustomTemplate1725629836652,
     AddArtifactsToChatMessage1726156258465,
     AddFollowUpPrompts1726666302024,
-    AddTypeToAssistant1733011290987
+    AddTypeToAssistant1733011290987,
+    AddExecutionEntity1738090872625
 ]

packages/server/src/database/migrations/postgres/1738090872625-AddExecutionEntity.ts

@@ -0,0 +1,31 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddExecutionEntity1738090872625 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS execution (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "executionData" text NOT NULL,
+                "action" text,
+                "state" varchar NOT NULL,
+                "agentflowId" uuid NOT NULL,
+                "sessionId" uuid NOT NULL,
+                "isPublic" boolean,
+                "createdDate" timestamp NOT NULL DEFAULT now(),
+                "updatedDate" timestamp NOT NULL DEFAULT now(),
+                "stoppedDate" timestamp,
+                CONSTRAINT "PK_936a419c3b8044598d72d95da61" PRIMARY KEY (id)
+            );`
+        )
+
+        const columnExists = await queryRunner.hasColumn('chat_message', 'executionId')
+        if (!columnExists) {
+            await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN "executionId" uuid;`)
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE execution`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "executionId";`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -28,6 +28,7 @@ import { AddCustomTemplate1725629836652 } from './1725629836652-AddCustomTemplat
 import { AddArtifactsToChatMessage1726156258465 } from './1726156258465-AddArtifactsToChatMessage'
 import { AddFollowUpPrompts1726666309552 } from './1726666309552-AddFollowUpPrompts'
 import { AddTypeToAssistant1733011290987 } from './1733011290987-AddTypeToAssistant'
+import { AddExecutionEntity1738090872625 } from './1738090872625-AddExecutionEntity'
 
 export const postgresMigrations = [
     Init1693891895163,
@@ -59,5 +60,6 @@ export const postgresMigrations = [
     AddCustomTemplate1725629836652,
     AddArtifactsToChatMessage1726156258465,
     AddFollowUpPrompts1726666309552,
-    AddTypeToAssistant1733011290987
+    AddTypeToAssistant1733011290987,
+    AddExecutionEntity1738090872625
 ]

packages/server/src/database/migrations/sqlite/1738090872625-AddExecutionEntity.ts

@@ -0,0 +1,15 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddExecutionEntity1738090872625 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "execution" ("id" varchar PRIMARY KEY NOT NULL, "executionData" text NOT NULL, "action" text, "state" varchar NOT NULL, "agentflowId" varchar NOT NULL, "sessionId" varchar NOT NULL, "isPublic" boolean, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')), "stoppedDate" datetime);`
+        )
+        await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN "executionId" varchar;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE execution`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "executionId";`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -27,6 +27,7 @@ import { AddArtifactsToChatMessage1726156258465 } from './1726156258465-AddArtif
 import { AddCustomTemplate1725629836652 } from './1725629836652-AddCustomTemplate'
 import { AddFollowUpPrompts1726666294213 } from './1726666294213-AddFollowUpPrompts'
 import { AddTypeToAssistant1733011290987 } from './1733011290987-AddTypeToAssistant'
+import { AddExecutionEntity1738090872625 } from './1738090872625-AddExecutionEntity'
 
 export const sqliteMigrations = [
     Init1693835579790,
@@ -57,5 +58,6 @@ export const sqliteMigrations = [
     AddArtifactsToChatMessage1726156258465,
     AddCustomTemplate1725629836652,
     AddFollowUpPrompts1726666294213,
-    AddTypeToAssistant1733011290987
+    AddTypeToAssistant1733011290987,
+    AddExecutionEntity1738090872625
 ]