init migration poc

packages/server/src/DataSource.ts

@@ -1,26 +1,24 @@
 import 'reflect-metadata'
 import path from 'path'
 import { DataSource } from 'typeorm'
-import { ChatFlow } from './entity/ChatFlow'
-import { ChatMessage } from './entity/ChatMessage'
-import { Credential } from './entity/Credential'
-import { Tool } from './entity/Tool'
 import { getUserHome } from './utils'
+import { entities } from './database/entities'
+import { InitSqlite1693809869231 } from './database/migrations/1693809869231-initSqlite'
 
 let appDataSource: DataSource
 
 export const init = async (): Promise<void> => {
     let homePath
-    const synchronize = process.env.OVERRIDE_DATABASE === 'false' ? false : true
     switch (process.env.DATABASE_TYPE) {
         case 'sqlite':
             homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise')
             appDataSource = new DataSource({
                 type: 'sqlite',
                 database: path.resolve(homePath, 'database.sqlite'),
-                synchronize,
-                entities: [ChatFlow, ChatMessage, Tool, Credential],
-                migrations: []
+                synchronize: false,
+                migrationsRun: false,
+                entities: Object.values(entities),
+                migrations: [InitSqlite1693809869231]
             })
             break
         case 'mysql':
@@ -32,8 +30,9 @@ export const init = async (): Promise<void> => {
                 password: process.env.DATABASE_PASSWORD,
                 database: process.env.DATABASE_NAME,
                 charset: 'utf8mb4',
-                synchronize,
-                entities: [ChatFlow, ChatMessage, Tool, Credential],
+                synchronize: false,
+                migrationsRun: false,
+                entities: Object.values(entities),
                 migrations: []
             })
             break
@@ -45,8 +44,9 @@ export const init = async (): Promise<void> => {
                 username: process.env.DATABASE_USER,
                 password: process.env.DATABASE_PASSWORD,
                 database: process.env.DATABASE_NAME,
-                synchronize,
-                entities: [ChatFlow, ChatMessage, Tool, Credential],
+                synchronize: false,
+                migrationsRun: false,
+                entities: Object.values(entities),
                 migrations: []
             })
             break
@@ -55,9 +55,10 @@ export const init = async (): Promise<void> => {
             appDataSource = new DataSource({
                 type: 'sqlite',
                 database: path.resolve(homePath, 'database.sqlite'),
-                synchronize,
-                entities: [ChatFlow, ChatMessage, Tool, Credential],
-                migrations: []
+                synchronize: false,
+                migrationsRun: false,
+                entities: Object.values(entities),
+                migrations: [InitSqlite1693809869231]
             })
             break
     }

packages/server/src/database/entities/ChatFlow.ts

@@ -1,6 +1,6 @@
 /* eslint-disable */
 import { Entity, Column, CreateDateColumn, UpdateDateColumn, PrimaryGeneratedColumn } from 'typeorm'
-import { IChatFlow } from '../Interface'
+import { IChatFlow } from '../../Interface'
 
 @Entity()
 export class ChatFlow implements IChatFlow {

packages/server/src/database/entities/ChatMessage.ts

@@ -1,6 +1,6 @@
 /* eslint-disable */
 import { Entity, Column, CreateDateColumn, PrimaryGeneratedColumn, Index } from 'typeorm'
-import { IChatMessage, MessageType } from '../Interface'
+import { IChatMessage, MessageType } from '../../Interface'
 
 @Entity()
 export class ChatMessage implements IChatMessage {

packages/server/src/database/entities/Credential.ts

@@ -1,6 +1,6 @@
 /* eslint-disable */
 import { Entity, Column, PrimaryGeneratedColumn, Index, CreateDateColumn, UpdateDateColumn } from 'typeorm'
-import { ICredential } from '../Interface'
+import { ICredential } from '../../Interface'
 
 @Entity()
 export class Credential implements ICredential {

packages/server/src/database/entities/Tool.ts

@@ -1,6 +1,6 @@
 /* eslint-disable */
 import { Entity, Column, CreateDateColumn, UpdateDateColumn, PrimaryGeneratedColumn } from 'typeorm'
-import { ITool } from '../Interface'
+import { ITool } from '../../Interface'
 
 @Entity()
 export class Tool implements ITool {

packages/server/src/database/entities/index.ts

@@ -0,0 +1,11 @@
+import { ChatFlow } from './ChatFlow'
+import { ChatMessage } from './ChatMessage'
+import { Credential } from './Credential'
+import { Tool } from './Tool'
+
+export const entities = {
+    ChatFlow,
+    ChatMessage,
+    Credential,
+    Tool
+}

packages/server/src/database/migrations/1693809869231-initSqlite.ts

@@ -0,0 +1,29 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class InitSqlite1693809869231 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        console.info('started migration query')
+        await queryRunner.query(
+            `CREATE TABLE "chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" boolean, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));`
+        )
+        await queryRunner.query(
+            `CREATE TABLE "chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')));`
+        )
+        await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`)
+        await queryRunner.query(
+            `CREATE TABLE "credential" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "credentialName" varchar NOT NULL, "encryptedData" varchar NOT NULL, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));`
+        )
+        await queryRunner.query(
+            `CREATE TABLE "tool" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "description" text NOT NULL, "color" varchar NOT NULL, "iconSrc" varchar, "schema" varchar, "func" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));`
+        )
+        console.info('Finish migration query')
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE "chat_flow"`)
+        await queryRunner.query(`DROP INDEX "IDX_chat_messagee574527322272fd838f4f0f3d3"`)
+        await queryRunner.query(`DROP TABLE "chat_message"`)
+        await queryRunner.query(`DROP TABLE "credential"`)
+        await queryRunner.query(`DROP TABLE "tool"`)
+    }
+}

packages/server/src/index.ts

@@ -48,10 +48,10 @@ import {
 import { cloneDeep, omit } from 'lodash'
 import { getDataSource } from './DataSource'
 import { NodesPool } from './NodesPool'
-import { ChatFlow } from './entity/ChatFlow'
-import { ChatMessage } from './entity/ChatMessage'
-import { Credential } from './entity/Credential'
-import { Tool } from './entity/Tool'
+import { ChatFlow } from './database/entities/ChatFlow'
+import { ChatMessage } from './database/entities/ChatMessage'
+import { Credential } from './database/entities/Credential'
+import { Tool } from './database/entities/Tool'
 import { ChatflowPool } from './ChatflowPool'
 import { ICommonObject, INodeOptionsValue } from 'flowise-components'
 
@@ -71,6 +71,11 @@ export class App {
             .then(async () => {
                 logger.info('📦 [server]: Data Source has been initialized!')
 
+                //Migrations
+                console.info(`start migration`)
+                await this.AppDataSource.runMigrations({ transaction: 'each' })
+                console.info(`finish migration`)
+
                 // Initialize nodes pool
                 this.nodesPool = new NodesPool()
                 await this.nodesPool.initialize()

packages/server/src/utils/index.ts

@@ -30,10 +30,10 @@ import {
 import { scryptSync, randomBytes, timingSafeEqual } from 'crypto'
 import { lib, PBKDF2, AES, enc } from 'crypto-js'
 
-import { ChatFlow } from '../entity/ChatFlow'
-import { ChatMessage } from '../entity/ChatMessage'
-import { Credential } from '../entity/Credential'
-import { Tool } from '../entity/Tool'
+import { ChatFlow } from '../database/entities/ChatFlow'
+import { ChatMessage } from '../database/entities/ChatMessage'
+import { Credential } from '../database/entities/Credential'
+import { Tool } from '../database/entities/Tool'
 import { DataSource } from 'typeorm'
 
 const QUESTION_VAR_PREFIX = 'question'