From c5eff32cf05519b99fe15d7c2a3cf214178bff9c Mon Sep 17 00:00:00 2001 From: Atish Amte Date: Tue, 18 Jul 2023 23:52:16 +0530 Subject: [PATCH 1/4] Multiple Database Support --- packages/server/.env.example | 14 +++++- packages/server/README.md | 9 +++- packages/server/package.json | 2 + packages/server/src/ChildProcess.ts | 60 ++++++++++++++++++++--- packages/server/src/DataSource.ts | 59 ++++++++++++++++++---- packages/server/src/entity/ChatFlow.ts | 2 +- packages/server/src/entity/ChatMessage.ts | 2 +- packages/server/src/entity/Tool.ts | 2 +- 8 files changed, 128 insertions(+), 22 deletions(-) diff --git a/packages/server/.env.example b/packages/server/.env.example index d9b2da76..699960bc 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -2,8 +2,20 @@ PORT=3000 # FLOWISE_USERNAME=user # FLOWISE_PASSWORD=1234 # DEBUG=true -# DATABASE_PATH=/your_database_path/.flowise # APIKEY_PATH=/your_api_key_path/.flowise # LOG_PATH=/your_log_path/logs # LOG_LEVEL=debug (error | warn | info | verbose | debug) # EXECUTION_MODE=main (child | main) + +OVERRIDE_DATABASE="true" +DATABASE_TYPE="sqlite" # sqlite, mysql, postgres + +# When database is sqlite +# DATABASE_PATH=/your_database_path/.flowise + +# When database is not sqlite +# DATABASE_PORT="3306" +# DATABASE_HOST="127.0.0.1" +# DATABASE_NAME="flowise" +# DATABASE_USER="root" +# DATABASE_PASSWORD="atish123" \ No newline at end of file diff --git a/packages/server/README.md b/packages/server/README.md index fb3a0c12..ed0c6a4c 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -41,9 +41,16 @@ Flowise support different environment variables to configure your instance. You | DEBUG | Print logs from components | Boolean | | LOG_PATH | Location where log files are stored | String | `your-path/Flowise/logs` | | LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | -| DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | | APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | | EXECUTION_MODE | Whether predictions run in their own process or the main process | Enum String: `child`, `main` | `main` | +| OVERRIDE_DATABASE| Override current database with default | Enum String: `true`, `false` | `true` | +| DATABASE_TYPE | Type of database to store the flowise data | Enum String: `sqlite`, `mysql`, `postgres` | `sqlite` | +| DATABASE_PATH | Location where database is saved (When DATABASE_TYPE is sqlite) | String | `your-home-dir/.flowise` | +| DATABASE_HOST | Host URL or IP address (When DATABASE_TYPE is not sqlite) | String | | +| DATABASE_PORT | Database port (When DATABASE_TYPE is not sqlite) | String | | +| DATABASE_USERNAME| Database username (When DATABASE_TYPE is not sqlite) | String | | +| DATABASE_PASSWORD| Database password (When DATABASE_TYPE is not sqlite) | String | | +| DATABASE_NAME | Database name (When DATABASE_TYPE is not sqlite) | String | | You can also specify the env variables when using `npx`. For example: diff --git a/packages/server/package.json b/packages/server/package.json index daed16d8..73acc64c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -55,6 +55,8 @@ "flowise-ui": "*", "moment-timezone": "^0.5.34", "multer": "^1.4.5-lts.1", + "mysql": "^2.18.1", + "pg": "^8.11.1", "reflect-metadata": "^0.1.13", "socket.io": "^4.6.1", "sqlite3": "^5.1.6", diff --git a/packages/server/src/ChildProcess.ts b/packages/server/src/ChildProcess.ts index 27629480..c14a4977 100644 --- a/packages/server/src/ChildProcess.ts +++ b/packages/server/src/ChildProcess.ts @@ -142,14 +142,58 @@ export class ChildProcess { * @returns {DataSource} */ async function initDB() { - const homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') - const childAppDataSource = new DataSource({ - type: 'sqlite', - database: path.resolve(homePath, 'database.sqlite'), - synchronize: true, - entities: [ChatFlow, ChatMessage, Tool], - migrations: [] - }) + let childAppDataSource; + let homePath; + switch (process.env.DATABASE_TYPE) { + case 'sqlite': + homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') + childAppDataSource = new DataSource({ + type: 'sqlite', + database: path.resolve(homePath, 'database.sqlite'), + synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + entities: [ChatFlow, ChatMessage, Tool], + migrations: [] + }) + break; + case 'mysql': + childAppDataSource = new DataSource({ + type: 'mysql', + host: process.env.DATABASE_HOST, + port: parseInt(process.env.DATABASE_PORT || '3306'), + username: process.env.DATABASE_USER, + password: process.env.DATABASE_PASSWORD, + database: process.env.DATABASE_NAME, + charset: 'utf8mb4', + synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + entities: [ChatFlow, ChatMessage, Tool], + migrations: [] + }) + break; + case 'postgres': + childAppDataSource = new DataSource({ + type: 'postgres', + host: process.env.DATABASE_HOST, + port: parseInt(process.env.DATABASE_PORT || '5432'), + username: process.env.DATABASE_USER, + password: process.env.DATABASE_PASSWORD, + database: process.env.DATABASE_NAME, + synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + entities: [ChatFlow, ChatMessage, Tool], + migrations: [] + }) + break; + default: + homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') + childAppDataSource = new DataSource({ + type: 'sqlite', + database: path.resolve(homePath, 'database.sqlite'), + synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + entities: [ChatFlow, ChatMessage, Tool], + migrations: [] + }) + break; + } + return await childAppDataSource.initialize() } diff --git a/packages/server/src/DataSource.ts b/packages/server/src/DataSource.ts index 03b9d5ce..b2ae5cd9 100644 --- a/packages/server/src/DataSource.ts +++ b/packages/server/src/DataSource.ts @@ -9,15 +9,56 @@ import { getUserHome } from './utils' let appDataSource: DataSource export const init = async (): Promise => { - const homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') - - appDataSource = new DataSource({ - type: 'sqlite', - database: path.resolve(homePath, 'database.sqlite'), - synchronize: true, - entities: [ChatFlow, ChatMessage, Tool], - migrations: [] - }) + let homePath; + switch (process.env.DATABASE_TYPE) { + case 'sqlite': + homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') + appDataSource = new DataSource({ + type: 'sqlite', + database: path.resolve(homePath, 'database.sqlite'), + synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + entities: [ChatFlow, ChatMessage, Tool], + migrations: [] + }) + break; + case 'mysql': + appDataSource = new DataSource({ + type: 'mysql', + host: process.env.DATABASE_HOST, + port: parseInt(process.env.DATABASE_PORT || '3306'), + username: process.env.DATABASE_USER, + password: process.env.DATABASE_PASSWORD, + database: process.env.DATABASE_NAME, + charset: 'utf8mb4', + synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + entities: [ChatFlow, ChatMessage, Tool], + migrations: [] + }) + break; + case 'postgres': + appDataSource = new DataSource({ + type: 'postgres', + host: process.env.DATABASE_HOST, + port: parseInt(process.env.DATABASE_PORT || '5432'), + username: process.env.DATABASE_USER, + password: process.env.DATABASE_PASSWORD, + database: process.env.DATABASE_NAME, + synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + entities: [ChatFlow, ChatMessage, Tool], + migrations: [] + }) + break; + default: + homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') + appDataSource = new DataSource({ + type: 'sqlite', + database: path.resolve(homePath, 'database.sqlite'), + synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + entities: [ChatFlow, ChatMessage, Tool], + migrations: [] + }) + break; + } } export function getDataSource(): DataSource { diff --git a/packages/server/src/entity/ChatFlow.ts b/packages/server/src/entity/ChatFlow.ts index e1d212cf..adaf0917 100644 --- a/packages/server/src/entity/ChatFlow.ts +++ b/packages/server/src/entity/ChatFlow.ts @@ -10,7 +10,7 @@ export class ChatFlow implements IChatFlow { @Column() name: string - @Column() + @Column({ type: "text" }) flowData: string @Column({ nullable: true }) diff --git a/packages/server/src/entity/ChatMessage.ts b/packages/server/src/entity/ChatMessage.ts index 3e4e41d2..46972f6a 100644 --- a/packages/server/src/entity/ChatMessage.ts +++ b/packages/server/src/entity/ChatMessage.ts @@ -14,7 +14,7 @@ export class ChatMessage implements IChatMessage { @Column() chatflowid: string - @Column() + @Column({ type: "text" }) content: string @Column({ nullable: true }) diff --git a/packages/server/src/entity/Tool.ts b/packages/server/src/entity/Tool.ts index 222fd766..4fc89519 100644 --- a/packages/server/src/entity/Tool.ts +++ b/packages/server/src/entity/Tool.ts @@ -10,7 +10,7 @@ export class Tool implements ITool { @Column() name: string - @Column() + @Column({ type: "text" }) description: string @Column() From 318bd516d79e0bc40799130f8ae10a5b90337359 Mon Sep 17 00:00:00 2001 From: Atish Amte Date: Tue, 18 Jul 2023 23:57:44 +0530 Subject: [PATCH 2/4] Update .env.example --- packages/server/.env.example | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/.env.example b/packages/server/.env.example index 57deb431..923c62a0 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -14,11 +14,11 @@ DATABASE_TYPE="sqlite" # sqlite, mysql, postgres # DATABASE_PATH=/your_database_path/.flowise # When database is not sqlite -# DATABASE_PORT="3306" -# DATABASE_HOST="127.0.0.1" +# DATABASE_PORT="" +# DATABASE_HOST="" # DATABASE_NAME="flowise" -# DATABASE_USER="root" -# DATABASE_PASSWORD="atish123" +# DATABASE_USER="" +# DATABASE_PASSWORD="" # TOOL_FUNCTION_BUILTIN_DEP=crypto,fs # TOOL_FUNCTION_EXTERNAL_DEP=moment,lodash From 4c4fd6ff6794e7fc2b5861d8e1d7bd44cc48a620 Mon Sep 17 00:00:00 2001 From: Atish Amte Date: Wed, 19 Jul 2023 00:03:48 +0530 Subject: [PATCH 3/4] Resolved linting error --- packages/server/src/ChildProcess.ts | 20 ++++++++++---------- packages/server/src/DataSource.ts | 18 +++++++++--------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/server/src/ChildProcess.ts b/packages/server/src/ChildProcess.ts index c14a4977..a93906c0 100644 --- a/packages/server/src/ChildProcess.ts +++ b/packages/server/src/ChildProcess.ts @@ -142,19 +142,19 @@ export class ChildProcess { * @returns {DataSource} */ async function initDB() { - let childAppDataSource; - let homePath; + let childAppDataSource + let homePath switch (process.env.DATABASE_TYPE) { case 'sqlite': homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') childAppDataSource = new DataSource({ type: 'sqlite', database: path.resolve(homePath, 'database.sqlite'), - synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + synchronize: process.env.OVERRIDE_DATABASE == 'true', entities: [ChatFlow, ChatMessage, Tool], migrations: [] }) - break; + break case 'mysql': childAppDataSource = new DataSource({ type: 'mysql', @@ -164,11 +164,11 @@ async function initDB() { password: process.env.DATABASE_PASSWORD, database: process.env.DATABASE_NAME, charset: 'utf8mb4', - synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + synchronize: process.env.OVERRIDE_DATABASE == 'true', entities: [ChatFlow, ChatMessage, Tool], migrations: [] }) - break; + break case 'postgres': childAppDataSource = new DataSource({ type: 'postgres', @@ -177,21 +177,21 @@ async function initDB() { username: process.env.DATABASE_USER, password: process.env.DATABASE_PASSWORD, database: process.env.DATABASE_NAME, - synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + synchronize: process.env.OVERRIDE_DATABASE == 'true', entities: [ChatFlow, ChatMessage, Tool], migrations: [] }) - break; + break default: homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') childAppDataSource = new DataSource({ type: 'sqlite', database: path.resolve(homePath, 'database.sqlite'), - synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + synchronize: process.env.OVERRIDE_DATABASE == 'true', entities: [ChatFlow, ChatMessage, Tool], migrations: [] }) - break; + break } return await childAppDataSource.initialize() diff --git a/packages/server/src/DataSource.ts b/packages/server/src/DataSource.ts index b2ae5cd9..958074cc 100644 --- a/packages/server/src/DataSource.ts +++ b/packages/server/src/DataSource.ts @@ -9,18 +9,18 @@ import { getUserHome } from './utils' let appDataSource: DataSource export const init = async (): Promise => { - let homePath; + let homePath switch (process.env.DATABASE_TYPE) { case 'sqlite': homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') appDataSource = new DataSource({ type: 'sqlite', database: path.resolve(homePath, 'database.sqlite'), - synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + synchronize: process.env.OVERRIDE_DATABASE == 'true', entities: [ChatFlow, ChatMessage, Tool], migrations: [] }) - break; + break case 'mysql': appDataSource = new DataSource({ type: 'mysql', @@ -30,11 +30,11 @@ export const init = async (): Promise => { password: process.env.DATABASE_PASSWORD, database: process.env.DATABASE_NAME, charset: 'utf8mb4', - synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + synchronize: process.env.OVERRIDE_DATABASE == 'true', entities: [ChatFlow, ChatMessage, Tool], migrations: [] }) - break; + break case 'postgres': appDataSource = new DataSource({ type: 'postgres', @@ -43,21 +43,21 @@ export const init = async (): Promise => { username: process.env.DATABASE_USER, password: process.env.DATABASE_PASSWORD, database: process.env.DATABASE_NAME, - synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + synchronize: process.env.OVERRIDE_DATABASE == 'true', entities: [ChatFlow, ChatMessage, Tool], migrations: [] }) - break; + break default: homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') appDataSource = new DataSource({ type: 'sqlite', database: path.resolve(homePath, 'database.sqlite'), - synchronize: (process.env.OVERRIDE_DATABASE == 'true'), + synchronize: process.env.OVERRIDE_DATABASE == 'true', entities: [ChatFlow, ChatMessage, Tool], migrations: [] }) - break; + break } } From 7f5acf32aaf585f2810bbf8a00d5a148e27f832e Mon Sep 17 00:00:00 2001 From: Atish Amte Date: Wed, 19 Jul 2023 00:07:42 +0530 Subject: [PATCH 4/4] Readme updated --- README.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b4b7fe76..951c1198 100644 --- a/README.md +++ b/README.md @@ -131,18 +131,25 @@ FLOWISE_PASSWORD=1234 Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. Read [more](https://docs.flowiseai.com/environment-variables) | Variable | Description | Type | Default | -| -------------------------- | ---------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | +| ---------------- | ---------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | | PORT | The HTTP port Flowise runs on | Number | 3000 | -| FLOWISE_USERNAME | Username to login | String | -| FLOWISE_PASSWORD | Password to login | String | -| DEBUG | Print logs onto terminal/console | Boolean | -| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/packages/server` | -| LOG_LEVEL | Different log levels for loggers to be saved | Enum String: `error`, `info`, `verbose`, `debug` | `info` | -| DATABASE_PATH | Location where database is saved | String | `your-home-dir/.flowise` | +| FLOWISE_USERNAME | Username to login | String | | +| FLOWISE_PASSWORD | Password to login | String | | +| DEBUG | Print logs from components | Boolean | | +| LOG_PATH | Location where log files are stored | String | `your-path/Flowise/logs` | +| LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | | APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | | EXECUTION_MODE | Whether predictions run in their own process or the main process | Enum String: `child`, `main` | `main` | | TOOL_FUNCTION_BUILTIN_DEP | NodeJS built-in modules to be used for Tool Function | String | | | TOOL_FUNCTION_EXTERNAL_DEP | External modules to be used for Tool Function | String | | +| OVERRIDE_DATABASE | Override current database with default | Enum String: `true`, `false` | `true` | +| DATABASE_TYPE | Type of database to store the flowise data | Enum String: `sqlite`, `mysql`, `postgres` | `sqlite` | +| DATABASE_PATH | Location where database is saved (When DATABASE_TYPE is sqlite) | String | `your-home-dir/.flowise` | +| DATABASE_HOST | Host URL or IP address (When DATABASE_TYPE is not sqlite) | String | | +| DATABASE_PORT | Database port (When DATABASE_TYPE is not sqlite) | String | | +| DATABASE_USERNAME | Database username (When DATABASE_TYPE is not sqlite) | String | | +| DATABASE_PASSWORD | Database password (When DATABASE_TYPE is not sqlite) | String | | +| DATABASE_NAME | Database name (When DATABASE_TYPE is not sqlite) | String | | You can also specify the env variables when using `npx`. For example: