From 3f0c6339e13497cd578c1f2306b6fbd98f63c574 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Sat, 2 Sep 2023 21:52:20 +0800 Subject: [PATCH 01/35] swap URL to get Confluence access token directly --- packages/components/credentials/ConfluenceApi.credential.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/credentials/ConfluenceApi.credential.ts b/packages/components/credentials/ConfluenceApi.credential.ts index a1d32e9c..371201a0 100644 --- a/packages/components/credentials/ConfluenceApi.credential.ts +++ b/packages/components/credentials/ConfluenceApi.credential.ts @@ -12,7 +12,7 @@ class ConfluenceApi implements INodeCredential { this.name = 'confluenceApi' this.version = 1.0 this.description = - 'Refer to official guide on how to get accessToken on Confluence' + 'Refer here to get accessToken on Confluence' this.inputs = [ { label: 'Access Token', From a05d73d83b245e29cc51d723c8d998aa445923eb Mon Sep 17 00:00:00 2001 From: Yongtae Date: Sun, 3 Sep 2023 19:12:42 +0900 Subject: [PATCH 02/35] Add input option for Model Name in GoogleVertexAIEmbedding --- .../GoogleVertexAIEmbedding.ts | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts b/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts index fdf7ea97..ad86ff5f 100644 --- a/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts +++ b/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts @@ -33,11 +33,35 @@ class GoogleVertexAIEmbedding_Embeddings implements INode { description: 'Google Vertex AI credential. If you are using a GCP service like Cloud Run, or if you have installed default credentials on your local machine, you do not need to set this credential.' } - this.inputs = [] + this.inputs = [ + { + label: 'Model Name', + name: 'modelName', + type: 'options', + options: [ + { + label: 'textembedding-gecko@001', + name: 'textembedding-gecko@001' + }, + { + label: 'textembedding-gecko@latest', + name: 'textembedding-gecko@latest' + }, + { + label: 'textembedding-gecko-multilingual@latest', + name: 'textembedding-gecko-multilingual@latest' + } + ], + default: 'textembedding-gecko@001', + optional: true, + additionalParams: true + } + ] } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const modelName = nodeData.inputs?.modelName ?? 'textembedding-gecko@001' const googleApplicationCredentialFilePath = getCredentialParam('googleApplicationCredentialFilePath', credentialData, nodeData) const googleApplicationCredential = getCredentialParam('googleApplicationCredential', credentialData, nodeData) const projectID = getCredentialParam('projectID', credentialData, nodeData) @@ -58,7 +82,9 @@ class GoogleVertexAIEmbedding_Embeddings implements INode { if (projectID) authOptions.projectId = projectID } - const obj: GoogleVertexAIEmbeddingsParams = {} + const obj: GoogleVertexAIEmbeddingsParams = { + model: modelName + } if (Object.keys(authOptions).length !== 0) obj.authOptions = authOptions const model = new GoogleVertexAIEmbeddings(obj) From 0b67afe54667744d8f3895eaeccc02bf04facb4a Mon Sep 17 00:00:00 2001 From: Yongtae Date: Sun, 3 Sep 2023 19:16:45 +0900 Subject: [PATCH 03/35] Refactor modelName assignment in GoogleVertexAIEmbedding.ts --- .../GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts b/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts index ad86ff5f..61518ceb 100644 --- a/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts +++ b/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts @@ -61,7 +61,7 @@ class GoogleVertexAIEmbedding_Embeddings implements INode { async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { const credentialData = await getCredentialData(nodeData.credential ?? '', options) - const modelName = nodeData.inputs?.modelName ?? 'textembedding-gecko@001' + const modelName = nodeData.inputs?.modelName as string const googleApplicationCredentialFilePath = getCredentialParam('googleApplicationCredentialFilePath', credentialData, nodeData) const googleApplicationCredential = getCredentialParam('googleApplicationCredential', credentialData, nodeData) const projectID = getCredentialParam('projectID', credentialData, nodeData) From 4d8b24180c048cbaf1dd2746c72fd416d747245b Mon Sep 17 00:00:00 2001 From: Yongtae Date: Sun, 3 Sep 2023 19:27:00 +0900 Subject: [PATCH 04/35] add 32k model --- .../ChatGoogleVertexAI/ChatGoogleVertexAI.ts | 8 ++++++++ .../nodes/llms/GoogleVertexAI/GoogleVertexAI.ts | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts b/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts index a7ac4259..1bc06f4b 100644 --- a/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts +++ b/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts @@ -46,6 +46,14 @@ class GoogleVertexAI_ChatModels implements INode { { label: 'codechat-bison', name: 'codechat-bison' + }, + { + label: 'chat-bison-32k', + name: 'chat-bison-32k' + }, + { + label: 'codechat-bison-32k', + name: 'codechat-bison-32k' } ], default: 'chat-bison', diff --git a/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts b/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts index b379ad47..4d19d04f 100644 --- a/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts +++ b/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts @@ -50,6 +50,18 @@ class GoogleVertexAI_LLMs implements INode { { label: 'code-gecko', name: 'code-gecko' + }, + { + label: 'text-bison-32k', + name: 'text-bison-32k' + }, + { + label: 'code-bison-32k', + name: 'code-bison-32k' + }, + { + label: 'code-gecko-32k', + name: 'code-gecko-32k' } ], default: 'text-bison' From 1505e72dd1908dd0eae85b32163727a6892737c9 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Mon, 4 Sep 2023 08:57:48 +0800 Subject: [PATCH 05/35] add API Key URL instead of swap URL --- packages/components/credentials/ConfluenceApi.credential.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/credentials/ConfluenceApi.credential.ts b/packages/components/credentials/ConfluenceApi.credential.ts index 371201a0..656ecf6b 100644 --- a/packages/components/credentials/ConfluenceApi.credential.ts +++ b/packages/components/credentials/ConfluenceApi.credential.ts @@ -12,7 +12,7 @@ class ConfluenceApi implements INodeCredential { this.name = 'confluenceApi' this.version = 1.0 this.description = - 'Refer here to get accessToken on Confluence' + 'Refer to official guide on how to get Access Token or API Token on Confluence' this.inputs = [ { label: 'Access Token', From 95740a84647d597b96ef8d9b78e21d7761874164 Mon Sep 17 00:00:00 2001 From: Yongtae Date: Mon, 4 Sep 2023 11:01:56 +0900 Subject: [PATCH 06/35] Refactor GoogleVertexAIEmbedding.ts to remove unnecessary additionalParams field --- .../GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts b/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts index 61518ceb..3cbd600a 100644 --- a/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts +++ b/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts @@ -53,8 +53,7 @@ class GoogleVertexAIEmbedding_Embeddings implements INode { } ], default: 'textembedding-gecko@001', - optional: true, - additionalParams: true + optional: true } ] } From a373422abf93fdb2ca92d52f5c9e4dc42f008fde Mon Sep 17 00:00:00 2001 From: Yongtae Date: Mon, 4 Sep 2023 11:02:35 +0900 Subject: [PATCH 07/35] Refactor GoogleVertexAIEmbedding.ts for handling modelName in GoogleVertexAIEmbeddingsParams --- .../GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts b/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts index 3cbd600a..7d086e0c 100644 --- a/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts +++ b/packages/components/nodes/embeddings/GoogleVertexAIEmbedding/GoogleVertexAIEmbedding.ts @@ -81,9 +81,8 @@ class GoogleVertexAIEmbedding_Embeddings implements INode { if (projectID) authOptions.projectId = projectID } - const obj: GoogleVertexAIEmbeddingsParams = { - model: modelName - } + const obj: GoogleVertexAIEmbeddingsParams = {} + if (modelName) obj.model = modelName if (Object.keys(authOptions).length !== 0) obj.authOptions = authOptions const model = new GoogleVertexAIEmbeddings(obj) From 008f1a95f15a09b2892a1071526919f9605bde1d Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Mon, 4 Sep 2023 17:14:49 +0800 Subject: [PATCH 08/35] init migration poc --- packages/server/src/DataSource.ts | 31 ++++++++++--------- .../{entity => database/entities}/ChatFlow.ts | 2 +- .../entities}/ChatMessage.ts | 2 +- .../entities}/Credential.ts | 2 +- .../src/{entity => database/entities}/Tool.ts | 2 +- .../server/src/database/entities/index.ts | 11 +++++++ .../migrations/1693809869231-initSqlite.ts | 29 +++++++++++++++++ packages/server/src/index.ts | 13 +++++--- packages/server/src/utils/index.ts | 8 ++--- 9 files changed, 73 insertions(+), 27 deletions(-) rename packages/server/src/{entity => database/entities}/ChatFlow.ts (93%) rename packages/server/src/{entity => database/entities}/ChatMessage.ts (88%) rename packages/server/src/{entity => database/entities}/Credential.ts (90%) rename packages/server/src/{entity => database/entities}/Tool.ts (93%) create mode 100644 packages/server/src/database/entities/index.ts create mode 100644 packages/server/src/database/migrations/1693809869231-initSqlite.ts diff --git a/packages/server/src/DataSource.ts b/packages/server/src/DataSource.ts index b0d56477..a1d37996 100644 --- a/packages/server/src/DataSource.ts +++ b/packages/server/src/DataSource.ts @@ -1,26 +1,24 @@ import 'reflect-metadata' import path from 'path' import { DataSource } from 'typeorm' -import { ChatFlow } from './entity/ChatFlow' -import { ChatMessage } from './entity/ChatMessage' -import { Credential } from './entity/Credential' -import { Tool } from './entity/Tool' import { getUserHome } from './utils' +import { entities } from './database/entities' +import { InitSqlite1693809869231 } from './database/migrations/1693809869231-initSqlite' let appDataSource: DataSource export const init = async (): Promise => { let homePath - const synchronize = process.env.OVERRIDE_DATABASE === 'false' ? false : true switch (process.env.DATABASE_TYPE) { case 'sqlite': homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') appDataSource = new DataSource({ type: 'sqlite', database: path.resolve(homePath, 'database.sqlite'), - synchronize, - entities: [ChatFlow, ChatMessage, Tool, Credential], - migrations: [] + synchronize: false, + migrationsRun: false, + entities: Object.values(entities), + migrations: [InitSqlite1693809869231] }) break case 'mysql': @@ -32,8 +30,9 @@ export const init = async (): Promise => { password: process.env.DATABASE_PASSWORD, database: process.env.DATABASE_NAME, charset: 'utf8mb4', - synchronize, - entities: [ChatFlow, ChatMessage, Tool, Credential], + synchronize: false, + migrationsRun: false, + entities: Object.values(entities), migrations: [] }) break @@ -45,8 +44,9 @@ export const init = async (): Promise => { username: process.env.DATABASE_USER, password: process.env.DATABASE_PASSWORD, database: process.env.DATABASE_NAME, - synchronize, - entities: [ChatFlow, ChatMessage, Tool, Credential], + synchronize: false, + migrationsRun: false, + entities: Object.values(entities), migrations: [] }) break @@ -55,9 +55,10 @@ export const init = async (): Promise => { appDataSource = new DataSource({ type: 'sqlite', database: path.resolve(homePath, 'database.sqlite'), - synchronize, - entities: [ChatFlow, ChatMessage, Tool, Credential], - migrations: [] + synchronize: false, + migrationsRun: false, + entities: Object.values(entities), + migrations: [InitSqlite1693809869231] }) break } diff --git a/packages/server/src/entity/ChatFlow.ts b/packages/server/src/database/entities/ChatFlow.ts similarity index 93% rename from packages/server/src/entity/ChatFlow.ts rename to packages/server/src/database/entities/ChatFlow.ts index 4c37e083..a1a32a88 100644 --- a/packages/server/src/entity/ChatFlow.ts +++ b/packages/server/src/database/entities/ChatFlow.ts @@ -1,6 +1,6 @@ /* eslint-disable */ import { Entity, Column, CreateDateColumn, UpdateDateColumn, PrimaryGeneratedColumn } from 'typeorm' -import { IChatFlow } from '../Interface' +import { IChatFlow } from '../../Interface' @Entity() export class ChatFlow implements IChatFlow { diff --git a/packages/server/src/entity/ChatMessage.ts b/packages/server/src/database/entities/ChatMessage.ts similarity index 88% rename from packages/server/src/entity/ChatMessage.ts rename to packages/server/src/database/entities/ChatMessage.ts index 8123020c..23804846 100644 --- a/packages/server/src/entity/ChatMessage.ts +++ b/packages/server/src/database/entities/ChatMessage.ts @@ -1,6 +1,6 @@ /* eslint-disable */ import { Entity, Column, CreateDateColumn, PrimaryGeneratedColumn, Index } from 'typeorm' -import { IChatMessage, MessageType } from '../Interface' +import { IChatMessage, MessageType } from '../../Interface' @Entity() export class ChatMessage implements IChatMessage { diff --git a/packages/server/src/entity/Credential.ts b/packages/server/src/database/entities/Credential.ts similarity index 90% rename from packages/server/src/entity/Credential.ts rename to packages/server/src/database/entities/Credential.ts index b724eed6..e77711dc 100644 --- a/packages/server/src/entity/Credential.ts +++ b/packages/server/src/database/entities/Credential.ts @@ -1,6 +1,6 @@ /* eslint-disable */ import { Entity, Column, PrimaryGeneratedColumn, Index, CreateDateColumn, UpdateDateColumn } from 'typeorm' -import { ICredential } from '../Interface' +import { ICredential } from '../../Interface' @Entity() export class Credential implements ICredential { diff --git a/packages/server/src/entity/Tool.ts b/packages/server/src/database/entities/Tool.ts similarity index 93% rename from packages/server/src/entity/Tool.ts rename to packages/server/src/database/entities/Tool.ts index 011bf957..d459eee3 100644 --- a/packages/server/src/entity/Tool.ts +++ b/packages/server/src/database/entities/Tool.ts @@ -1,6 +1,6 @@ /* eslint-disable */ import { Entity, Column, CreateDateColumn, UpdateDateColumn, PrimaryGeneratedColumn } from 'typeorm' -import { ITool } from '../Interface' +import { ITool } from '../../Interface' @Entity() export class Tool implements ITool { diff --git a/packages/server/src/database/entities/index.ts b/packages/server/src/database/entities/index.ts new file mode 100644 index 00000000..ff109863 --- /dev/null +++ b/packages/server/src/database/entities/index.ts @@ -0,0 +1,11 @@ +import { ChatFlow } from './ChatFlow' +import { ChatMessage } from './ChatMessage' +import { Credential } from './Credential' +import { Tool } from './Tool' + +export const entities = { + ChatFlow, + ChatMessage, + Credential, + Tool +} diff --git a/packages/server/src/database/migrations/1693809869231-initSqlite.ts b/packages/server/src/database/migrations/1693809869231-initSqlite.ts new file mode 100644 index 00000000..78af741d --- /dev/null +++ b/packages/server/src/database/migrations/1693809869231-initSqlite.ts @@ -0,0 +1,29 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class InitSqlite1693809869231 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + console.info('started migration query') + await queryRunner.query( + `CREATE TABLE "chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" boolean, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + ) + await queryRunner.query( + `CREATE TABLE "chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')));` + ) + await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`) + await queryRunner.query( + `CREATE TABLE "credential" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "credentialName" varchar NOT NULL, "encryptedData" varchar NOT NULL, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + ) + await queryRunner.query( + `CREATE TABLE "tool" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "description" text NOT NULL, "color" varchar NOT NULL, "iconSrc" varchar, "schema" varchar, "func" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + ) + console.info('Finish migration query') + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "chat_flow"`) + await queryRunner.query(`DROP INDEX "IDX_chat_messagee574527322272fd838f4f0f3d3"`) + await queryRunner.query(`DROP TABLE "chat_message"`) + await queryRunner.query(`DROP TABLE "credential"`) + await queryRunner.query(`DROP TABLE "tool"`) + } +} diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 66c7b000..1e8dc83c 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -48,10 +48,10 @@ import { import { cloneDeep, omit } from 'lodash' import { getDataSource } from './DataSource' import { NodesPool } from './NodesPool' -import { ChatFlow } from './entity/ChatFlow' -import { ChatMessage } from './entity/ChatMessage' -import { Credential } from './entity/Credential' -import { Tool } from './entity/Tool' +import { ChatFlow } from './database/entities/ChatFlow' +import { ChatMessage } from './database/entities/ChatMessage' +import { Credential } from './database/entities/Credential' +import { Tool } from './database/entities/Tool' import { ChatflowPool } from './ChatflowPool' import { ICommonObject, INodeOptionsValue } from 'flowise-components' @@ -71,6 +71,11 @@ export class App { .then(async () => { logger.info('📦 [server]: Data Source has been initialized!') + //Migrations + console.info(`start migration`) + await this.AppDataSource.runMigrations({ transaction: 'each' }) + console.info(`finish migration`) + // Initialize nodes pool this.nodesPool = new NodesPool() await this.nodesPool.initialize() diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index b95dd301..73c748a3 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -30,10 +30,10 @@ import { import { scryptSync, randomBytes, timingSafeEqual } from 'crypto' import { lib, PBKDF2, AES, enc } from 'crypto-js' -import { ChatFlow } from '../entity/ChatFlow' -import { ChatMessage } from '../entity/ChatMessage' -import { Credential } from '../entity/Credential' -import { Tool } from '../entity/Tool' +import { ChatFlow } from '../database/entities/ChatFlow' +import { ChatMessage } from '../database/entities/ChatMessage' +import { Credential } from '../database/entities/Credential' +import { Tool } from '../database/entities/Tool' import { DataSource } from 'typeorm' const QUESTION_VAR_PREFIX = 'question' From 3849169a7670b6c648d8ded545a98f343c2c3514 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Mon, 4 Sep 2023 22:36:38 +0800 Subject: [PATCH 09/35] cleanup sqlite migration scripts --- package.json | 3 ++- packages/server/src/DataSource.ts | 6 +++--- .../1693835579790-Init.ts} | 8 +++----- packages/server/src/database/migrations/sqlite/index.ts | 3 +++ packages/server/src/index.ts | 4 +--- 5 files changed, 12 insertions(+), 12 deletions(-) rename packages/server/src/database/migrations/{1693809869231-initSqlite.ts => sqlite/1693835579790-Init.ts} (85%) create mode 100644 packages/server/src/database/migrations/sqlite/index.ts diff --git a/package.json b/package.json index 5ef0ecdb..dae9d098 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "lint": "eslint \"**/*.{js,jsx,ts,tsx,json,md}\"", "lint-fix": "yarn lint --fix", "quick": "pretty-quick --staged", - "postinstall": "husky install" + "postinstall": "husky install", + "migration:create": "yarn typeorm migration:create" }, "lint-staged": { "*.{js,jsx,ts,tsx,json,md}": "eslint --fix" diff --git a/packages/server/src/DataSource.ts b/packages/server/src/DataSource.ts index a1d37996..11ddc8ee 100644 --- a/packages/server/src/DataSource.ts +++ b/packages/server/src/DataSource.ts @@ -3,7 +3,7 @@ import path from 'path' import { DataSource } from 'typeorm' import { getUserHome } from './utils' import { entities } from './database/entities' -import { InitSqlite1693809869231 } from './database/migrations/1693809869231-initSqlite' +import { sqliteMigrations } from './database/migrations/sqlite' let appDataSource: DataSource @@ -18,7 +18,7 @@ export const init = async (): Promise => { synchronize: false, migrationsRun: false, entities: Object.values(entities), - migrations: [InitSqlite1693809869231] + migrations: sqliteMigrations }) break case 'mysql': @@ -58,7 +58,7 @@ export const init = async (): Promise => { synchronize: false, migrationsRun: false, entities: Object.values(entities), - migrations: [InitSqlite1693809869231] + migrations: sqliteMigrations }) break } diff --git a/packages/server/src/database/migrations/1693809869231-initSqlite.ts b/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts similarity index 85% rename from packages/server/src/database/migrations/1693809869231-initSqlite.ts rename to packages/server/src/database/migrations/sqlite/1693835579790-Init.ts index 78af741d..2ae72b63 100644 --- a/packages/server/src/database/migrations/1693809869231-initSqlite.ts +++ b/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts @@ -1,27 +1,25 @@ import { MigrationInterface, QueryRunner } from 'typeorm' -export class InitSqlite1693809869231 implements MigrationInterface { +export class Init1693835579790 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { - console.info('started migration query') await queryRunner.query( `CREATE TABLE "chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" boolean, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` ) await queryRunner.query( `CREATE TABLE "chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')));` ) - await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`) + await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid");`) await queryRunner.query( `CREATE TABLE "credential" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "credentialName" varchar NOT NULL, "encryptedData" varchar NOT NULL, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` ) await queryRunner.query( `CREATE TABLE "tool" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "description" text NOT NULL, "color" varchar NOT NULL, "iconSrc" varchar, "schema" varchar, "func" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` ) - console.info('Finish migration query') } public async down(queryRunner: QueryRunner): Promise { await queryRunner.query(`DROP TABLE "chat_flow"`) - await queryRunner.query(`DROP INDEX "IDX_chat_messagee574527322272fd838f4f0f3d3"`) + await queryRunner.query(`DROP INDEX "IDX_e574527322272fd838f4f0f3d3"`) await queryRunner.query(`DROP TABLE "chat_message"`) await queryRunner.query(`DROP TABLE "credential"`) await queryRunner.query(`DROP TABLE "tool"`) diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts new file mode 100644 index 00000000..31fef34a --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -0,0 +1,3 @@ +import { Init1693835579790 } from './1693835579790-Init' + +export const sqliteMigrations = [Init1693835579790] diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 1e8dc83c..81dbbcb4 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -71,10 +71,8 @@ export class App { .then(async () => { logger.info('📦 [server]: Data Source has been initialized!') - //Migrations - console.info(`start migration`) + // Run Migrations Scripts await this.AppDataSource.runMigrations({ transaction: 'each' }) - console.info(`finish migration`) // Initialize nodes pool this.nodesPool = new NodesPool() From 02924f96c8205a50990949b273954358f17be01f Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 00:26:47 +0800 Subject: [PATCH 10/35] add text into potential large column --- packages/server/src/database/entities/ChatFlow.ts | 2 +- packages/server/src/database/entities/ChatMessage.ts | 2 +- packages/server/src/database/entities/Credential.ts | 2 +- packages/server/src/database/entities/Tool.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/database/entities/ChatFlow.ts b/packages/server/src/database/entities/ChatFlow.ts index a1a32a88..a4fdd130 100644 --- a/packages/server/src/database/entities/ChatFlow.ts +++ b/packages/server/src/database/entities/ChatFlow.ts @@ -22,7 +22,7 @@ export class ChatFlow implements IChatFlow { @Column({ nullable: true }) apikeyid?: string - @Column({ nullable: true }) + @Column({ nullable: true, type: 'text' }) chatbotConfig?: string @CreateDateColumn() diff --git a/packages/server/src/database/entities/ChatMessage.ts b/packages/server/src/database/entities/ChatMessage.ts index 23804846..4b5306ee 100644 --- a/packages/server/src/database/entities/ChatMessage.ts +++ b/packages/server/src/database/entities/ChatMessage.ts @@ -17,7 +17,7 @@ export class ChatMessage implements IChatMessage { @Column({ type: 'text' }) content: string - @Column({ nullable: true }) + @Column({ nullable: true, type: 'text' }) sourceDocuments?: string @CreateDateColumn() diff --git a/packages/server/src/database/entities/Credential.ts b/packages/server/src/database/entities/Credential.ts index e77711dc..822f2584 100644 --- a/packages/server/src/database/entities/Credential.ts +++ b/packages/server/src/database/entities/Credential.ts @@ -13,7 +13,7 @@ export class Credential implements ICredential { @Column() credentialName: string - @Column() + @Column({ type: 'text' }) encryptedData: string @CreateDateColumn() diff --git a/packages/server/src/database/entities/Tool.ts b/packages/server/src/database/entities/Tool.ts index d459eee3..8e675eb0 100644 --- a/packages/server/src/database/entities/Tool.ts +++ b/packages/server/src/database/entities/Tool.ts @@ -19,10 +19,10 @@ export class Tool implements ITool { @Column({ nullable: true }) iconSrc?: string - @Column({ nullable: true }) + @Column({ nullable: true, type: 'text' }) schema?: string - @Column({ nullable: true }) + @Column({ nullable: true, type: 'text' }) func?: string @CreateDateColumn() From 17c2309454963482a5f333c8f1859ce238c1ba1e Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 00:36:37 +0800 Subject: [PATCH 11/35] add mysqlMigrations --- packages/server/src/DataSource.ts | 3 +- .../migrations/mysql/1693840429259-Init.ts | 64 +++++++++++++++++++ .../src/database/migrations/mysql/index.ts | 3 + 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/database/migrations/mysql/1693840429259-Init.ts create mode 100644 packages/server/src/database/migrations/mysql/index.ts diff --git a/packages/server/src/DataSource.ts b/packages/server/src/DataSource.ts index 11ddc8ee..6f8e56fd 100644 --- a/packages/server/src/DataSource.ts +++ b/packages/server/src/DataSource.ts @@ -4,6 +4,7 @@ import { DataSource } from 'typeorm' import { getUserHome } from './utils' import { entities } from './database/entities' import { sqliteMigrations } from './database/migrations/sqlite' +import { mysqlMigrations } from './database/migrations/mysql' let appDataSource: DataSource @@ -33,7 +34,7 @@ export const init = async (): Promise => { synchronize: false, migrationsRun: false, entities: Object.values(entities), - migrations: [] + migrations: mysqlMigrations }) break case 'postgres': diff --git a/packages/server/src/database/migrations/mysql/1693840429259-Init.ts b/packages/server/src/database/migrations/mysql/1693840429259-Init.ts new file mode 100644 index 00000000..8d034680 --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1693840429259-Init.ts @@ -0,0 +1,64 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class Init1693840429259 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE \`chat_flow\` ( + \`id\` varchar(36) NOT NULL, + \`name\` varchar(255) NOT NULL, + \`flowData\` text NOT NULL, + \`deployed\` tinyint DEFAULT NULL, + \`isPublic\` tinyint DEFAULT NULL, + \`apikeyid\` varchar(255) DEFAULT NULL, + \`chatbotConfig\` text, + \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;` + ) + await queryRunner.query( + `CREATE TABLE \`chat_message\` ( + \`id\` varchar(36) NOT NULL, + \`role\` varchar(255) NOT NULL, + \`chatflowid\` varchar(255) NOT NULL, + \`content\` text NOT NULL, + \`sourceDocuments\` text, + \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (\`id\`), + KEY \`IDX_e574527322272fd838f4f0f3d3\` (\`chatflowid\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;` + ) + await queryRunner.query( + `CREATE TABLE \`credential\` ( + \`id\` varchar(36) NOT NULL, + \`name\` varchar(255) NOT NULL, + \`credentialName\` varchar(255) NOT NULL, + \`encryptedData\` text NOT NULL, + \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;` + ) + await queryRunner.query( + `CREATE TABLE \`tool\` ( + \`id\` varchar(36) NOT NULL, + \`name\` varchar(255) NOT NULL, + \`description\` text NOT NULL, + \`color\` varchar(255) NOT NULL, + \`iconSrc\` varchar(255) DEFAULT NULL, + \`schema\` text, + \`func\` text, + \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;` + ) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "chat_flow"`) + await queryRunner.query(`DROP TABLE "chat_message"`) + await queryRunner.query(`DROP TABLE "credential"`) + await queryRunner.query(`DROP TABLE "tool"`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts new file mode 100644 index 00000000..79fa17ad --- /dev/null +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -0,0 +1,3 @@ +import { Init1693840429259 } from './1693840429259-Init' + +export const mysqlMigrations = [Init1693840429259] From b688f962363e2f8aefcbff51055ee79b49532c20 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 00:55:39 +0800 Subject: [PATCH 12/35] update sqlite init query --- .../migrations/mysql/1693840429259-Init.ts | 8 ++++---- .../migrations/sqlite/1693835579790-Init.ts | 19 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/packages/server/src/database/migrations/mysql/1693840429259-Init.ts b/packages/server/src/database/migrations/mysql/1693840429259-Init.ts index 8d034680..6ae611dc 100644 --- a/packages/server/src/database/migrations/mysql/1693840429259-Init.ts +++ b/packages/server/src/database/migrations/mysql/1693840429259-Init.ts @@ -56,9 +56,9 @@ export class Init1693840429259 implements MigrationInterface { } public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP TABLE "chat_flow"`) - await queryRunner.query(`DROP TABLE "chat_message"`) - await queryRunner.query(`DROP TABLE "credential"`) - await queryRunner.query(`DROP TABLE "tool"`) + await queryRunner.query(`DROP TABLE chat_flow`) + await queryRunner.query(`DROP TABLE chat_message`) + await queryRunner.query(`DROP TABLE credential`) + await queryRunner.query(`DROP TABLE tool`) } } diff --git a/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts b/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts index 2ae72b63..2c199881 100644 --- a/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts +++ b/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts @@ -3,25 +3,24 @@ import { MigrationInterface, QueryRunner } from 'typeorm' export class Init1693835579790 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { await queryRunner.query( - `CREATE TABLE "chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" boolean, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + `CREATE TABLE "chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" boolean, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` ) await queryRunner.query( - `CREATE TABLE "chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')));` + `CREATE TABLE "chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')));` ) - await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid");`) + await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`) await queryRunner.query( - `CREATE TABLE "credential" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "credentialName" varchar NOT NULL, "encryptedData" varchar NOT NULL, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + `CREATE TABLE "credential" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "credentialName" varchar NOT NULL, "encryptedData" text NOT NULL, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` ) await queryRunner.query( - `CREATE TABLE "tool" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "description" text NOT NULL, "color" varchar NOT NULL, "iconSrc" varchar, "schema" varchar, "func" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + `CREATE TABLE "tool" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "description" text NOT NULL, "color" varchar NOT NULL, "iconSrc" varchar, "schema" text, "func" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` ) } public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP TABLE "chat_flow"`) - await queryRunner.query(`DROP INDEX "IDX_e574527322272fd838f4f0f3d3"`) - await queryRunner.query(`DROP TABLE "chat_message"`) - await queryRunner.query(`DROP TABLE "credential"`) - await queryRunner.query(`DROP TABLE "tool"`) + await queryRunner.query(`DROP TABLE chat_flow`) + await queryRunner.query(`DROP TABLE chat_message`) + await queryRunner.query(`DROP TABLE credential`) + await queryRunner.query(`DROP TABLE tool`) } } From ae66f163de47c69209e517bef3bc0309c8918441 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 13:49:50 +0800 Subject: [PATCH 13/35] add postgresMigrations --- packages/server/src/DataSource.ts | 3 +- .../migrations/postgres/1693891895163-Init.ts | 64 +++++++++++++++++++ .../src/database/migrations/postgres/index.ts | 3 + 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/database/migrations/postgres/1693891895163-Init.ts create mode 100644 packages/server/src/database/migrations/postgres/index.ts diff --git a/packages/server/src/DataSource.ts b/packages/server/src/DataSource.ts index 6f8e56fd..9265e55f 100644 --- a/packages/server/src/DataSource.ts +++ b/packages/server/src/DataSource.ts @@ -5,6 +5,7 @@ import { getUserHome } from './utils' import { entities } from './database/entities' import { sqliteMigrations } from './database/migrations/sqlite' import { mysqlMigrations } from './database/migrations/mysql' +import { postgresMigrations } from './database/migrations/postgres' let appDataSource: DataSource @@ -48,7 +49,7 @@ export const init = async (): Promise => { synchronize: false, migrationsRun: false, entities: Object.values(entities), - migrations: [] + migrations: postgresMigrations }) break default: diff --git a/packages/server/src/database/migrations/postgres/1693891895163-Init.ts b/packages/server/src/database/migrations/postgres/1693891895163-Init.ts new file mode 100644 index 00000000..defb5f87 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1693891895163-Init.ts @@ -0,0 +1,64 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class Init1693891895163 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE chat_flow ( + id uuid NOT NULL DEFAULT uuid_generate_v4(), + "name" varchar NOT NULL, + "flowData" text NOT NULL, + deployed bool NULL, + "isPublic" bool NULL, + apikeyid varchar NULL, + "chatbotConfig" text NULL, + "createdDate" timestamp NOT NULL DEFAULT now(), + "updatedDate" timestamp NOT NULL DEFAULT now(), + CONSTRAINT "PK_3c7cea7d047ac4b91764574cdbf" PRIMARY KEY (id) + );` + ) + await queryRunner.query( + `CREATE TABLE chat_message ( + id uuid NOT NULL DEFAULT uuid_generate_v4(), + "role" varchar NOT NULL, + chatflowid varchar NOT NULL, + "content" text NOT NULL, + "sourceDocuments" text NULL, + "createdDate" timestamp NOT NULL DEFAULT now(), + CONSTRAINT "PK_3cc0d85193aade457d3077dd06b" PRIMARY KEY (id) + );` + ) + await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON chat_message USING btree (chatflowid);`) + await queryRunner.query( + `CREATE TABLE credential ( + id uuid NOT NULL DEFAULT uuid_generate_v4(), + "name" varchar NOT NULL, + "credentialName" varchar NOT NULL, + "encryptedData" text NOT NULL, + "createdDate" timestamp NOT NULL DEFAULT now(), + "updatedDate" timestamp NOT NULL DEFAULT now(), + CONSTRAINT "PK_3a5169bcd3d5463cefeec78be82" PRIMARY KEY (id) + );` + ) + await queryRunner.query( + `CREATE TABLE tool ( + id uuid NOT NULL DEFAULT uuid_generate_v4(), + "name" varchar NOT NULL, + description text NOT NULL, + color varchar NOT NULL, + "iconSrc" varchar NULL, + "schema" text NULL, + func text NULL, + "createdDate" timestamp NOT NULL DEFAULT now(), + "updatedDate" timestamp NOT NULL DEFAULT now(), + CONSTRAINT "PK_3bf5b1016a384916073184f99b7" PRIMARY KEY (id) + );` + ) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE chat_flow`) + await queryRunner.query(`DROP TABLE chat_message`) + await queryRunner.query(`DROP TABLE credential`) + await queryRunner.query(`DROP TABLE tool`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts new file mode 100644 index 00000000..c8805785 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -0,0 +1,3 @@ +import { Init1693891895163 } from './1693891895163-Init' + +export const postgresMigrations = [Init1693891895163] From 0e891bfb3aa1cbad492c9947ff4569aaec6ae0ec Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 16:13:04 +0800 Subject: [PATCH 14/35] preserve previous user data --- .../server/src/database/entities/ChatFlow.ts | 2 +- .../src/database/entities/ChatMessage.ts | 2 +- .../src/database/entities/Credential.ts | 2 +- packages/server/src/database/entities/Tool.ts | 4 ++-- .../migrations/mysql/1693840429259-Init.ts | 18 ++++++++--------- .../migrations/postgres/1693891895163-Init.ts | 20 +++++++++---------- .../migrations/sqlite/1693835579790-Init.ts | 10 +++++----- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/packages/server/src/database/entities/ChatFlow.ts b/packages/server/src/database/entities/ChatFlow.ts index a4fdd130..a1a32a88 100644 --- a/packages/server/src/database/entities/ChatFlow.ts +++ b/packages/server/src/database/entities/ChatFlow.ts @@ -22,7 +22,7 @@ export class ChatFlow implements IChatFlow { @Column({ nullable: true }) apikeyid?: string - @Column({ nullable: true, type: 'text' }) + @Column({ nullable: true }) chatbotConfig?: string @CreateDateColumn() diff --git a/packages/server/src/database/entities/ChatMessage.ts b/packages/server/src/database/entities/ChatMessage.ts index 4b5306ee..23804846 100644 --- a/packages/server/src/database/entities/ChatMessage.ts +++ b/packages/server/src/database/entities/ChatMessage.ts @@ -17,7 +17,7 @@ export class ChatMessage implements IChatMessage { @Column({ type: 'text' }) content: string - @Column({ nullable: true, type: 'text' }) + @Column({ nullable: true }) sourceDocuments?: string @CreateDateColumn() diff --git a/packages/server/src/database/entities/Credential.ts b/packages/server/src/database/entities/Credential.ts index 822f2584..e77711dc 100644 --- a/packages/server/src/database/entities/Credential.ts +++ b/packages/server/src/database/entities/Credential.ts @@ -13,7 +13,7 @@ export class Credential implements ICredential { @Column() credentialName: string - @Column({ type: 'text' }) + @Column() encryptedData: string @CreateDateColumn() diff --git a/packages/server/src/database/entities/Tool.ts b/packages/server/src/database/entities/Tool.ts index 8e675eb0..d459eee3 100644 --- a/packages/server/src/database/entities/Tool.ts +++ b/packages/server/src/database/entities/Tool.ts @@ -19,10 +19,10 @@ export class Tool implements ITool { @Column({ nullable: true }) iconSrc?: string - @Column({ nullable: true, type: 'text' }) + @Column({ nullable: true }) schema?: string - @Column({ nullable: true, type: 'text' }) + @Column({ nullable: true }) func?: string @CreateDateColumn() diff --git a/packages/server/src/database/migrations/mysql/1693840429259-Init.ts b/packages/server/src/database/migrations/mysql/1693840429259-Init.ts index 6ae611dc..9d07206d 100644 --- a/packages/server/src/database/migrations/mysql/1693840429259-Init.ts +++ b/packages/server/src/database/migrations/mysql/1693840429259-Init.ts @@ -3,51 +3,51 @@ import { MigrationInterface, QueryRunner } from 'typeorm' export class Init1693840429259 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { await queryRunner.query( - `CREATE TABLE \`chat_flow\` ( + `CREATE TABLE IF NOT EXISTS \`chat_flow\` ( \`id\` varchar(36) NOT NULL, \`name\` varchar(255) NOT NULL, \`flowData\` text NOT NULL, \`deployed\` tinyint DEFAULT NULL, \`isPublic\` tinyint DEFAULT NULL, \`apikeyid\` varchar(255) DEFAULT NULL, - \`chatbotConfig\` text, + \`chatbotConfig\` varchar(255) DEFAULT NULL, \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (\`id\`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;` ) await queryRunner.query( - `CREATE TABLE \`chat_message\` ( + `CREATE TABLE IF NOT EXISTS \`chat_message\` ( \`id\` varchar(36) NOT NULL, \`role\` varchar(255) NOT NULL, \`chatflowid\` varchar(255) NOT NULL, \`content\` text NOT NULL, - \`sourceDocuments\` text, + \`sourceDocuments\` varchar(255) DEFAULT NULL, \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), PRIMARY KEY (\`id\`), KEY \`IDX_e574527322272fd838f4f0f3d3\` (\`chatflowid\`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;` ) await queryRunner.query( - `CREATE TABLE \`credential\` ( + `CREATE TABLE IF NOT EXISTS \`credential\` ( \`id\` varchar(36) NOT NULL, \`name\` varchar(255) NOT NULL, \`credentialName\` varchar(255) NOT NULL, - \`encryptedData\` text NOT NULL, + \`encryptedData\` varchar(255) NOT NULL, \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (\`id\`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;` ) await queryRunner.query( - `CREATE TABLE \`tool\` ( + `CREATE TABLE IF NOT EXISTS \`tool\` ( \`id\` varchar(36) NOT NULL, \`name\` varchar(255) NOT NULL, \`description\` text NOT NULL, \`color\` varchar(255) NOT NULL, \`iconSrc\` varchar(255) DEFAULT NULL, - \`schema\` text, - \`func\` text, + \`schema\` varchar(255) DEFAULT NULL, + \`func\` varchar(255) DEFAULT NULL, \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (\`id\`) diff --git a/packages/server/src/database/migrations/postgres/1693891895163-Init.ts b/packages/server/src/database/migrations/postgres/1693891895163-Init.ts index defb5f87..3ffe055c 100644 --- a/packages/server/src/database/migrations/postgres/1693891895163-Init.ts +++ b/packages/server/src/database/migrations/postgres/1693891895163-Init.ts @@ -3,51 +3,51 @@ import { MigrationInterface, QueryRunner } from 'typeorm' export class Init1693891895163 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { await queryRunner.query( - `CREATE TABLE chat_flow ( + `CREATE TABLE IF NOT EXISTS chat_flow ( id uuid NOT NULL DEFAULT uuid_generate_v4(), "name" varchar NOT NULL, "flowData" text NOT NULL, deployed bool NULL, "isPublic" bool NULL, apikeyid varchar NULL, - "chatbotConfig" text NULL, + "chatbotConfig" varchar NULL, "createdDate" timestamp NOT NULL DEFAULT now(), "updatedDate" timestamp NOT NULL DEFAULT now(), CONSTRAINT "PK_3c7cea7d047ac4b91764574cdbf" PRIMARY KEY (id) );` ) await queryRunner.query( - `CREATE TABLE chat_message ( + `CREATE TABLE IF NOT EXISTS chat_message ( id uuid NOT NULL DEFAULT uuid_generate_v4(), "role" varchar NOT NULL, chatflowid varchar NOT NULL, "content" text NOT NULL, - "sourceDocuments" text NULL, + "sourceDocuments" varchar NULL, "createdDate" timestamp NOT NULL DEFAULT now(), CONSTRAINT "PK_3cc0d85193aade457d3077dd06b" PRIMARY KEY (id) );` ) - await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON chat_message USING btree (chatflowid);`) + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_e574527322272fd838f4f0f3d3" ON chat_message USING btree (chatflowid);`) await queryRunner.query( - `CREATE TABLE credential ( + `CREATE TABLE IF NOT EXISTS credential ( id uuid NOT NULL DEFAULT uuid_generate_v4(), "name" varchar NOT NULL, "credentialName" varchar NOT NULL, - "encryptedData" text NOT NULL, + "encryptedData" varchar NOT NULL, "createdDate" timestamp NOT NULL DEFAULT now(), "updatedDate" timestamp NOT NULL DEFAULT now(), CONSTRAINT "PK_3a5169bcd3d5463cefeec78be82" PRIMARY KEY (id) );` ) await queryRunner.query( - `CREATE TABLE tool ( + `CREATE TABLE IF NOT EXISTS tool ( id uuid NOT NULL DEFAULT uuid_generate_v4(), "name" varchar NOT NULL, description text NOT NULL, color varchar NOT NULL, "iconSrc" varchar NULL, - "schema" text NULL, - func text NULL, + "schema" varchar NULL, + func varchar NULL, "createdDate" timestamp NOT NULL DEFAULT now(), "updatedDate" timestamp NOT NULL DEFAULT now(), CONSTRAINT "PK_3bf5b1016a384916073184f99b7" PRIMARY KEY (id) diff --git a/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts b/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts index 2c199881..04b2e660 100644 --- a/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts +++ b/packages/server/src/database/migrations/sqlite/1693835579790-Init.ts @@ -3,17 +3,17 @@ import { MigrationInterface, QueryRunner } from 'typeorm' export class Init1693835579790 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { await queryRunner.query( - `CREATE TABLE "chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" boolean, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + `CREATE TABLE IF NOT EXISTS "chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" boolean, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` ) await queryRunner.query( - `CREATE TABLE "chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')));` + `CREATE TABLE IF NOT EXISTS "chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')));` ) - await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`) + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`) await queryRunner.query( - `CREATE TABLE "credential" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "credentialName" varchar NOT NULL, "encryptedData" text NOT NULL, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + `CREATE TABLE IF NOT EXISTS "credential" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "credentialName" varchar NOT NULL, "encryptedData" varchar NOT NULL, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` ) await queryRunner.query( - `CREATE TABLE "tool" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "description" text NOT NULL, "color" varchar NOT NULL, "iconSrc" varchar, "schema" text, "func" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + `CREATE TABLE IF NOT EXISTS "tool" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "description" text NOT NULL, "color" varchar NOT NULL, "iconSrc" varchar, "schema" varchar, "func" varchar, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` ) } From 427ec32dc60b1cba57ff8182bd6d725df54f67e0 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 21:50:00 +0800 Subject: [PATCH 15/35] add sqlite modify chatflow --- .../server/src/database/entities/ChatFlow.ts | 2 +- .../sqlite/1693920824108-ModifyChatFlow.ts | 18 ++++++++++++++++++ .../src/database/migrations/sqlite/index.ts | 3 ++- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 packages/server/src/database/migrations/sqlite/1693920824108-ModifyChatFlow.ts diff --git a/packages/server/src/database/entities/ChatFlow.ts b/packages/server/src/database/entities/ChatFlow.ts index a1a32a88..a4fdd130 100644 --- a/packages/server/src/database/entities/ChatFlow.ts +++ b/packages/server/src/database/entities/ChatFlow.ts @@ -22,7 +22,7 @@ export class ChatFlow implements IChatFlow { @Column({ nullable: true }) apikeyid?: string - @Column({ nullable: true }) + @Column({ nullable: true, type: 'text' }) chatbotConfig?: string @CreateDateColumn() diff --git a/packages/server/src/database/migrations/sqlite/1693920824108-ModifyChatFlow.ts b/packages/server/src/database/migrations/sqlite/1693920824108-ModifyChatFlow.ts new file mode 100644 index 00000000..429dc0ee --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1693920824108-ModifyChatFlow.ts @@ -0,0 +1,18 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyChatFlow1693920824108 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "temp_chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" boolean, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + ) + await queryRunner.query( + `INSERT INTO "temp_chat_flow" ("id", "name", "flowData", "deployed", "isPublic", "apikeyid", "chatbotConfig", "createdDate", "updatedDate") SELECT "id", "name", "flowData", "deployed", "isPublic", "apikeyid", "chatbotConfig", "createdDate", "updatedDate" FROM "chat_flow";` + ) + await queryRunner.query(`DROP TABLE chat_flow;`) + await queryRunner.query(`ALTER TABLE temp_chat_flow RENAME TO chat_flow;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE temp_chat_flow`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index 31fef34a..376a7746 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -1,3 +1,4 @@ import { Init1693835579790 } from './1693835579790-Init' +import { ModifyChatFlow1693920824108 } from './1693920824108-ModifyChatFlow' -export const sqliteMigrations = [Init1693835579790] +export const sqliteMigrations = [Init1693835579790, ModifyChatFlow1693920824108] From 22984618ca7a8428963da060db47e70aa8d3d34f Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 22:02:51 +0800 Subject: [PATCH 16/35] add sqlite modify chatmessage --- .../src/database/entities/ChatMessage.ts | 2 +- .../sqlite/1693921865247-ModifyChatMessage.ts | 18 ++++++++++++++++++ .../src/database/migrations/sqlite/index.ts | 3 ++- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 packages/server/src/database/migrations/sqlite/1693921865247-ModifyChatMessage.ts diff --git a/packages/server/src/database/entities/ChatMessage.ts b/packages/server/src/database/entities/ChatMessage.ts index 23804846..4b5306ee 100644 --- a/packages/server/src/database/entities/ChatMessage.ts +++ b/packages/server/src/database/entities/ChatMessage.ts @@ -17,7 +17,7 @@ export class ChatMessage implements IChatMessage { @Column({ type: 'text' }) content: string - @Column({ nullable: true }) + @Column({ nullable: true, type: 'text' }) sourceDocuments?: string @CreateDateColumn() diff --git a/packages/server/src/database/migrations/sqlite/1693921865247-ModifyChatMessage.ts b/packages/server/src/database/migrations/sqlite/1693921865247-ModifyChatMessage.ts new file mode 100644 index 00000000..fc327e00 --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1693921865247-ModifyChatMessage.ts @@ -0,0 +1,18 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyChatMessage1693921865247 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')));` + ) + await queryRunner.query( + `INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "createdDate") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "createdDate" FROM "chat_message";` + ) + await queryRunner.query(`DROP TABLE chat_message;`) + await queryRunner.query(`ALTER TABLE temp_chat_message RENAME TO chat_message;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE temp_chat_message`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index 376a7746..6e24be67 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -1,4 +1,5 @@ import { Init1693835579790 } from './1693835579790-Init' import { ModifyChatFlow1693920824108 } from './1693920824108-ModifyChatFlow' +import { ModifyChatMessage1693921865247 } from './1693921865247-ModifyChatMessage' -export const sqliteMigrations = [Init1693835579790, ModifyChatFlow1693920824108] +export const sqliteMigrations = [Init1693835579790, ModifyChatFlow1693920824108, ModifyChatMessage1693921865247] From a7fa3f48e2b44291b7f8de1b1240e8d8b92d8dcf Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 22:18:28 +0800 Subject: [PATCH 17/35] add chat_message index --- .../migrations/sqlite/1693921865247-ModifyChatMessage.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/database/migrations/sqlite/1693921865247-ModifyChatMessage.ts b/packages/server/src/database/migrations/sqlite/1693921865247-ModifyChatMessage.ts index fc327e00..94cd6e0e 100644 --- a/packages/server/src/database/migrations/sqlite/1693921865247-ModifyChatMessage.ts +++ b/packages/server/src/database/migrations/sqlite/1693921865247-ModifyChatMessage.ts @@ -10,6 +10,7 @@ export class ModifyChatMessage1693921865247 implements MigrationInterface { ) await queryRunner.query(`DROP TABLE chat_message;`) await queryRunner.query(`ALTER TABLE temp_chat_message RENAME TO chat_message;`) + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`) } public async down(queryRunner: QueryRunner): Promise { From 1c1b057512dd7dda4747abac82aebab6577929fd Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 22:27:17 +0800 Subject: [PATCH 18/35] add sqlite modify credential --- .../server/src/database/entities/Credential.ts | 2 +- .../sqlite/1693923551694-ModifyCredential.ts | 18 ++++++++++++++++++ .../src/database/migrations/sqlite/index.ts | 8 +++++++- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 packages/server/src/database/migrations/sqlite/1693923551694-ModifyCredential.ts diff --git a/packages/server/src/database/entities/Credential.ts b/packages/server/src/database/entities/Credential.ts index e77711dc..822f2584 100644 --- a/packages/server/src/database/entities/Credential.ts +++ b/packages/server/src/database/entities/Credential.ts @@ -13,7 +13,7 @@ export class Credential implements ICredential { @Column() credentialName: string - @Column() + @Column({ type: 'text' }) encryptedData: string @CreateDateColumn() diff --git a/packages/server/src/database/migrations/sqlite/1693923551694-ModifyCredential.ts b/packages/server/src/database/migrations/sqlite/1693923551694-ModifyCredential.ts new file mode 100644 index 00000000..2a906a83 --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1693923551694-ModifyCredential.ts @@ -0,0 +1,18 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyCredential1693923551694 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "temp_credential" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "credentialName" varchar NOT NULL, "encryptedData" text NOT NULL, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + ) + await queryRunner.query( + `INSERT INTO "temp_credential" ("id", "name", "credentialName", "encryptedData", "createdDate", "updatedDate") SELECT "id", "name", "credentialName", "encryptedData", "createdDate", "updatedDate" FROM "credential";` + ) + await queryRunner.query(`DROP TABLE credential;`) + await queryRunner.query(`ALTER TABLE temp_credential RENAME TO credential;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE credential`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index 6e24be67..4a8251a6 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -1,5 +1,11 @@ import { Init1693835579790 } from './1693835579790-Init' import { ModifyChatFlow1693920824108 } from './1693920824108-ModifyChatFlow' import { ModifyChatMessage1693921865247 } from './1693921865247-ModifyChatMessage' +import { ModifyCredential1693923551694 } from './1693923551694-ModifyCredential' -export const sqliteMigrations = [Init1693835579790, ModifyChatFlow1693920824108, ModifyChatMessage1693921865247] +export const sqliteMigrations = [ + Init1693835579790, + ModifyChatFlow1693920824108, + ModifyChatMessage1693921865247, + ModifyCredential1693923551694 +] From aa1122641caf3b8fff5bc6810f161778a18d5cd1 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Tue, 5 Sep 2023 22:41:28 +0800 Subject: [PATCH 19/35] add sqlite modify tool --- packages/server/src/database/entities/Tool.ts | 4 ++-- .../sqlite/1693923551694-ModifyCredential.ts | 2 +- .../sqlite/1693924207475-ModifyTool.ts | 18 ++++++++++++++++++ .../src/database/migrations/sqlite/index.ts | 4 +++- 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 packages/server/src/database/migrations/sqlite/1693924207475-ModifyTool.ts diff --git a/packages/server/src/database/entities/Tool.ts b/packages/server/src/database/entities/Tool.ts index d459eee3..8e675eb0 100644 --- a/packages/server/src/database/entities/Tool.ts +++ b/packages/server/src/database/entities/Tool.ts @@ -19,10 +19,10 @@ export class Tool implements ITool { @Column({ nullable: true }) iconSrc?: string - @Column({ nullable: true }) + @Column({ nullable: true, type: 'text' }) schema?: string - @Column({ nullable: true }) + @Column({ nullable: true, type: 'text' }) func?: string @CreateDateColumn() diff --git a/packages/server/src/database/migrations/sqlite/1693923551694-ModifyCredential.ts b/packages/server/src/database/migrations/sqlite/1693923551694-ModifyCredential.ts index 2a906a83..fccb3c18 100644 --- a/packages/server/src/database/migrations/sqlite/1693923551694-ModifyCredential.ts +++ b/packages/server/src/database/migrations/sqlite/1693923551694-ModifyCredential.ts @@ -13,6 +13,6 @@ export class ModifyCredential1693923551694 implements MigrationInterface { } public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP TABLE credential`) + await queryRunner.query(`DROP TABLE temp_credential`) } } diff --git a/packages/server/src/database/migrations/sqlite/1693924207475-ModifyTool.ts b/packages/server/src/database/migrations/sqlite/1693924207475-ModifyTool.ts new file mode 100644 index 00000000..ad27eb12 --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1693924207475-ModifyTool.ts @@ -0,0 +1,18 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyTool1693924207475 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "temp_tool" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "description" text NOT NULL, "color" varchar NOT NULL, "iconSrc" varchar, "schema" text, "func" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));` + ) + await queryRunner.query( + `INSERT INTO "temp_tool" ("id", "name", "description", "color", "iconSrc", "schema", "func", "createdDate", "updatedDate") SELECT "id", "name", "description", "color", "iconSrc", "schema", "func", "createdDate", "updatedDate" FROM "tool";` + ) + await queryRunner.query(`DROP TABLE tool;`) + await queryRunner.query(`ALTER TABLE temp_tool RENAME TO tool;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE temp_tool`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index 4a8251a6..c3c1fe7a 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -2,10 +2,12 @@ import { Init1693835579790 } from './1693835579790-Init' import { ModifyChatFlow1693920824108 } from './1693920824108-ModifyChatFlow' import { ModifyChatMessage1693921865247 } from './1693921865247-ModifyChatMessage' import { ModifyCredential1693923551694 } from './1693923551694-ModifyCredential' +import { ModifyTool1693924207475 } from './1693924207475-ModifyTool' export const sqliteMigrations = [ Init1693835579790, ModifyChatFlow1693920824108, ModifyChatMessage1693921865247, - ModifyCredential1693923551694 + ModifyCredential1693923551694, + ModifyTool1693924207475 ] From 86e09d726ab54715f3b91207e2f16bc8ebe25a98 Mon Sep 17 00:00:00 2001 From: Pooria Arab <42897904+pooriaarab@users.noreply.github.com> Date: Tue, 5 Sep 2023 11:23:08 -0700 Subject: [PATCH 20/35] Update Chroma_Existing.ts to add metadatafilter --- .../nodes/vectorstores/Chroma/Chroma_Existing.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts index f55faa40..1d360182 100644 --- a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts +++ b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts @@ -51,6 +51,13 @@ class Chroma_Existing_VectorStores implements INode { type: 'string', optional: true }, + { + label: 'Chroma Metadata Filter', + name: 'chromaMetadataFilter', + type: 'json', + optional: true, + additionalParams: true + }, { label: 'Top K', name: 'topK', @@ -86,13 +93,18 @@ class Chroma_Existing_VectorStores implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData) + const chromaMetadataFilter = nodeData.inputs?.chromaMetadataFilter; + const metadataFilter = chromaMetadataFilter ? JSON.parse(chromaMetadataFilter) : {}; + const obj: { collectionName: string url?: string chromaApiKey?: string + metadataFilter?: any } = { collectionName } if (chromaURL) obj.url = chromaURL if (chromaApiKey) obj.chromaApiKey = chromaApiKey + if (chromaMetadataFilter) obj.metadataFilter = metadataFilter const vectorStore = await ChromaExtended.fromExistingCollection(embeddings, obj) From e5167f3e20de169c8363777cb2a1106ed040715f Mon Sep 17 00:00:00 2001 From: Martin Andrews Date: Tue, 5 Sep 2023 18:43:14 +0000 Subject: [PATCH 21/35] First draft of PaLM API LLM component --- .../GoogleMakerSuite.credential.ts | 23 +++ .../nodes/llms/GooglePaLM/GooglePaLM.ts | 158 ++++++++++++++++++ .../llms/GooglePaLM/Google_PaLM_Logo.svg | 67 ++++++++ packages/components/package.json | 1 + 4 files changed, 249 insertions(+) create mode 100644 packages/components/credentials/GoogleMakerSuite.credential.ts create mode 100644 packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts create mode 100644 packages/components/nodes/llms/GooglePaLM/Google_PaLM_Logo.svg diff --git a/packages/components/credentials/GoogleMakerSuite.credential.ts b/packages/components/credentials/GoogleMakerSuite.credential.ts new file mode 100644 index 00000000..83b850a3 --- /dev/null +++ b/packages/components/credentials/GoogleMakerSuite.credential.ts @@ -0,0 +1,23 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class GoogleMakerSuite implements INodeCredential { + label: string + name: string + version: number + inputs: INodeParams[] + + constructor() { + this.label = 'Google MakerSuite' + this.name = 'googleMakerSuite' + this.version = 1.0 + this.inputs = [ + { + label: 'MakerSuite API Key', + name: 'googleMakerSuiteKey', + type: 'password' + } + ] + } +} + +module.exports = { credClass: GoogleMakerSuite } diff --git a/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts new file mode 100644 index 00000000..c286531c --- /dev/null +++ b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts @@ -0,0 +1,158 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { GooglePaLM, GooglePaLMTextInput } from 'langchain/llms/googlepalm' + +class GooglePaLM_LLMs implements INode { + label: string + name: string + version: number + type: string + icon: string + category: string + description: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'GooglePaLM' + this.name = 'GooglePaLM' + this.version = 1.0 + this.type = 'GooglePaLM' + this.icon = 'Google_PaLM_Logo.svg' + this.category = 'LLMs' + this.description = 'Wrapper around Google MakerSuite PaLM large language models' + this.baseClasses = [this.type, ...getBaseClasses(GooglePaLM)] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['googleMakerSuite'], + description: + 'Google MakerSuite API credential. Get this from https://makersuite.google.com/app/apikey' + + } + this.inputs = [ + { + label: 'Model Name', + name: 'modelName', + type: 'options', + options: [ + { + label: 'models/text-bison-001', + name: 'models/text-bison-001' + } + ], + default: 'models/text-bison-001', + optional: true + }, + { + label: 'Temperature', + name: 'temperature', + type: 'number', + step: 0.1, + default: 0.7, + optional: true, + description: "Controls the randomness of the output.\n"+ + "Values can range from [0.0,1.0], inclusive. A value closer to 1.0 "+ + "will produce responses that are more varied and creative, while"+ + "a value closer to 0.0 will typically result in more straightforward"+ + "responses from the model." + }, + { + label: 'Max Output Tokens', + name: 'maxOutputTokens', + type: 'number', + step: 1, + optional: true, + additionalParams: true, + description: "Maximum number of tokens to generate in the completion." + }, + { + label: 'Top Probability', + name: 'topP', + type: 'number', + step: 0.1, + optional: true, + additionalParams: true, + description: "Top-p changes how the model selects tokens for output.\n"+ + "Tokens are selected from most probable to least until "+ + "the sum of their probabilities equals the top-p value.\n"+ + "For example, if tokens A, B, and C have a probability of .3, .2, and .1 "+ + "and the top-p value is .5, then the model will select either A or B "+ + "as the next token (using temperature)." + }, + { + label: 'Top-k', + name: 'topK', + type: 'number', + step: 1, + optional: true, + additionalParams: true, + description: "Top-k changes how the model selects tokens for output.\n"+ + "A top-k of 1 means the selected token is the most probable among "+ + "all tokens in the model’s vocabulary (also called greedy decoding), "+ + "while a top-k of 3 means that the next token is selected from "+ + "among the 3 most probable tokens (using temperature)." + }, + { + label: 'Stop Sequences', + name: 'stopSequencesObj', + type: 'json', + optional: true, + additionalParams: true + //default: { list:[] }, + //description: + // "The 'list' field should contain a list of character strings (up to 5) that will stop output generation.\n"+ + // " * If specified, the API will stop at the first appearance of a stop sequence.\n"+ + // "Note: The stop sequence will not be included as part of the response." + } + /* + { + label: 'Safety Settings', + name: 'safetySettings', + type: 'json', + optional: true, + additionalParams: true + } + */ + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const modelName = nodeData.inputs?.modelName as string + const temperature = nodeData.inputs?.temperature as string + const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string + const topP = nodeData.inputs?.topP as string + const topK = nodeData.inputs?.topK as string + const stopSequencesObj = nodeData.inputs?.stopSequencesObj + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const googleMakerSuiteKey = getCredentialParam('googleMakerSuiteKey', credentialData, nodeData) + + const obj: Partial = { + modelName: modelName, + temperature: parseFloat(temperature), + apiKey: googleMakerSuiteKey + } + + if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10) + if (topP) obj.topP = parseFloat(topP) + if (topK) obj.topK = parseFloat(topK) + + let parsedStopSequences: any | undefined = undefined + if (stopSequencesObj) { + try { + parsedStopSequences = typeof stopSequencesObj === 'object' ? stopSequencesObj : JSON.parse(stopSequencesObj) + obj.stopSequences = parsedStopSequences.list || [] + } catch (exception) { + throw new Error("Invalid JSON in the GooglePaLM's stopSequences: " + exception) + } + } + + const model = new GooglePaLM(obj) + return model + } +} + +module.exports = { nodeClass: GooglePaLM_LLMs } diff --git a/packages/components/nodes/llms/GooglePaLM/Google_PaLM_Logo.svg b/packages/components/nodes/llms/GooglePaLM/Google_PaLM_Logo.svg new file mode 100644 index 00000000..5c345fe1 --- /dev/null +++ b/packages/components/nodes/llms/GooglePaLM/Google_PaLM_Logo.svg @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/components/package.json b/packages/components/package.json index 1837609e..d3dd4e53 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -19,6 +19,7 @@ "@aws-sdk/client-dynamodb": "^3.360.0", "@dqbd/tiktoken": "^1.0.7", "@getzep/zep-js": "^0.6.3", + "@google-ai/generativelanguage": "^0.2.1", "@huggingface/inference": "^2.6.1", "@notionhq/client": "^2.2.8", "@opensearch-project/opensearch": "^1.2.0", From 19824bf3399b7ad9fb00639a39885f096cc14df3 Mon Sep 17 00:00:00 2001 From: Martin Andrews Date: Wed, 6 Sep 2023 08:41:12 +0000 Subject: [PATCH 22/35] Moved APIkey helper text, and fixed linter complaints --- .../GoogleMakerSuite.credential.ts | 3 + .../nodes/llms/GooglePaLM/GooglePaLM.ts | 58 +++++++++---------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/packages/components/credentials/GoogleMakerSuite.credential.ts b/packages/components/credentials/GoogleMakerSuite.credential.ts index 83b850a3..f37cbf88 100644 --- a/packages/components/credentials/GoogleMakerSuite.credential.ts +++ b/packages/components/credentials/GoogleMakerSuite.credential.ts @@ -4,12 +4,15 @@ class GoogleMakerSuite implements INodeCredential { label: string name: string version: number + description: string inputs: INodeParams[] constructor() { this.label = 'Google MakerSuite' this.name = 'googleMakerSuite' this.version = 1.0 + this.description = + 'Use the Google MakerSuite API credential site to get this key.' this.inputs = [ { label: 'MakerSuite API Key', diff --git a/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts index c286531c..24630360 100644 --- a/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts +++ b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts @@ -27,12 +27,9 @@ class GooglePaLM_LLMs implements INode { label: 'Connect Credential', name: 'credential', type: 'credential', - credentialNames: ['googleMakerSuite'], - description: - 'Google MakerSuite API credential. Get this from https://makersuite.google.com/app/apikey' - + credentialNames: ['googleMakerSuite'] } - this.inputs = [ + this.inputs = [ { label: 'Model Name', name: 'modelName', @@ -52,12 +49,13 @@ class GooglePaLM_LLMs implements INode { type: 'number', step: 0.1, default: 0.7, - optional: true, - description: "Controls the randomness of the output.\n"+ - "Values can range from [0.0,1.0], inclusive. A value closer to 1.0 "+ - "will produce responses that are more varied and creative, while"+ - "a value closer to 0.0 will typically result in more straightforward"+ - "responses from the model." + optional: true, + description: + 'Controls the randomness of the output.\n' + + 'Values can range from [0.0,1.0], inclusive. A value closer to 1.0 ' + + 'will produce responses that are more varied and creative, while ' + + 'a value closer to 0.0 will typically result in more straightforward ' + + 'responses from the model.' }, { label: 'Max Output Tokens', @@ -66,7 +64,7 @@ class GooglePaLM_LLMs implements INode { step: 1, optional: true, additionalParams: true, - description: "Maximum number of tokens to generate in the completion." + description: 'Maximum number of tokens to generate in the completion.' }, { label: 'Top Probability', @@ -74,13 +72,14 @@ class GooglePaLM_LLMs implements INode { type: 'number', step: 0.1, optional: true, - additionalParams: true, - description: "Top-p changes how the model selects tokens for output.\n"+ - "Tokens are selected from most probable to least until "+ - "the sum of their probabilities equals the top-p value.\n"+ - "For example, if tokens A, B, and C have a probability of .3, .2, and .1 "+ - "and the top-p value is .5, then the model will select either A or B "+ - "as the next token (using temperature)." + additionalParams: true, + description: + 'Top-p changes how the model selects tokens for output.\n' + + 'Tokens are selected from most probable to least until ' + + 'the sum of their probabilities equals the top-p value.\n' + + 'For example, if tokens A, B, and C have a probability of .3, .2, and .1 ' + + 'and the top-p value is .5, then the model will select either A or B ' + + 'as the next token (using temperature).' }, { label: 'Top-k', @@ -89,11 +88,12 @@ class GooglePaLM_LLMs implements INode { step: 1, optional: true, additionalParams: true, - description: "Top-k changes how the model selects tokens for output.\n"+ - "A top-k of 1 means the selected token is the most probable among "+ - "all tokens in the model’s vocabulary (also called greedy decoding), "+ - "while a top-k of 3 means that the next token is selected from "+ - "among the 3 most probable tokens (using temperature)." + description: + 'Top-k changes how the model selects tokens for output.\n' + + 'A top-k of 1 means the selected token is the most probable among ' + + 'all tokens in the model vocabulary (also called greedy decoding), ' + + 'while a top-k of 3 means that the next token is selected from ' + + 'among the 3 most probable tokens (using temperature).' }, { label: 'Stop Sequences', @@ -102,11 +102,11 @@ class GooglePaLM_LLMs implements INode { optional: true, additionalParams: true //default: { list:[] }, - //description: - // "The 'list' field should contain a list of character strings (up to 5) that will stop output generation.\n"+ - // " * If specified, the API will stop at the first appearance of a stop sequence.\n"+ - // "Note: The stop sequence will not be included as part of the response." - } + //description: + // 'The "list" field should contain a list of character strings (up to 5) that will stop output generation.\n' + + // ' * If specified, the API will stop at the first appearance of a stop sequence.\n' + + // 'Note: The stop sequence will not be included as part of the response.' + } /* { label: 'Safety Settings', From 6f1f5ef06b58afd1c6780e46e2e1aeaae1c26e54 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 6 Sep 2023 18:37:30 +0800 Subject: [PATCH 23/35] add postgres modify chatflow --- .../postgres/1693995626941-ModifyChatFlow.ts | 11 +++++++++++ .../server/src/database/migrations/postgres/index.ts | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/database/migrations/postgres/1693995626941-ModifyChatFlow.ts diff --git a/packages/server/src/database/migrations/postgres/1693995626941-ModifyChatFlow.ts b/packages/server/src/database/migrations/postgres/1693995626941-ModifyChatFlow.ts new file mode 100644 index 00000000..86f09430 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1693995626941-ModifyChatFlow.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyChatFlow1693995626941 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" ALTER COLUMN "chatbotConfig" TYPE TEXT`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" ALTER COLUMN "chatbotConfig" TYPE VARCHAR`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts index c8805785..ac03fcc4 100644 --- a/packages/server/src/database/migrations/postgres/index.ts +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -1,3 +1,4 @@ import { Init1693891895163 } from './1693891895163-Init' +import { ModifyChatFlow1693995626941 } from './1693995626941-ModifyChatFlow' -export const postgresMigrations = [Init1693891895163] +export const postgresMigrations = [Init1693891895163, ModifyChatFlow1693995626941] From 5facb8fba855713a3b0e756c8854a053f0a7e8b1 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 6 Sep 2023 18:43:52 +0800 Subject: [PATCH 24/35] add postgres modify chatmessage --- .../postgres/1693996694528-ModifyChatMessage.ts | 11 +++++++++++ .../server/src/database/migrations/postgres/index.ts | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/database/migrations/postgres/1693996694528-ModifyChatMessage.ts diff --git a/packages/server/src/database/migrations/postgres/1693996694528-ModifyChatMessage.ts b/packages/server/src/database/migrations/postgres/1693996694528-ModifyChatMessage.ts new file mode 100644 index 00000000..4933e8f7 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1693996694528-ModifyChatMessage.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyChatMessage1693996694528 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "sourceDocuments" TYPE TEXT`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "sourceDocuments" TYPE VARCHAR`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts index ac03fcc4..be15aab0 100644 --- a/packages/server/src/database/migrations/postgres/index.ts +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -1,4 +1,5 @@ import { Init1693891895163 } from './1693891895163-Init' import { ModifyChatFlow1693995626941 } from './1693995626941-ModifyChatFlow' +import { ModifyChatMessage1693996694528 } from './1693996694528-ModifyChatMessage' -export const postgresMigrations = [Init1693891895163, ModifyChatFlow1693995626941] +export const postgresMigrations = [Init1693891895163, ModifyChatFlow1693995626941, ModifyChatMessage1693996694528] From d92ca34b00a21fcafbbc9ae5f5101dd5006ab6d6 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 6 Sep 2023 18:48:18 +0800 Subject: [PATCH 25/35] add postgres modify credential --- .../postgres/1693997070000-ModifyCredential.ts | 11 +++++++++++ .../server/src/database/migrations/postgres/index.ts | 8 +++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/database/migrations/postgres/1693997070000-ModifyCredential.ts diff --git a/packages/server/src/database/migrations/postgres/1693997070000-ModifyCredential.ts b/packages/server/src/database/migrations/postgres/1693997070000-ModifyCredential.ts new file mode 100644 index 00000000..b19ec428 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1693997070000-ModifyCredential.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyCredential1693997070000 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "credential" ALTER COLUMN "encryptedData" TYPE TEXT`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "credential" ALTER COLUMN "encryptedData" TYPE VARCHAR`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts index be15aab0..55fe514b 100644 --- a/packages/server/src/database/migrations/postgres/index.ts +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -1,5 +1,11 @@ import { Init1693891895163 } from './1693891895163-Init' import { ModifyChatFlow1693995626941 } from './1693995626941-ModifyChatFlow' import { ModifyChatMessage1693996694528 } from './1693996694528-ModifyChatMessage' +import { ModifyCredential1693997070000 } from './1693997070000-ModifyCredential' -export const postgresMigrations = [Init1693891895163, ModifyChatFlow1693995626941, ModifyChatMessage1693996694528] +export const postgresMigrations = [ + Init1693891895163, + ModifyChatFlow1693995626941, + ModifyChatMessage1693996694528, + ModifyCredential1693997070000 +] From 0d6d15ecf445948cf33dd28617942cba22b252e9 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 6 Sep 2023 18:55:35 +0800 Subject: [PATCH 26/35] add postgres modify tool --- .../postgres/1693995626941-ModifyChatFlow.ts | 4 ++-- .../postgres/1693996694528-ModifyChatMessage.ts | 4 ++-- .../postgres/1693997070000-ModifyCredential.ts | 4 ++-- .../migrations/postgres/1693997339912-ModifyTool.ts | 11 +++++++++++ .../server/src/database/migrations/postgres/index.ts | 4 +++- 5 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 packages/server/src/database/migrations/postgres/1693997339912-ModifyTool.ts diff --git a/packages/server/src/database/migrations/postgres/1693995626941-ModifyChatFlow.ts b/packages/server/src/database/migrations/postgres/1693995626941-ModifyChatFlow.ts index 86f09430..aee6d3fe 100644 --- a/packages/server/src/database/migrations/postgres/1693995626941-ModifyChatFlow.ts +++ b/packages/server/src/database/migrations/postgres/1693995626941-ModifyChatFlow.ts @@ -2,10 +2,10 @@ import { MigrationInterface, QueryRunner } from 'typeorm' export class ModifyChatFlow1693995626941 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "chat_flow" ALTER COLUMN "chatbotConfig" TYPE TEXT`) + await queryRunner.query(`ALTER TABLE "chat_flow" ALTER COLUMN "chatbotConfig" TYPE TEXT;`) } public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "chat_flow" ALTER COLUMN "chatbotConfig" TYPE VARCHAR`) + await queryRunner.query(`ALTER TABLE "chat_flow" ALTER COLUMN "chatbotConfig" TYPE VARCHAR;`) } } diff --git a/packages/server/src/database/migrations/postgres/1693996694528-ModifyChatMessage.ts b/packages/server/src/database/migrations/postgres/1693996694528-ModifyChatMessage.ts index 4933e8f7..da288783 100644 --- a/packages/server/src/database/migrations/postgres/1693996694528-ModifyChatMessage.ts +++ b/packages/server/src/database/migrations/postgres/1693996694528-ModifyChatMessage.ts @@ -2,10 +2,10 @@ import { MigrationInterface, QueryRunner } from 'typeorm' export class ModifyChatMessage1693996694528 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "sourceDocuments" TYPE TEXT`) + await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "sourceDocuments" TYPE TEXT;`) } public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "sourceDocuments" TYPE VARCHAR`) + await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "sourceDocuments" TYPE VARCHAR;`) } } diff --git a/packages/server/src/database/migrations/postgres/1693997070000-ModifyCredential.ts b/packages/server/src/database/migrations/postgres/1693997070000-ModifyCredential.ts index b19ec428..ef45201a 100644 --- a/packages/server/src/database/migrations/postgres/1693997070000-ModifyCredential.ts +++ b/packages/server/src/database/migrations/postgres/1693997070000-ModifyCredential.ts @@ -2,10 +2,10 @@ import { MigrationInterface, QueryRunner } from 'typeorm' export class ModifyCredential1693997070000 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "credential" ALTER COLUMN "encryptedData" TYPE TEXT`) + await queryRunner.query(`ALTER TABLE "credential" ALTER COLUMN "encryptedData" TYPE TEXT;`) } public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "credential" ALTER COLUMN "encryptedData" TYPE VARCHAR`) + await queryRunner.query(`ALTER TABLE "credential" ALTER COLUMN "encryptedData" TYPE VARCHAR;`) } } diff --git a/packages/server/src/database/migrations/postgres/1693997339912-ModifyTool.ts b/packages/server/src/database/migrations/postgres/1693997339912-ModifyTool.ts new file mode 100644 index 00000000..a4bf95c4 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1693997339912-ModifyTool.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyTool1693997339912 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "tool" ALTER COLUMN "schema" TYPE TEXT, ALTER COLUMN "func" TYPE TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "tool" ALTER COLUMN "schema" TYPE VARCHAR, ALTER COLUMN "func" TYPE VARCHAR;`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts index 55fe514b..2bac9f33 100644 --- a/packages/server/src/database/migrations/postgres/index.ts +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -2,10 +2,12 @@ import { Init1693891895163 } from './1693891895163-Init' import { ModifyChatFlow1693995626941 } from './1693995626941-ModifyChatFlow' import { ModifyChatMessage1693996694528 } from './1693996694528-ModifyChatMessage' import { ModifyCredential1693997070000 } from './1693997070000-ModifyCredential' +import { ModifyTool1693997339912 } from './1693997339912-ModifyTool' export const postgresMigrations = [ Init1693891895163, ModifyChatFlow1693995626941, ModifyChatMessage1693996694528, - ModifyCredential1693997070000 + ModifyCredential1693997070000, + ModifyTool1693997339912 ] From fa1060f9eb025781f65f5a63b197430794eb8eb1 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 6 Sep 2023 19:15:53 +0800 Subject: [PATCH 27/35] add mysql modify chat_flow --- .../migrations/mysql/1693997791471-ModifyChatFlow.ts | 11 +++++++++++ .../server/src/database/migrations/mysql/index.ts | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/database/migrations/mysql/1693997791471-ModifyChatFlow.ts diff --git a/packages/server/src/database/migrations/mysql/1693997791471-ModifyChatFlow.ts b/packages/server/src/database/migrations/mysql/1693997791471-ModifyChatFlow.ts new file mode 100644 index 00000000..d0023b67 --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1693997791471-ModifyChatFlow.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyChatFlow1693997791471 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`chat_flow\` MODIFY \`chatbotConfig\` TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`chat_flow\` MODIFY \`chatbotConfig\` VARCHAR;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index 79fa17ad..45102969 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -1,3 +1,4 @@ import { Init1693840429259 } from './1693840429259-Init' +import { ModifyChatFlow1693997791471 } from './1693997791471-ModifyChatFlow' -export const mysqlMigrations = [Init1693840429259] +export const mysqlMigrations = [Init1693840429259, ModifyChatFlow1693997791471] From 1d4761dc2cc579db2e1368af74285002b5905dfa Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 6 Sep 2023 19:20:15 +0800 Subject: [PATCH 28/35] add mysql modify chat_message --- .../mysql/1693999022236-ModifyChatMessage.ts | 11 +++++++++++ .../server/src/database/migrations/mysql/index.ts | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/database/migrations/mysql/1693999022236-ModifyChatMessage.ts diff --git a/packages/server/src/database/migrations/mysql/1693999022236-ModifyChatMessage.ts b/packages/server/src/database/migrations/mysql/1693999022236-ModifyChatMessage.ts new file mode 100644 index 00000000..3f6eaa4e --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1693999022236-ModifyChatMessage.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyChatMessage1693999022236 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`chat_message\` MODIFY \`sourceDocuments\` TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`chat_message\` MODIFY \`sourceDocuments\` VARCHAR;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index 45102969..d62545bf 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -1,4 +1,5 @@ import { Init1693840429259 } from './1693840429259-Init' import { ModifyChatFlow1693997791471 } from './1693997791471-ModifyChatFlow' +import { ModifyChatMessage1693999022236 } from './1693999022236-ModifyChatMessage' -export const mysqlMigrations = [Init1693840429259, ModifyChatFlow1693997791471] +export const mysqlMigrations = [Init1693840429259, ModifyChatFlow1693997791471, ModifyChatMessage1693999022236] From b9bdf8435c6a7cabcbf5a884343cf93faeed0b7a Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 6 Sep 2023 19:56:18 +0800 Subject: [PATCH 29/35] add mysql modify credential --- .../mysql/1693999261583-ModifyCredential.ts | 11 +++++++++++ .../server/src/database/migrations/mysql/index.ts | 8 +++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/database/migrations/mysql/1693999261583-ModifyCredential.ts diff --git a/packages/server/src/database/migrations/mysql/1693999261583-ModifyCredential.ts b/packages/server/src/database/migrations/mysql/1693999261583-ModifyCredential.ts new file mode 100644 index 00000000..54c33af1 --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1693999261583-ModifyCredential.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyCredential1693999261583 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`credential\` MODIFY \`encryptedData\` TEXT NOT NULL;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`credential\` MODIFY \`encryptedData\` VARCHAR NOT NULL;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index d62545bf..f946cb5f 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -1,5 +1,11 @@ import { Init1693840429259 } from './1693840429259-Init' import { ModifyChatFlow1693997791471 } from './1693997791471-ModifyChatFlow' import { ModifyChatMessage1693999022236 } from './1693999022236-ModifyChatMessage' +import { ModifyCredential1693999261583 } from './1693999261583-ModifyCredential' -export const mysqlMigrations = [Init1693840429259, ModifyChatFlow1693997791471, ModifyChatMessage1693999022236] +export const mysqlMigrations = [ + Init1693840429259, + ModifyChatFlow1693997791471, + ModifyChatMessage1693999022236, + ModifyCredential1693999261583 +] From 2574b02dd7717ad2184b59bcdee89648311942f1 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 6 Sep 2023 20:03:06 +0800 Subject: [PATCH 30/35] add mysql modify tool --- .../migrations/mysql/1694001465232-ModifyTool.ts | 11 +++++++++++ .../server/src/database/migrations/mysql/index.ts | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/database/migrations/mysql/1694001465232-ModifyTool.ts diff --git a/packages/server/src/database/migrations/mysql/1694001465232-ModifyTool.ts b/packages/server/src/database/migrations/mysql/1694001465232-ModifyTool.ts new file mode 100644 index 00000000..934506cb --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1694001465232-ModifyTool.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class ModifyTool1694001465232 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`tool\` MODIFY \`schema\` TEXT, MODIFY \`func\` TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`tool\` MODIFY \`schema\` VARCHAR, MODIFY \`func\` VARCHAR;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index f946cb5f..124eb70f 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -2,10 +2,12 @@ import { Init1693840429259 } from './1693840429259-Init' import { ModifyChatFlow1693997791471 } from './1693997791471-ModifyChatFlow' import { ModifyChatMessage1693999022236 } from './1693999022236-ModifyChatMessage' import { ModifyCredential1693999261583 } from './1693999261583-ModifyCredential' +import { ModifyTool1694001465232 } from './1694001465232-ModifyTool' export const mysqlMigrations = [ Init1693840429259, ModifyChatFlow1693997791471, ModifyChatMessage1693999022236, - ModifyCredential1693999261583 + ModifyCredential1693999261583, + ModifyTool1694001465232 ] From bf459bf5abb507df9e3d7b4c22ce92ac93b4d91d Mon Sep 17 00:00:00 2001 From: Pooria Arab <42897904+pooriaarab@users.noreply.github.com> Date: Thu, 7 Sep 2023 08:00:09 -0700 Subject: [PATCH 31/35] Update Chroma_Existing.ts - first OSS PR that works!! --- .../nodes/vectorstores/Chroma/Chroma_Existing.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts index 1d360182..30662d61 100644 --- a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts +++ b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts @@ -93,18 +93,20 @@ class Chroma_Existing_VectorStores implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData) - const chromaMetadataFilter = nodeData.inputs?.chromaMetadataFilter; - const metadataFilter = chromaMetadataFilter ? JSON.parse(chromaMetadataFilter) : {}; + const chromaMetadataFilter = nodeData.inputs?.chromaMetadataFilter + const metadataFilter = chromaMetadataFilter ? JSON.parse(chromaMetadataFilter) : {} const obj: { collectionName: string url?: string chromaApiKey?: string - metadataFilter?: any + filter?: object | undefined } = { collectionName } if (chromaURL) obj.url = chromaURL - if (chromaApiKey) obj.chromaApiKey = chromaApiKey - if (chromaMetadataFilter) obj.metadataFilter = metadataFilter + if (chromaMetadataFilter) { + const metadatafilter = typeof chromaMetadataFilter === 'object' ? chromaMetadataFilter : JSON.parse(chromaMetadataFilter) + obj.filter = metadatafilter + } const vectorStore = await ChromaExtended.fromExistingCollection(embeddings, obj) From ad8141e0788814bf43c77e09e59471e38c057a70 Mon Sep 17 00:00:00 2001 From: Henry Date: Fri, 8 Sep 2023 11:49:22 +0100 Subject: [PATCH 32/35] Apache License 2.0 --- LICENSE.md | 189 ++++++++++++++++++++++++++++--- README-ZH.md | 10 +- README.md | 4 +- packages/components/README-ZH.md | 6 +- packages/components/README.md | 4 +- packages/server/README-ZH.md | 2 +- packages/server/README.md | 4 +- packages/ui/README-ZH.md | 6 +- packages/ui/README.md | 4 +- 9 files changed, 192 insertions(+), 37 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 56552bdf..4a27187d 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,21 +1,176 @@ -The MIT License + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Copyright (c) 2023 FlowiseAI +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +1. Definitions. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/README-ZH.md b/README-ZH.md index 10efce8b..2805ef9b 100644 --- a/README-ZH.md +++ b/README-ZH.md @@ -2,7 +2,7 @@ -# Flowise - 轻松构建LLM应用程序 +# Flowise - 轻松构建 LLM 应用程序 [![发布说明](https://img.shields.io/github/release/FlowiseAI/Flowise)](https://github.com/FlowiseAI/Flowise/releases) [![Discord](https://img.shields.io/discord/1087698854775881778?label=Discord&logo=discord)](https://discord.gg/jbaHfsRVBW) @@ -10,13 +10,13 @@ [![GitHub星图](https://img.shields.io/github/stars/FlowiseAI/Flowise?style=social)](https://star-history.com/#FlowiseAI/Flowise) [![GitHub分支](https://img.shields.io/github/forks/FlowiseAI/Flowise?style=social)](https://github.com/FlowiseAI/Flowise/fork) -[English](<./README.md>) | 中文 +[English](./README.md) | 中文

拖放界面构建定制化的LLM流程

-## ⚡快速入门 +## ⚡ 快速入门 下载并安装 [NodeJS](https://nodejs.org/en/download) >= 18.15.0 @@ -67,7 +67,7 @@ ## 👨‍💻 开发者 -Flowise 在一个单一的代码库中有3个不同的模块。 +Flowise 在一个单一的代码库中有 3 个不同的模块。 - `server`：用于提供 API 逻辑的 Node 后端 - `ui`：React 前端 @@ -185,4 +185,4 @@ Flowise 支持不同的环境变量来配置您的实例。您可以在 `package ## 📄 许可证 -此代码库中的源代码在[MIT许可证](LICENSE.md)下提供。 +此代码库中的源代码在[Apache License Version 2.0 许可证](LICENSE.md)下提供。 diff --git a/README.md b/README.md index 36c53b4b..9f685846 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![GitHub star chart](https://img.shields.io/github/stars/FlowiseAI/Flowise?style=social)](https://star-history.com/#FlowiseAI/Flowise) [![GitHub fork](https://img.shields.io/github/forks/FlowiseAI/Flowise?style=social)](https://github.com/FlowiseAI/Flowise/fork) -English | [中文](<./README-ZH.md>) +English | [中文](./README-ZH.md)

Drag & drop UI to build your customized LLM flow

@@ -186,4 +186,4 @@ See [contributing guide](CONTRIBUTING.md). Reach out to us at [Discord](https:// ## 📄 License -Source code in this repository is made available under the [MIT License](LICENSE.md). +Source code in this repository is made available under the [Apache License Version 2.0](LICENSE.md). diff --git a/packages/components/README-ZH.md b/packages/components/README-ZH.md index 12cb240b..52d43eb2 100644 --- a/packages/components/README-ZH.md +++ b/packages/components/README-ZH.md @@ -2,9 +2,9 @@ # 流式组件 -[English](<./README.md>) | 中文 +[English](./README.md) | 中文 -Flowise的应用集成。包含节点和凭据。 +Flowise 的应用集成。包含节点和凭据。 ![Flowise](https://github.com/FlowiseAI/Flowise/blob/main/images/flowise.gif?raw=true) @@ -16,4 +16,4 @@ npm i flowise-components ## 许可证 -此存储库中的源代码在[MIT许可证](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md)下提供。 +此存储库中的源代码在[Apache License Version 2.0 许可证](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md)下提供。 diff --git a/packages/components/README.md b/packages/components/README.md index 8d561e34..f8e08374 100644 --- a/packages/components/README.md +++ b/packages/components/README.md @@ -2,7 +2,7 @@ # Flowise Components -English | [中文](<./README-ZH.md>) +English | [中文](./README-ZH.md) Apps integration for Flowise. Contain Nodes and Credentials. @@ -16,4 +16,4 @@ npm i flowise-components ## License -Source code in this repository is made available under the [MIT License](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md). +Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md). diff --git a/packages/server/README-ZH.md b/packages/server/README-ZH.md index e58f08bf..8c40fd11 100644 --- a/packages/server/README-ZH.md +++ b/packages/server/README-ZH.md @@ -97,4 +97,4 @@ npx flowise start --PORT=3000 --DEBUG=true ## 📄 许可证 -本仓库中的源代码在[MIT 许可证](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md)下提供。 +本仓库中的源代码在[Apache License Version 2.0 许可证](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md)下提供。 diff --git a/packages/server/README.md b/packages/server/README.md index b561b0b0..049e7261 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -2,7 +2,7 @@ # Flowise - Low-Code LLM apps builder -English | [中文](<./README-ZH.md>) +English | [中文](./README-ZH.md) ![Flowise](https://github.com/FlowiseAI/Flowise/blob/main/images/flowise.gif?raw=true) @@ -77,4 +77,4 @@ See [contributing guide](https://github.com/FlowiseAI/Flowise/blob/master/CONTRI ## 📄 License -Source code in this repository is made available under the [MIT License](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md). +Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md). diff --git a/packages/ui/README-ZH.md b/packages/ui/README-ZH.md index c6307935..e34c3de0 100644 --- a/packages/ui/README-ZH.md +++ b/packages/ui/README-ZH.md @@ -2,9 +2,9 @@ # 流程界面 -[English](<./README.md>) | 中文 +[English](./README.md) | 中文 -Flowise的React前端界面。 +Flowise 的 React 前端界面。 ![Flowise](https://github.com/FlowiseAI/Flowise/blob/main/images/flowise.gif?raw=true) @@ -16,4 +16,4 @@ npm i flowise-ui ## 许可证 -本仓库中的源代码在[MIT许可证](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md)下提供。 +本仓库中的源代码在[Apache License Version 2.0 许可证](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md)下提供。 diff --git a/packages/ui/README.md b/packages/ui/README.md index 487172f8..b476d47f 100644 --- a/packages/ui/README.md +++ b/packages/ui/README.md @@ -2,7 +2,7 @@ # Flowise UI -English | [中文](<./README-ZH.md>) +English | [中文](./README-ZH.md) React frontend ui for Flowise. @@ -16,4 +16,4 @@ npm i flowise-ui ## License -Source code in this repository is made available under the [MIT License](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md). +Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md). From 06a5a71d2677054fad3c331c1951ead70c58300b Mon Sep 17 00:00:00 2001 From: Henry Date: Fri, 8 Sep 2023 12:42:26 +0100 Subject: [PATCH 33/35] update apify for metadata --- .../ApifyWebsiteContentCrawler.ts | 53 +++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts b/packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts index a5e6a6e0..9ecaa594 100644 --- a/packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts +++ b/packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts @@ -21,11 +21,17 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode { this.name = 'apifyWebsiteContentCrawler' this.type = 'Document' this.icon = 'apify-symbol-transparent.svg' - this.version = 1.0 + this.version = 2.0 this.category = 'Document Loaders' this.description = 'Load data from Apify Website Content Crawler' this.baseClasses = [this.type] this.inputs = [ + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, { label: 'Start URLs', name: 'urls', @@ -64,14 +70,16 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode { name: 'maxCrawlDepth', type: 'number', optional: true, - default: 1 + default: 1, + additionalParams: true }, { label: 'Max crawl pages', name: 'maxCrawlPages', type: 'number', optional: true, - default: 3 + default: 3, + additionalParams: true }, { label: 'Additional input', @@ -80,13 +88,15 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode { default: JSON.stringify({}), description: 'For additional input options for the crawler see documentation.', - optional: true + optional: true, + additionalParams: true }, { - label: 'Text Splitter', - name: 'textSplitter', - type: 'TextSplitter', - optional: true + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] this.credential = { @@ -99,6 +109,7 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode { async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const metadata = nodeData.inputs?.metadata // Get input options and merge with additional input const urls = nodeData.inputs?.urls as string @@ -132,7 +143,31 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode { } }) - return textSplitter ? loader.loadAndSplit(textSplitter) : loader.load() + let docs = [] + + if (textSplitter) { + docs = await loader.loadAndSplit(textSplitter) + } else { + docs = await loader.load() + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs } } From f9f18b6d2c7631b6e2e97b0f54d2a0c39a7e1173 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Fri, 8 Sep 2023 14:32:46 +0100 Subject: [PATCH 34/35] Update Chroma_Existing.ts add accidentally removed chromaApiKey --- packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts index 30662d61..e268434c 100644 --- a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts +++ b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts @@ -103,6 +103,7 @@ class Chroma_Existing_VectorStores implements INode { filter?: object | undefined } = { collectionName } if (chromaURL) obj.url = chromaURL + if (chromaApiKey) obj.chromaApiKey = chromaApiKey if (chromaMetadataFilter) { const metadatafilter = typeof chromaMetadataFilter === 'object' ? chromaMetadataFilter : JSON.parse(chromaMetadataFilter) obj.filter = metadatafilter From ba9c1cd5682f3a120f56246c058fb7b35cdf6fcf Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Fri, 8 Sep 2023 14:34:07 +0100 Subject: [PATCH 35/35] Update Chroma_Existing.ts remove not needed code --- packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts index e268434c..ff929ef1 100644 --- a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts +++ b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts @@ -94,7 +94,6 @@ class Chroma_Existing_VectorStores implements INode { const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData) const chromaMetadataFilter = nodeData.inputs?.chromaMetadataFilter - const metadataFilter = chromaMetadataFilter ? JSON.parse(chromaMetadataFilter) : {} const obj: { collectionName: string