From bee16d4982819741fcf5a78308bc79bf32b55a17 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Wed, 21 Jun 2023 19:29:32 +0100 Subject: [PATCH 1/4] Update FUNDING.yml --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index cd443a19..fa9d527b 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,6 +1,6 @@ # These are supported funding model platforms -github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +github: [FlowiseAI] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username From 7ce0f71e2fc53cf9180c5af8659611b1b39d4c0f Mon Sep 17 00:00:00 2001 From: ivalkshfoeif Date: Wed, 21 Jun 2023 18:56:37 -0700 Subject: [PATCH 2/4] add JSONLines Loader --- .../documentloaders/Jsonlines/Jsonlines.ts | 105 ++++++++++++++++++ .../documentloaders/Jsonlines/jsonlines.svg | 16 +++ 2 files changed, 121 insertions(+) create mode 100644 packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts create mode 100644 packages/components/nodes/documentloaders/Jsonlines/jsonlines.svg diff --git a/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts new file mode 100644 index 00000000..e92a97aa --- /dev/null +++ b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts @@ -0,0 +1,105 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { JSONLinesLoader } from 'langchain/document_loaders/fs/json' + +class Jsonlines_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Json Lines File' + this.name = 'jsonlinesFile' + this.type = 'Document' + this.icon = 'jsonlines.svg' + this.category = 'Document Loaders' + this.description = `Load data from JSON Lines files` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Jsonlines File', + name: 'jsonlinesFile', + type: 'file', + fileType: '.jsonl' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Pointer Extraction', + name: 'pointerName', + type: 'string', + description: 'Extracting the pointer', + placeholder: 'Enter pointer name', + optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const jsonLinesFileBase64 = nodeData.inputs?.jsonlinesFile as string + const pointerName = nodeData.inputs?.pointerName as string + const metadata = nodeData.inputs?.metadata + + let alldocs = [] + let files: string[] = [] + + if (jsonLinesFileBase64.startsWith('[') && jsonLinesFileBase64.endsWith(']')) { + files = JSON.parse(jsonLinesFileBase64) + } else { + files = [jsonLinesFileBase64] + } + + for (const file of files) { + const splitDataURI = file.split(',') + splitDataURI.pop() + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + const blob = new Blob([bf]) + const loader = new JSONLinesLoader(blob, pointerName) + + if (textSplitter) { + const docs = await loader.loadAndSplit(textSplitter) + alldocs.push(...docs) + } else { + const docs = await loader.load() + alldocs.push(...docs) + } + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of alldocs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return alldocs + } +} + +module.exports = { nodeClass: Jsonlines_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/Jsonlines/jsonlines.svg b/packages/components/nodes/documentloaders/Jsonlines/jsonlines.svg new file mode 100644 index 00000000..f3686f0c --- /dev/null +++ b/packages/components/nodes/documentloaders/Jsonlines/jsonlines.svg @@ -0,0 +1,16 @@ + + + + + background + + + + + + + Layer 1 + JSON + Lines + + \ No newline at end of file From 7646e973e3a77d8f4500f82d3e70ba51bd203048 Mon Sep 17 00:00:00 2001 From: ivalkshfoeif Date: Wed, 21 Jun 2023 19:34:30 -0700 Subject: [PATCH 3/4] update pointer attribute and logic --- .../nodes/documentloaders/Jsonlines/Jsonlines.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts index e92a97aa..4af8c2ce 100644 --- a/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts +++ b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts @@ -37,9 +37,8 @@ class Jsonlines_DocumentLoaders implements INode { label: 'Pointer Extraction', name: 'pointerName', type: 'string', - description: 'Extracting the pointer', placeholder: 'Enter pointer name', - optional: true + optional: false }, { label: 'Metadata', @@ -60,6 +59,8 @@ class Jsonlines_DocumentLoaders implements INode { let alldocs = [] let files: string[] = [] + let pointer = '/' + pointerName.trim() + if (jsonLinesFileBase64.startsWith('[') && jsonLinesFileBase64.endsWith(']')) { files = JSON.parse(jsonLinesFileBase64) } else { @@ -71,7 +72,7 @@ class Jsonlines_DocumentLoaders implements INode { splitDataURI.pop() const bf = Buffer.from(splitDataURI.pop() || '', 'base64') const blob = new Blob([bf]) - const loader = new JSONLinesLoader(blob, pointerName) + const loader = new JSONLinesLoader(blob, pointer) if (textSplitter) { const docs = await loader.loadAndSplit(textSplitter) From dd328dcd51f73cb2666e35805f684f8a15d1b843 Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 22 Jun 2023 15:47:03 +0100 Subject: [PATCH 4/4] add db_path --- docker/.env.example | 2 ++ docker/docker-compose.yml | 2 ++ packages/server/.env.example | 1 + packages/server/README.md | 24 +++++++++++++++++++----- packages/server/src/DataSource.ts | 2 +- packages/server/src/commands/start.ts | 2 ++ 6 files changed, 27 insertions(+), 6 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index f3211196..e313316d 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1,3 +1,5 @@ PORT=3000 # FLOWISE_USERNAME=user # FLOWISE_PASSWORD=1234 +# DATABASE_PATH=/your_database_path/.flowise +# EXECUTION_MODE=child or main \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c776f96e..0bb68097 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -8,6 +8,8 @@ services: - PORT=${PORT} - FLOWISE_USERNAME=${FLOWISE_USERNAME} - FLOWISE_PASSWORD=${FLOWISE_PASSWORD} + - DATABASE_PATH=${DATABASE_PATH} + - EXECUTION_MODE=${EXECUTION_MODE} ports: - '${PORT}:${PORT}' volumes: diff --git a/packages/server/.env.example b/packages/server/.env.example index fd82c096..e313316d 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -1,4 +1,5 @@ PORT=3000 # FLOWISE_USERNAME=user # FLOWISE_PASSWORD=1234 +# DATABASE_PATH=/your_database_path/.flowise # EXECUTION_MODE=child or main \ No newline at end of file diff --git a/packages/server/README.md b/packages/server/README.md index 2cdf41d1..e4d1e439 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -31,14 +31,28 @@ FLOWISE_PASSWORD=1234 ## 📖 Documentation -Coming Soon - -## 💻 Cloud Hosted - -Coming Soon +[Flowise Docs](https://docs.flowiseai.com/) ## 🌐 Self Host +### [Railway](https://docs.flowiseai.com/deployment/railway) + +[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/YK7J0v) + +### [Render](https://docs.flowiseai.com/deployment/render) + +[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://docs.flowiseai.com/deployment/render) + +### [AWS](https://docs.flowiseai.com/deployment/aws) + +### [Azure](https://docs.flowiseai.com/deployment/azure) + +### [DigitalOcean](https://docs.flowiseai.com/deployment/digital-ocean) + +### [GCP](https://docs.flowiseai.com/deployment/gcp) + +## 💻 Cloud Hosted + Coming Soon ## 🙋 Support diff --git a/packages/server/src/DataSource.ts b/packages/server/src/DataSource.ts index 76c8e144..19396315 100644 --- a/packages/server/src/DataSource.ts +++ b/packages/server/src/DataSource.ts @@ -8,7 +8,7 @@ import { getUserHome } from './utils' let appDataSource: DataSource export const init = async (): Promise => { - const homePath = path.join(getUserHome(), '.flowise') + const homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') appDataSource = new DataSource({ type: 'sqlite', diff --git a/packages/server/src/commands/start.ts b/packages/server/src/commands/start.ts index 94b8d995..9066f1cf 100644 --- a/packages/server/src/commands/start.ts +++ b/packages/server/src/commands/start.ts @@ -18,6 +18,7 @@ export default class Start extends Command { FLOWISE_USERNAME: Flags.string(), FLOWISE_PASSWORD: Flags.string(), PORT: Flags.string(), + DATABASE_PATH: Flags.string(), EXECUTION_MODE: Flags.string() } @@ -53,6 +54,7 @@ export default class Start extends Command { if (flags.FLOWISE_USERNAME) process.env.FLOWISE_USERNAME = flags.FLOWISE_USERNAME if (flags.FLOWISE_PASSWORD) process.env.FLOWISE_PASSWORD = flags.FLOWISE_PASSWORD if (flags.PORT) process.env.PORT = flags.PORT + if (flags.DATABASE_PATH) process.env.DATABASE_PATH = flags.DATABASE_PATH if (flags.EXECUTION_MODE) process.env.EXECUTION_MODE = flags.EXECUTION_MODE await (async () => {