diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index cd443a19..fa9d527b 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,6 +1,6 @@ # These are supported funding model platforms -github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +github: [FlowiseAI] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username diff --git a/docker/.env.example b/docker/.env.example index f3211196..e313316d 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1,3 +1,5 @@ PORT=3000 # FLOWISE_USERNAME=user # FLOWISE_PASSWORD=1234 +# DATABASE_PATH=/your_database_path/.flowise +# EXECUTION_MODE=child or main \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c776f96e..0bb68097 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -8,6 +8,8 @@ services: - PORT=${PORT} - FLOWISE_USERNAME=${FLOWISE_USERNAME} - FLOWISE_PASSWORD=${FLOWISE_PASSWORD} + - DATABASE_PATH=${DATABASE_PATH} + - EXECUTION_MODE=${EXECUTION_MODE} ports: - '${PORT}:${PORT}' volumes: diff --git a/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts new file mode 100644 index 00000000..4af8c2ce --- /dev/null +++ b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts @@ -0,0 +1,106 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { JSONLinesLoader } from 'langchain/document_loaders/fs/json' + +class Jsonlines_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Json Lines File' + this.name = 'jsonlinesFile' + this.type = 'Document' + this.icon = 'jsonlines.svg' + this.category = 'Document Loaders' + this.description = `Load data from JSON Lines files` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Jsonlines File', + name: 'jsonlinesFile', + type: 'file', + fileType: '.jsonl' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Pointer Extraction', + name: 'pointerName', + type: 'string', + placeholder: 'Enter pointer name', + optional: false + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const jsonLinesFileBase64 = nodeData.inputs?.jsonlinesFile as string + const pointerName = nodeData.inputs?.pointerName as string + const metadata = nodeData.inputs?.metadata + + let alldocs = [] + let files: string[] = [] + + let pointer = '/' + pointerName.trim() + + if (jsonLinesFileBase64.startsWith('[') && jsonLinesFileBase64.endsWith(']')) { + files = JSON.parse(jsonLinesFileBase64) + } else { + files = [jsonLinesFileBase64] + } + + for (const file of files) { + const splitDataURI = file.split(',') + splitDataURI.pop() + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + const blob = new Blob([bf]) + const loader = new JSONLinesLoader(blob, pointer) + + if (textSplitter) { + const docs = await loader.loadAndSplit(textSplitter) + alldocs.push(...docs) + } else { + const docs = await loader.load() + alldocs.push(...docs) + } + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of alldocs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return alldocs + } +} + +module.exports = { nodeClass: Jsonlines_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/Jsonlines/jsonlines.svg b/packages/components/nodes/documentloaders/Jsonlines/jsonlines.svg new file mode 100644 index 00000000..f3686f0c --- /dev/null +++ b/packages/components/nodes/documentloaders/Jsonlines/jsonlines.svg @@ -0,0 +1,16 @@ + + + + + background + + + + + + + Layer 1 + JSON + Lines + + \ No newline at end of file diff --git a/packages/server/.env.example b/packages/server/.env.example index fd82c096..e313316d 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -1,4 +1,5 @@ PORT=3000 # FLOWISE_USERNAME=user # FLOWISE_PASSWORD=1234 +# DATABASE_PATH=/your_database_path/.flowise # EXECUTION_MODE=child or main \ No newline at end of file diff --git a/packages/server/README.md b/packages/server/README.md index 2cdf41d1..e4d1e439 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -31,14 +31,28 @@ FLOWISE_PASSWORD=1234 ## 📖 Documentation -Coming Soon - -## 💻 Cloud Hosted - -Coming Soon +[Flowise Docs](https://docs.flowiseai.com/) ## 🌐 Self Host +### [Railway](https://docs.flowiseai.com/deployment/railway) + +[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/YK7J0v) + +### [Render](https://docs.flowiseai.com/deployment/render) + +[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://docs.flowiseai.com/deployment/render) + +### [AWS](https://docs.flowiseai.com/deployment/aws) + +### [Azure](https://docs.flowiseai.com/deployment/azure) + +### [DigitalOcean](https://docs.flowiseai.com/deployment/digital-ocean) + +### [GCP](https://docs.flowiseai.com/deployment/gcp) + +## 💻 Cloud Hosted + Coming Soon ## 🙋 Support diff --git a/packages/server/src/DataSource.ts b/packages/server/src/DataSource.ts index 2ec8104a..03b9d5ce 100644 --- a/packages/server/src/DataSource.ts +++ b/packages/server/src/DataSource.ts @@ -9,7 +9,7 @@ import { getUserHome } from './utils' let appDataSource: DataSource export const init = async (): Promise => { - const homePath = path.join(getUserHome(), '.flowise') + const homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise') appDataSource = new DataSource({ type: 'sqlite', diff --git a/packages/server/src/commands/start.ts b/packages/server/src/commands/start.ts index 94b8d995..9066f1cf 100644 --- a/packages/server/src/commands/start.ts +++ b/packages/server/src/commands/start.ts @@ -18,6 +18,7 @@ export default class Start extends Command { FLOWISE_USERNAME: Flags.string(), FLOWISE_PASSWORD: Flags.string(), PORT: Flags.string(), + DATABASE_PATH: Flags.string(), EXECUTION_MODE: Flags.string() } @@ -53,6 +54,7 @@ export default class Start extends Command { if (flags.FLOWISE_USERNAME) process.env.FLOWISE_USERNAME = flags.FLOWISE_USERNAME if (flags.FLOWISE_PASSWORD) process.env.FLOWISE_PASSWORD = flags.FLOWISE_PASSWORD if (flags.PORT) process.env.PORT = flags.PORT + if (flags.DATABASE_PATH) process.env.DATABASE_PATH = flags.DATABASE_PATH if (flags.EXECUTION_MODE) process.env.EXECUTION_MODE = flags.EXECUTION_MODE await (async () => {