diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..cd443a19 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,13 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d087fb93..759f195f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,8 @@ jobs: platform: [ubuntu-latest] node-version: [18.15.0] runs-on: ${{ matrix.platform }} - + env: + PUPPETEER_SKIP_DOWNLOAD: true steps: - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} diff --git a/.github/workflows/test_docker_build.yml b/.github/workflows/test_docker_build.yml index a698c247..a27cf22d 100644 --- a/.github/workflows/test_docker_build.yml +++ b/.github/workflows/test_docker_build.yml @@ -7,12 +7,13 @@ on: pull_request: branches: - - "*" + - '*' jobs: build: runs-on: ubuntu-latest - + env: + PUPPETEER_SKIP_DOWNLOAD: true steps: - uses: actions/checkout@v3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a09051f3..03211d51 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,7 +42,7 @@ Flowise has 3 different modules in a single mono repository. #### Prerequisite -- Install Yarn +- Install [Yarn v1](https://classic.yarnpkg.com/en/docs/install) ```bash npm i -g yarn ``` @@ -84,7 +84,11 @@ Flowise has 3 different modules in a single mono repository. yarn start ``` -9. For development, run +9. For development: + + - Create `.env` file and specify the `PORT` (refer to `.env.example`) in `packages/ui` + - Create `.env` file and specify the `PORT` (refer to `.env.example`) in `packages/server` + - Run ```bash yarn dev diff --git a/Dockerfile b/Dockerfile index fc76cd00..fe01ed8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,9 @@ FROM node:18-alpine RUN apk add --update libc6-compat python3 make g++ +# needed for pdfjs-dist +RUN apk add --no-cache build-base cairo-dev pango-dev +ENV PUPPETEER_SKIP_DOWNLOAD=true WORKDIR /usr/src/packages diff --git a/README.md b/README.md index 05221f47..dbce8f3b 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Flowise has 3 different modules in a single mono repository. ### Prerequisite -- Install Yarn +- Install [Yarn v1](https://classic.yarnpkg.com/en/docs/install) ```bash npm i -g yarn ``` @@ -107,9 +107,13 @@ Flowise has 3 different modules in a single mono repository. 6. For development build: - ```bash - yarn dev - ``` + - Create `.env` file and specify the `PORT` (refer to `.env.example`) in `packages/ui` + - Create `.env` file and specify the `PORT` (refer to `.env.example`) in `packages/server` + - Run + + ```bash + yarn dev + ``` Any code changes will reload the app automatically on [http://localhost:8080](http://localhost:8080) @@ -138,6 +142,8 @@ FLOWISE_PASSWORD=1234 ### [AWS](https://docs.flowiseai.com/deployment/aws) +### [Azure](https://docs.flowiseai.com/deployment/azure) + ### [DigitalOcean](https://docs.flowiseai.com/deployment/digital-ocean) ### [GCP](https://docs.flowiseai.com/deployment/gcp) diff --git a/docker/Dockerfile b/docker/Dockerfile index e4bf704a..2203af11 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,6 +4,9 @@ USER root RUN apk add --no-cache git RUN apk add --no-cache python3 py3-pip make g++ +# needed for pdfjs-dist +RUN apk add --no-cache build-base cairo-dev pango-dev +ENV PUPPETEER_SKIP_DOWNLOAD=true # You can install a specific version like: flowise@1.0.0 RUN npm install -g flowise diff --git a/packages/components/nodes/documentloaders/Figma/Figma.ts b/packages/components/nodes/documentloaders/Figma/Figma.ts new file mode 100644 index 00000000..388c4ee0 --- /dev/null +++ b/packages/components/nodes/documentloaders/Figma/Figma.ts @@ -0,0 +1,81 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { FigmaFileLoader, FigmaLoaderParams } from 'langchain/document_loaders/web/figma' + +class Figma_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Figma' + this.name = 'figma' + this.type = 'Document' + this.icon = 'figma.png' + this.category = 'Document Loaders' + this.description = 'Load data from a Figma file' + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Access Token', + name: 'accessToken', + type: 'password', + placeholder: '' + }, + { + label: 'File Key', + name: 'fileKey', + type: 'string', + placeholder: 'key' + }, + { + label: 'Node IDs', + name: 'nodeIds', + type: 'string', + placeholder: '0, 1, 2' + }, + { + label: 'Recursive', + name: 'recursive', + type: 'boolean', + optional: true + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const accessToken = nodeData.inputs?.accessToken as string + const nodeIds = (nodeData.inputs?.nodeIds as string)?.split(',') || [] + const fileKey = nodeData.inputs?.fileKey as string + + const options: FigmaLoaderParams = { + accessToken, + nodeIds, + fileKey + } + + const loader = new FigmaFileLoader(options) + const docs = await loader.load() + + return docs + } +} + +module.exports = { nodeClass: Figma_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/Figma/figma.png b/packages/components/nodes/documentloaders/Figma/figma.png new file mode 100644 index 00000000..72372ddf Binary files /dev/null and b/packages/components/nodes/documentloaders/Figma/figma.png differ diff --git a/packages/components/nodes/documentloaders/Pdf/Pdf.ts b/packages/components/nodes/documentloaders/Pdf/Pdf.ts index bc36f8cb..ddb7edb8 100644 --- a/packages/components/nodes/documentloaders/Pdf/Pdf.ts +++ b/packages/components/nodes/documentloaders/Pdf/Pdf.ts @@ -49,6 +49,13 @@ class Pdf_DocumentLoaders implements INode { ], default: 'perPage' }, + { + label: 'Use Legacy Build', + name: 'legacyBuild', + type: 'boolean', + optional: true, + additionalParams: true + }, { label: 'Metadata', name: 'metadata', @@ -64,6 +71,7 @@ class Pdf_DocumentLoaders implements INode { const pdfFileBase64 = nodeData.inputs?.pdfFile as string const usage = nodeData.inputs?.usage as string const metadata = nodeData.inputs?.metadata + const legacyBuild = nodeData.inputs?.legacyBuild as boolean let alldocs = [] let files: string[] = [] @@ -81,8 +89,9 @@ class Pdf_DocumentLoaders implements INode { if (usage === 'perFile') { const loader = new PDFLoader(new Blob([bf]), { splitPages: false, - // @ts-ignore - pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') + pdfjs: () => + // @ts-ignore + legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }) if (textSplitter) { const docs = await loader.loadAndSplit(textSplitter) @@ -92,8 +101,11 @@ class Pdf_DocumentLoaders implements INode { alldocs.push(...docs) } } else { - // @ts-ignore - const loader = new PDFLoader(new Blob([bf]), { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }) + const loader = new PDFLoader(new Blob([bf]), { + pdfjs: () => + // @ts-ignore + legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') + }) if (textSplitter) { const docs = await loader.loadAndSplit(textSplitter) alldocs.push(...docs) diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts new file mode 100644 index 00000000..6b7790af --- /dev/null +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -0,0 +1,117 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { PlaywrightWebBaseLoader } from 'langchain/document_loaders/web/playwright' +import { test } from 'linkifyjs' +import { getAvailableURLs } from '../../../src' + +class Playwright_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Playwright Web Scraper' + this.name = 'playwrightWebScraper' + this.type = 'Document' + this.icon = 'playwright.svg' + this.category = 'Document Loaders' + this.description = `Load data from webpages` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'URL', + name: 'url', + type: 'string' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Web Scrap for Relative Links', + name: 'webScrap', + type: 'boolean', + optional: true, + additionalParams: true + }, + { + label: 'Web Scrap Links Limit', + name: 'limit', + type: 'number', + default: 10, + optional: true, + additionalParams: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const metadata = nodeData.inputs?.metadata + const webScrap = nodeData.inputs?.webScrap as boolean + let limit = nodeData.inputs?.limit as string + + let url = nodeData.inputs?.url as string + url = url.trim() + if (!test(url)) { + throw new Error('Invalid URL') + } + + const playwrightLoader = async (url: string): Promise => { + let docs = [] + const loader = new PlaywrightWebBaseLoader(url) + if (textSplitter) { + docs = await loader.loadAndSplit(textSplitter) + } else { + docs = await loader.load() + } + return docs + } + + let availableUrls: string[] + let docs = [] + if (webScrap) { + if (!limit) limit = '10' + availableUrls = await getAvailableURLs(url, parseInt(limit)) + for (let i = 0; i < availableUrls.length; i++) { + docs.push(...(await playwrightLoader(availableUrls[i]))) + } + } else { + docs = await playwrightLoader(url) + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs + } +} + +module.exports = { nodeClass: Playwright_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/Playwright/playwright.svg b/packages/components/nodes/documentloaders/Playwright/playwright.svg new file mode 100644 index 00000000..0992832d --- /dev/null +++ b/packages/components/nodes/documentloaders/Playwright/playwright.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/packages/components/nodes/documentloaders/Subtitles/Subtitles.ts b/packages/components/nodes/documentloaders/Subtitles/Subtitles.ts new file mode 100644 index 00000000..0f60e151 --- /dev/null +++ b/packages/components/nodes/documentloaders/Subtitles/Subtitles.ts @@ -0,0 +1,95 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { SRTLoader } from 'langchain/document_loaders/fs/srt' + +class Subtitles_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Subtitles File' + this.name = 'subtitlesFile' + this.type = 'Document' + this.icon = 'subtitlesFile.svg' + this.category = 'Document Loaders' + this.description = `Load data from subtitles files` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Subtitles File', + name: 'subtitlesFile', + type: 'file', + fileType: '.srt' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const subtitlesFileBase64 = nodeData.inputs?.subtitlesFile as string + const metadata = nodeData.inputs?.metadata + + let alldocs = [] + let files: string[] = [] + + if (subtitlesFileBase64.startsWith('[') && subtitlesFileBase64.endsWith(']')) { + files = JSON.parse(subtitlesFileBase64) + } else { + files = [subtitlesFileBase64] + } + + for (const file of files) { + const splitDataURI = file.split(',') + splitDataURI.pop() + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + const blob = new Blob([bf]) + const loader = new SRTLoader(blob) + + if (textSplitter) { + const docs = await loader.loadAndSplit(textSplitter) + alldocs.push(...docs) + } else { + const docs = await loader.load() + alldocs.push(...docs) + } + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of alldocs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + return alldocs + } +} + +module.exports = { nodeClass: Subtitles_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/Subtitles/subtitlesFile.svg b/packages/components/nodes/documentloaders/Subtitles/subtitlesFile.svg new file mode 100644 index 00000000..a6ee925b --- /dev/null +++ b/packages/components/nodes/documentloaders/Subtitles/subtitlesFile.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/components/nodes/embeddings/AzureOpenAIEmbedding/AzureOpenAIEmbedding.ts b/packages/components/nodes/embeddings/AzureOpenAIEmbedding/AzureOpenAIEmbedding.ts index 7fe61e62..2a211622 100644 --- a/packages/components/nodes/embeddings/AzureOpenAIEmbedding/AzureOpenAIEmbedding.ts +++ b/packages/components/nodes/embeddings/AzureOpenAIEmbedding/AzureOpenAIEmbedding.ts @@ -42,18 +42,10 @@ class AzureOpenAIEmbedding_Embeddings implements INode { { label: 'Azure OpenAI Api Version', name: 'azureOpenAIApiVersion', - type: 'options', - options: [ - { - label: '2023-03-15-preview', - name: '2023-03-15-preview' - }, - { - label: '2022-12-01', - name: '2022-12-01' - } - ], - default: '2023-03-15-preview' + type: 'string', + placeholder: 'YOUR-API-VERSION', + description: + 'Description of Supported API Versions. Please refer examples' }, { label: 'Batch Size', diff --git a/packages/components/nodes/textsplitters/CodeTextSplitter/CodeTextSplitter.ts b/packages/components/nodes/textsplitters/CodeTextSplitter/CodeTextSplitter.ts new file mode 100644 index 00000000..b14655b8 --- /dev/null +++ b/packages/components/nodes/textsplitters/CodeTextSplitter/CodeTextSplitter.ts @@ -0,0 +1,128 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses } from '../../../src/utils' +import { + RecursiveCharacterTextSplitter, + RecursiveCharacterTextSplitterParams, + SupportedTextSplitterLanguage +} from 'langchain/text_splitter' + +class CodeTextSplitter_TextSplitters implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + constructor() { + this.label = 'Code Text Splitter' + this.name = 'codeTextSplitter' + this.type = 'CodeTextSplitter' + this.icon = 'codeTextSplitter.svg' + this.category = 'Text Splitters' + this.description = `Split documents based on language-specific syntax` + this.baseClasses = [this.type, ...getBaseClasses(RecursiveCharacterTextSplitter)] + this.inputs = [ + { + label: 'Language', + name: 'language', + type: 'options', + options: [ + { + label: 'cpp', + name: 'cpp' + }, + { + label: 'go', + name: 'go' + }, + { + label: 'java', + name: 'java' + }, + { + label: 'js', + name: 'js' + }, + { + label: 'php', + name: 'php' + }, + { + label: 'proto', + name: 'proto' + }, + { + label: 'python', + name: 'python' + }, + { + label: 'rst', + name: 'rst' + }, + { + label: 'ruby', + name: 'ruby' + }, + { + label: 'rust', + name: 'rust' + }, + { + label: 'scala', + name: 'scala' + }, + { + label: 'swift', + name: 'swift' + }, + { + label: 'markdown', + name: 'markdown' + }, + { + label: 'latex', + name: 'latex' + }, + { + label: 'html', + name: 'html' + }, + { + label: 'sol', + name: 'sol' + } + ] + }, + { + label: 'Chunk Size', + name: 'chunkSize', + type: 'number', + default: 1000, + optional: true + }, + { + label: 'Chunk Overlap', + name: 'chunkOverlap', + type: 'number', + optional: true + } + ] + } + async init(nodeData: INodeData): Promise { + const chunkSize = nodeData.inputs?.chunkSize as string + const chunkOverlap = nodeData.inputs?.chunkOverlap as string + const language = nodeData.inputs?.language as SupportedTextSplitterLanguage + + const obj = {} as RecursiveCharacterTextSplitterParams + + if (chunkSize) obj.chunkSize = parseInt(chunkSize, 10) + if (chunkOverlap) obj.chunkOverlap = parseInt(chunkOverlap, 10) + + const splitter = RecursiveCharacterTextSplitter.fromLanguage(language, obj) + + return splitter + } +} +module.exports = { nodeClass: CodeTextSplitter_TextSplitters } diff --git a/packages/components/nodes/textsplitters/CodeTextSplitter/codeTextSplitter.svg b/packages/components/nodes/textsplitters/CodeTextSplitter/codeTextSplitter.svg new file mode 100644 index 00000000..d3b3d188 --- /dev/null +++ b/packages/components/nodes/textsplitters/CodeTextSplitter/codeTextSplitter.svg @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/packages/components/package.json b/packages/components/package.json index b0a58161..738c7752 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -32,16 +32,19 @@ "faiss-node": "^0.2.1", "form-data": "^4.0.0", "graphql": "^16.6.0", + "html-to-text": "^9.0.5", "langchain": "^0.0.94", "linkifyjs": "^4.1.1", "mammoth": "^1.5.1", "moment": "^2.29.3", "node-fetch": "^2.6.11", "pdf-parse": "^1.1.1", + "pdfjs-dist": "^3.7.107", + "playwright": "^1.35.0", "puppeteer": "^20.7.1", + "srt-parser-2": "^1.2.3", "weaviate-ts-client": "^1.1.0", - "ws": "^8.9.0", - "html-to-text": "^9.0.5" + "ws": "^8.9.0" }, "devDependencies": { "@types/gulp": "4.0.9", diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index b248f22c..40bd75cd 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -507,8 +507,8 @@ export class App { }) if (!chatflow) return res.status(404).send(`Chatflow ${chatflowid} not found`) - const chatId = await getChatId(chatflow.id) - if (!chatId) return res.status(500).send(`Chatflow ${chatflowid} first message not found`) + let chatId = await getChatId(chatflow.id) + if (!chatId) chatId = Date.now().toString() if (!isInternal) { await this.validateKey(req, res, chatflow) diff --git a/packages/ui/src/ui-component/dialog/APICodeDialog.js b/packages/ui/src/ui-component/dialog/APICodeDialog.js index e2e7438d..e64f4bf8 100644 --- a/packages/ui/src/ui-component/dialog/APICodeDialog.js +++ b/packages/ui/src/ui-component/dialog/APICodeDialog.js @@ -135,6 +135,9 @@ const embedCodeCustomization = (chatflowid) => { Chatbot.init({ chatflowid: "${chatflowid}", apiHost: "${baseURL}", + chatflowConfig: { + // topK: 2 + }, theme: { button: { backgroundColor: "#3B81F6", @@ -149,6 +152,7 @@ const embedCodeCustomization = (chatflowid) => { backgroundColor: "#ffffff", height: 700, width: 400, + fontSize: 16, poweredByTextColor: "#303235", botMessage: { backgroundColor: "#f7f8ff", @@ -189,6 +193,7 @@ const APICodeDialog = ({ show, dialogProps, onCancel }) => { const getAllAPIKeysApi = useApi(apiKeyApi.getAllAPIKeys) const updateChatflowApi = useApi(chatflowsApi.updateChatflow) + const getIsChatflowStreamingApi = useApi(chatflowsApi.getIsChatflowStreaming) const getConfigApi = useApi(configApi.getConfig) const onCheckBoxChanged = (newVal) => { @@ -553,6 +558,7 @@ query({ useEffect(() => { if (show) { getAllAPIKeysApi.request() + getIsChatflowStreamingApi.request(dialogProps.chatflowid) } // eslint-disable-next-line react-hooks/exhaustive-deps @@ -665,6 +671,15 @@ query({ wrapLines /> )} + {value !== 0 && getIsChatflowStreamingApi.data?.isStreaming && ( +

+ Read  + + here + +  on how to stream response back to application +

+ )} ))}