Chore/refractor (#4454)

* markdown files and env examples cleanup

* components update

* update jsonlines description

* server refractor

* update telemetry

* add execute custom node

* add ui refractor

* add username and password authenticate

* correctly retrieve past images in agentflowv2

* disable e2e temporarily

* add existing username and password authenticate

* update migration to default workspace

* update todo

* blob storage migrating

* throw error on agent tool call error

* add missing execution import

* add referral

* chore: add error message when importData is undefined

* migrate api keys to db

* fix: data too long for column executionData

* migrate api keys from json to db at init

* add info on account setup

* update docstore missing fields

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>

.github/workflows/autoSyncMergedPullRequest.yml

@@ -1,33 +0,0 @@
-name: autoSyncMergedPullRequest
-on:
-    pull_request_target:
-        types:
-            - closed
-        branches: ['main']
-jobs:
-    autoSyncMergedPullRequest:
-        if: github.event.pull_request.merged == true
-        runs-on: ubuntu-latest
-        permissions:
-            contents: write
-        steps:
-            - uses: actions/checkout@v4
-            - name: Show PR info
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-              run: |
-                  echo The PR #${{ github.event.pull_request.number }} was merged on main branch!
-            - name: Repository Dispatch
-              uses: peter-evans/repository-dispatch@v3
-              with:
-                  token: ${{ secrets.AUTOSYNC_TOKEN }}
-                  repository: ${{ secrets.AUTOSYNC_CH_URL }}
-                  event-type: ${{ secrets.AUTOSYNC_PR_EVENT_TYPE }}
-                  client-payload: >-
-                      {
-                        "ref": "${{ github.ref }}",
-                        "prNumber": "${{ github.event.pull_request.number }}",
-                        "prTitle": "${{ github.event.pull_request.title }}",
-                        "prDescription": "",
-                        "sha": "${{ github.sha }}"
-                      }

.github/workflows/autoSyncSingleCommit.yml

@@ -1,36 +0,0 @@
-name: autoSyncSingleCommit
-on:
-    push:
-        branches:
-            - main
-jobs:
-    doNotAutoSyncSingleCommit:
-        if: github.event.commits[1] != null
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v3
-            - name: IGNORE autoSyncSingleCommit
-              run: |
-                  echo This single commit has came from a merged commit. We will ignore it. This case is handled in autoSyncMergedPullRequest workflow for merge commits comming from merged pull requests only! Beware, the regular merge commits are not handled by any workflow for the moment.
-    autoSyncSingleCommit:
-        if: github.event.commits[1] == null
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v3
-            - name: autoSyncSingleCommit
-              env:
-                  GITHUB_CONTEXT: ${{ toJSON(github) }}
-              run: |
-                  echo Autosync a single commit with id: ${{ github.sha }} from openSource main branch towards cloud hosted version.
-            - name: Repository Dispatch
-              uses: peter-evans/repository-dispatch@v3
-              with:
-                  token: ${{ secrets.AUTOSYNC_TOKEN }}
-                  repository: ${{ secrets.AUTOSYNC_CH_URL }}
-                  event-type: ${{ secrets.AUTOSYNC_SC_EVENT_TYPE }}
-                  client-payload: >-
-                      {
-                        "ref": "${{ github.ref }}",
-                        "sha": "${{ github.sha }}",
-                        "commitMessage": "${{ github.event.commits[0].id }}"
-                      }

.github/workflows/docker-image.yml

@@ -3,6 +3,22 @@ name: Docker Image CI
 on:
     workflow_dispatch:
         inputs:
+            registry:
+                description: 'Container Registry to push the image to.'
+                type: choice
+                required: true
+                default: 'aws_ecr'
+                options:
+                    - 'docker_hub'
+                    - 'aws_ecr'
+            environment:
+                description: 'Environment to push the image to.'
+                required: true
+                default: 'dev'
+                type: choice
+                options:
+                    - dev
+                    - prod
             node_version:
                 description: 'Node.js version to build this image with.'
                 type: choice
@@ -19,25 +35,57 @@ on:
 jobs:
     docker:
         runs-on: ubuntu-latest
+        environment: ${{ github.event.inputs.environment }}
         steps:
+            - name: Set default values
+              id: defaults
+              run: |
+                  echo "registry=${{ github.event.inputs.registry || 'aws_ecr' }}" >> $GITHUB_OUTPUT
+                  echo "node_version=${{ github.event.inputs.node_version || '20' }}" >> $GITHUB_OUTPUT
+                  echo "tag_version=${{ github.event.inputs.tag_version || 'latest' }}" >> $GITHUB_OUTPUT
+
             - name: Checkout
               uses: actions/checkout@v4.1.1
+
             - name: Set up QEMU
               uses: docker/setup-qemu-action@v3.0.0
+
             - name: Set up Docker Buildx
               uses: docker/setup-buildx-action@v3.0.0
+
+            # ------------------------
+            # Login Steps (conditional)
+            # ------------------------
             - name: Login to Docker Hub
+              if: steps.defaults.outputs.registry == 'docker_hub'
               uses: docker/login-action@v3
               with:
                   username: ${{ secrets.DOCKERHUB_USERNAME }}
                   password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+            - name: Configure AWS Credentials
+              if: steps.defaults.outputs.registry == 'aws_ecr'
+              uses: aws-actions/configure-aws-credentials@v3
+              with:
+                  aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+                  aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                  aws-region: ${{ secrets.AWS_REGION }}
+
+            - name: Login to Amazon ECR
+              if: steps.defaults.outputs.registry == 'aws_ecr'
+              uses: aws-actions/amazon-ecr-login@v1
+
+            # -------------------------
+            # Build and push (conditional tags)
+            # -------------------------
             - name: Build and push
               uses: docker/build-push-action@v5.3.0
               with:
                   context: .
-                  file: ./docker/Dockerfile
+                  file: Dockerfile
                   build-args: |
-                      NODE_VERSION=${{github.event.inputs.node_version}}
+                      NODE_VERSION=${{ steps.defaults.outputs.node_version }}
                   platforms: linux/amd64,linux/arm64
                   push: true
-                  tags: flowiseai/flowise:${{github.event.inputs.tag_version}}
+                  tags: |
+                      ${{ steps.defaults.outputs.registry == 'docker_hub' && format('flowiseai/flowise:{0}', steps.defaults.outputs.tag_version) || format('{0}.dkr.ecr.{1}.amazonaws.com/flowise:{2}', secrets.AWS_ACCOUNT_ID, secrets.AWS_REGION, steps.defaults.outputs.tag_version) }}

.github/workflows/main.yml

@@ -6,6 +6,7 @@ on:
     pull_request:
         branches:
             - '*'
+    workflow_dispatch:
 permissions:
     contents: read
 jobs:
@@ -31,6 +32,8 @@ jobs:
             - run: pnpm install
             - run: pnpm lint
             - run: pnpm build
+              env:
+                  NODE_OPTIONS: '--max_old_space_size=4096'
             - name: Cypress install
               run: pnpm cypress install
             - name: Install dependencies (Cypress Action)

.github/workflows/test_docker_build.yml

@@ -8,13 +8,12 @@ on:
     pull_request:
         branches:
             - '*'
-
+    workflow_dispatch:
 jobs:
     build:
         runs-on: ubuntu-latest
         env:
             PUPPETEER_SKIP_DOWNLOAD: true
         steps:
-            - uses: actions/checkout@v3
-
+            - uses: actions/checkout@v4
             - run: docker build --no-cache -t flowise .