From 177e7f5c0fa83ecbae2509f09c3101cb20381c27 Mon Sep 17 00:00:00 2001 From: rkeshwani Date: Fri, 11 Aug 2023 00:20:04 +0000 Subject: [PATCH 1/6] Add additional optional input parameter for adding additional file loaders. --- .../nodes/documentloaders/Folder/Folder.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index 83bffd18..4ffdce91 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -46,6 +46,13 @@ class Folder_DocumentLoaders implements INode { type: 'json', optional: true, additionalParams: true + }, + { + label: 'Additional File Loaders', + name: 'additionalLoaders', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -54,6 +61,8 @@ class Folder_DocumentLoaders implements INode { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const folderPath = nodeData.inputs?.folderPath as string const metadata = nodeData.inputs?.metadata + const additionalLoaders = nodeData.inputs?.additionalLoaders + const parsedLoaders = additionalLoaders ? ( typeof metadata === 'object' ? additionalLoaders: JSON.parse( additionalLoaders ) ) : [] const loader = new DirectoryLoader(folderPath, { '.json': (path) => new JSONLoader(path), @@ -61,7 +70,8 @@ class Folder_DocumentLoaders implements INode { '.csv': (path) => new CSVLoader(path), '.docx': (path) => new DocxLoader(path), // @ts-ignore - '.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }) + '.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }), + ...parsedLoaders }) let docs = [] From ff7ee41758dd4fc24893cadb8c7551706f111529 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 14 Aug 2023 00:31:07 +0200 Subject: [PATCH 2/6] Added postgres, cockroachdb, mssql, mysql, mariadb, mongodb and oracle to SqlDatabaseChain_Chains --- .../SqlDatabaseChain/SqlDatabaseChain.ts | 68 +++++++++++++++---- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts b/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts index 9416371b..6bfc2f8a 100644 --- a/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts +++ b/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts @@ -5,6 +5,9 @@ import { DataSource } from 'typeorm' import { SqlDatabase } from 'langchain/sql_db' import { BaseLanguageModel } from 'langchain/base_language' import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler' +import { DataSourceOptions } from 'typeorm/data-source' + +type DatabaseType = 'sqlite' | 'postgres' | 'cockroachdb' | 'mssql' | 'mysql' | 'mariadb' | 'mongodb' | 'oracle' class SqlDatabaseChain_Chains implements INode { label: string @@ -38,36 +41,64 @@ class SqlDatabaseChain_Chains implements INode { type: 'options', options: [ { - label: 'SQlite', + label: 'SQLite', name: 'sqlite' + }, + { + label: 'PostgreSQL', + name: 'postgres' + }, + { + label: 'CockroachDB', + name: 'cockroachdb' + }, + { + label: 'MSSQL', + name: 'mssql' + }, + { + label: 'MySQL', + name: 'mysql' + }, + { + label: 'MariaDB', + name: 'mariadb' + }, + { + label: 'MongoDB', + name: 'mongodb' + }, + { + label: 'Oracle', + name: 'oracle' } ], default: 'sqlite' }, { - label: 'Database File Path', - name: 'dbFilePath', + label: 'Connection string or file path (sqlite only)', + name: 'url', type: 'string', - placeholder: 'C:/Users/chinook.db' + placeholder: '1270.0.0.1:5432/chinook' } ] } async init(nodeData: INodeData): Promise { - const databaseType = nodeData.inputs?.database as 'sqlite' + const databaseType = nodeData.inputs?.database as DatabaseType const model = nodeData.inputs?.model as BaseLanguageModel - const dbFilePath = nodeData.inputs?.dbFilePath + const url = nodeData.inputs?.url - const chain = await getSQLDBChain(databaseType, dbFilePath, model) + const chain = await getSQLDBChain(databaseType, url, model) return chain } async run(nodeData: INodeData, input: string, options: ICommonObject): Promise { - const databaseType = nodeData.inputs?.database as 'sqlite' + const databaseType = nodeData.inputs?.database as DatabaseType const model = nodeData.inputs?.model as BaseLanguageModel - const dbFilePath = nodeData.inputs?.dbFilePath + const url = nodeData.inputs?.url - const chain = await getSQLDBChain(databaseType, dbFilePath, model) + const chain = await getSQLDBChain(databaseType, url, model) const loggerHandler = new ConsoleCallbackHandler(options.logger) if (options.socketIO && options.socketIOClientId) { @@ -81,11 +112,18 @@ class SqlDatabaseChain_Chains implements INode { } } -const getSQLDBChain = async (databaseType: 'sqlite', dbFilePath: string, llm: BaseLanguageModel) => { - const datasource = new DataSource({ - type: databaseType, - database: dbFilePath - }) +const getSQLDBChain = async (databaseType: DatabaseType, url: string, llm: BaseLanguageModel) => { + const datasource = new DataSource( + databaseType === 'sqlite' + ? { + type: databaseType, + database: url + } + : ({ + type: databaseType, + url: url + } as DataSourceOptions) + ) const db = await SqlDatabase.fromDataSourceParams({ appDataSource: datasource From 1e90db79890b53bb1b81747acb3021d87c917765 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 14 Aug 2023 16:18:10 +0200 Subject: [PATCH 3/6] Remove unsupported databases --- .../SqlDatabaseChain/SqlDatabaseChain.ts | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts b/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts index 6bfc2f8a..2a0c71cf 100644 --- a/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts +++ b/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts @@ -7,7 +7,7 @@ import { BaseLanguageModel } from 'langchain/base_language' import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler' import { DataSourceOptions } from 'typeorm/data-source' -type DatabaseType = 'sqlite' | 'postgres' | 'cockroachdb' | 'mssql' | 'mysql' | 'mariadb' | 'mongodb' | 'oracle' +type DatabaseType = 'sqlite' | 'postgres' | 'mssql' | 'mysql' class SqlDatabaseChain_Chains implements INode { label: string @@ -48,10 +48,6 @@ class SqlDatabaseChain_Chains implements INode { label: 'PostgreSQL', name: 'postgres' }, - { - label: 'CockroachDB', - name: 'cockroachdb' - }, { label: 'MSSQL', name: 'mssql' @@ -59,18 +55,6 @@ class SqlDatabaseChain_Chains implements INode { { label: 'MySQL', name: 'mysql' - }, - { - label: 'MariaDB', - name: 'mariadb' - }, - { - label: 'MongoDB', - name: 'mongodb' - }, - { - label: 'Oracle', - name: 'oracle' } ], default: 'sqlite' From f6933b592d934fc2530ddd621066c0a7a30fea3c Mon Sep 17 00:00:00 2001 From: rkeshwani Date: Tue, 15 Aug 2023 00:13:38 +0000 Subject: [PATCH 4/6] Added additional file extensions and removed abstracted inputs. --- .../nodes/documentloaders/Folder/Folder.ts | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index 4ffdce91..7b90d9ed 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -47,13 +47,6 @@ class Folder_DocumentLoaders implements INode { optional: true, additionalParams: true }, - { - label: 'Additional File Loaders', - name: 'additionalLoaders', - type: 'json', - optional: true, - additionalParams: true - } ] } @@ -61,8 +54,6 @@ class Folder_DocumentLoaders implements INode { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const folderPath = nodeData.inputs?.folderPath as string const metadata = nodeData.inputs?.metadata - const additionalLoaders = nodeData.inputs?.additionalLoaders - const parsedLoaders = additionalLoaders ? ( typeof metadata === 'object' ? additionalLoaders: JSON.parse( additionalLoaders ) ) : [] const loader = new DirectoryLoader(folderPath, { '.json': (path) => new JSONLoader(path), @@ -71,7 +62,39 @@ class Folder_DocumentLoaders implements INode { '.docx': (path) => new DocxLoader(path), // @ts-ignore '.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }), - ...parsedLoaders + '.aspx': (path) => new TextLoader(path), + '.asp': (path) => new TextLoader(path), + '.cpp': (path) => new TextLoader(path), // C++ + '.c': (path) => new TextLoader(path), + '.cs': (path) => new TextLoader(path), + '.css': (path) => new TextLoader(path), + '.go': (path) => new TextLoader(path), // Go + '.h': (path) => new TextLoader(path), // C++ Header files + '.java': (path) => new TextLoader(path), // Java + '.js': (path) => new TextLoader(path), // JavaScript + '.less': (path) => new TextLoader(path), // Less files + '.ts': (path) => new TextLoader(path), // TypeScript + '.php': (path) => new TextLoader(path), // PHP + '.proto': (path) => new TextLoader(path), // Protocol Buffers + '.python': (path) => new TextLoader(path), // Python + '.py': (path) => new TextLoader(path), // Python + '.rst': (path) => new TextLoader(path), // reStructuredText + '.ruby': (path) => new TextLoader(path), // Ruby + '.rb': (path) => new TextLoader(path), // Ruby + '.rs': (path) => new TextLoader(path), // Rust + '.scala': (path) => new TextLoader(path), // Scala + '.sc': (path) => new TextLoader(path), // Scala + '.scss': (path) => new TextLoader(path),// Sass + '.sol': (path) => new TextLoader(path), // Solidity + '.sql': (path) => new TextLoader(path),//SQL + '.swift': (path) => new TextLoader(path), // Swift + '.markdown': (path) => new TextLoader(path), // Markdown + '.md': (path) => new TextLoader(path), // Markdown + '.tex': (path) => new TextLoader(path), // LaTeX + '.ltx': (path) => new TextLoader(path), // LaTeX + '.html': (path) => new TextLoader(path), // HTML + '.vb': (path) => new TextLoader(path), // Visual Basic + '.xml': (path) => new TextLoader(path)// XML }) let docs = [] From d10f3800e6a66cf201f63f993865bcb5fb3f96fa Mon Sep 17 00:00:00 2001 From: rkeshwani Date: Tue, 15 Aug 2023 23:36:50 +0000 Subject: [PATCH 5/6] Fixed spaces and comma issue. --- .../components/nodes/documentloaders/Folder/Folder.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index 7b90d9ed..f5d0c640 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -46,7 +46,7 @@ class Folder_DocumentLoaders implements INode { type: 'json', optional: true, additionalParams: true - }, + } ] } @@ -84,9 +84,9 @@ class Folder_DocumentLoaders implements INode { '.rs': (path) => new TextLoader(path), // Rust '.scala': (path) => new TextLoader(path), // Scala '.sc': (path) => new TextLoader(path), // Scala - '.scss': (path) => new TextLoader(path),// Sass + '.scss': (path) => new TextLoader(path), // Sass '.sol': (path) => new TextLoader(path), // Solidity - '.sql': (path) => new TextLoader(path),//SQL + '.sql': (path) => new TextLoader(path), //SQL '.swift': (path) => new TextLoader(path), // Swift '.markdown': (path) => new TextLoader(path), // Markdown '.md': (path) => new TextLoader(path), // Markdown @@ -94,7 +94,7 @@ class Folder_DocumentLoaders implements INode { '.ltx': (path) => new TextLoader(path), // LaTeX '.html': (path) => new TextLoader(path), // HTML '.vb': (path) => new TextLoader(path), // Visual Basic - '.xml': (path) => new TextLoader(path)// XML + '.xml': (path) => new TextLoader(path) // XML }) let docs = [] From 8034076361d82dd12dae8b36f36f7ad27702e88d Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 16 Aug 2023 02:02:49 +0100 Subject: [PATCH 6/6] update Vectara template --- .../chatflows/Vectara LLM Chain Upload.json | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/server/marketplaces/chatflows/Vectara LLM Chain Upload.json b/packages/server/marketplaces/chatflows/Vectara LLM Chain Upload.json index 2146aa12..0758ec9a 100644 --- a/packages/server/marketplaces/chatflows/Vectara LLM Chain Upload.json +++ b/packages/server/marketplaces/chatflows/Vectara LLM Chain Upload.json @@ -4,14 +4,14 @@ { "width": 300, "height": 408, - "id": "vectaraExisting_0", + "id": "vectaraUpsert_0", "position": { "x": 438, "y": 214 }, "type": "customNode", "data": { - "id": "vectaraExisting_0", + "id": "vectaraUpsert_0", "label": "Vectara Upsert Document", "version": 1, - "name": "vectaraExisting", + "name": "vectaraUpsert", "type": "Vectara", "baseClasses": ["Vectara", "VectorStoreRetriever", "BaseRetriever"], "category": "Vector Stores", @@ -22,7 +22,7 @@ "name": "credential", "type": "credential", "credentialNames": ["vectaraApi"], - "id": "vectaraExisting_0-input-credential-credential" + "id": "vectaraUpsert_0-input-credential-credential" }, { "label": "Filter", @@ -30,7 +30,7 @@ "type": "json", "additionalParams": true, "optional": true, - "id": "vectaraExisting_0-input-filter-json" + "id": "vectaraUpsert_0-input-filter-json" }, { "label": "Lambda", @@ -38,7 +38,7 @@ "type": "number", "additionalParams": true, "optional": true, - "id": "vectaraExisting_0-input-lambda-number" + "id": "vectaraUpsert_0-input-lambda-number" }, { "label": "Top K", @@ -48,7 +48,7 @@ "type": "number", "additionalParams": true, "optional": true, - "id": "vectaraExisting_0-input-topK-number" + "id": "vectaraUpsert_0-input-topK-number" } ], "inputAnchors": [ @@ -57,7 +57,7 @@ "name": "document", "type": "Document", "list": true, - "id": "vectaraExisting_0-input-document-Document" + "id": "vectaraUpsert_0-input-document-Document" } ], "inputs": { @@ -73,13 +73,13 @@ "type": "options", "options": [ { - "id": "vectaraExisting_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever", + "id": "vectaraUpsert_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever", "name": "retriever", "label": "Vectara Retriever", "type": "Vectara | VectorStoreRetriever | BaseRetriever" }, { - "id": "vectaraExisting_0-output-vectorStore-Vectara|VectorStore", + "id": "vectaraUpsert_0-output-vectorStore-Vectara|VectorStore", "name": "vectorStore", "label": "Vectara Vector Store", "type": "Vectara | VectorStore" @@ -392,7 +392,7 @@ ], "inputs": { "model": "{{chatOpenAI_0.data.instance}}", - "vectorStoreRetriever": "{{vectaraExisting_0.data.instance}}", + "vectorStoreRetriever": "{{vectaraUpsert_0.data.instance}}", "memory": "", "returnSourceDocuments": "", "systemMessagePrompt": "", @@ -418,19 +418,19 @@ { "source": "pdfFile_0", "sourceHandle": "pdfFile_0-output-pdfFile-Document", - "target": "vectaraExisting_0", - "targetHandle": "vectaraExisting_0-input-document-Document", + "target": "vectaraUpsert_0", + "targetHandle": "vectaraUpsert_0-input-document-Document", "type": "buttonedge", - "id": "pdfFile_0-pdfFile_0-output-pdfFile-Document-vectaraExisting_0-vectaraExisting_0-input-document-Document", + "id": "pdfFile_0-pdfFile_0-output-pdfFile-Document-vectaraUpsert_0-vectaraUpsert_0-input-document-Document", "data": { "label": "" } }, { - "source": "vectaraExisting_0", - "sourceHandle": "vectaraExisting_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever", + "source": "vectaraUpsert_0", + "sourceHandle": "vectaraUpsert_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever", "target": "conversationalRetrievalQAChain_0", "targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", "type": "buttonedge", - "id": "vectaraExisting_0-vectaraExisting_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", + "id": "vectaraUpsert_0-vectaraUpsert_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", "data": { "label": "" } }, {