diff --git a/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts b/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts index 9416371b..2a0c71cf 100644 --- a/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts +++ b/packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts @@ -5,6 +5,9 @@ import { DataSource } from 'typeorm' import { SqlDatabase } from 'langchain/sql_db' import { BaseLanguageModel } from 'langchain/base_language' import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler' +import { DataSourceOptions } from 'typeorm/data-source' + +type DatabaseType = 'sqlite' | 'postgres' | 'mssql' | 'mysql' class SqlDatabaseChain_Chains implements INode { label: string @@ -38,36 +41,48 @@ class SqlDatabaseChain_Chains implements INode { type: 'options', options: [ { - label: 'SQlite', + label: 'SQLite', name: 'sqlite' + }, + { + label: 'PostgreSQL', + name: 'postgres' + }, + { + label: 'MSSQL', + name: 'mssql' + }, + { + label: 'MySQL', + name: 'mysql' } ], default: 'sqlite' }, { - label: 'Database File Path', - name: 'dbFilePath', + label: 'Connection string or file path (sqlite only)', + name: 'url', type: 'string', - placeholder: 'C:/Users/chinook.db' + placeholder: '1270.0.0.1:5432/chinook' } ] } async init(nodeData: INodeData): Promise { - const databaseType = nodeData.inputs?.database as 'sqlite' + const databaseType = nodeData.inputs?.database as DatabaseType const model = nodeData.inputs?.model as BaseLanguageModel - const dbFilePath = nodeData.inputs?.dbFilePath + const url = nodeData.inputs?.url - const chain = await getSQLDBChain(databaseType, dbFilePath, model) + const chain = await getSQLDBChain(databaseType, url, model) return chain } async run(nodeData: INodeData, input: string, options: ICommonObject): Promise { - const databaseType = nodeData.inputs?.database as 'sqlite' + const databaseType = nodeData.inputs?.database as DatabaseType const model = nodeData.inputs?.model as BaseLanguageModel - const dbFilePath = nodeData.inputs?.dbFilePath + const url = nodeData.inputs?.url - const chain = await getSQLDBChain(databaseType, dbFilePath, model) + const chain = await getSQLDBChain(databaseType, url, model) const loggerHandler = new ConsoleCallbackHandler(options.logger) if (options.socketIO && options.socketIOClientId) { @@ -81,11 +96,18 @@ class SqlDatabaseChain_Chains implements INode { } } -const getSQLDBChain = async (databaseType: 'sqlite', dbFilePath: string, llm: BaseLanguageModel) => { - const datasource = new DataSource({ - type: databaseType, - database: dbFilePath - }) +const getSQLDBChain = async (databaseType: DatabaseType, url: string, llm: BaseLanguageModel) => { + const datasource = new DataSource( + databaseType === 'sqlite' + ? { + type: databaseType, + database: url + } + : ({ + type: databaseType, + url: url + } as DataSourceOptions) + ) const db = await SqlDatabase.fromDataSourceParams({ appDataSource: datasource diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index 83bffd18..f5d0c640 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -61,7 +61,40 @@ class Folder_DocumentLoaders implements INode { '.csv': (path) => new CSVLoader(path), '.docx': (path) => new DocxLoader(path), // @ts-ignore - '.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }) + '.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }), + '.aspx': (path) => new TextLoader(path), + '.asp': (path) => new TextLoader(path), + '.cpp': (path) => new TextLoader(path), // C++ + '.c': (path) => new TextLoader(path), + '.cs': (path) => new TextLoader(path), + '.css': (path) => new TextLoader(path), + '.go': (path) => new TextLoader(path), // Go + '.h': (path) => new TextLoader(path), // C++ Header files + '.java': (path) => new TextLoader(path), // Java + '.js': (path) => new TextLoader(path), // JavaScript + '.less': (path) => new TextLoader(path), // Less files + '.ts': (path) => new TextLoader(path), // TypeScript + '.php': (path) => new TextLoader(path), // PHP + '.proto': (path) => new TextLoader(path), // Protocol Buffers + '.python': (path) => new TextLoader(path), // Python + '.py': (path) => new TextLoader(path), // Python + '.rst': (path) => new TextLoader(path), // reStructuredText + '.ruby': (path) => new TextLoader(path), // Ruby + '.rb': (path) => new TextLoader(path), // Ruby + '.rs': (path) => new TextLoader(path), // Rust + '.scala': (path) => new TextLoader(path), // Scala + '.sc': (path) => new TextLoader(path), // Scala + '.scss': (path) => new TextLoader(path), // Sass + '.sol': (path) => new TextLoader(path), // Solidity + '.sql': (path) => new TextLoader(path), //SQL + '.swift': (path) => new TextLoader(path), // Swift + '.markdown': (path) => new TextLoader(path), // Markdown + '.md': (path) => new TextLoader(path), // Markdown + '.tex': (path) => new TextLoader(path), // LaTeX + '.ltx': (path) => new TextLoader(path), // LaTeX + '.html': (path) => new TextLoader(path), // HTML + '.vb': (path) => new TextLoader(path), // Visual Basic + '.xml': (path) => new TextLoader(path) // XML }) let docs = [] diff --git a/packages/server/marketplaces/chatflows/Vectara LLM Chain Upload.json b/packages/server/marketplaces/chatflows/Vectara LLM Chain Upload.json index 2146aa12..784ad240 100644 --- a/packages/server/marketplaces/chatflows/Vectara LLM Chain Upload.json +++ b/packages/server/marketplaces/chatflows/Vectara LLM Chain Upload.json @@ -1,186 +1,11 @@ { "description": "A simple LLM chain that uses Vectara to enable conversations with uploaded documents", "nodes": [ - { - "width": 300, - "height": 408, - "id": "vectaraExisting_0", - "position": { "x": 438, "y": 214 }, - "type": "customNode", - "data": { - "id": "vectaraExisting_0", - "label": "Vectara Upsert Document", - "version": 1, - "name": "vectaraExisting", - "type": "Vectara", - "baseClasses": ["Vectara", "VectorStoreRetriever", "BaseRetriever"], - "category": "Vector Stores", - "description": "Upsert documents to Vectara", - "inputParams": [ - { - "label": "Connect Credential", - "name": "credential", - "type": "credential", - "credentialNames": ["vectaraApi"], - "id": "vectaraExisting_0-input-credential-credential" - }, - { - "label": "Filter", - "name": "filter", - "type": "json", - "additionalParams": true, - "optional": true, - "id": "vectaraExisting_0-input-filter-json" - }, - { - "label": "Lambda", - "name": "lambda", - "type": "number", - "additionalParams": true, - "optional": true, - "id": "vectaraExisting_0-input-lambda-number" - }, - { - "label": "Top K", - "name": "topK", - "description": "Number of top results to fetch. Defaults to 4", - "placeholder": "4", - "type": "number", - "additionalParams": true, - "optional": true, - "id": "vectaraExisting_0-input-topK-number" - } - ], - "inputAnchors": [ - { - "label": "Document", - "name": "document", - "type": "Document", - "list": true, - "id": "vectaraExisting_0-input-document-Document" - } - ], - "inputs": { - "document": ["{{pdfFile_0.data.instance}}"], - "filter": "", - "lambda": "", - "topK": "" - }, - "outputAnchors": [ - { - "name": "output", - "label": "Output", - "type": "options", - "options": [ - { - "id": "vectaraExisting_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever", - "name": "retriever", - "label": "Vectara Retriever", - "type": "Vectara | VectorStoreRetriever | BaseRetriever" - }, - { - "id": "vectaraExisting_0-output-vectorStore-Vectara|VectorStore", - "name": "vectorStore", - "label": "Vectara Vector Store", - "type": "Vectara | VectorStore" - } - ], - "default": "retriever" - } - ], - "outputs": { "output": "retriever" }, - "selected": false - }, - "selected": false, - "dragging": false, - "positionAbsolute": { "x": 438, "y": 214 } - }, - { - "width": 300, - "height": 509, - "id": "pdfFile_0", - "position": { "x": 68.3013317598369, "y": 199.60454731299677 }, - "type": "customNode", - "data": { - "id": "pdfFile_0", - "label": "Pdf File", - "version": 1, - "name": "pdfFile", - "type": "Document", - "baseClasses": ["Document"], - "category": "Document Loaders", - "description": "Load data from PDF files", - "inputParams": [ - { - "label": "Pdf File", - "name": "pdfFile", - "type": "file", - "fileType": ".pdf", - "id": "pdfFile_0-input-pdfFile-file" - }, - { - "label": "Usage", - "name": "usage", - "type": "options", - "options": [ - { "label": "One document per page", "name": "perPage" }, - { "label": "One document per file", "name": "perFile" } - ], - "default": "perPage", - "id": "pdfFile_0-input-usage-options" - }, - { - "label": "Use Legacy Build", - "name": "legacyBuild", - "type": "boolean", - "optional": true, - "additionalParams": true, - "id": "pdfFile_0-input-legacyBuild-boolean" - }, - { - "label": "Metadata", - "name": "metadata", - "type": "json", - "optional": true, - "additionalParams": true, - "id": "pdfFile_0-input-metadata-json" - } - ], - "inputAnchors": [ - { - "label": "Text Splitter", - "name": "textSplitter", - "type": "TextSplitter", - "optional": true, - "id": "pdfFile_0-input-textSplitter-TextSplitter" - } - ], - "inputs": { - "textSplitter": "", - "usage": "perPage", - "legacyBuild": "", - "metadata": "" - }, - "outputAnchors": [ - { - "id": "pdfFile_0-output-pdfFile-Document", - "name": "pdfFile", - "label": "Document", - "type": "Document" - } - ], - "outputs": {}, - "selected": false - }, - "selected": false, - "positionAbsolute": { "x": 68.3013317598369, "y": 199.60454731299677 }, - "dragging": false - }, { "width": 300, "height": 525, "id": "chatOpenAI_0", - "position": { "x": 804.3889791707068, "y": 195.11620799951592 }, + "position": { "x": 514.1088940275924, "y": 199.574479681537 }, "type": "customNode", "data": { "id": "chatOpenAI_0", @@ -211,10 +36,7 @@ { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" }, { "label": "gpt-3.5-turbo-16k", "name": "gpt-3.5-turbo-16k" }, - { - "label": "gpt-3.5-turbo-16k-0613", - "name": "gpt-3.5-turbo-16k-0613" - } + { "label": "gpt-3.5-turbo-16k-0613", "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", "optional": true, @@ -286,7 +108,7 @@ "inputAnchors": [], "inputs": { "modelName": "gpt-3.5-turbo", - "temperature": "0.2", + "temperature": "0.5", "maxTokens": "", "topP": "", "frequencyPenalty": "", @@ -306,14 +128,14 @@ "selected": false }, "selected": false, - "positionAbsolute": { "x": 804.3889791707068, "y": 195.11620799951592 }, + "positionAbsolute": { "x": 514.1088940275924, "y": 199.574479681537 }, "dragging": false }, { "width": 300, "height": 481, "id": "conversationalRetrievalQAChain_0", - "position": { "x": 1160.4877473512795, "y": 259.2799138505109 }, + "position": { "x": 900.4793407261002, "y": 205.9476004518217 }, "type": "customNode", "data": { "id": "conversationalRetrievalQAChain_0", @@ -392,7 +214,7 @@ ], "inputs": { "model": "{{chatOpenAI_0.data.instance}}", - "vectorStoreRetriever": "{{vectaraExisting_0.data.instance}}", + "vectorStoreRetriever": "{{vectaraUpsert_0.data.instance}}", "memory": "", "returnSourceDocuments": "", "systemMessagePrompt": "", @@ -410,29 +232,191 @@ "selected": false }, "selected": false, - "positionAbsolute": { "x": 1160.4877473512795, "y": 259.2799138505109 }, + "positionAbsolute": { "x": 900.4793407261002, "y": 205.9476004518217 }, "dragging": false + }, + { + "width": 300, + "height": 509, + "id": "pdfFile_0", + "position": { "x": -210.44158723479913, "y": 236.6627524951051 }, + "type": "customNode", + "data": { + "id": "pdfFile_0", + "label": "Pdf File", + "version": 1, + "name": "pdfFile", + "type": "Document", + "baseClasses": ["Document"], + "category": "Document Loaders", + "description": "Load data from PDF files", + "inputParams": [ + { "label": "Pdf File", "name": "pdfFile", "type": "file", "fileType": ".pdf", "id": "pdfFile_0-input-pdfFile-file" }, + { + "label": "Usage", + "name": "usage", + "type": "options", + "options": [ + { "label": "One document per page", "name": "perPage" }, + { "label": "One document per file", "name": "perFile" } + ], + "default": "perPage", + "id": "pdfFile_0-input-usage-options" + }, + { + "label": "Use Legacy Build", + "name": "legacyBuild", + "type": "boolean", + "optional": true, + "additionalParams": true, + "id": "pdfFile_0-input-legacyBuild-boolean" + }, + { + "label": "Metadata", + "name": "metadata", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "pdfFile_0-input-metadata-json" + } + ], + "inputAnchors": [ + { + "label": "Text Splitter", + "name": "textSplitter", + "type": "TextSplitter", + "optional": true, + "id": "pdfFile_0-input-textSplitter-TextSplitter" + } + ], + "inputs": { "textSplitter": "", "usage": "perPage", "legacyBuild": "", "metadata": "" }, + "outputAnchors": [ + { "id": "pdfFile_0-output-pdfFile-Document", "name": "pdfFile", "label": "Document", "type": "Document" } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { "x": -210.44158723479913, "y": 236.6627524951051 }, + "dragging": false + }, + { + "width": 300, + "height": 408, + "id": "vectaraUpsert_0", + "position": { "x": 172.06946164914868, "y": 373.11406233089934 }, + "type": "customNode", + "data": { + "id": "vectaraUpsert_0", + "label": "Vectara Upsert Document", + "version": 1, + "name": "vectaraUpsert", + "type": "Vectara", + "baseClasses": ["Vectara", "VectorStoreRetriever", "BaseRetriever"], + "category": "Vector Stores", + "description": "Upsert documents to Vectara", + "inputParams": [ + { + "label": "Connect Credential", + "name": "credential", + "type": "credential", + "credentialNames": ["vectaraApi"], + "id": "vectaraUpsert_0-input-credential-credential" + }, + { + "label": "Vectara Metadata Filter", + "name": "filter", + "description": "Filter to apply to Vectara metadata. Refer to the documentation on how to use Vectara filters with Flowise.", + "type": "string", + "additionalParams": true, + "optional": true, + "id": "vectaraUpsert_0-input-filter-string" + }, + { + "label": "Sentences Before", + "name": "sentencesBefore", + "description": "Number of sentences to fetch before the matched sentence. Defaults to 2.", + "type": "number", + "additionalParams": true, + "optional": true, + "id": "vectaraUpsert_0-input-sentencesBefore-number" + }, + { + "label": "Sentences After", + "name": "sentencesAfter", + "description": "Number of sentences to fetch after the matched sentence. Defaults to 2.", + "type": "number", + "additionalParams": true, + "optional": true, + "id": "vectaraUpsert_0-input-sentencesAfter-number" + }, + { + "label": "Lambda", + "name": "lambda", + "description": "Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.", + "type": "number", + "additionalParams": true, + "optional": true, + "id": "vectaraUpsert_0-input-lambda-number" + }, + { + "label": "Top K", + "name": "topK", + "description": "Number of top results to fetch. Defaults to 4", + "placeholder": "4", + "type": "number", + "additionalParams": true, + "optional": true, + "id": "vectaraUpsert_0-input-topK-number" + } + ], + "inputAnchors": [ + { + "label": "Document", + "name": "document", + "type": "Document", + "list": true, + "id": "vectaraUpsert_0-input-document-Document" + } + ], + "inputs": { + "document": ["{{pdfFile_0.data.instance}}"], + "filter": "", + "sentencesBefore": "", + "sentencesAfter": "", + "lambda": "", + "topK": "" + }, + "outputAnchors": [ + { + "name": "output", + "label": "Output", + "type": "options", + "options": [ + { + "id": "vectaraUpsert_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever", + "name": "retriever", + "label": "Vectara Retriever", + "type": "Vectara | VectorStoreRetriever | BaseRetriever" + }, + { + "id": "vectaraUpsert_0-output-vectorStore-Vectara|VectorStore", + "name": "vectorStore", + "label": "Vectara Vector Store", + "type": "Vectara | VectorStore" + } + ], + "default": "retriever" + } + ], + "outputs": { "output": "retriever" }, + "selected": false + }, + "positionAbsolute": { "x": 172.06946164914868, "y": 373.11406233089934 }, + "selected": false } ], "edges": [ - { - "source": "pdfFile_0", - "sourceHandle": "pdfFile_0-output-pdfFile-Document", - "target": "vectaraExisting_0", - "targetHandle": "vectaraExisting_0-input-document-Document", - "type": "buttonedge", - "id": "pdfFile_0-pdfFile_0-output-pdfFile-Document-vectaraExisting_0-vectaraExisting_0-input-document-Document", - "data": { "label": "" } - }, - { - "source": "vectaraExisting_0", - "sourceHandle": "vectaraExisting_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever", - "target": "conversationalRetrievalQAChain_0", - "targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", - "type": "buttonedge", - "id": "vectaraExisting_0-vectaraExisting_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", - "data": { "label": "" } - }, { "source": "chatOpenAI_0", "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel", @@ -441,6 +425,24 @@ "type": "buttonedge", "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-model-BaseLanguageModel", "data": { "label": "" } + }, + { + "source": "pdfFile_0", + "sourceHandle": "pdfFile_0-output-pdfFile-Document", + "target": "vectaraUpsert_0", + "targetHandle": "vectaraUpsert_0-input-document-Document", + "type": "buttonedge", + "id": "pdfFile_0-pdfFile_0-output-pdfFile-Document-vectaraUpsert_0-vectaraUpsert_0-input-document-Document", + "data": { "label": "" } + }, + { + "source": "vectaraUpsert_0", + "sourceHandle": "vectaraUpsert_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever", + "target": "conversationalRetrievalQAChain_0", + "targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", + "type": "buttonedge", + "id": "vectaraUpsert_0-vectaraUpsert_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", + "data": { "label": "" } } ] }