From 177e7f5c0fa83ecbae2509f09c3101cb20381c27 Mon Sep 17 00:00:00 2001 From: rkeshwani Date: Fri, 11 Aug 2023 00:20:04 +0000 Subject: [PATCH 1/3] Add additional optional input parameter for adding additional file loaders. --- .../nodes/documentloaders/Folder/Folder.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index 83bffd18..4ffdce91 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -46,6 +46,13 @@ class Folder_DocumentLoaders implements INode { type: 'json', optional: true, additionalParams: true + }, + { + label: 'Additional File Loaders', + name: 'additionalLoaders', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -54,6 +61,8 @@ class Folder_DocumentLoaders implements INode { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const folderPath = nodeData.inputs?.folderPath as string const metadata = nodeData.inputs?.metadata + const additionalLoaders = nodeData.inputs?.additionalLoaders + const parsedLoaders = additionalLoaders ? ( typeof metadata === 'object' ? additionalLoaders: JSON.parse( additionalLoaders ) ) : [] const loader = new DirectoryLoader(folderPath, { '.json': (path) => new JSONLoader(path), @@ -61,7 +70,8 @@ class Folder_DocumentLoaders implements INode { '.csv': (path) => new CSVLoader(path), '.docx': (path) => new DocxLoader(path), // @ts-ignore - '.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }) + '.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }), + ...parsedLoaders }) let docs = [] From f6933b592d934fc2530ddd621066c0a7a30fea3c Mon Sep 17 00:00:00 2001 From: rkeshwani Date: Tue, 15 Aug 2023 00:13:38 +0000 Subject: [PATCH 2/3] Added additional file extensions and removed abstracted inputs. --- .../nodes/documentloaders/Folder/Folder.ts | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index 4ffdce91..7b90d9ed 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -47,13 +47,6 @@ class Folder_DocumentLoaders implements INode { optional: true, additionalParams: true }, - { - label: 'Additional File Loaders', - name: 'additionalLoaders', - type: 'json', - optional: true, - additionalParams: true - } ] } @@ -61,8 +54,6 @@ class Folder_DocumentLoaders implements INode { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const folderPath = nodeData.inputs?.folderPath as string const metadata = nodeData.inputs?.metadata - const additionalLoaders = nodeData.inputs?.additionalLoaders - const parsedLoaders = additionalLoaders ? ( typeof metadata === 'object' ? additionalLoaders: JSON.parse( additionalLoaders ) ) : [] const loader = new DirectoryLoader(folderPath, { '.json': (path) => new JSONLoader(path), @@ -71,7 +62,39 @@ class Folder_DocumentLoaders implements INode { '.docx': (path) => new DocxLoader(path), // @ts-ignore '.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }), - ...parsedLoaders + '.aspx': (path) => new TextLoader(path), + '.asp': (path) => new TextLoader(path), + '.cpp': (path) => new TextLoader(path), // C++ + '.c': (path) => new TextLoader(path), + '.cs': (path) => new TextLoader(path), + '.css': (path) => new TextLoader(path), + '.go': (path) => new TextLoader(path), // Go + '.h': (path) => new TextLoader(path), // C++ Header files + '.java': (path) => new TextLoader(path), // Java + '.js': (path) => new TextLoader(path), // JavaScript + '.less': (path) => new TextLoader(path), // Less files + '.ts': (path) => new TextLoader(path), // TypeScript + '.php': (path) => new TextLoader(path), // PHP + '.proto': (path) => new TextLoader(path), // Protocol Buffers + '.python': (path) => new TextLoader(path), // Python + '.py': (path) => new TextLoader(path), // Python + '.rst': (path) => new TextLoader(path), // reStructuredText + '.ruby': (path) => new TextLoader(path), // Ruby + '.rb': (path) => new TextLoader(path), // Ruby + '.rs': (path) => new TextLoader(path), // Rust + '.scala': (path) => new TextLoader(path), // Scala + '.sc': (path) => new TextLoader(path), // Scala + '.scss': (path) => new TextLoader(path),// Sass + '.sol': (path) => new TextLoader(path), // Solidity + '.sql': (path) => new TextLoader(path),//SQL + '.swift': (path) => new TextLoader(path), // Swift + '.markdown': (path) => new TextLoader(path), // Markdown + '.md': (path) => new TextLoader(path), // Markdown + '.tex': (path) => new TextLoader(path), // LaTeX + '.ltx': (path) => new TextLoader(path), // LaTeX + '.html': (path) => new TextLoader(path), // HTML + '.vb': (path) => new TextLoader(path), // Visual Basic + '.xml': (path) => new TextLoader(path)// XML }) let docs = [] From d10f3800e6a66cf201f63f993865bcb5fb3f96fa Mon Sep 17 00:00:00 2001 From: rkeshwani Date: Tue, 15 Aug 2023 23:36:50 +0000 Subject: [PATCH 3/3] Fixed spaces and comma issue. --- .../components/nodes/documentloaders/Folder/Folder.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index 7b90d9ed..f5d0c640 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -46,7 +46,7 @@ class Folder_DocumentLoaders implements INode { type: 'json', optional: true, additionalParams: true - }, + } ] } @@ -84,9 +84,9 @@ class Folder_DocumentLoaders implements INode { '.rs': (path) => new TextLoader(path), // Rust '.scala': (path) => new TextLoader(path), // Scala '.sc': (path) => new TextLoader(path), // Scala - '.scss': (path) => new TextLoader(path),// Sass + '.scss': (path) => new TextLoader(path), // Sass '.sol': (path) => new TextLoader(path), // Solidity - '.sql': (path) => new TextLoader(path),//SQL + '.sql': (path) => new TextLoader(path), //SQL '.swift': (path) => new TextLoader(path), // Swift '.markdown': (path) => new TextLoader(path), // Markdown '.md': (path) => new TextLoader(path), // Markdown @@ -94,7 +94,7 @@ class Folder_DocumentLoaders implements INode { '.ltx': (path) => new TextLoader(path), // LaTeX '.html': (path) => new TextLoader(path), // HTML '.vb': (path) => new TextLoader(path), // Visual Basic - '.xml': (path) => new TextLoader(path)// XML + '.xml': (path) => new TextLoader(path) // XML }) let docs = []